diff --git a/README.md b/README.md index c93f28cf..a8d1c692 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,13 @@ The following options are available for running the Any-Sync Node: - `-v` — current version. - `-h` — help message. +## Changing the node set (resharding) + +Adding and removing tree nodes on a running network is supported when the +nodes share an archive bucket — see the operator guide: +[docs/resharding-operations.md](docs/resharding-operations.md) +(design: [docs/resharding-plan.md](docs/resharding-plan.md)). + ## Graph example of using Any-Sync Nodes group ```mermaid diff --git a/archive/adopter/adopter.go b/archive/adopter/adopter.go new file mode 100644 index 00000000..118efbdb --- /dev/null +++ b/archive/adopter/adopter.go @@ -0,0 +1,226 @@ +//go:generate mockgen -destination mock_adopter/mock_adopter.go github.com/anyproto/any-sync-node/archive/adopter Adopter +package adopter + +import ( + "context" + "errors" + + anystore "github.com/anyproto/any-store" + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/metric" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/nodeconf" + "go.uber.org/zap" + + "github.com/anyproto/any-sync-node/archive" + "github.com/anyproto/any-sync-node/archive/archivestore" + "github.com/anyproto/any-sync-node/nodehead" + "github.com/anyproto/any-sync-node/nodestorage" + "github.com/anyproto/any-sync-node/nodesync/nodesyncproto" +) + +const CName = "node.archive.adopter" + +var log = logger.NewNamed(CName) + +func New() Adopter { + return new(adopter) +} + +// Adopter handles AdoptArchive requests: another node hands over an archived +// space snapshot parked in the shared bucket; we server-side copy the object +// into our own prefix and register the space as archived without ever +// materializing the SQLite DB. +type Adopter interface { + app.Component + AdoptArchive(ctx context.Context, req *nodesyncproto.AdoptArchiveRequest) (resp *nodesyncproto.AdoptArchiveResponse, err error) +} + +// historyPeerEpochs is how many retained configuration epochs back a sender +// may be recognized as a network node: a node removed from the configuration +// must still be able to hand its spaces off while it drains. +const historyPeerEpochs = 5 + +type adopter struct { + storage nodestorage.NodeStorage + archiveStore archivestore.ArchiveStore + archive archive.Archive + nodeHead nodehead.NodeHead + nodeConf nodeconf.Service + confHistory nodeconf.HistoryStore + stat *adopterStat +} + +func (ad *adopter) Init(a *app.App) (err error) { + ad.storage = a.MustComponent(nodestorage.CName).(nodestorage.NodeStorage) + ad.archiveStore = a.MustComponent(archivestore.CName).(archivestore.ArchiveStore) + ad.archive = a.MustComponent(archive.CName).(archive.Archive) + ad.nodeHead = a.MustComponent(nodehead.CName).(nodehead.NodeHead) + ad.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.Service) + ad.confHistory, _ = a.Component(nodeconf.CNameStore).(nodeconf.HistoryStore) + ad.stat = new(adopterStat) + if m := a.Component(metric.CName); m != nil { + registerMetric(ad.stat, m.(metric.Metric).Registry()) + } + return +} + +func (ad *adopter) Name() (name string) { + return CName +} + +func (ad *adopter) AdoptArchive(ctx context.Context, req *nodesyncproto.AdoptArchiveRequest) (resp *nodesyncproto.AdoptArchiveResponse, err error) { + defer func() { + switch { + case err != nil: + ad.stat.rejected.Add(1) + case resp.Result == nodesyncproto.AdoptArchiveResult_AdoptArchiveOk: + ad.stat.adopted.Add(1) + case resp.Result == nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveSame: + ad.stat.alreadyHaveSame.Add(1) + case resp.Result == nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveDiverged: + ad.stat.alreadyHaveDiverged.Add(1) + } + }() + if !ad.archiveStore.Shared() { + return nil, nodesyncproto.ErrArchiveUnavailable + } + if err = ad.checkPeerIsNode(ctx); err != nil { + return nil, err + } + if !ad.nodeConf.IsResponsible(req.SpaceId) { + // the sender acts on a different (stale or newer) configuration: + // never adopt spaces we don't own — an ACK from a non-owner must not + // justify the sender's deletion + return nil, nodesyncproto.ErrNotResponsible + } + entry, entryErr := ad.storage.IndexStorage().SpaceStatusEntry(ctx, req.SpaceId) + switch { + case entryErr == nil: + switch entry.Status { + case nodestorage.SpaceStatusRemove: + return nil, nodesyncproto.ErrSpaceDeleted + case nodestorage.SpaceStatusRemovePrepare: + // pending deletion is cancellable: don't let the sender drop its + // copy, and don't adopt data into a deletion-flow status either; + // the sender parks the space until the deletion resolves + return nil, nodesyncproto.ErrSpacePendingDeletion + case nodestorage.SpaceStatusOk: + if ad.storage.SpaceExists(req.SpaceId) { + return alreadyHaveResponse(entry, req), nil + } + // index entry says ok, but there is no local db: adopt to repair + case nodestorage.SpaceStatusArchived: + ok, hErr := ad.archiveStore.Exists(ctx, req.SpaceId) + if hErr != nil { + return nil, hErr + } + if ok { + return alreadyHaveResponse(entry, req), nil + } + // index says archived, but our object is gone: adopt to repair + case nodestorage.SpaceStatusError: + // an Error entry needs an operator: the local db may hold data + // that failed to archive — never overwrite it with a snapshot and + // never ACK the sender's deletion with a broken copy + return &nodesyncproto.AdoptArchiveResponse{ + Result: nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveDiverged, + }, nil + } + case errors.Is(entryErr, anystore.ErrDocNotFound): + if ad.storage.SpaceExists(req.SpaceId) { + // a local db without an index entry: heads unknown, force convergence + return &nodesyncproto.AdoptArchiveResponse{ + Result: nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveDiverged, + }, nil + } + default: + return nil, entryErr + } + + if err = ad.archiveStore.CopyFrom(ctx, req.SrcKey, req.SpaceId); err != nil { + if errors.Is(err, archivestore.ErrNotFound) { + return nil, nodesyncproto.ErrArchiveObjectMissing + } + return nil, err + } + ok, err := ad.archiveStore.Exists(ctx, req.SpaceId) + if err != nil { + return nil, err + } + if !ok { + return nil, nodesyncproto.ErrArchiveObjectMissing + } + if err = ad.storage.IndexStorage().MarkArchivedRemote(ctx, req.SpaceId, req.OldHash, req.NewHash, req.CompressedSize, req.UncompressedSize); err != nil { + return nil, err + } + if _, err = ad.nodeHead.SetHead(req.SpaceId, req.OldHash, req.NewHash); err != nil { + log.Warn("can't set nodehead after adoption", zap.String("spaceId", req.SpaceId), zap.Error(err)) + err = nil + } + log.Info("adopted archived space", zap.String("spaceId", req.SpaceId), zap.Bool("eager", req.Eager)) + if req.Eager { + ad.archive.QueueRestore(req.SpaceId) + } + return &nodesyncproto.AdoptArchiveResponse{ + Result: nodesyncproto.AdoptArchiveResult_AdoptArchiveOk, + }, nil +} + +// alreadyHaveResponse reports whether our existing copy matches the offered +// heads: only an exact match counts as a durable ACK for the sender — a +// diverged copy could be older than the sender's and must not justify its +// deletion. +func alreadyHaveResponse(entry nodestorage.SpaceStatusEntry, req *nodesyncproto.AdoptArchiveRequest) *nodesyncproto.AdoptArchiveResponse { + if entry.NewHash == req.NewHash && entry.OldHash == req.OldHash { + return &nodesyncproto.AdoptArchiveResponse{ + Result: nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveSame, + } + } + return &nodesyncproto.AdoptArchiveResponse{ + Result: nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveDiverged, + } +} + +func (ad *adopter) checkPeerIsNode(ctx context.Context) (err error) { + peerId, err := peer.CtxPeerId(ctx) + if err != nil { + return err + } + if len(ad.nodeConf.NodeTypes(peerId)) > 0 { + return nil + } + // a node removed from the configuration keeps draining after the removal: + // recognize senders that were tree nodes in recently retained epochs + if ad.wasRecentTreeNode(ctx, peerId) { + return nil + } + return nodesyncproto.ErrPeerIsNotNode +} + +func (ad *adopter) wasRecentTreeNode(ctx context.Context, peerId string) bool { + if ad.confHistory == nil { + return false + } + netId := ad.nodeConf.Configuration().NetworkId + epochs, err := ad.confHistory.Epochs(ctx, netId) + if err != nil || len(epochs) == 0 { + return false + } + if len(epochs) > historyPeerEpochs { + epochs = epochs[len(epochs)-historyPeerEpochs:] + } + for i := len(epochs) - 1; i >= 0; i-- { + conf, gErr := ad.confHistory.GetByEpoch(ctx, netId, epochs[i]) + if gErr != nil { + continue + } + for _, n := range conf.Nodes { + if n.PeerId == peerId && n.HasType(nodeconf.NodeTypeTree) { + return true + } + } + } + return false +} diff --git a/archive/adopter/adopter_test.go b/archive/adopter/adopter_test.go new file mode 100644 index 00000000..f3e04000 --- /dev/null +++ b/archive/adopter/adopter_test.go @@ -0,0 +1,219 @@ +package adopter + +import ( + "context" + "testing" + + anystore "github.com/anyproto/any-store" + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/nodeconf" + "github.com/anyproto/any-sync/nodeconf/mock_nodeconf" + "github.com/anyproto/any-sync/testutil/anymock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync-node/archive" + "github.com/anyproto/any-sync-node/archive/archivestore" + "github.com/anyproto/any-sync-node/archive/archivestore/mock_archivestore" + "github.com/anyproto/any-sync-node/archive/mock_archive" + "github.com/anyproto/any-sync-node/nodehead" + "github.com/anyproto/any-sync-node/nodehead/mock_nodehead" + "github.com/anyproto/any-sync-node/nodestorage" + "github.com/anyproto/any-sync-node/nodestorage/mock_nodestorage" + "github.com/anyproto/any-sync-node/nodesync/nodesyncproto" +) + +var ctx = peer.CtxWithPeerId(context.Background(), "peer1") + +func adoptReq() *nodesyncproto.AdoptArchiveRequest { + return &nodesyncproto.AdoptArchiveRequest{ + SpaceId: "space.id", + SrcKey: "othernode/space.id", + OldHash: "oldHash", + NewHash: "newHash", + CompressedSize: 10, + UncompressedSize: 20, + } +} + +func TestAdopter_AdoptArchive(t *testing.T) { + t.Run("not shared", func(t *testing.T) { + fx := newFixture(t) + fx.archiveStore.EXPECT().Shared().Return(false) + _, err := fx.AdoptArchive(ctx, adoptReq()) + assert.ErrorIs(t, err, nodesyncproto.ErrArchiveUnavailable) + }) + t.Run("client peer rejected", func(t *testing.T) { + fx := newFixture(t) + fx.archiveStore.EXPECT().Shared().Return(true) + fx.nodeConf.EXPECT().NodeTypes("peer1").Return(nil) + _, err := fx.AdoptArchive(ctx, adoptReq()) + assert.ErrorIs(t, err, nodesyncproto.ErrPeerIsNotNode) + }) + t.Run("adopt ok", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + fx.expectNodePeer() + fx.indexStorage.EXPECT().SpaceStatusEntry(ctx, req.SpaceId).Return(nodestorage.SpaceStatusEntry{}, anystore.ErrDocNotFound) + fx.storage.EXPECT().SpaceExists(req.SpaceId).Return(false) + fx.archiveStore.EXPECT().CopyFrom(ctx, req.SrcKey, req.SpaceId).Return(nil) + fx.archiveStore.EXPECT().Exists(ctx, req.SpaceId).Return(true, nil) + fx.indexStorage.EXPECT().MarkArchivedRemote(ctx, req.SpaceId, "oldHash", "newHash", int64(10), int64(20)).Return(nil) + fx.nodeHead.EXPECT().SetHead(req.SpaceId, "oldHash", "newHash").Return(0, nil) + + resp, err := fx.AdoptArchive(ctx, req) + require.NoError(t, err) + assert.Equal(t, nodesyncproto.AdoptArchiveResult_AdoptArchiveOk, resp.Result) + }) + t.Run("adopt eager queues restore", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + req.Eager = true + fx.expectNodePeer() + fx.indexStorage.EXPECT().SpaceStatusEntry(ctx, req.SpaceId).Return(nodestorage.SpaceStatusEntry{}, anystore.ErrDocNotFound) + fx.storage.EXPECT().SpaceExists(req.SpaceId).Return(false) + fx.archiveStore.EXPECT().CopyFrom(ctx, req.SrcKey, req.SpaceId).Return(nil) + fx.archiveStore.EXPECT().Exists(ctx, req.SpaceId).Return(true, nil) + fx.indexStorage.EXPECT().MarkArchivedRemote(ctx, req.SpaceId, "oldHash", "newHash", int64(10), int64(20)).Return(nil) + fx.nodeHead.EXPECT().SetHead(req.SpaceId, "oldHash", "newHash").Return(0, nil) + fx.archive.EXPECT().QueueRestore(req.SpaceId) + + resp, err := fx.AdoptArchive(ctx, req) + require.NoError(t, err) + assert.Equal(t, nodesyncproto.AdoptArchiveResult_AdoptArchiveOk, resp.Result) + }) + t.Run("already have live", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + fx.expectNodePeer() + fx.indexStorage.EXPECT().SpaceStatusEntry(ctx, req.SpaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusOk, OldHash: "oldHash", NewHash: "newHash"}, nil) + fx.storage.EXPECT().SpaceExists(req.SpaceId).Return(true) + + resp, err := fx.AdoptArchive(ctx, req) + require.NoError(t, err) + assert.Equal(t, nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveSame, resp.Result) + }) + t.Run("already have archived with object", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + fx.expectNodePeer() + fx.indexStorage.EXPECT().SpaceStatusEntry(ctx, req.SpaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusArchived, OldHash: "oldHash", NewHash: "newHash"}, nil) + fx.archiveStore.EXPECT().Exists(ctx, req.SpaceId).Return(true, nil) + + resp, err := fx.AdoptArchive(ctx, req) + require.NoError(t, err) + assert.Equal(t, nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveSame, resp.Result) + }) + t.Run("archived but object missing: re-adopt", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + fx.expectNodePeer() + fx.indexStorage.EXPECT().SpaceStatusEntry(ctx, req.SpaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusArchived}, nil) + fx.archiveStore.EXPECT().Exists(ctx, req.SpaceId).Return(false, nil) + fx.archiveStore.EXPECT().CopyFrom(ctx, req.SrcKey, req.SpaceId).Return(nil) + fx.archiveStore.EXPECT().Exists(ctx, req.SpaceId).Return(true, nil) + fx.indexStorage.EXPECT().MarkArchivedRemote(ctx, req.SpaceId, "oldHash", "newHash", int64(10), int64(20)).Return(nil) + fx.nodeHead.EXPECT().SetHead(req.SpaceId, "oldHash", "newHash").Return(0, nil) + + resp, err := fx.AdoptArchive(ctx, req) + require.NoError(t, err) + assert.Equal(t, nodesyncproto.AdoptArchiveResult_AdoptArchiveOk, resp.Result) + }) + t.Run("deleted space rejected", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + fx.expectNodePeer() + fx.indexStorage.EXPECT().SpaceStatusEntry(ctx, req.SpaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusRemove}, nil) + _, err := fx.AdoptArchive(ctx, req) + assert.ErrorIs(t, err, nodesyncproto.ErrSpaceDeleted) + }) + t.Run("pending deletion parks the sender", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + fx.expectNodePeer() + fx.indexStorage.EXPECT().SpaceStatusEntry(ctx, req.SpaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusRemovePrepare}, nil) + _, err := fx.AdoptArchive(ctx, req) + assert.ErrorIs(t, err, nodesyncproto.ErrSpacePendingDeletion) + }) + t.Run("error status: never adopted, never acked", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + fx.expectNodePeer() + fx.indexStorage.EXPECT().SpaceStatusEntry(ctx, req.SpaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusError}, nil) + resp, err := fx.AdoptArchive(ctx, req) + require.NoError(t, err) + assert.Equal(t, nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveDiverged, resp.Result) + }) + t.Run("not responsible rejected", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + fx.archiveStore.EXPECT().Shared().Return(true) + fx.nodeConf.EXPECT().NodeTypes("peer1").Return([]nodeconf.NodeType{nodeconf.NodeTypeTree}) + fx.nodeConf.EXPECT().IsResponsible(req.SpaceId).Return(false) + _, err := fx.AdoptArchive(ctx, req) + assert.ErrorIs(t, err, nodesyncproto.ErrNotResponsible) + }) + t.Run("source object missing", func(t *testing.T) { + fx := newFixture(t) + req := adoptReq() + fx.expectNodePeer() + fx.indexStorage.EXPECT().SpaceStatusEntry(ctx, req.SpaceId).Return(nodestorage.SpaceStatusEntry{}, anystore.ErrDocNotFound) + fx.storage.EXPECT().SpaceExists(req.SpaceId).Return(false) + fx.archiveStore.EXPECT().CopyFrom(ctx, req.SrcKey, req.SpaceId).Return(archivestore.ErrNotFound) + _, err := fx.AdoptArchive(ctx, req) + assert.ErrorIs(t, err, nodesyncproto.ErrArchiveObjectMissing) + }) +} + +type fixture struct { + Adopter + a *app.App + storage *mock_nodestorage.MockNodeStorage + indexStorage *mock_nodestorage.MockIndexStorage + archiveStore *mock_archivestore.MockArchiveStore + archive *mock_archive.MockArchive + nodeHead *mock_nodehead.MockNodeHead + nodeConf *mock_nodeconf.MockService +} + +func (fx *fixture) expectNodePeer() { + fx.archiveStore.EXPECT().Shared().Return(true) + fx.nodeConf.EXPECT().NodeTypes("peer1").Return([]nodeconf.NodeType{nodeconf.NodeTypeTree}) + fx.nodeConf.EXPECT().IsResponsible("space.id").Return(true) +} + +func newFixture(t *testing.T) *fixture { + ctrl := gomock.NewController(t) + fx := &fixture{ + a: new(app.App), + storage: mock_nodestorage.NewMockNodeStorage(ctrl), + indexStorage: mock_nodestorage.NewMockIndexStorage(ctrl), + archiveStore: mock_archivestore.NewMockArchiveStore(ctrl), + archive: mock_archive.NewMockArchive(ctrl), + nodeHead: mock_nodehead.NewMockNodeHead(ctrl), + nodeConf: mock_nodeconf.NewMockService(ctrl), + Adopter: New(), + } + anymock.ExpectComp(fx.storage.EXPECT(), nodestorage.CName) + anymock.ExpectComp(fx.archiveStore.EXPECT(), archivestore.CName) + anymock.ExpectComp(fx.archive.EXPECT(), archive.CName) + anymock.ExpectComp(fx.nodeHead.EXPECT(), nodehead.CName) + anymock.ExpectComp(fx.nodeConf.EXPECT(), nodeconf.CName) + fx.storage.EXPECT().IndexStorage().AnyTimes().Return(fx.indexStorage) + + fx.a.Register(fx.storage). + Register(fx.archiveStore). + Register(fx.archive). + Register(fx.nodeHead). + Register(fx.nodeConf). + Register(fx.Adopter) + + require.NoError(t, fx.a.Start(context.Background())) + t.Cleanup(func() { + require.NoError(t, fx.a.Close(context.Background())) + ctrl.Finish() + }) + return fx +} diff --git a/archive/adopter/mock_adopter/mock_adopter.go b/archive/adopter/mock_adopter/mock_adopter.go new file mode 100644 index 00000000..784a52fb --- /dev/null +++ b/archive/adopter/mock_adopter/mock_adopter.go @@ -0,0 +1,86 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anyproto/any-sync-node/archive/adopter (interfaces: Adopter) +// +// Generated by this command: +// +// mockgen -destination mock_adopter/mock_adopter.go github.com/anyproto/any-sync-node/archive/adopter Adopter +// + +// Package mock_adopter is a generated GoMock package. +package mock_adopter + +import ( + context "context" + reflect "reflect" + + nodesyncproto "github.com/anyproto/any-sync-node/nodesync/nodesyncproto" + app "github.com/anyproto/any-sync/app" + gomock "go.uber.org/mock/gomock" +) + +// MockAdopter is a mock of Adopter interface. +type MockAdopter struct { + ctrl *gomock.Controller + recorder *MockAdopterMockRecorder + isgomock struct{} +} + +// MockAdopterMockRecorder is the mock recorder for MockAdopter. +type MockAdopterMockRecorder struct { + mock *MockAdopter +} + +// NewMockAdopter creates a new mock instance. +func NewMockAdopter(ctrl *gomock.Controller) *MockAdopter { + mock := &MockAdopter{ctrl: ctrl} + mock.recorder = &MockAdopterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAdopter) EXPECT() *MockAdopterMockRecorder { + return m.recorder +} + +// AdoptArchive mocks base method. +func (m *MockAdopter) AdoptArchive(ctx context.Context, req *nodesyncproto.AdoptArchiveRequest) (*nodesyncproto.AdoptArchiveResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdoptArchive", ctx, req) + ret0, _ := ret[0].(*nodesyncproto.AdoptArchiveResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AdoptArchive indicates an expected call of AdoptArchive. +func (mr *MockAdopterMockRecorder) AdoptArchive(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdoptArchive", reflect.TypeOf((*MockAdopter)(nil).AdoptArchive), ctx, req) +} + +// Init mocks base method. +func (m *MockAdopter) Init(a *app.App) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init", a) + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockAdopterMockRecorder) Init(a any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockAdopter)(nil).Init), a) +} + +// Name mocks base method. +func (m *MockAdopter) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockAdopterMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockAdopter)(nil).Name)) +} diff --git a/archive/adopter/stat.go b/archive/adopter/stat.go new file mode 100644 index 00000000..28820a64 --- /dev/null +++ b/archive/adopter/stat.go @@ -0,0 +1,42 @@ +package adopter + +import ( + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" +) + +type adopterStat struct { + // adopted is the number of spaces copied into our prefix and registered + adopted atomic.Uint32 + // alreadyHaveSame: requests ACKed because our copy matches the offered heads + alreadyHaveSame atomic.Uint32 + // alreadyHaveDiverged: requests answered with a diverged copy (sender must converge) + alreadyHaveDiverged atomic.Uint32 + // rejected: requests refused or failed (deleted/pending spaces, non-owners, + // non-node peers, transfer errors) + rejected atomic.Uint32 +} + +func registerMetric(s *adopterStat, registry *prometheus.Registry) { + gauge := func(name, help string, value func() float64) { + registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "node", + Subsystem: "adopter", + Name: name, + Help: help, + }, value)) + } + gauge("adopted", "spaces adopted (copied into our prefix and registered) since start", func() float64 { + return float64(s.adopted.Load()) + }) + gauge("already_have_same", "adopt requests ACKed with an equal local copy since start", func() float64 { + return float64(s.alreadyHaveSame.Load()) + }) + gauge("already_have_diverged", "adopt requests answered with a diverged local copy since start", func() float64 { + return float64(s.alreadyHaveDiverged.Load()) + }) + gauge("rejected", "adopt requests refused or failed since start", func() float64 { + return float64(s.rejected.Load()) + }) +} diff --git a/archive/archive.go b/archive/archive.go index 95682235..246d1cbd 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "sync" "time" anystore "github.com/anyproto/any-store" @@ -20,11 +21,19 @@ import ( "github.com/anyproto/any-sync-node/archive/archivestore" "github.com/anyproto/any-sync-node/nodestorage" - "github.com/anyproto/any-sync-node/nodesync" ) const CName = "node.archive" +// nodeSyncCName is nodesync.CName; the nodesync package is referenced by name +// to avoid an import cycle (nodesync -> nodehead -> ... -> archive). +const nodeSyncCName = "node.nodesync" + +// syncWaiter is implemented by nodesync.NodeSync. +type syncWaiter interface { + WaitSyncOnStart() <-chan struct{} +} + var log = logger.NewNamed(CName) func New() Archive { @@ -34,6 +43,19 @@ func New() Archive { type Archive interface { app.ComponentRunnable Restore(ctx context.Context, spaceId string) (err error) + // ForceArchive uploads a snapshot of the space to the archive store while + // keeping the local DB and its status intact. Used to ship live spaces + // through the shared archive store during migration. The caller must ensure + // the space is not already archived: opening an archived space triggers a + // restore. Returns the heads read from the snapshot itself (they exactly + // describe the uploaded object, unlike the asynchronously updated index) + // and the snapshot sizes. + // The return types are primitives on purpose: a struct would make the + // generated mock import this package and create test-only import cycles. + ForceArchive(ctx context.Context, spaceId string) (oldHash, newHash string, compressedSize, uncompressedSize int64, err error) + // QueueRestore schedules a background restore of an archived space + // (used for eager adoption of migrated spaces). + QueueRestore(spaceId string) } type archive struct { @@ -41,11 +63,18 @@ type archive struct { archiveStore archivestore.ArchiveStore config Config checker periodicsync.PeriodicSync + sweeper periodicsync.PeriodicSync accessDurCutoff time.Duration stat *archiveStat syncWaiter <-chan struct{} runCtx context.Context runCtxCancel context.CancelFunc + + restoreQueue chan string + restoreQueued sync.Map + // archiveMu serializes the periodic archiver with the prefix sweeper so a + // sweep decision can't race a concurrent re-archive of the same object + archiveMu sync.Mutex } func (a *archive) Init(ap *app.App) (err error) { @@ -56,14 +85,20 @@ func (a *archive) Init(ap *app.App) (err error) { a.config.ArchiveAfterDays = 7 } a.accessDurCutoff = time.Duration(a.config.ArchiveAfterDays) * time.Hour * 24 - a.syncWaiter = ap.MustComponent(nodesync.CName).(nodesync.NodeSync).WaitSyncOnStart() + a.syncWaiter = ap.MustComponent(nodeSyncCName).(syncWaiter).WaitSyncOnStart() a.runCtx, a.runCtxCancel = context.WithCancel(context.Background()) if a.config.CheckPeriodMinutes <= 0 { a.config.CheckPeriodMinutes = 2 } period := time.Minute * time.Duration(a.config.CheckPeriodMinutes) a.checker = periodicsync.NewPeriodicSyncDuration(period, time.Hour, a.check, log) + if a.config.SweepPeriodHours <= 0 { + a.config.SweepPeriodHours = 24 + } + a.sweeper = periodicsync.NewPeriodicSyncDuration( + time.Duration(a.config.SweepPeriodHours)*time.Hour, time.Hour, a.sweep, log) a.stat = new(archiveStat) + a.restoreQueue = make(chan string, 1000) if m := ap.Component(metric.CName); m != nil { registerMetric(a.stat, m.(metric.Metric).Registry()) } @@ -75,6 +110,7 @@ func (a *archive) Name() (name string) { } func (a *archive) Run(_ context.Context) (err error) { + go a.restoreWorker() if !a.config.Enabled { return } @@ -85,13 +121,49 @@ func (a *archive) Run(_ context.Context) (err error) { case <-a.syncWaiter: } a.checker.Run() + a.sweeper.Run() }() return } +func (a *archive) QueueRestore(spaceId string) { + if _, loaded := a.restoreQueued.LoadOrStore(spaceId, struct{}{}); loaded { + return + } + select { + case a.restoreQueue <- spaceId: + default: + // queue is full: drop, the space will be restored lazily on first access + a.restoreQueued.Delete(spaceId) + } +} + +func (a *archive) restoreWorker() { + for { + select { + case <-a.runCtx.Done(): + return + case spaceId := <-a.restoreQueue: + ctx, cancel := context.WithTimeout(a.runCtx, time.Minute*10) + // opening the space storage restores it if archived; reuses the + // storage cache's single-flight so concurrent lazy restores are safe + err := a.storageProvider.TryLockAndOpenDb(ctx, spaceId, func(db anystore.DB) error { + return nil + }) + cancel() + a.restoreQueued.Delete(spaceId) + if err != nil && !errors.Is(err, nodestorage.ErrLocked) && !errors.Is(err, context.Canceled) { + log.Warn("eager restore failed, will restore lazily on access", zap.String("spaceId", spaceId), zap.Error(err)) + } + } + } +} + var errArchived = errors.New("archived") func (a *archive) Archive(ctx context.Context, spaceId string) (err error) { + a.archiveMu.Lock() + defer a.archiveMu.Unlock() var gzSize, dbSize int64 tmpDir, err := os.MkdirTemp("", spaceId) if err != nil { @@ -141,6 +213,64 @@ func (a *archive) Archive(ctx context.Context, spaceId string) (err error) { return } +func (a *archive) ForceArchive(ctx context.Context, spaceId string) (oldHash, newHash string, compressedSize, uncompressedSize int64, err error) { + // DumpStorage backups the db into a temp dir; it works for open spaces too + err = a.storageProvider.DumpStorage(ctx, spaceId, func(path string) error { + // read the heads from the snapshot: they describe exactly what the + // uploaded object contains + oldHash, newHash, err = readSnapshotHeads(ctx, spaceId, filepath.Join(path, "store.db")) + if err != nil { + return err + } + gzPath, gzSize, dbSize, err := a.createGzipFromStore(path) + if err != nil { + return err + } + compressedSize, uncompressedSize = gzSize, dbSize + + r, err := os.Open(gzPath) + if err != nil { + return err + } + defer func() { + _ = r.Close() + }() + return a.archiveStore.Put(ctx, spaceId, r) + }) + if err == nil { + a.stat.forceArchived.Add(1) + } + return +} + +// readSnapshotHeads reads the space state directly from the snapshot db +// (schema of commonspace/headsync/statestorage, including the legacy +// single-hash fallback). +func readSnapshotHeads(ctx context.Context, spaceId, dbPath string) (oldHash, newHash string, err error) { + db, err := anystore.Open(ctx, dbPath, nil) + if err != nil { + return + } + defer func() { + _ = db.Close() + }() + coll, err := db.OpenCollection(ctx, "state") + if err != nil { + return + } + doc, err := coll.FindId(ctx, spaceId) + if err != nil { + return + } + oldHash = doc.Value().GetString("oh") + newHash = doc.Value().GetString("nh") + if oldHash == "" || newHash == "" { + oldHash = doc.Value().GetString("h") + newHash = oldHash + } + return oldHash, newHash, nil +} + // createGzipFromStore creates store.gz from store.db inside spaceDir. // Returns path to .gz, its size and original db size. func (a *archive) createGzipFromStore(spaceDir string) (gzPath string, gzSize, dbSize int64, err error) { @@ -189,14 +319,21 @@ func (a *archive) createGzipFromStore(spaceDir string) (gzPath string, gzSize, d func (a *archive) Restore(ctx context.Context, spaceId string) (err error) { if err = a.restoreFile(ctx, spaceId); err != nil { - _ = os.RemoveAll(a.storageProvider.StoreDir(spaceId)) + if !errors.Is(err, archivestore.ErrNotFound) { + // clean up a partially extracted db; a missing archive object wrote + // nothing, and the dir may hold a pre-existing db that must survive + _ = os.RemoveAll(a.storageProvider.StoreDir(spaceId)) + } return err } if err = a.storageProvider.IndexStorage().SetSpaceStatus(ctx, spaceId, nodestorage.SpaceStatusOk, ""); err != nil { return } a.stat.restored.Add(1) - return a.archiveStore.Delete(ctx, spaceId) + // the archive object is intentionally kept (restore is a copy, not a move): + // it is overwritten by the next archive cycle and makes restore idempotent; + // stale objects of live spaces are garbage-collected by the archive sweeper + return nil } func (a *archive) restoreFile(ctx context.Context, spaceId string) (err error) { @@ -281,6 +418,9 @@ func (a *archive) Close(_ context.Context) (err error) { if a.checker != nil { a.checker.Close() } + if a.sweeper != nil { + a.sweeper.Close() + } if a.runCtxCancel != nil { a.runCtxCancel() } diff --git a/archive/archive_test.go b/archive/archive_test.go index 1c3f48db..614af156 100644 --- a/archive/archive_test.go +++ b/archive/archive_test.go @@ -6,8 +6,10 @@ import ( "os" "path/filepath" "testing" + "time" anystore "github.com/anyproto/any-store" + "github.com/anyproto/any-store/anyenc" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/testutil/anymock" "github.com/stretchr/testify/assert" @@ -68,7 +70,7 @@ func TestArchive_Archive(t *testing.T) { }) fx.indexStorage.EXPECT().SetSpaceStatus(ctx, spaceId, nodestorage.SpaceStatusOk, "") - fx.archiveStore.EXPECT().Delete(ctx, spaceId) + // restore keeps the archive object: restore is a copy, not a move require.NoError(t, fx.Restore(ctx, spaceId)) @@ -136,3 +138,68 @@ func (t testConfig) Name() string { func (t testConfig) GetArchive() Config { return Config{} } + +func TestArchive_ForceArchive(t *testing.T) { + fx := newFixture(t) + tmpDir := t.TempDir() + + spaceId := "force.id" + // prepare a dump dir with a store.db like DumpStorage produces, + // including the space state the snapshot heads are read from + db, err := anystore.Open(ctx, filepath.Join(tmpDir, "store.db"), nil) + require.NoError(t, err) + _, err = db.CreateCollection(ctx, "test") + require.NoError(t, err) + stateColl, err := db.Collection(ctx, "state") + require.NoError(t, err) + arena := &anyenc.Arena{} + stateDoc := arena.NewObject() + stateDoc.Set("id", arena.NewString(spaceId)) + stateDoc.Set("oh", arena.NewString("snap-old")) + stateDoc.Set("nh", arena.NewString("snap-new")) + require.NoError(t, stateColl.Insert(ctx, stateDoc)) + require.NoError(t, db.Close()) + fx.storage.EXPECT(). + DumpStorage(ctx, spaceId, gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, do func(path string) error) error { + return do(tmpDir) + }) + var uploaded int + fx.archiveStore.EXPECT().Put(ctx, spaceId, gomock.Any()).DoAndReturn(func(_ context.Context, _ string, rd io.ReadSeeker) error { + data, err := io.ReadAll(rd) + require.NoError(t, err) + uploaded = len(data) + return nil + }) + + // no MarkArchived, no local deletion: the space stays live + oldHash, newHash, compressedSize, uncompressedSize, err := fx.ForceArchive(ctx, spaceId) + require.NoError(t, err) + assert.Equal(t, "snap-old", oldHash) + assert.Equal(t, "snap-new", newHash) + assert.Greater(t, compressedSize, int64(0)) + assert.Greater(t, uncompressedSize, int64(0)) + assert.Equal(t, int(compressedSize), uploaded) + _, err = os.Stat(filepath.Join(tmpDir, "store.db")) + assert.NoError(t, err) +} + +func TestArchive_QueueRestore(t *testing.T) { + fx := newFixture(t) + spaceId := "eager.id" + + restored := make(chan struct{}) + fx.storage.EXPECT(). + TryLockAndOpenDb(gomock.Any(), spaceId, gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, _ nodestorage.DoAfterOpenFunc) error { + close(restored) + return nil + }) + + fx.QueueRestore(spaceId) + select { + case <-restored: + case <-time.After(time.Second * 5): + t.Fatal("eager restore was not triggered") + } +} diff --git a/archive/archivestore/archivestore.go b/archive/archivestore/archivestore.go index 12d10f07..a94665b0 100644 --- a/archive/archivestore/archivestore.go +++ b/archive/archivestore/archivestore.go @@ -6,11 +6,15 @@ import ( "errors" "fmt" "io" + "net/http" + "net/url" "strings" + "time" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/logger" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -34,6 +38,20 @@ type ArchiveStore interface { Get(ctx context.Context, name string) (data io.ReadCloser, err error) Put(ctx context.Context, name string, data io.ReadSeeker) (err error) Delete(ctx context.Context, name string) (err error) + // Exists checks the object presence in this node's prefix without fetching it. + Exists(ctx context.Context, name string) (ok bool, err error) + // Key returns the full bucket key for the given name in this node's prefix. + Key(name string) string + // CopyFrom server-side copies an object from an absolute bucket key + // (typically another node's prefix in the shared bucket) into this node's prefix. + CopyFrom(ctx context.Context, srcKey, name string) (err error) + // Shared reports whether the bucket is shared across the network's tree nodes, + // enabling S3-mediated space migration. + Shared() bool + // List iterates all objects in this node's prefix; iter receives the object + // name (key without the prefix) and its last-modified time, returning false + // to stop the iteration. + List(ctx context.Context, iter func(name string, lastModified time.Time) (bool, error)) (err error) } type archiveStore struct { @@ -42,6 +60,7 @@ type archiveStore struct { client *s3.S3 keyPrefix string enabled bool + shared bool } func (as *archiveStore) Init(a *app.App) (err error) { @@ -89,6 +108,7 @@ func (as *archiveStore) Init(a *app.App) (err error) { as.client = s3.New(as.sess) as.keyPrefix = conf.KeyPrefix + "/" as.enabled = true + as.shared = conf.Shared return } @@ -138,3 +158,96 @@ func (as *archiveStore) Delete(ctx context.Context, name string) (err error) { }) return } + +// Exists deliberately uses ListObjectsV2 instead of HeadObject: GCS's +// S3-interop layer can serve stale HEAD responses for several seconds after a +// mutation (observed: HEAD returns 200 for a just-deleted object when the same +// key was HEAD'ed before), while listing is read-after-write consistent. +// Exists backs the durable-ACK check of the resharding handoff, where a stale +// positive could acknowledge an object that is already gone. +func (as *archiveStore) Exists(ctx context.Context, name string) (ok bool, err error) { + if !as.enabled { + return false, ErrDisabled + } + key := as.keyPrefix + name + out, err := as.client.ListObjectsV2WithContext(ctx, &s3.ListObjectsV2Input{ + Bucket: as.bucket, + Prefix: aws.String(key), + MaxKeys: aws.Int64(1), + }) + if err != nil { + return false, err + } + for _, obj := range out.Contents { + if obj.Key != nil && *obj.Key == key { + return true, nil + } + } + return false, nil +} + +func (as *archiveStore) Key(name string) string { + return as.keyPrefix + name +} + +func (as *archiveStore) CopyFrom(ctx context.Context, srcKey, name string) (err error) { + if !as.enabled { + return ErrDisabled + } + _, err = as.client.CopyObjectWithContext(ctx, &s3.CopyObjectInput{ + Bucket: as.bucket, + CopySource: aws.String(url.PathEscape(*as.bucket + "/" + srcKey)), + Key: aws.String(as.keyPrefix + name), + }) + if err != nil && isNotFoundErr(err) { + return ErrNotFound + } + return +} + +func (as *archiveStore) Shared() bool { + return as.enabled && as.shared +} + +func (as *archiveStore) List(ctx context.Context, iter func(name string, lastModified time.Time) (bool, error)) (err error) { + if !as.enabled { + return ErrDisabled + } + var iterErr error + err = as.client.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{ + Bucket: as.bucket, + Prefix: aws.String(as.keyPrefix), + }, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + for _, obj := range page.Contents { + if obj.Key == nil { + continue + } + name := strings.TrimPrefix(*obj.Key, as.keyPrefix) + var lastModified time.Time + if obj.LastModified != nil { + lastModified = *obj.LastModified + } + cont, iErr := iter(name, lastModified) + if iErr != nil { + iterErr = iErr + return false + } + if !cont { + return false + } + } + return !lastPage + }) + if err == nil { + err = iterErr + } + return +} + +func isNotFoundErr(err error) bool { + var awsErr awserr.RequestFailure + if errors.As(err, &awsErr) { + return awsErr.StatusCode() == http.StatusNotFound || awsErr.Code() == s3.ErrCodeNoSuchKey + } + return strings.HasPrefix(err.Error(), s3.ErrCodeNoSuchKey) || strings.HasPrefix(err.Error(), "NotFound") +} diff --git a/archive/archivestore/config.go b/archive/archivestore/config.go index 0ac18252..5f3c8c06 100644 --- a/archive/archivestore/config.go +++ b/archive/archivestore/config.go @@ -18,4 +18,8 @@ type Config struct { Credentials Credentials `yaml:"credentials"` ForcePathStyle bool `yaml:"forcePathStyle"` KeyPrefix string `yaml:"keyPrefix"` + // Shared marks the bucket as shared across all tree nodes of the network + // (each node under its own KeyPrefix). Enables S3-mediated space migration + // (resharding): snapshots are handed between nodes via server-side copy. + Shared bool `yaml:"shared"` } diff --git a/archive/archivestore/mock_archivestore/mock_archivestore.go b/archive/archivestore/mock_archivestore/mock_archivestore.go index 1f92d90b..0b6eca00 100644 --- a/archive/archivestore/mock_archivestore/mock_archivestore.go +++ b/archive/archivestore/mock_archivestore/mock_archivestore.go @@ -13,6 +13,7 @@ import ( context "context" io "io" reflect "reflect" + time "time" app "github.com/anyproto/any-sync/app" gomock "go.uber.org/mock/gomock" @@ -42,6 +43,20 @@ func (m *MockArchiveStore) EXPECT() *MockArchiveStoreMockRecorder { return m.recorder } +// CopyFrom mocks base method. +func (m *MockArchiveStore) CopyFrom(ctx context.Context, srcKey, name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CopyFrom", ctx, srcKey, name) + ret0, _ := ret[0].(error) + return ret0 +} + +// CopyFrom indicates an expected call of CopyFrom. +func (mr *MockArchiveStoreMockRecorder) CopyFrom(ctx, srcKey, name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyFrom", reflect.TypeOf((*MockArchiveStore)(nil).CopyFrom), ctx, srcKey, name) +} + // Delete mocks base method. func (m *MockArchiveStore) Delete(ctx context.Context, name string) error { m.ctrl.T.Helper() @@ -56,6 +71,21 @@ func (mr *MockArchiveStoreMockRecorder) Delete(ctx, name any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockArchiveStore)(nil).Delete), ctx, name) } +// Exists mocks base method. +func (m *MockArchiveStore) Exists(ctx context.Context, name string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, name) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockArchiveStoreMockRecorder) Exists(ctx, name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockArchiveStore)(nil).Exists), ctx, name) +} + // Get mocks base method. func (m *MockArchiveStore) Get(ctx context.Context, name string) (io.ReadCloser, error) { m.ctrl.T.Helper() @@ -85,6 +115,34 @@ func (mr *MockArchiveStoreMockRecorder) Init(a any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockArchiveStore)(nil).Init), a) } +// Key mocks base method. +func (m *MockArchiveStore) Key(name string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Key", name) + ret0, _ := ret[0].(string) + return ret0 +} + +// Key indicates an expected call of Key. +func (mr *MockArchiveStoreMockRecorder) Key(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Key", reflect.TypeOf((*MockArchiveStore)(nil).Key), name) +} + +// List mocks base method. +func (m *MockArchiveStore) List(ctx context.Context, iter func(string, time.Time) (bool, error)) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, iter) + ret0, _ := ret[0].(error) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockArchiveStoreMockRecorder) List(ctx, iter any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockArchiveStore)(nil).List), ctx, iter) +} + // Name mocks base method. func (m *MockArchiveStore) Name() string { m.ctrl.T.Helper() @@ -112,3 +170,17 @@ func (mr *MockArchiveStoreMockRecorder) Put(ctx, name, data any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockArchiveStore)(nil).Put), ctx, name, data) } + +// Shared mocks base method. +func (m *MockArchiveStore) Shared() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Shared") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Shared indicates an expected call of Shared. +func (mr *MockArchiveStoreMockRecorder) Shared() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shared", reflect.TypeOf((*MockArchiveStore)(nil).Shared)) +} diff --git a/archive/archivestore/realstore_test.go b/archive/archivestore/realstore_test.go new file mode 100644 index 00000000..aa768d77 --- /dev/null +++ b/archive/archivestore/realstore_test.go @@ -0,0 +1,135 @@ +package archivestore + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/anyproto/any-sync/app" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRealS3 exercises the store against a real S3-compatible service. +// Opt-in: set ARCHIVE_TEST_S3_ACCESS_KEY, ARCHIVE_TEST_S3_SECRET_KEY, +// ARCHIVE_TEST_S3_BUCKET and optionally ARCHIVE_TEST_S3_ENDPOINT +// (e.g. https://storage.googleapis.com for GCS interop). +func TestRealS3(t *testing.T) { + accessKey := os.Getenv("ARCHIVE_TEST_S3_ACCESS_KEY") + secretKey := os.Getenv("ARCHIVE_TEST_S3_SECRET_KEY") + bucket := os.Getenv("ARCHIVE_TEST_S3_BUCKET") + if accessKey == "" || secretKey == "" || bucket == "" { + t.Skip("real S3 credentials are not configured") + } + endpoint := os.Getenv("ARCHIVE_TEST_S3_ENDPOINT") + region := os.Getenv("ARCHIVE_TEST_S3_REGION") + if region == "" { + region = "auto" + } + + newStore := func(t *testing.T, prefix string) ArchiveStore { + s := New() + a := new(app.App) + a.Register(realTestConfig{cfg: Config{ + Enabled: true, + Region: region, + Bucket: bucket, + Endpoint: endpoint, + ForcePathStyle: true, + KeyPrefix: prefix, + Shared: true, + Credentials: Credentials{AccessKey: accessKey, SecretKey: secretKey}, + }}).Register(s) + require.NoError(t, a.Start(context.Background())) + t.Cleanup(func() { _ = a.Close(context.Background()) }) + return s + } + + runId := fmt.Sprintf("reshardtest-%d", time.Now().UnixNano()) + nodeA := newStore(t, runId+"/nodeA") + nodeB := newStore(t, runId+"/nodeB") + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*3) + defer cancel() + + const name = "space.test" + payload := []byte("resharding-archive-test-payload") + + t.Cleanup(func() { + cctx, ccancel := context.WithTimeout(context.Background(), time.Minute) + defer ccancel() + _ = nodeA.Delete(cctx, name) + _ = nodeB.Delete(cctx, name) + }) + + // missing object: Exists false, Get/CopyFrom -> ErrNotFound + ok, err := nodeA.Exists(ctx, name) + require.NoError(t, err) + assert.False(t, ok) + _, err = nodeA.Get(ctx, name) + assert.ErrorIs(t, err, ErrNotFound) + err = nodeB.CopyFrom(ctx, nodeA.Key(name), name) + assert.ErrorIs(t, err, ErrNotFound) + + // put + head + get on node A + require.NoError(t, nodeA.Put(ctx, name, bytes.NewReader(payload))) + ok, err = nodeA.Exists(ctx, name) + require.NoError(t, err) + assert.True(t, ok) + rd, err := nodeA.Get(ctx, name) + require.NoError(t, err) + got, err := io.ReadAll(rd) + require.NoError(t, err) + require.NoError(t, rd.Close()) + assert.Equal(t, payload, got) + + // server-side copy A -> B (the AdoptArchive transfer path) + require.NoError(t, nodeB.CopyFrom(ctx, nodeA.Key(name), name)) + ok, err = nodeB.Exists(ctx, name) + require.NoError(t, err) + assert.True(t, ok) + rd, err = nodeB.Get(ctx, name) + require.NoError(t, err) + got, err = io.ReadAll(rd) + require.NoError(t, err) + require.NoError(t, rd.Close()) + assert.Equal(t, payload, got) + + // list both prefixes (the sweeper path) + var listedA, listedB []string + require.NoError(t, nodeA.List(ctx, func(n string, lastModified time.Time) (bool, error) { + listedA = append(listedA, n) + assert.WithinDuration(t, time.Now(), lastModified, time.Minute*10) + return true, nil + })) + require.NoError(t, nodeB.List(ctx, func(n string, _ time.Time) (bool, error) { + listedB = append(listedB, n) + return true, nil + })) + assert.Equal(t, []string{name}, listedA) + assert.Equal(t, []string{name}, listedB) + + // delete on A must not affect B (per-node prefixes isolate blast radius); + // this also guards the GCS interop quirk: HEAD can be stale after DELETE, + // which is why Exists is list-based + require.NoError(t, nodeA.Delete(ctx, name)) + ok, err = nodeA.Exists(ctx, name) + require.NoError(t, err) + assert.False(t, ok) + ok, err = nodeB.Exists(ctx, name) + require.NoError(t, err) + assert.True(t, ok) + + require.NoError(t, nodeB.Delete(ctx, name)) +} + +type realTestConfig struct { + cfg Config +} + +func (c realTestConfig) Init(_ *app.App) error { return nil } +func (c realTestConfig) Name() string { return "config" } +func (c realTestConfig) GetS3Store() Config { return c.cfg } diff --git a/archive/config.go b/archive/config.go index dc86e48f..d4e68220 100644 --- a/archive/config.go +++ b/archive/config.go @@ -8,4 +8,7 @@ type Config struct { Enabled bool `yaml:"enabled"` ArchiveAfterDays int `yaml:"archiveAfterDays"` CheckPeriodMinutes int `yaml:"checkPeriodMinutes"` + // SweepPeriodHours is the period of the archive-prefix garbage collector + // (removes stale objects that no longer back an archived space), default 24. + SweepPeriodHours int `yaml:"sweepPeriodHours"` } diff --git a/archive/mock_archive/mock_archive.go b/archive/mock_archive/mock_archive.go index 2c7080d0..b687db09 100644 --- a/archive/mock_archive/mock_archive.go +++ b/archive/mock_archive/mock_archive.go @@ -55,6 +55,24 @@ func (mr *MockArchiveMockRecorder) Close(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockArchive)(nil).Close), ctx) } +// ForceArchive mocks base method. +func (m *MockArchive) ForceArchive(ctx context.Context, spaceId string) (string, string, int64, int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ForceArchive", ctx, spaceId) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(int64) + ret3, _ := ret[3].(int64) + ret4, _ := ret[4].(error) + return ret0, ret1, ret2, ret3, ret4 +} + +// ForceArchive indicates an expected call of ForceArchive. +func (mr *MockArchiveMockRecorder) ForceArchive(ctx, spaceId any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ForceArchive", reflect.TypeOf((*MockArchive)(nil).ForceArchive), ctx, spaceId) +} + // Init mocks base method. func (m *MockArchive) Init(a *app.App) error { m.ctrl.T.Helper() @@ -83,6 +101,18 @@ func (mr *MockArchiveMockRecorder) Name() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockArchive)(nil).Name)) } +// QueueRestore mocks base method. +func (m *MockArchive) QueueRestore(spaceId string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "QueueRestore", spaceId) +} + +// QueueRestore indicates an expected call of QueueRestore. +func (mr *MockArchiveMockRecorder) QueueRestore(spaceId any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueueRestore", reflect.TypeOf((*MockArchive)(nil).QueueRestore), spaceId) +} + // Restore mocks base method. func (m *MockArchive) Restore(ctx context.Context, spaceId string) error { m.ctrl.T.Helper() diff --git a/archive/stat.go b/archive/stat.go index 41af3b40..099b86db 100644 --- a/archive/stat.go +++ b/archive/stat.go @@ -7,9 +7,11 @@ import ( ) type archiveStat struct { - archived atomic.Uint32 - archiveError atomic.Uint32 - restored atomic.Uint32 + archived atomic.Uint32 + archiveError atomic.Uint32 + restored atomic.Uint32 + forceArchived atomic.Uint32 + swept atomic.Uint32 } func registerMetric(s *archiveStat, registry *prometheus.Registry) { @@ -34,4 +36,18 @@ func registerMetric(s *archiveStat, registry *prometheus.Registry) { }, func() float64 { return float64(s.archiveError.Load()) })) + registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "node", + Subsystem: "archive", + Name: "force_archived", + }, func() float64 { + return float64(s.forceArchived.Load()) + })) + registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "node", + Subsystem: "archive", + Name: "swept", + }, func() float64 { + return float64(s.swept.Load()) + })) } diff --git a/archive/sweeper.go b/archive/sweeper.go new file mode 100644 index 00000000..555ab65c --- /dev/null +++ b/archive/sweeper.go @@ -0,0 +1,98 @@ +package archive + +import ( + "context" + "errors" + "time" + + anystore "github.com/anyproto/any-store" + "go.uber.org/zap" + + "github.com/anyproto/any-sync-node/nodestorage" +) + +const ( + // sweepMinObjectAge protects fresh objects from the sweeper: an object may + // legitimately exist for a live space for a while (restore keeps objects, + // ForceArchive parks handoff snapshots), so only stale ones are collected. + sweepMinObjectAge = time.Hour * 24 * 7 + // sweepMinDiskAge disables collection of objects without an index entry + // until the storage root is old enough: after a disk replacement the index + // starts empty while the node's prefix still holds all its archived spaces, + // and those objects may be the node's only copies until anti-entropy + // rebuilds the index. + sweepMinDiskAge = time.Hour * 24 * 30 +) + +// sweep garbage-collects this node's archive prefix: objects that don't back +// an Archived index entry (leftovers of restores, finished handoffs, deleted +// spaces) are removed once they are old enough. It shares archiveMu with the +// periodic archiver so an object can't be re-archived while being judged. +func (a *archive) sweep(ctx context.Context) (err error) { + index := a.storageProvider.IndexStorage() + diskAge := time.Since(a.storageProvider.DiskGen().CreatedTime) + var checked, deleted, kept int + err = a.archiveStore.List(ctx, func(name string, lastModified time.Time) (bool, error) { + checked++ + if time.Since(lastModified) <= sweepMinObjectAge { + return true, nil + } + // the archiver can't flip the status or overwrite the object while we + // hold the lock, so the decision and the delete can't race a re-archive + a.archiveMu.Lock() + remove, rErr := a.sweepDecision(ctx, index, name, diskAge) + if rErr != nil { + a.archiveMu.Unlock() + return false, rErr + } + if !remove { + a.archiveMu.Unlock() + kept++ + return true, nil + } + dErr := a.archiveStore.Delete(ctx, name) + a.archiveMu.Unlock() + if dErr != nil { + log.Warn("archive sweep: can't delete object", zap.String("name", name), zap.Error(dErr)) + } else { + deleted++ + a.stat.swept.Add(1) + } + return true, nil + }) + if err != nil { + return + } + if deleted > 0 || kept > 0 { + log.Info("archive sweep finished", zap.Int("objects", checked), zap.Int("deleted", deleted), zap.Int("keptStale", kept)) + } + return +} + +func (a *archive) sweepDecision(ctx context.Context, index nodestorage.IndexStorage, name string, diskAge time.Duration) (remove bool, err error) { + entry, entryErr := index.SpaceStatusEntry(ctx, name) + switch { + case errors.Is(entryErr, anystore.ErrDocNotFound): + // unknown space: a leak from an aborted adoption — but after a disk + // replacement the empty index makes every object look unknown while + // the objects may be the only copies; wait until the disk is old + // enough for anti-entropy to have rebuilt the index + return diskAge > sweepMinDiskAge, nil + case entryErr != nil: + return false, entryErr + } + switch entry.Status { + case nodestorage.SpaceStatusRemove, nodestorage.SpaceStatusMoved: + // deleted or handed off: the object is garbage + return true, nil + case nodestorage.SpaceStatusOk: + // restore/handoff leftover — but only when the live db actually exists + // locally; a status of Ok without a db (e.g. a cancelled deletion of an + // archived space) means the object may be our only copy + return a.storageProvider.SpaceExists(name), nil + default: + // Archived backs the space; RemovePrepare may be cancelled; + // Error/NotResponsible are left for the operator + return false, nil + } +} diff --git a/cmd/any-sync-node.go b/cmd/any-sync-node.go index f6875192..10cfa7c7 100644 --- a/cmd/any-sync-node.go +++ b/cmd/any-sync-node.go @@ -30,6 +30,7 @@ import ( "github.com/anyproto/any-sync/util/syncqueues" "github.com/anyproto/any-sync-node/archive" + "github.com/anyproto/any-sync-node/archive/adopter" "github.com/anyproto/any-sync-node/archive/archivestore" "github.com/anyproto/any-sync-node/nodehead" "github.com/anyproto/any-sync-node/nodespace/migrator" @@ -39,6 +40,8 @@ import ( "github.com/anyproto/any-sync-node/nodesync/coldsync" "github.com/anyproto/any-sync-node/nodesync/hotsync" "github.com/anyproto/any-sync-node/oldstorage" + "github.com/anyproto/any-sync-node/repairer" + "github.com/anyproto/any-sync-node/resharder" // import this to keep govvv in go.mod on mod tidy _ "github.com/ahmetb/govvv/integration-test/app-different-package/mypkg" @@ -157,6 +160,9 @@ func Bootstrap(a *app.App) { Register(nodedebugrpc.New()). Register(archivestore.New()). Register(archive.New()). + Register(adopter.New()). + Register(resharder.New()). + Register(repairer.New()). Register(quic.New()). Register(yamux.New()) } diff --git a/config/config.go b/config/config.go index fe75cc65..e0aa810d 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,8 @@ import ( "github.com/anyproto/any-sync-node/nodestorage" "github.com/anyproto/any-sync-node/nodesync" "github.com/anyproto/any-sync-node/nodesync/hotsync" + "github.com/anyproto/any-sync-node/repairer" + "github.com/anyproto/any-sync-node/resharder" ) const CName = "config" @@ -56,6 +58,8 @@ type Config struct { Quic quic.Config `yaml:"quic"` S3Store archivestore.Config `yaml:"s3Store"` Archive archive.Config `yaml:"archive"` + Resharder resharder.Config `yaml:"resharder"` + Repairer repairer.Config `yaml:"repairer"` Secure secureservice.Config `yaml:"secure"` } @@ -142,3 +146,11 @@ func (c Config) GetArchive() archive.Config { func (c Config) GetSecureService() secureservice.Config { return c.Secure } + +func (c Config) GetResharder() resharder.Config { + return c.Resharder +} + +func (c Config) GetRepairer() repairer.Config { + return c.Repairer +} diff --git a/docs/resharding-e2e.md b/docs/resharding-e2e.md new file mode 100644 index 00000000..39f561e1 --- /dev/null +++ b/docs/resharding-e2e.md @@ -0,0 +1,68 @@ +# Resharding: manual e2e runbook + +The in-process integration test (`resharder/integration_test.go`) covers the +data plane (drain → adopt → eager restore) with real storage and archive +stacks. This runbook exercises the full network path with real processes. + +## Prerequisites + +- mongodb on `localhost:27017` (coordinator storage) +- MinIO (or any S3-compatible store) with one bucket, e.g. `anysync-archive` +- binaries built from the resharding branches: `any-sync-coordinator`, + `confapply`, `any-sync-node` +- network configs generated with `any-sync-network` (any-sync-tools), + 4 tree nodes + coordinator + +## Node configuration + +Every tree node gets the shared bucket with its own prefix and archiving on: + +```yaml +s3Store: + enabled: true + endpoint: http://127.0.0.1:9000 + bucket: anysync-archive + forcePathStyle: true + keyPrefix: # unique per node + shared: true # gates the resharding machinery + credentials: { accessKey: ..., secretKey: ... } +archive: + enabled: true + archiveAfterDays: 1 +resharder: + drainIntervalMinutes: 5 +``` + +## Scenario 1: add a node + +1. Start coordinator + tree nodes 1–3 with an epoch-1 config + (`confapply -c coordinator.yml -n network.yml -e` → prints `epoch: 1`). +2. Create spaces via an any-sync-sdk2 client (`e2e/local.yml` shape) and let + them sync; optionally wait for archiving to kick in. +3. Publish an epoch-2 config adding tree node 4 (`confapply ... -e`). +4. Within the nodeconf poll interval every node logs + `net configuration changed` with `afterEpoch: 2` and drain cycles start. +5. Watch metrics: `node_resharder_draining` falls to 0 (`node_resharder_state` + back to idle); `node_resharder_moved` and `node_adopter_adopted` grow. Spaces whose partitions moved to node 4 + appear in its index as Archived (dormant) or restored (hot, eager). +6. Verify a moved space: client reads it through node 4; the drainer answers + `ErrPeerIsNotResponsible` for client requests and its copy is gone + (index status Moved). + +## Scenario 2: unsafe config rejected + +Publish a config replacing all tree nodes at once: `confapply` must fail with +`unsafe configuration: N of 3000 partitions lose all current replicas`. +Re-run with `-force` only to confirm the escape hatch works. + +## Scenario 3: node decommission + +Same as scenario 1 but epoch-2 removes a node; watch its `draining` gauge go +to zero, then it can be shut down. Its S3 prefix empties as handoffs complete +(leftovers are collected by the daily archive sweep after the 7-day age guard). + +## Scenario 4: deletion during drain + +Delete a space (client flow → coordinator deletion log) while its old owner +drains: the owner receives `ErrSpaceDeleted` from the adopters and drops its +copy without resurrecting the space. diff --git a/docs/resharding-operations.md b/docs/resharding-operations.md new file mode 100644 index 00000000..55669350 --- /dev/null +++ b/docs/resharding-operations.md @@ -0,0 +1,209 @@ +# Resharding: operator guide + +How to change the set of tree nodes of a running network. Two supported +operations: **adding node(s)** and **removing node(s)**. Everything else +(hardware replacement) is a combination of the two. + +Covered by tests: `resharder/scenarios_test.go` (`TestScenario_AddNode`, +`TestScenario_RemoveNode`) runs both flows in-process over real chash rings; +`resharder/integration_test.go` runs the handoff against a real S3 bucket when +`ARCHIVE_TEST_S3_*` env vars are set. + +## Prerequisites + +Resharding is gated on a **shared archive store**. Every tree node (including +new ones) needs, in `any-sync-node.yml`: + +```yaml +s3Store: + enabled: true + bucket: + keyPrefix: + shared: true # gates the resharding machinery + # endpoint/credentials as usual (GCS interop: endpoint https://storage.googleapis.com, + # region us-east-1, forcePathStyle: true) +archive: + enabled: true +``` + +Without `shared: true` nodes never drain — topology changes then behave as +before this feature (data stays on the old nodes; only anti-entropy fills the +new ones). + +Network configurations are published with `confapply` (coordinator repo): + +``` +confapply -c coordinator.yml -n network.yml -e +# prints: id: epoch: +``` + +Every publish gets a monotonically increasing **epoch**. `confapply` refuses +configurations where any chash partition would lose *all* of its current +replicas (`unsafe configuration: ...`); `-force` overrides this check — +emergencies only, it can produce partitions with no live source. + +## First deployment (one-time preparations) + +1. **Merge/release order**: any-sync → tag → bump the dependency in + coordinator and node → release both. Old node builds are compatible with + new ones (an old receiver answers `AdoptArchive` with "unknown rpc"; the + drainer parks and retries), so binaries can roll gradually — but publish + no topology change until the whole tree fleet runs the new build. +2. **Audit the S3 layout before enabling anything**: + - every tree node must use the **same bucket** (server-side copy does not + cross buckets) and a **unique keyPrefix** — verify no two nodes share a + prefix (a shared prefix was already subtly broken before this feature); + - node credentials must allow `GetObject` bucket-wide (they become + CopyObject *sources* for keys other nodes hand them) plus write/list on + their own prefix. +3. **Coordinator deploy**: on start it creates a unique partial index on + `nodeConf.epoch` — the mongo user needs `createIndex`. Existing configs + carry no epoch (treated as 0); the first `confapply` mints epoch 1. +4. **Deploy nodes with `shared: false` first** (or leave the flag out): the + machinery stays dormant. Expect a one-time `fresh disk or new node` + warning per node — the `.diskgen` marker is created on first boot after + the upgrade. Its creation time also arms the sweeper's 30-day protection + for unindexed legacy objects automatically. +5. **Dashboards/alerts before enabling**: panel the `node_resharder_*` and + `node_adopter_*` gauges; alert on `state == 2` lasting days, + `parked`/`errors` growing without `moved` growing, and coordinator + `unsafe configuration` rejections. +6. **Mint a baseline epoch**: republish the *current unchanged* topology + (`confapply ... -e`). Tree membership is identical, so no drains are + scheduled — it just assigns epoch 1 everywhere and starts the config + history that removed-node draining later relies on. +7. **Flip `s3Store.shared: true` fleet-wide.** Note: the first drain cycles + will also digest *legacy* orphans (spaces from past topology changes that + were never cleaned up) — expect `draining > 0` and background handoffs + without any new topology change. That is the intended cleanup; it is + verified the same way as a live drain. A legacy `notresponsible/` dir + from the old spacechecker tool shows up as an unindexable id in logs — + harmless; remove those dirs manually at leisure. +8. **Optional but recommended for the first real reshard**: enable bucket + object versioning with a 30-day expiry of noncurrent versions — cheap + insurance while trust in the machinery is being established; drop it + later. Snapshot the coordinator's `nodeConf` collection before the first + topology publish. +9. **Dry-run in staging** first: docs/resharding-e2e.md walks the full + add/remove/guardrail/delete-during-drain scenarios on a local network. + +## Adding node(s) + +1. Deploy the new node(s) with the shared `s3Store` config; start them. They + join with an empty storage root (logged as `fresh disk or new node`). +2. Add the node(s) to the network YAML with type `tree` and publish: + `confapply ... -e`. If the guardrail rejects the change (too many + partitions would change hands at once), add fewer nodes per step. +3. Within the nodeconf poll interval (`networkUpdateIntervalSec`, default + 10 min) every node logs `net configuration changed` and starts a drain + cycle. Nodes that lost partitions hand the affected spaces to the new + owner(s) through the bucket (archived spaces: server-side copy; live + spaces: snapshot + eager restore on the new node) and delete their local + copies only after **two current owners confirm the same heads**. +4. Watch progress per node (Prometheus): + - `node_resharder_state` — 0 disabled (bucket not shared), 1 idle, + 2 draining; the whole fleet back at 1 = resharding complete + - `node_resharder_epoch` — configuration epoch the node acts on; shows + config propagation across the fleet after a publish + - `node_resharder_draining` — spaces still to hand off (goal: 0) + - `node_resharder_moved` — spaces handed off and deleted locally + - `node_resharder_parked` / `node_resharder_errors` — handoffs postponed + to the next cycle (persistently high = an owner is unreachable or + diverged) and failed drain attempts + - `node_resharder_last_cycle_unix` — when the last drain cycle finished + - `node_adopter_adopted` on the new node — spaces received; + `node_adopter_already_have_same` on surviving owners — head-verified + ACKs; `node_adopter_rejected` — refused/failed adopt requests +5. Done when `node_resharder_state` is back to 1 (idle) and + `node_resharder_draining` is 0 on all nodes. No client action is + needed at any point: clients are bounced to the new owners by the normal + `ErrPeerIsNotResponsible` flow when they refresh the configuration. + +## Removing node(s) + +1. Publish a configuration without the node(s): `confapply ... -e`. The + guardrail ensures each partition keeps at least one current replica — + remove at most one member of any replica set per step (in practice: + remove one node at a time, or unrelated nodes together). +2. **Keep the removed node running.** It keeps polling the coordinator, sees + the new epoch, and starts draining everything it holds to the current + owners. Receivers recognize it through the retained configuration history + (last 100 epochs), so being absent from the current config does not block + the handoff. +3. Watch the removed node: `node_resharder_state` → 1 (idle), + `node_resharder_draining` → 0 and + `node_resharder_moved` settles. Spot-check with the debug API if desired: + all its spaces end in status `Moved` and its bucket prefix empties. +4. Shut the node down and decommission it. Leftover objects in its prefix + (if any) are collected by the daily archive sweep after the age guard. + +## Replacing a node + +Replace = add the new node (step above), wait for draining to finish, then +remove the old one. For a **dead** node (hardware loss, no drain possible): +publish the config without it; the surviving replicas hold every partition +(guardrail invariant) and the normal `nodesync` anti-entropy repopulates the +new owner(s). The dead node's bucket prefix can be deleted manually after +recovery is confirmed. + +## Spaces in Error state (missing or corrupted data) + +Resharding deliberately routes **around** broken copies instead of touching +them; redundancy is maintained by the healthy replicas. + +How a broken copy behaves during a reshard, by kind: + +- **Index status `Error`** (failed archive, drain found neither object nor + db, ...): excluded from drain candidates, its archive object is never + swept, and an owner holding an `Error` entry answers adopt requests with + *diverged* — it neither adopts a snapshot over the possibly-valuable local + data nor counts as a durable ACK for anyone's deletion. +- **Corrupted db with status `Ok`**: the drain's snapshot fails to open it — + the space parks and retries every cycle; `node_resharder_errors` grows and + the failure is logged with the space id. Nothing is deleted. +- **Status `Ok` with no local data at all**: nothing to hand off — the entry + is marked `Moved`; the current owners get the space from the healthy + replicas via the normal handoff/anti-entropy. + +Why this is safe: the drain deletion rule needs **2 of 3 current owners** to +confirm the heads, so one broken owner never blocks a reshard (the two +healthy ones ACK) and never enables a wrong deletion (it refuses to ACK). +The space stays fully available through its healthy replicas throughout. + +**Automatic self-healing (repairer).** A periodic loop (default hourly, +`repairer.repairIntervalMinutes`) repairs `Error` spaces this node is +responsible for: + +1. *Repair in place*: if the local db opens and indexes fine, the error was + transient (e.g. one failed archive upload) — the status is cleared without + touching data. +2. *Quarantine + re-pull*: a genuinely broken db is moved to + `/.quarantine/-` (data preserved for the + operator, invisible to the node) and a valid copy is pulled from a + responsible neighbor with the regular coldsync — works on every network, + shared bucket or not. The pulled copy is validated before the status + clears; on failure the space stays `Error` and is retried next cycle. + +Metrics: `node_repairer_errored` (responsible Error spaces left after the +last cycle — the alerting signal), `repaired`, `repaired_in_place`, +`quarantined`. + +Operator involvement is only needed for what the repairer refuses to touch: +`Error` spaces the node is *not* responsible for (drain territory — the +healthy owners already serve them; inspect with `spacechecker`, then clear +or delete), persistent `node_repairer_errored > 0` (all replicas +unreachable/broken — investigate the peers), and the `.quarantine` dir, +which is never cleaned automatically: delete old entries after confirming +the repaired spaces are healthy. + +## Safety properties (what the machinery guarantees) + +- A drained space is deleted locally only after ≥2 current owners durably + hold **exactly the advertised heads** (verified against the snapshot, with + an S3 existence check for archived copies). +- Writes landing during a handoff park the space for the next cycle — never + lost, never handed off stale. +- Spaces pending deletion are not drained; cancelled deletions cannot lose + the archived copy. +- A configuration can always be published mid-migration: nodes simply + retarget to the newest owners. Epochs never wait for drains to finish. diff --git a/docs/resharding-plan.md b/docs/resharding-plan.md new file mode 100644 index 00000000..2a89543c --- /dev/null +++ b/docs/resharding-plan.md @@ -0,0 +1,193 @@ +# Resharding: research & implementation plan + +*Drafted 2026-07-04. Based on a code audit of `any-sync-node` + `any-sync@v0.11.20` and an external design consult. Status: proposal.* + +## 1. Current state (what the code actually does) + +Placement is a **pure function** of `spaceId` and the network configuration: + +- `nodeconf` builds a consistent-hash ring (`go-chash`) over **tree nodes only**: `PartitionCount = 3000`, `ReplicationFactor = 3` (`any-sync/nodeconf/service.go:22`). Hash key is `ReplKey(spaceId)` — the suffix after the last `.` — so derived spaces co-locate (`nodeconf/nodeconf.go:126`). +- Everyone (nodes *and* clients) evaluates the same function locally. Config comes from the coordinator (`NetworkConfiguration(currentId)`), identified by an **opaque `ConfigurationId`** — no ordering/epoch semantics. Nodes poll every ~10 min (`nodeconf/service.go:112-150`) and atomically rebuild the chash on change; **no hook/event fires** on a config change. +- `IsResponsible` is enforced only against **client** peers (`nodespace/checks.go:14`); node→node requests are never rejected. +- Storage: one anystore/SQLite DB per space at `//store.db`; a per-node index DB holds `{spaceId → heads hash, SpaceStatus}` (`nodestorage/indexstorage.go`); `nodehead` maintains per-partition `ldiff` trees for anti-entropy. +- **Archive tier**: spaces not accessed for `ArchiveAfterDays` (default **7**) are backed up, gzipped, uploaded to S3 (`/`, per-node bucket/prefix config), marked `SpaceStatusArchived` (index keeps last heads hash + sizes), and the **local DB dir is deleted** (`archive/archive.go:94`). Restore is lazy: any open of an archived space — including `DumpStorage`, the coldsync server path — downloads from S3, sets status Ok, and **deletes the S3 object** (`storageservice.go:278`, `archive/archive.go:190`). Archived spaces **remain in the partition ldiff** with their archive-time heads (`indexstorage.go:125` — `ReadHashes` includes Ok + Archived). Each replica archives independently on its own last-access clock, so a dormant space typically exists as up to 3 separate gz objects. Given mostly-dormant user data, **the majority of spaces on a production node are expected to be Archived**. +- Data movement is a **repair loop only** (`nodesync/`): on start + every N hours, for each of the 3000 partitions where the node is a member, run ldiff against the other members; missing spaces → `coldsync` (full SQLite backup stream, refuses if space exists locally), diverged spaces → `hotsync` (normal tree sync, ~300 concurrent). + +**The gaps that make topology changes unsafe today:** + +1. Nothing reacts to a config change — sync is timer-driven, there is no old-vs-new membership diff. +2. Nothing ever **removes or hands off** spaces a node lost responsibility for. Orphaned copies accumulate (years' worth already). The only tool is the manual per-space `debug/spacechecker` `Fix`, which parks the dir under `notresponsible/` **without verifying new owners have the data**. +3. A new owner serves (rejects nothing from nodes, and for clients returns errors) before it has synced; an old owner keeps accepting stale-client writes; RF is silently violated in both directions during the transition window. +4. `coldsync` can't overwrite/merge into an existing stale local copy and has no resume for GB-size DBs. +5. Node replacement with the same identity but an empty disk is a **read black hole**: chash considers it fully operational immediately. +6. Anti-entropy can **resurrect deleted spaces**: a peer that lacks a space (because it processed the deletion log) will happily receive it again via coldsync from a node that still holds an orphaned copy. +7. **Archive amplification**: any naive migration of a mostly-archived partition becomes a restore storm — each coldsync of an archived space forces the source to S3-GET + unpack + disk-spike + S3-DELETE, stream the SQLite DB, and re-archive (S3 PUT) days later; a draining node would restore nearly its whole archived population just to push and then delete it. Node replacement also leaks: a rebuilt node's old S3 prefix keeps millions of orphaned gz objects nothing will ever clean. + +## 2. Design direction + +Given client-evaluated placement, weakly-synchronized config distribution, and CRDT space data, a strongly-consistent coordinator-driven migration state machine (Ceph/CockroachDB style) fights the architecture. The right shape is a **decentralized, epoch-aware handoff** (Dynamo/Cassandra bootstrap+drain style), leaning on CRDT convergence: + +> Placement stays a pure function. The coordinator versions configs with a monotonic **epoch** and enforces safety invariants before publishing. Each node, on observing an epoch change, locally diffs the old and new rings and runs a per-partition **Bootstrapping** (gained) or **Draining** (lost) state machine. The deletion invariant is **transport-agnostic**: a draining node may delete a space only after its state is **durably held by ≥2 current owners** — *how* the data got there doesn't matter. Bulk data movement is the **S3 data plane** ("force-archive + rename", §2.5): archived snapshots copied server-side between node prefixes plus a pointer-handoff RPC; the CRDT tree sync converges any delta. Deletions are protected by **tombstones** with a coordinator fallback. + +**Scope: resharding is a production-network feature.** Archiving is optional and most self-hosted networks run without S3 — but they don't need resharding either: with RF=3, any network of ≤3 tree nodes has every node owning every partition, so there is nothing to reshard; topology changes at that scale are handled by today's lazy `nodesync` (which the epoch/safety rails below still make safer). The migration machinery activates only when the network has a shared archive store configured (self-hosters who grow past 3 nodes can opt in with any S3-compatible store, e.g. MinIO). No node-to-node bulk-streaming transport is built for resharding — that was evaluated and dropped as serving a user that doesn't exist. + +### 2.1 Protocol/config changes + +- **Epoch**: add a monotonically increasing integer to `nodeconf.Configuration`; coordinator increments on every topology change. Keep `ConfigurationId` for compatibility. +- **Config history**: nodes persist the last K configs locally (tiny); coordinator additionally serves history by epoch. A node that adopts epoch N can always compute the N-1 ring (whom to pull from / push to), even after a restart mid-migration. +- **Coordinator guardrail (hard rule)**: before publishing epoch N+1, simulate the new ring and verify **replica overlap ≥1 (target 2) surviving member per partition** vs epoch N. Reject configs that replace all replicas of any partition at once. This is what makes "pull from current members" always possible and removes any need to serialize epochs — a new epoch may be published mid-migration; nodes simply retarget. +- **DiskGenID (replacement safety)**: a tree node initializing an empty storage dir generates a UUID and registers it with the coordinator. If a node restarts with a new/missing DiskGenID, it enters global Bootstrapping for **all** its partitions: refuse client reads for unsynced spaces (transient error → client retries another replica) until anti-entropy completes. Closes the black-hole-on-replacement trap. + +### 2.2 Node-local state machine + +On observing epoch N → N+1, diff the rings per partition: + +**Draining (lost partition P) — the sender drives the migration:** +1. Adopt the new ring: client writes for P are now rejected (`IsResponsible == false`); stragglers via legacy paths are still covered by step 3. +2. Per space S, **metadata first**: ask the current owners whether they hold S's heads (their index answers for live *and* archived copies — archived heads are frozen at archive time and stay in the index/ldiff). If ≥2 owners give a *durable ACK* (§2.5), skip to step 5 with **zero data transfer** — the common case, since RF=3 means the survivors already hold dormant spaces. +3. On divergence: `hotsync`-**push** latest local state to ≥2 surviving replicas and wait for their ACKs ("peer needs nothing after my push" *is* the superset proof; no new DAG-traversal RPC). For an archived local copy this needs a restore first, but a dormant space can't have diverged — rare path. +4. Hand the space to the new owner via the S3 data plane (§2.5): if archived — `CopyObject` own gz → target prefix, then `AdoptArchive` RPC; if live — `ForceArchive` (snapshot, DB stays live) → `CopyObject` → `AdoptArchive(eager=true)`. +5. **Delete locally only after ≥2 current owners durably hold the state** (metadata ACK, push ACK, or adopted archive). Overlap guarantee means ≥1 owner already had the data; 2 ACKs keep quorum durability even if one new owner dies immediately. "Delete locally" for an archived space means the index entry **and** the own-prefix S3 object. +6. If the network moves to epoch N+2 mid-drain, retarget to the newest owners; the rule is always "≥2 from *current* owners". + +**Bootstrapping (gained partition P) — mostly passive:** +- Normal case: the drainer pushes `AdoptArchive` pointers; the node verifies the object (S3 `HEAD`), registers `Archived` + heads in its index, and restores lazily on first access — or eagerly (background queue) when flagged, so hot spaces don't pay S3-GET latency on the next client request. After an eager restore, hotsync against peers converges the post-snapshot delta. +- Pull mode (drainer dead / unplanned loss): drive the same engine from the receiver — ask a surviving replica to `ForceArchive`/`CopyObject` + `AdoptArchive`. (The existing lazy `nodesync` coldsync also still converges missing spaces, as today.) +- Client cache-miss on a not-yet-adopted space → enqueue high-priority pull + return the existing transient error so the client retries a warm replica. Post-MVP: synchronous pull-on-demand for small spaces. + +### 2.3 Deletion tombstones (stop resurrection) + +- On processing a deletion-log record, do not just remove the DB: write a tombstone `{spaceId, status=Deleted, expiresAt = now + 30d}` in the index; GC tombstones after TTL. +- `coldsync`/handoff prep-request: receiver checks its index; if tombstoned → reply `ErrSpaceDeleted`; sender applies the tombstone locally and treats a drain handoff as **successful**. +- Fallback for expired tombstones / fresh nodes: on an incoming coldsync of an unknown *orphan-age* space, the receiver may ask the coordinator "is S in the deletion log?" before accepting. Keeps local tombstone state bounded; the coordinator's deletion log stays the global authority. + +### 2.4 Legacy orphan sweeper (years of accumulated copies) + +Low-priority, rate-limited background worker; for each local space with `IsResponsible == false`: + +1. Deleted? (local tombstone, else coordinator deletion-log check) → yes: delete local DB, done. +2. Integrity: DB fails to open/verify → move to `/quarantine/`, never sync corrupted data. +3. Resolve current 3 owners from the *current* ring. +4. Push & verify with the drain logic (hotsync push, need ≥2 ACKs) → delete local DB. +5. <2 ACKs (owners unreachable, disk full, timeouts) → **park**, retry next sweep cycle. Never delete unverified. + +This also subsumes and retires the manual `spacechecker` flow. + +**Archived orphans** (index entry + S3 object, no local DB): same flow, but the verify step is the metadata check (§2.5); on success delete both the index entry and the S3 object; if owners lack the heads, transfer the archive object (`ColdsyncArchive`) instead of restore+push. + +### 2.5 The S3 data plane ("force-archive + rename") + +Resharding never streams space data node-to-node on the main network: **S3 is the universal migration channel**, reusing the battle-tested archive machinery. This is the Cassandra snapshot-ship model adapted to CRDTs — the key property making it safe: *restoring from ANY snapshot (even stale) + tree-sync always converges; a snapshot is never wrong, only old.* + +Primitives: + +- **`ForceArchive(spaceId)`**: like `Archive()` but leaves the local DB intact and the status `Ok` — a pure snapshot-to-S3 (Backup + gzip + PUT). Used to ship live spaces; the post-snapshot delta converges via normal tree sync with the surviving replicas. +- **S3 rename**: server-side `CopyObject` from the source node's prefix to the target node's prefix (+ later delete of the source object). No bytes flow through node NICs; millions of copies cost dollars. A shared bucket across tree nodes is a **prerequisite** of the migration machinery (see scope note in §2); networks without one simply don't run it. +- **`AdoptArchive(spaceId, s3Key, headsHash, eager)` RPC** (control plane): tells the new owner "your copy is parked at this key". Receiver `HEAD`-verifies the object, registers `Archived` + heads in its index; `eager=true` queues an immediate background restore + hotsync (for spaces that were live on the drainer). +- **Durable ACK**: when an owner is asked "do you have heads H of space S?" and its index says `Archived`, it must S3-`HEAD` the object before ACKing (milliseconds, near-free). On a miss it repairs its index and NACKs, forcing a real transfer. Closes the "index says archived but the object is gone" hole — indexes are self-reported; disks and buckets are not trusted. +- **Per-node prefixes stay** (shared content-addressed namespace with cross-node dedup was considered and **rejected for now**): decentralized GC of shared objects is the classic data-loss vector — one buggy sweeper deleting a shared gz destroys the archive for *all* replicas at once, whereas per-node prefixes isolate the blast radius to one replica. If the ~3x archive storage cost ever becomes a problem, dedup can be added later with a strictly **centralized** GC that cross-references all node indexes. +- **Restore becomes copy, not move**: stop deleting the S3 object inside `Restore`; delete on the next successful re-archive instead. Idempotent restores, and a parked snapshot survives a botched restore. Optional hardening: content-addressed keys within the node prefix (`//.gz`) making objects immutable. +- **S3 prefix sweeper**: low-priority job listing the node's own prefix, deleting objects not matching the local index. Without it, node replacement (DiskGenID bootstrap-from-scratch) leaks the old prefix's millions of gz objects forever. +- **Tombstones delete archives too**: deletion-log processing for an `Archived` space issues the S3 DELETE directly (today's path would pointlessly restore the space just to delete it). +- **Throttling**: the bottleneck moves from node NIC/disk IOPS to S3 API limits. Token-bucket the migration workers' S3 calls (PUT/COPY) cluster-wide (e.g. hundreds/sec); a 100k-space partition handoff is 100k cheap index checks + a bounded stream of copies, with PUTs only for the live minority. + +Failure semantics are the big win: a node dying mid-migration leaves data safely parked in S3; the target just resumes pointer adoption. Source and target never need to be online simultaneously. + +### 2.6 Client-facing behavior + +**MVP requires zero client protocol changes.** Existing behavior suffices: + +- Stale client → drained node: gets `ErrPeerIsNotResponsible` (existing). Verified against the client code: there is **no reactive config refresh on this error** — the client converges via its periodic nodeconf poll (≤ `networkUpdateIntervalSec`, default 10 min; immediate on app start). This stays safe because the guardrail keeps ≥2 of any stale 3-owner list valid, clients resolve node ids from nodeconf **per call** (no per-space caching — verified in commonspace/diffsyncer and the sdk2 peer manager), and writes broadcast to all responsible peers, so one rejecting node never blocks the others. +- Fresh client → new owner that lacks the space: `getSpaceStorageFromRemote` tries every responsible peer in turn, so pulls fall through to the warm replicas; adopted-but-archived spaces restore lazily on first access on the node itself. + +Accepted MVP risks: up to one poll interval of degraded head-sync for affected spaces per epoch change (errors in logs, delayed convergence — not incorrectness); +1 RTT / one lazy restore for clients hitting a cold replica; the rare "cold replica + both warm replicas down" outage until adoption completes. Note the guardrail is therefore also a **client-correctness invariant**: `-force` must never be used while the stale-client window is open. + +Post-MVP client protocol (in priority order): +1. **Pull-on-demand** on bootstrap nodes (node-side only, biggest UX win). +2. **Epoch in client RPCs** + `ErrConfigStale(latestConfig)` fast-bounce: instant client convergence, no waiting on polls. Node treats `clientEpoch > nodeEpoch` as a single-flighted, rate-limited (≥5s) trigger to refetch from the coordinator, returning a transient error meanwhile; never trust the claimed epoch value itself (fetch, then compare against the coordinator's real latest). + +## 3. Failure modes explicitly covered + +- **Node dies mid-reshard / epoch published mid-migration**: no global "reshard in progress" state to corrupt; nodes recompute diffs against the newest ring and retarget. Config history makes restart-safe. +- **All replicas replaced at once**: rejected by the coordinator overlap guardrail. +- **Replaced hardware, same identity**: DiskGenID forces bootstrap mode instead of serving empty. +- **Offline old owner reconnecting with un-handed-off writes**: drain verification happens *after* pushing local heads, so late writes are pushed before deletion; CRDT merge handles concurrency. +- **Deleted-space resurrection via anti-entropy**: tombstones + coordinator fallback. +- **ReplKey co-location**: locks for on-demand pulls must be **per-spaceId**, not per-partition/replKey, so one giant space doesn't block siblings. +- **Corrupted local copy**: quarantined, never propagated. +- **Archived-but-missing S3 object**: durable ACK (`HEAD` before ACK) detects it; index gets repaired and the drainer keeps its copy. +- **Hot space migrated via snapshot**: writes between snapshot (T0) and write-cutoff (T1) are hotsync-pushed to the 2 surviving replicas before local delete; the adopter restores the T0 snapshot and converges the T0→T1 delta (and anything newer) from the survivors. No lost-delta window. +- **Stale gz vs newer live DB**: `AdoptArchive` carries the heads hash and only the copy's owner initiates handoff from its own index state; restore-as-copy + (optional) content-addressed keys close the rest. +- **Buggy sweeper / rogue node**: per-node S3 prefixes confine damage to that replica's archive; RF=3 peers unaffected (this is why shared-namespace dedup was rejected). +- **S3 storage leaks**: per-node S3 sweeper (replacement scenario) + tombstone-time S3 delete (deletion scenario). + +## 4. Observability & "done" definition + +Per-node Prometheus metrics: + +- `partitions_bootstrapping_total`, `partitions_draining_total` +- `spaces_orphaned_total`, `spaces_parked_total`, `spaces_quarantined_total` +- handoff throughput (bytes/sec), ACK counts, sweep progress + +A reshard for epoch N is **complete** when `sum(bootstrapping) == 0 && sum(draining) == 0` across the fleet. This is an operational signal, not a coordinator precondition — epochs never block on it. + +## 5. Phased plan + +**Phase 1 — Epochs & safety rails** *(coordinator + nodeconf changes)* +- Epoch field + config history (coordinator API + local persistence in `nodeconfstore`). +- Coordinator overlap guardrail on config publish. +- DiskGenID registration + global-bootstrap mode on mismatch. +- Config-change hook in `nodeconf.Service` (`setLastConfiguration` currently swaps silently — add a subscription). + +**Phase 2 — The S3 engine** *(data plane)* +- `ForceArchive` (snapshot without deleting/flipping the live DB). +- S3 `Transfer(src, dst)`: server-side `CopyObject` between node prefixes in the shared bucket (gated on the migration-machinery prerequisite; no streaming fallback is built). +- `AdoptArchive(spaceId, s3Key, headsHash, eager)` RPC + `HEAD` verification + eager-restore queue. +- Restore-as-copy (move S3 delete from `Restore` to re-archive; consider content-addressed keys here since it changes the key scheme). +- Cluster-wide S3-call token bucket for migration workers. +- Metrics. + +**Phase 3 — The state machine** *(bootstrap & drain, makes add/remove/replace safe)* +- Ring-diff on epoch change → per-partition Bootstrapping/Draining (today `nodesync/nodesync.go` `getRelatePartitions` only looks at the current ring, timer-driven). +- Drain sequence: metadata verification (durable ACK incl. S3 `HEAD`) → hotsync-push on divergence → ForceArchive/CopyObject/AdoptArchive handoff → ≥2-owner rule → local delete of DB/index/S3 object (reuse `SpaceStatus` transitions in `nodestorage/indexstorage.go`). +- Bootstrap: adopt pointers, eager-restore hot spaces, pull mode for dead-drainer recovery. +- Deletion tombstones + `ErrSpaceDeleted` handshake in handoff prep; tombstone execution deletes the S3 object for archived spaces directly. + +**Phase 4 — Cleanup & polish** +- Legacy orphan sweeper (with quarantine + park + archived branch), retiring the manual `spacechecker` flow — same logic as drain, run lazily. +- Per-node S3 prefix sweeper (orphaned gz objects after replacements/migrations). +- Tombstone TTL GC + coordinator deletion-log fallback. +- Pull-on-demand on bootstrap nodes. +- (Later) client epochs + `ErrConfigStale`; capacity weights in chash (lib already supports them — weight changes are just topology changes to this machinery); RF increase uses the same bootstrap path. + +Ordering rationale: the S3 engine is deliberately built *before* the state machine — it's small, independently testable (ForceArchive + CopyObject + AdoptArchive can be exercised manually on single spaces), and every later phase rides on it. What this plan **no longer contains** (killed by the S3 data plane): the `ColdsyncArchive` streaming RPC, coldsync overwrite/merge/resume changes, node-to-node streaming queues and their throttling. + +## 6. Open items to settle before implementation + +- Where epoch lives in the coordinator's data model and how existing `ConfigurationId` consumers migrate. +- Exact ACK semantics in the hotsync push session (today sync is symmetric; need an explicit "peer needs nothing" signal surfaced to the drain controller). +- Throttle defaults (bytes/sec per node, sweep rate) — derive from production disk/network headroom. +- Exact gating of the migration machinery: config flag vs auto-detect "all tree nodes share an archive bucket" via nodeconf metadata. Safety rails (epochs, guardrail, DiskGenID, tombstones) ship everywhere regardless; only drain/bootstrap data movement is gated. +- **Verify production S3 layout**: the design assumes per-node bucket/prefix. If any replicas currently share a prefix, today's restore-deletes-object behavior (`archive/archive.go:199`) is a live bug independent of resharding — one replica's restore breaks the others' archived state. +- Whether `DumpStorage`-triggered restores should be blocked once the S3 data plane exists (a node→node request for an archived space should go through `AdoptArchive`/`Transfer`, never force a restore on the source). +- Content-addressed S3 keys require a migration for existing objects (or dual-scheme lookup during a transition window). +- How a node detects "shared S3 bucket" for the `Transfer` fast path (nodeconf metadata vs static node config), and the IAM policy allowing cross-prefix `CopyObject` within the bucket. +- Sizing: measure the live-vs-archived ratio per partition in production to predict PUT volume (`ForceArchive` of the live minority) and migration wall-clock for a typical node decommission. + +## 7. Implementation status (2026-07-04, branch `resharding`) + +Implemented across three branches (any-sync, any-sync-coordinator, any-sync-node — all named `resharding`): + +- **Phase 1**: `Configuration.Epoch` + proto field; `nodeconf.Service.ObserveChanges`; `nodeconf.HistoryStore` (last 100 epochs on disk); coordinator epoch assignment + partition-overlap guardrail in `nodeconfsource.Add` (`confapply -force` escape hatch); `.diskgen` fresh-storage marker. +- **Phase 2**: `archivestore.Exists/Key/CopyFrom/List` + `shared` config flag; `archive.ForceArchive` (snapshot, DB stays live); restore-as-copy; `AdoptArchive` RPC + adopter component (HEAD-verified server-side copy, head-aware AlreadyHave responses, node-only); eager-restore queue. +- **Phase 3**: `resharder` component — drain cycle on config change + periodic; metadata-first verification, ≥2-current-owner ACK rule, heads recheck before deletion, hotsync convergence for diverged owners, `SpaceStatusMoved`; spacedeleter deletes S3 objects directly for archived spaces. +- **Phase 4**: archive prefix sweeper (daily, 7-day age guard) + unindexed-dir reconciliation. +- **Tests**: unit coverage per package + in-process integration test (`resharder/integration_test.go`) driving drain → adopt → eager-restore over two real node stacks and a shared in-memory bucket. Manual full-network runbook: `docs/resharding-e2e.md`. + +Deviations from the plan, deliberate: +- DiskGenID is a local marker only (no coordinator registration): a wiped disk simply looks like an empty node — reads of missing spaces already error (clients retry replicas), nodesync repopulates, and the guardrail prevents multi-replica replacement. Registration can be added later if operators want alerting. +- Drain verification uses AdoptArchive responses (Ok / AlreadyHaveSame / Diverged by index heads) instead of a separate heads-query RPC — one round-trip does both handoff and verification. Divergence resolves through hotsync (space load syncs with current owners) rather than an explicit push session. +- Bootstrapping is push-driven by drainers (plus existing lazy nodesync as pull-mode repair); no separate bootstrap state machine was needed. +- Quarantine of corrupted DBs is not automated (failed handoffs park and are visible via `node_resharder_parked`; `spacechecker` remains the operator tool). + +Rollout order: release any-sync → bump + release coordinator (guardrail is active immediately; epochs appear on the next `confapply`) → bump + roll nodes (machinery stays dormant until `s3Store.shared: true`). diff --git a/go.mod b/go.mod index 678954a6..bca83b7b 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module github.com/anyproto/any-sync-node -go 1.25.0 +go 1.25.7 require ( github.com/ahmetb/govvv v0.3.0 github.com/akrylysov/pogreb v0.10.3-0.20240803013244-523613e335e9 - github.com/anyproto/any-store v0.4.6 - github.com/anyproto/any-sync v0.11.20 + github.com/anyproto/any-store v0.4.7 + github.com/anyproto/any-sync v0.12.15-0.20260704195630-ae5feaa3bd4d github.com/anyproto/go-chash v0.1.0 github.com/aws/aws-sdk-go v1.55.8 github.com/cheggaaa/mb/v3 v3.0.2 @@ -16,12 +16,12 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/mock v0.6.0 go.uber.org/multierr v1.11.0 - go.uber.org/zap v1.27.1 - golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a - golang.org/x/net v0.52.0 + go.uber.org/zap v1.28.0 + golang.org/x/exp v0.0.0-20260603202125-055de637280b + golang.org/x/net v0.56.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - storj.io/drpc v0.0.34 + storj.io/drpc v1.0.0 ) require ( @@ -39,8 +39,9 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/disintegration/imaging v1.6.2 // indirect + github.com/dunglas/httpsfv v1.1.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/flopp/go-findfont v0.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect @@ -51,21 +52,21 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/yamux v0.1.2 // indirect github.com/huandu/skiplist v1.2.1 // indirect - github.com/ipfs/boxo v0.37.0 // indirect + github.com/ipfs/boxo v0.41.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect - github.com/ipfs/go-cid v0.6.0 // indirect + github.com/ipfs/go-cid v0.6.1 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect - github.com/libp2p/go-libp2p v0.47.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/libp2p/go-libp2p v0.48.0 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect + github.com/mr-tron/base58 v1.3.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.16.1 // indirect - github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multibase v0.3.0 // indirect github.com/multiformats/go-multicodec v0.10.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.6.1 // indirect @@ -74,22 +75,24 @@ require ( github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.17.0 // indirect - github.com/quic-go/quic-go v0.59.0 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.20.1 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.60.0 // indirect + github.com/quic-go/webtransport-go v0.11.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tetratelabs/wazero v1.10.1 // indirect - github.com/valyala/fastjson v1.6.7 // indirect + github.com/valyala/fastjson v1.6.10 // indirect github.com/zeebo/blake3 v0.2.4 // indirect github.com/zeebo/errs v1.3.0 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/crypto v0.49.0 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + golang.org/x/crypto v0.53.0 // indirect golang.org/x/image v0.21.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.35.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.43.0 // indirect + golang.org/x/tools v0.47.0 // indirect lukechampine.com/blake3 v1.4.1 // indirect modernc.org/libc v1.66.8 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index b1c42b52..839bf222 100644 --- a/go.sum +++ b/go.sum @@ -7,10 +7,10 @@ github.com/ahmetb/govvv v0.3.0 h1:YGLGwEyiUwHFy5eh/RUhdupbuaCGBYn5T5GWXp+WJB0= github.com/ahmetb/govvv v0.3.0/go.mod h1:4WRFpdWtc/YtKgPFwa1dr5+9hiRY5uKAL08bOlxOR6s= github.com/akrylysov/pogreb v0.10.3-0.20240803013244-523613e335e9 h1:GnBlbnor8geJB4j7GGXqVHiSG0cHNEkIYNmpYDAPa+Y= github.com/akrylysov/pogreb v0.10.3-0.20240803013244-523613e335e9/go.mod h1:fPb3n+7H42SxX84B4/os7POrR+UGRKSRr3kNS5+xq/c= -github.com/anyproto/any-store v0.4.6 h1:opnTUGfuHa/VA9Incu6iLEy3WsQ4mb2o1oAXGDysH0s= -github.com/anyproto/any-store v0.4.6/go.mod h1:Npi35qMUVZ8ouiV4o9AqpZDs6LbDOF+5ZLlVijXofFM= -github.com/anyproto/any-sync v0.11.20 h1:/yv38s5KJh1A9SmYj1zD5k8Fqp1v0UZae3n8KNBd5Z0= -github.com/anyproto/any-sync v0.11.20/go.mod h1:qR5leKtd4j6cnmFiaI4ULe+Az3qjhKpdFtScV7aFCHY= +github.com/anyproto/any-store v0.4.7 h1:329NWY/xUzGdwKSqFgjAFaPAVkTJbR+WdJirPrvTm/w= +github.com/anyproto/any-store v0.4.7/go.mod h1:8cqb52gjZSaYnlybugqpSqSG1RQygY96D2vWMbSJsLo= +github.com/anyproto/any-sync v0.12.15-0.20260704195630-ae5feaa3bd4d h1:7z5s2ywwj91DCvUw3k9Do/JGJZi8UqEDtBEGnrHyoEE= +github.com/anyproto/any-sync v0.12.15-0.20260704195630-ae5feaa3bd4d/go.mod h1:HObtLnsHG20r8rYfwDkpHF7FHzLMjQ/USGhEDLPBVc8= github.com/anyproto/go-bip39 v1.0.0 h1:T6/7WowKYDeyuX/QyXtt98ZX0XXaoOh17M/LFF2M5yk= github.com/anyproto/go-bip39 v1.0.0/go.mod h1:l0rcxmXRyiWAYzE1noMAc4qbeNrbhUwxM3rqSO9ILwo= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= @@ -57,10 +57,12 @@ github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54= +github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/flopp/go-findfont v0.1.0 h1:lPn0BymDUtJo+ZkV01VS3661HL6F4qFlkhcJN55u6mU= @@ -88,12 +90,12 @@ github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3 github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= github.com/huandu/skiplist v1.2.1 h1:dTi93MgjwErA/8idWTzIw4Y1kZsMWx35fmI2c8Rij7w= github.com/huandu/skiplist v1.2.1/go.mod h1:7v3iFjLcSAzO4fN5B8dvebvo/qsfumiLiDXMrPiHF9w= -github.com/ipfs/boxo v0.37.0 h1:2E3mZvydMI2t5IkAgtkmZ3sGsld0oS7o3I+xyzDk6uI= -github.com/ipfs/boxo v0.37.0/go.mod h1:8yyiRn54F2CsW13n0zwXEPrVsZix/gFj9SYIRYMZ6KE= +github.com/ipfs/boxo v0.41.0 h1:diKlFosOG2e1mgSO1CXqcMSnHvtn6ubUvaCf9iF8AIY= +github.com/ipfs/boxo v0.41.0/go.mod h1:1Fo36UVVvq3XAZwMDD82Cm4JTUi5x1k3AsJlg9DttOY= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= github.com/ipfs/go-block-format v0.2.3/go.mod h1:WJaQmPAKhD3LspLixqlqNFxiZ3BZ3xgqxxoSR/76pnA= -github.com/ipfs/go-cid v0.6.0 h1:DlOReBV1xhHBhhfy/gBNNTSyfOM6rLiIx9J7A4DGf30= -github.com/ipfs/go-cid v0.6.0/go.mod h1:NC4kS1LZjzfhK40UGmpXv5/qD2kcMzACYJNntCUiDhQ= +github.com/ipfs/go-cid v0.6.1 h1:T5TnNb08+ueovG76Z5gx1L4Y7QOaGTXHg1F6raWFxIc= +github.com/ipfs/go-cid v0.6.1/go.mod h1:zrY0SwOhjrrIdfPQ/kf+k1sXyJ0QE7cMxfCployLBs0= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -115,26 +117,26 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= -github.com/libp2p/go-libp2p v0.47.0 h1:qQpBjSCWNQFF0hjBbKirMXE9RHLtSuzTDkTfr1rw0yc= -github.com/libp2p/go-libp2p v0.47.0/go.mod h1:s8HPh7mMV933OtXzONaGFseCg/BE//m1V34p3x4EUOY= +github.com/libp2p/go-libp2p v0.48.0 h1:h2BrLAgrj7X8bEN05K7qmrjpNHYA+6tnsGRdprjTnvo= +github.com/libp2p/go-libp2p v0.48.0/go.mod h1:Q1fBZNdmC2Hf82husCTfkKJVfHm2we5zk+NWmOGEmWk= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= -github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= +github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= -github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= -github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multibase v0.3.0 h1:8helZD2+4Db7NNWFiktk2NePbF0boolBe6bDQvM4r68= +github.com/multiformats/go-multibase v0.3.0/go.mod h1:MoBLQPCkRTOL3eveIPO81860j2AQY8JwcnNlRkGRUfI= github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= @@ -160,12 +162,18 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= -github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= -github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= -github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= -github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= +github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0= +github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk= +github.com/quic-go/webtransport-go v0.11.0 h1:3afiZq7MHv3gmKCbMwZ8D5M1u0y/1RdONN9KlWp32J0= +github.com/quic-go/webtransport-go v0.11.0/go.mod h1:SHgEzUFVyj+9WUSuGB1P6Zd351Pww2leWV3SwlTovkA= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -179,8 +187,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8= github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU= -github.com/valyala/fastjson v1.6.7 h1:ZE4tRy0CIkh+qDc5McjatheGX2czdn8slQjomexVpBM= -github.com/valyala/fastjson v1.6.7/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4= +github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= @@ -197,44 +205,45 @@ go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= -golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= -golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q= +golang.org/x/exp v0.0.0-20260603202125-055de637280b/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.21.0 h1:c5qV36ajHpdj4Qi0GnE0jUc/yuo33OLFaa0d+crTD5s= golang.org/x/image v0.21.0/go.mod h1:vUbsLavqK/W303ZroQQVKQ+Af3Yl6Uz1Ppu5J/cLz78= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -276,5 +285,5 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -storj.io/drpc v0.0.34 h1:q9zlQKfJ5A7x8NQNFk8x7eKUF78FMhmAbZLnFK+og7I= -storj.io/drpc v0.0.34/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg= +storj.io/drpc v1.0.0 h1:1Xf1KCXXbV1viIfN56eqdJ3cNwpAL7OKwQkpS6ksing= +storj.io/drpc v1.0.0/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg= diff --git a/nodespace/spacedeleter/spacedeleter.go b/nodespace/spacedeleter/spacedeleter.go index f042387e..c11019fc 100644 --- a/nodespace/spacedeleter/spacedeleter.go +++ b/nodespace/spacedeleter/spacedeleter.go @@ -15,6 +15,7 @@ import ( "github.com/anyproto/any-sync/util/periodicsync" "go.uber.org/zap" + "github.com/anyproto/any-sync-node/archive/archivestore" "github.com/anyproto/any-sync-node/nodespace" "github.com/anyproto/any-sync-node/nodestorage" "github.com/anyproto/any-sync-node/nodesync" @@ -41,6 +42,7 @@ type spaceDeleter struct { spaceService nodespace.Service storageProvider nodestorage.NodeStorage nodeConf nodeconf.Service + archiveStore archivestore.ArchiveStore syncWaiter <-chan struct{} testOnce sync.Once @@ -54,6 +56,7 @@ func (s *spaceDeleter) Init(a *app.App) (err error) { s.storageProvider = a.MustComponent(nodestorage.CName).(nodestorage.NodeStorage) s.syncWaiter = a.MustComponent(nodesync.CName).(nodesync.NodeSync).WaitSyncOnStart() s.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.Service) + s.archiveStore = a.MustComponent(archivestore.CName).(archivestore.ArchiveStore) return } @@ -111,19 +114,34 @@ func (s *spaceDeleter) delete(ctx context.Context) (err error) { func (s *spaceDeleter) processDeletionRecord(ctx context.Context, rec *coordinatorproto.DeletionLogRecord) (err error) { log := log.With(zap.String("spaceId", rec.SpaceId), zap.String("deletionLogId", rec.Id), zap.String("status", rec.Status.String())) + + prevStatus, err := s.deletionStorage.SpaceStatus(ctx, rec.SpaceId) + if err != nil { + return err + } + deleteSpace := func() error { - // deleting space storage + if prevStatus == nodestorage.SpaceStatusArchived { + // the space lives in the archive store: delete the object directly + // instead of restoring it just to delete the db + if aErr := s.archiveStore.Delete(ctx, rec.SpaceId); aErr != nil && !errors.Is(aErr, archivestore.ErrNotFound) && !errors.Is(aErr, archivestore.ErrDisabled) { + return aErr + } + } + // remove the local db too: even for archived spaces a directory may + // linger (e.g. a crash between restore and the status flip). With the + // object already gone the restore attempt inside fails fast and the + // directory is removed regardless. err = s.storageProvider.DeleteSpaceStorage(ctx, rec.SpaceId) - if err != nil && !errors.Is(err, spacestorage.ErrSpaceStorageMissing) { + if err != nil && !errors.Is(err, spacestorage.ErrSpaceStorageMissing) && !errors.Is(err, archivestore.ErrNotFound) { return err } + // remove a leftover archive object if any (restore keeps objects) + if aErr := s.archiveStore.Delete(ctx, rec.SpaceId); aErr != nil && !errors.Is(aErr, archivestore.ErrNotFound) && !errors.Is(aErr, archivestore.ErrDisabled) { + log.Warn("can't delete archive object", zap.Error(aErr)) + } return s.deletionStorage.SetSpaceStatus(ctx, rec.SpaceId, nodestorage.SpaceStatusRemove, rec.Id) } - - prevStatus, err := s.deletionStorage.SpaceStatus(ctx, rec.SpaceId) - if err != nil { - return err - } if prevStatus == nodestorage.SpaceStatusRemove { log.Debug("space is already removed") err := s.deletionStorage.SetDeletionLogId(ctx, rec.Id) @@ -137,7 +155,12 @@ func (s *spaceDeleter) processDeletionRecord(ctx context.Context, rec *coordinat case coordinatorproto.DeletionLogRecordStatus_Ok: log.Debug("received deletion cancel record") status := nodestorage.SpaceStatusOk - if !s.nodeConf.IsResponsible(rec.SpaceId) { + if prevStatus == nodestorage.SpaceStatusArchived { + // an archived space has no local db: reverting to Ok would present + // the archive object as a stale leftover (and the sweeper would + // eventually collect the only copy); stay archived + status = nodestorage.SpaceStatusArchived + } else if !s.nodeConf.IsResponsible(rec.SpaceId) { status = nodestorage.SpaceStatusNotResponsible } err := s.deletionStorage.SetSpaceStatus(ctx, rec.SpaceId, status, rec.Id) diff --git a/nodespace/spacedeleter/spacedeleter_test.go b/nodespace/spacedeleter/spacedeleter_test.go index 7566fd18..ce0e2af0 100644 --- a/nodespace/spacedeleter/spacedeleter_test.go +++ b/nodespace/spacedeleter/spacedeleter_test.go @@ -12,7 +12,11 @@ import ( "github.com/anyproto/any-sync/testutil/anymock" "github.com/stretchr/testify/assert" + "github.com/anyproto/any-sync-node/archive/archivestore" + "github.com/anyproto/any-sync-node/archive/archivestore/mock_archivestore" "github.com/anyproto/any-sync-node/archive/mock_archive" + "github.com/anyproto/any-sync-node/nodehead" + "github.com/anyproto/any-sync-node/nodehead/mock_nodehead" "github.com/anyproto/any-sync-node/nodespace" "github.com/anyproto/any-sync-node/nodespace/mock_nodespace" "github.com/anyproto/any-sync-node/nodestorage" @@ -225,12 +229,18 @@ func newSpaceDeleterFixture(t *testing.T) *spaceDeleterFixture { nodeSync := mock_nodesync.NewMockNodeSync(ctrl) archive := mock_archive.NewMockArchive(ctrl) nodeConfMock := mock_nodeconf.NewMockService(ctrl) + archiveStore := mock_archivestore.NewMockArchiveStore(ctrl) + nodeHead := mock_nodehead.NewMockNodeHead(ctrl) storage := nodestorage.New() anymock.ExpectComp(coordClient.EXPECT(), coordinatorclient.CName) anymock.ExpectComp(spaceService.EXPECT(), nodespace.CName) anymock.ExpectComp(nodeSync.EXPECT(), nodesync.CName) anymock.ExpectComp(archive.EXPECT(), "node.archive") anymock.ExpectComp(nodeConfMock.EXPECT(), nodeconf.CName) + anymock.ExpectComp(archiveStore.EXPECT(), archivestore.CName) + anymock.ExpectComp(nodeHead.EXPECT(), nodehead.CName) + archiveStore.EXPECT().Delete(gomock.Any(), gomock.Any()).AnyTimes().Return(archivestore.ErrDisabled) + nodeHead.EXPECT().DeleteHeads(gomock.Any()).AnyTimes().Return(nil) nodeSync.EXPECT().WaitSyncOnStart().Return(waiterChan).AnyTimes() deleter := New().(*spaceDeleter) a.Register(storeConfig(dir)). @@ -240,6 +250,8 @@ func newSpaceDeleterFixture(t *testing.T) *spaceDeleterFixture { Register(archive). Register(nodeSync). Register(nodeConfMock). + Register(archiveStore). + Register(nodeHead). Register(deleter) err = a.Start(context.Background()) require.NoError(t, err) diff --git a/nodestorage/diskgen.go b/nodestorage/diskgen.go new file mode 100644 index 00000000..63b47093 --- /dev/null +++ b/nodestorage/diskgen.go @@ -0,0 +1,58 @@ +package nodestorage + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "time" +) + +const diskGenFileName = ".diskgen" + +// DiskGen marks a storage root with a generation id. The marker is created the +// first time a node starts on the given storage root; its absence on a node +// that expects to hold data means the disk was replaced (or wiped) and the +// node must treat all its content as missing rather than deleted. +type DiskGen struct { + // GenId is a random id generated when the storage root was initialized. + GenId string `json:"genId"` + // CreatedTime is when the storage root was initialized. + CreatedTime time.Time `json:"createdTime"` + // fresh is true if the marker was created by this process start. + fresh bool +} + +// Fresh reports whether the storage root was initialized on this start, +// i.e. no marker existed before (new node or replaced/wiped disk). +func (d DiskGen) Fresh() bool { + return d.fresh +} + +func loadOrCreateDiskGen(rootPath string) (gen DiskGen, err error) { + path := filepath.Join(rootPath, diskGenFileName) + data, readErr := os.ReadFile(path) + switch { + case readErr == nil: + if jsonErr := json.Unmarshal(data, &gen); jsonErr == nil && gen.GenId != "" { + return gen, nil + } + // corrupted marker: regenerate below, but this is not a fresh disk + case os.IsNotExist(readErr): + gen.fresh = true + default: + return gen, readErr + } + var buf [16]byte + if _, err = rand.Read(buf[:]); err != nil { + return + } + gen.GenId = hex.EncodeToString(buf[:]) + gen.CreatedTime = time.Now() + if data, err = json.Marshal(gen); err != nil { + return + } + err = os.WriteFile(path, data, 0o644) + return +} diff --git a/nodestorage/diskgen_test.go b/nodestorage/diskgen_test.go new file mode 100644 index 00000000..c6a28860 --- /dev/null +++ b/nodestorage/diskgen_test.go @@ -0,0 +1,32 @@ +package nodestorage + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadOrCreateDiskGen(t *testing.T) { + dir := t.TempDir() + + gen, err := loadOrCreateDiskGen(dir) + require.NoError(t, err) + assert.True(t, gen.Fresh()) + assert.NotEmpty(t, gen.GenId) + + gen2, err := loadOrCreateDiskGen(dir) + require.NoError(t, err) + assert.False(t, gen2.Fresh()) + assert.Equal(t, gen.GenId, gen2.GenId) + + // corrupted marker regenerates but is not treated as fresh + require.NoError(t, os.WriteFile(filepath.Join(dir, diskGenFileName), []byte("garbage"), 0o644)) + gen3, err := loadOrCreateDiskGen(dir) + require.NoError(t, err) + assert.False(t, gen3.Fresh()) + assert.NotEmpty(t, gen3.GenId) + assert.NotEqual(t, gen.GenId, gen3.GenId) +} diff --git a/nodestorage/indexstorage.go b/nodestorage/indexstorage.go index fcb6d6b6..e8ac6ee8 100644 --- a/nodestorage/indexstorage.go +++ b/nodestorage/indexstorage.go @@ -35,6 +35,9 @@ const ( SpaceStatusArchived SpaceStatusError SpaceStatusNotResponsible + // SpaceStatusMoved: the space was handed off to the current owners during + // resharding and its local data (db and/or archive object) was deleted. + SpaceStatusMoved ) var ( @@ -64,11 +67,14 @@ const ( type IndexStorage interface { UpdateHash(ctx context.Context, updates ...SpaceUpdate) (err error) ReadHashes(ctx context.Context, iterFunc func(update SpaceUpdate) (bool, error)) (err error) + ReadSpacesByStatus(ctx context.Context, status SpaceStatus, iterFunc func(spaceId string) (bool, error)) (err error) UpdateHashes(ctx context.Context, updateFunc func(spaceId, newHash, oldHash string) (newNewHash, newOldHash string, shouldUpdate bool)) (err error) SetSpaceStatus(ctx context.Context, spaceId string, status SpaceStatus, recId string) (err error) SpaceStatus(ctx context.Context, spaceId string) (status SpaceStatus, err error) SpaceStatusEntry(ctx context.Context, spaceId string) (entry SpaceStatusEntry, err error) MarkArchived(ctx context.Context, spaceId string, compressedSize, uncompressedSize int64) (err error) + DeleteSpaceEntry(ctx context.Context, spaceId string) (err error) + MarkArchivedRemote(ctx context.Context, spaceId, oldHash, newHash string, compressedSize, uncompressedSize int64) (err error) MarkError(ctx context.Context, spaceId string, errString string) (err error) DeletionLogId(ctx context.Context) (id string, err error) SetDeletionLogId(ctx context.Context, id string) (err error) @@ -151,6 +157,29 @@ func (d *indexStorage) ReadHashes(ctx context.Context, iterFunc func(update Spac return nil } +func (d *indexStorage) ReadSpacesByStatus(ctx context.Context, status SpaceStatus, iterFunc func(spaceId string) (bool, error)) (err error) { + filter := query.Key{ + Path: []string{statusKey}, + Filter: query.NewComp(query.CompOpEq, int(status)), + } + iter, err := d.spaceColl.Find(filter).Sort("id").Iter(ctx) + if err != nil { + return + } + defer iter.Close() + for iter.Next() { + doc, err := iter.Doc() + if err != nil { + return err + } + cont, err := iterFunc(doc.Value().GetString("id")) + if err != nil || !cont { + return err + } + } + return nil +} + func (d *indexStorage) SpaceStatus(ctx context.Context, spaceId string) (status SpaceStatus, err error) { doc, err := d.spaceColl.FindId(ctx, spaceId) if err != nil { @@ -219,6 +248,18 @@ func (d *indexStorage) SetSpaceStatus(ctx context.Context, spaceId string, statu return tx.Commit() } +// DeleteSpaceEntry removes the index entry entirely (unlike the deletion flow +// it leaves no tombstone). Used to garbage-collect entries of spaces that +// never durably existed anywhere, e.g. leftovers of an uncommitted space push. +func (d *indexStorage) DeleteSpaceEntry(ctx context.Context, spaceId string) (err error) { + err = d.spaceColl.DeleteId(ctx, spaceId) + if errors.Is(err, anystore.ErrDocNotFound) { + return nil + } + d.lastAccessCache.Delete(spaceId) + return err +} + func (d *indexStorage) MarkError(ctx context.Context, spaceId string, errString string) (err error) { _, err = d.spaceColl.UpdateId(ctx, spaceId, query.ModifyFunc(func(a *anyenc.Arena, v *anyenc.Value) (result *anyenc.Value, modified bool, err error) { v.Set(statusKey, a.NewNumberInt(int(SpaceStatusError))) @@ -238,6 +279,26 @@ func (d *indexStorage) MarkArchived(ctx context.Context, spaceId string, compres return err } +// MarkArchivedRemote registers a space adopted from another node as archived: +// unlike MarkArchived the space has never been opened locally, so the heads +// hashes come from the sender and the index entry may not exist yet. +func (d *indexStorage) MarkArchivedRemote(ctx context.Context, spaceId, oldHash, newHash string, compressedSize, uncompressedSize int64) (err error) { + _, err = d.spaceColl.UpsertId(ctx, spaceId, query.ModifyFunc(func(a *anyenc.Arena, v *anyenc.Value) (result *anyenc.Value, modified bool, err error) { + v.Set(oldHashKey, a.NewString(oldHash)) + v.Set(newHashKey, a.NewString(newHash)) + v.Set(lastAccessKey, a.NewNumberInt(int(time.Now().Unix()))) + v.Set(archiveSizeCompressedKey, a.NewNumberInt(int(compressedSize))) + v.Set(archiveSizeUncompressedKey, a.NewNumberInt(int(uncompressedSize))) + v.Set(statusKey, a.NewNumberInt(int(SpaceStatusArchived))) + return v, true, nil + })) + if err != nil { + return err + } + d.lastAccessCache.Store(spaceId, time.Now()) + return nil +} + func (d *indexStorage) DeletionLogId(ctx context.Context) (id string, err error) { doc, err := d.settingsColl.FindId(ctx, lastDeletionIdKey) if err != nil { diff --git a/nodestorage/mock_nodestorage/mock_nodestorage.go b/nodestorage/mock_nodestorage/mock_nodestorage.go index e7edc9c9..abbc442e 100644 --- a/nodestorage/mock_nodestorage/mock_nodestorage.go +++ b/nodestorage/mock_nodestorage/mock_nodestorage.go @@ -88,6 +88,20 @@ func (mr *MockNodeStorageMockRecorder) DeleteSpaceStorage(ctx, spaceId any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpaceStorage", reflect.TypeOf((*MockNodeStorage)(nil).DeleteSpaceStorage), ctx, spaceId) } +// DiskGen mocks base method. +func (m *MockNodeStorage) DiskGen() nodestorage.DiskGen { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiskGen") + ret0, _ := ret[0].(nodestorage.DiskGen) + return ret0 +} + +// DiskGen indicates an expected call of DiskGen. +func (mr *MockNodeStorageMockRecorder) DiskGen() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiskGen", reflect.TypeOf((*MockNodeStorage)(nil).DiskGen)) +} + // DumpStorage mocks base method. func (m *MockNodeStorage) DumpStorage(ctx context.Context, id string, do func(string) error) error { m.ctrl.T.Helper() @@ -212,6 +226,21 @@ func (mr *MockNodeStorageMockRecorder) OnWriteHash(onWrite any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnWriteHash", reflect.TypeOf((*MockNodeStorage)(nil).OnWriteHash), onWrite) } +// QuarantineSpace mocks base method. +func (m *MockNodeStorage) QuarantineSpace(ctx context.Context, spaceId string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QuarantineSpace", ctx, spaceId) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QuarantineSpace indicates an expected call of QuarantineSpace. +func (mr *MockNodeStorageMockRecorder) QuarantineSpace(ctx, spaceId any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QuarantineSpace", reflect.TypeOf((*MockNodeStorage)(nil).QuarantineSpace), ctx, spaceId) +} + // SpaceExists mocks base method. func (m *MockNodeStorage) SpaceExists(id string) bool { m.ctrl.T.Helper() @@ -336,6 +365,20 @@ func (mr *MockIndexStorageMockRecorder) Close() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockIndexStorage)(nil).Close)) } +// DeleteSpaceEntry mocks base method. +func (m *MockIndexStorage) DeleteSpaceEntry(ctx context.Context, spaceId string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteSpaceEntry", ctx, spaceId) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteSpaceEntry indicates an expected call of DeleteSpaceEntry. +func (mr *MockIndexStorageMockRecorder) DeleteSpaceEntry(ctx, spaceId any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpaceEntry", reflect.TypeOf((*MockIndexStorage)(nil).DeleteSpaceEntry), ctx, spaceId) +} + // DeletionLogId mocks base method. func (m *MockIndexStorage) DeletionLogId(ctx context.Context) (string, error) { m.ctrl.T.Helper() @@ -395,6 +438,20 @@ func (mr *MockIndexStorageMockRecorder) MarkArchived(ctx, spaceId, compressedSiz return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkArchived", reflect.TypeOf((*MockIndexStorage)(nil).MarkArchived), ctx, spaceId, compressedSize, uncompressedSize) } +// MarkArchivedRemote mocks base method. +func (m *MockIndexStorage) MarkArchivedRemote(ctx context.Context, spaceId, oldHash, newHash string, compressedSize, uncompressedSize int64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkArchivedRemote", ctx, spaceId, oldHash, newHash, compressedSize, uncompressedSize) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkArchivedRemote indicates an expected call of MarkArchivedRemote. +func (mr *MockIndexStorageMockRecorder) MarkArchivedRemote(ctx, spaceId, oldHash, newHash, compressedSize, uncompressedSize any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkArchivedRemote", reflect.TypeOf((*MockIndexStorage)(nil).MarkArchivedRemote), ctx, spaceId, oldHash, newHash, compressedSize, uncompressedSize) +} + // MarkError mocks base method. func (m *MockIndexStorage) MarkError(ctx context.Context, spaceId, errString string) error { m.ctrl.T.Helper() @@ -423,6 +480,20 @@ func (mr *MockIndexStorageMockRecorder) ReadHashes(ctx, iterFunc any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadHashes", reflect.TypeOf((*MockIndexStorage)(nil).ReadHashes), ctx, iterFunc) } +// ReadSpacesByStatus mocks base method. +func (m *MockIndexStorage) ReadSpacesByStatus(ctx context.Context, status nodestorage.SpaceStatus, iterFunc func(string) (bool, error)) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadSpacesByStatus", ctx, status, iterFunc) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReadSpacesByStatus indicates an expected call of ReadSpacesByStatus. +func (mr *MockIndexStorageMockRecorder) ReadSpacesByStatus(ctx, status, iterFunc any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadSpacesByStatus", reflect.TypeOf((*MockIndexStorage)(nil).ReadSpacesByStatus), ctx, status, iterFunc) +} + // RunMigrations mocks base method. func (m *MockIndexStorage) RunMigrations(ctx context.Context) error { m.ctrl.T.Helper() diff --git a/nodestorage/storageservice.go b/nodestorage/storageservice.go index fc40b7c2..6d233f11 100644 --- a/nodestorage/storageservice.go +++ b/nodestorage/storageservice.go @@ -77,6 +77,14 @@ type NodeStorage interface { DeleteSpaceStorage(ctx context.Context, spaceId string) error ForceRemove(id string) (err error) GetStats(ctx context.Context, id string, treeTop int) (spaceStats SpaceStats, err error) + // DiskGen returns the storage-root generation marker; DiskGen().Fresh() + // reports that the storage root was initialized on this start (new node or + // replaced/wiped disk). + DiskGen() DiskGen + // QuarantineSpace closes the space db and moves its directory aside + // (preserved under /.quarantine), e.g. before re-fetching a valid + // copy of a corrupted space from another node. + QuarantineSpace(ctx context.Context, spaceId string) (quarantinePath string, err error) } type StorageStats struct { @@ -108,6 +116,7 @@ type storageService struct { mu sync.Mutex statService debugstat.StatService archive archiveService + diskGen DiskGen } func (s *storageService) Init(a *app.App) (err error) { @@ -133,6 +142,12 @@ func (s *storageService) Init(a *app.App) (err error) { return err } } + if s.diskGen, err = loadOrCreateDiskGen(s.rootPath); err != nil { + return err + } + if s.diskGen.Fresh() { + log.Warn("storage root initialized on this start: fresh disk or new node", zap.String("diskGenId", s.diskGen.GenId)) + } comp, ok := a.Component(debugstat.CName).(debugstat.StatService) if !ok { comp = debugstat.NewNoOp() @@ -429,6 +444,10 @@ func (s *storageService) CreateSpaceStorage(ctx context.Context, payload spacest return newNodeStorage(st, cont, s.onHashChange), nil } +func (s *storageService) DiskGen() DiskGen { + return s.diskGen +} + func (s *storageService) GetStats(ctx context.Context, id string, treeTop int) (spaceStats SpaceStats, err error) { storage, err := s.WaitSpaceStorage(ctx, id) if err != nil { @@ -474,6 +493,28 @@ func (s *storageService) GetStats(ctx context.Context, id string, treeTop int) ( return } +// QuarantineSpace closes the space db and moves its directory into +// /.quarantine/- (dot-prefixed: invisible to +// AllSpaceIds). Used by the repairer to park a corrupted db before pulling a +// valid copy from another node; the data is preserved for the operator. +func (s *storageService) QuarantineSpace(ctx context.Context, spaceId string) (quarantinePath string, err error) { + if err = s.ForceRemove(spaceId); err != nil { + return + } + quarantineDir := filepath.Join(s.rootPath, ".quarantine") + if err = os.MkdirAll(quarantineDir, 0o755); err != nil { + return + } + quarantinePath = filepath.Join(quarantineDir, fmt.Sprintf("%s-%d", spaceId, time.Now().UnixNano())) + if err = os.Rename(s.StoreDir(spaceId), quarantinePath); err != nil { + return + } + if s.onDeleteStorage != nil { + s.onDeleteStorage(ctx, spaceId) + } + return +} + func (s *storageService) ForceRemove(id string) (err error) { ctx := context.Background() ss, err := s.cache.Pick(ctx, id) diff --git a/nodesync/nodesync.go b/nodesync/nodesync.go index e3c29771..5fc35bbb 100644 --- a/nodesync/nodesync.go +++ b/nodesync/nodesync.go @@ -72,10 +72,12 @@ func (n *nodeSync) Init(a *app.App) (err error) { registerMetric(n.syncStat, m.(metric.Metric).Registry()) } + adopter, _ := a.Component("node.archive.adopter").(archiveAdopter) return nodesyncproto.DRPCRegisterNodeSync(a.MustComponent(server.CName).(server.DRPCServer), &rpcHandler{ nodeRemoteDiffHandler: &nodeRemoteDiffHandler{nodehead: n.nodehead}, coldSync: n.coldsync, nodeSpace: n.nodespace, + adopter: adopter, }) } diff --git a/nodesync/nodesyncproto/errors.go b/nodesync/nodesyncproto/errors.go index f3fb2498..613d621c 100644 --- a/nodesync/nodesyncproto/errors.go +++ b/nodesync/nodesyncproto/errors.go @@ -12,4 +12,10 @@ var ( ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrCodes_Unexpected)) ErrExpectedCoordinator = errGroup.Register(errors.New("this request should be sent by coordinator"), uint64(ErrCodes_ExpectedCoordinator)) ErrUnsupportedStorageType = errGroup.Register(errors.New("unsupported storage"), uint64(ErrCodes_UnsupportedStorage)) + ErrSpaceDeleted = errGroup.Register(errors.New("space is deleted"), uint64(ErrCodes_SpaceDeleted)) + ErrArchiveUnavailable = errGroup.Register(errors.New("archive store is unavailable or not shared"), uint64(ErrCodes_ArchiveUnavailable)) + ErrArchiveObjectMissing = errGroup.Register(errors.New("archive object is missing"), uint64(ErrCodes_ArchiveObjectMissing)) + ErrPeerIsNotNode = errGroup.Register(errors.New("peer is not a network node"), uint64(ErrCodes_PeerIsNotNode)) + ErrNotResponsible = errGroup.Register(errors.New("node is not responsible for the space"), uint64(ErrCodes_NotResponsible)) + ErrSpacePendingDeletion = errGroup.Register(errors.New("space is pending deletion"), uint64(ErrCodes_SpacePendingDeletion)) ) diff --git a/nodesync/nodesyncproto/nodesync.pb.go b/nodesync/nodesyncproto/nodesync.pb.go index 1e7631ba..36217bfe 100644 --- a/nodesync/nodesyncproto/nodesync.pb.go +++ b/nodesync/nodesyncproto/nodesync.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.5 +// protoc-gen-go v1.36.11 // protoc v3.21.12 // source: nodesync/nodesyncproto/protos/nodesync.proto @@ -24,10 +24,16 @@ const ( type ErrCodes int32 const ( - ErrCodes_Unexpected ErrCodes = 0 - ErrCodes_ExpectedCoordinator ErrCodes = 1 - ErrCodes_UnsupportedStorage ErrCodes = 2 - ErrCodes_ErrorOffset ErrCodes = 1000 + ErrCodes_Unexpected ErrCodes = 0 + ErrCodes_ExpectedCoordinator ErrCodes = 1 + ErrCodes_UnsupportedStorage ErrCodes = 2 + ErrCodes_SpaceDeleted ErrCodes = 3 + ErrCodes_ArchiveUnavailable ErrCodes = 4 + ErrCodes_ArchiveObjectMissing ErrCodes = 5 + ErrCodes_PeerIsNotNode ErrCodes = 6 + ErrCodes_NotResponsible ErrCodes = 7 + ErrCodes_SpacePendingDeletion ErrCodes = 8 + ErrCodes_ErrorOffset ErrCodes = 1000 ) // Enum value maps for ErrCodes. @@ -36,13 +42,25 @@ var ( 0: "Unexpected", 1: "ExpectedCoordinator", 2: "UnsupportedStorage", + 3: "SpaceDeleted", + 4: "ArchiveUnavailable", + 5: "ArchiveObjectMissing", + 6: "PeerIsNotNode", + 7: "NotResponsible", + 8: "SpacePendingDeletion", 1000: "ErrorOffset", } ErrCodes_value = map[string]int32{ - "Unexpected": 0, - "ExpectedCoordinator": 1, - "UnsupportedStorage": 2, - "ErrorOffset": 1000, + "Unexpected": 0, + "ExpectedCoordinator": 1, + "UnsupportedStorage": 2, + "SpaceDeleted": 3, + "ArchiveUnavailable": 4, + "ArchiveObjectMissing": 5, + "PeerIsNotNode": 6, + "NotResponsible": 7, + "SpacePendingDeletion": 8, + "ErrorOffset": 1000, } ) @@ -119,6 +137,60 @@ func (ColdSyncProtocolType) EnumDescriptor() ([]byte, []int) { return file_nodesync_nodesyncproto_protos_nodesync_proto_rawDescGZIP(), []int{1} } +type AdoptArchiveResult int32 + +const ( + // the snapshot was copied and registered + AdoptArchiveResult_AdoptArchiveOk AdoptArchiveResult = 0 + // the receiver already holds a copy of the space with the same heads: + // counts as a durable ACK for the sender + AdoptArchiveResult_AdoptArchiveAlreadyHaveSame AdoptArchiveResult = 1 + // the receiver already holds a copy with different heads; the sender must + // converge via tree sync before the handoff can be acknowledged + AdoptArchiveResult_AdoptArchiveAlreadyHaveDiverged AdoptArchiveResult = 2 +) + +// Enum value maps for AdoptArchiveResult. +var ( + AdoptArchiveResult_name = map[int32]string{ + 0: "AdoptArchiveOk", + 1: "AdoptArchiveAlreadyHaveSame", + 2: "AdoptArchiveAlreadyHaveDiverged", + } + AdoptArchiveResult_value = map[string]int32{ + "AdoptArchiveOk": 0, + "AdoptArchiveAlreadyHaveSame": 1, + "AdoptArchiveAlreadyHaveDiverged": 2, + } +) + +func (x AdoptArchiveResult) Enum() *AdoptArchiveResult { + p := new(AdoptArchiveResult) + *p = x + return p +} + +func (x AdoptArchiveResult) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (AdoptArchiveResult) Descriptor() protoreflect.EnumDescriptor { + return file_nodesync_nodesyncproto_protos_nodesync_proto_enumTypes[2].Descriptor() +} + +func (AdoptArchiveResult) Type() protoreflect.EnumType { + return &file_nodesync_nodesyncproto_protos_nodesync_proto_enumTypes[2] +} + +func (x AdoptArchiveResult) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use AdoptArchiveResult.Descriptor instead. +func (AdoptArchiveResult) EnumDescriptor() ([]byte, []int) { + return file_nodesync_nodesyncproto_protos_nodesync_proto_rawDescGZIP(), []int{2} +} + // PartitionSyncRange presenting a request for one range type PartitionSyncRange struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -520,86 +592,209 @@ func (x *ColdSyncResponse) GetProtocolType() ColdSyncProtocolType { return ColdSyncProtocolType_Pogreb } +type AdoptArchiveRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + // srcKey is the full key of the gzipped snapshot within the shared bucket (sender's prefix) + SrcKey string `protobuf:"bytes,2,opt,name=srcKey,proto3" json:"srcKey,omitempty"` + // heads hashes of the space at snapshot time, as stored in the sender's index + OldHash string `protobuf:"bytes,3,opt,name=oldHash,proto3" json:"oldHash,omitempty"` + NewHash string `protobuf:"bytes,4,opt,name=newHash,proto3" json:"newHash,omitempty"` + // eager asks the receiver to restore the space in background right away + Eager bool `protobuf:"varint,5,opt,name=eager,proto3" json:"eager,omitempty"` + CompressedSize int64 `protobuf:"varint,6,opt,name=compressedSize,proto3" json:"compressedSize,omitempty"` + UncompressedSize int64 `protobuf:"varint,7,opt,name=uncompressedSize,proto3" json:"uncompressedSize,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AdoptArchiveRequest) Reset() { + *x = AdoptArchiveRequest{} + mi := &file_nodesync_nodesyncproto_protos_nodesync_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AdoptArchiveRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AdoptArchiveRequest) ProtoMessage() {} + +func (x *AdoptArchiveRequest) ProtoReflect() protoreflect.Message { + mi := &file_nodesync_nodesyncproto_protos_nodesync_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AdoptArchiveRequest.ProtoReflect.Descriptor instead. +func (*AdoptArchiveRequest) Descriptor() ([]byte, []int) { + return file_nodesync_nodesyncproto_protos_nodesync_proto_rawDescGZIP(), []int{7} +} + +func (x *AdoptArchiveRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *AdoptArchiveRequest) GetSrcKey() string { + if x != nil { + return x.SrcKey + } + return "" +} + +func (x *AdoptArchiveRequest) GetOldHash() string { + if x != nil { + return x.OldHash + } + return "" +} + +func (x *AdoptArchiveRequest) GetNewHash() string { + if x != nil { + return x.NewHash + } + return "" +} + +func (x *AdoptArchiveRequest) GetEager() bool { + if x != nil { + return x.Eager + } + return false +} + +func (x *AdoptArchiveRequest) GetCompressedSize() int64 { + if x != nil { + return x.CompressedSize + } + return 0 +} + +func (x *AdoptArchiveRequest) GetUncompressedSize() int64 { + if x != nil { + return x.UncompressedSize + } + return 0 +} + +type AdoptArchiveResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Result AdoptArchiveResult `protobuf:"varint,1,opt,name=result,proto3,enum=anyNodeSync.AdoptArchiveResult" json:"result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AdoptArchiveResponse) Reset() { + *x = AdoptArchiveResponse{} + mi := &file_nodesync_nodesyncproto_protos_nodesync_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AdoptArchiveResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AdoptArchiveResponse) ProtoMessage() {} + +func (x *AdoptArchiveResponse) ProtoReflect() protoreflect.Message { + mi := &file_nodesync_nodesyncproto_protos_nodesync_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AdoptArchiveResponse.ProtoReflect.Descriptor instead. +func (*AdoptArchiveResponse) Descriptor() ([]byte, []int) { + return file_nodesync_nodesyncproto_protos_nodesync_proto_rawDescGZIP(), []int{8} +} + +func (x *AdoptArchiveResponse) GetResult() AdoptArchiveResult { + if x != nil { + return x.Result + } + return AdoptArchiveResult_AdoptArchiveOk +} + var File_nodesync_nodesyncproto_protos_nodesync_proto protoreflect.FileDescriptor -var file_nodesync_nodesyncproto_protos_nodesync_proto_rawDesc = string([]byte{ - 0x0a, 0x2c, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x73, - 0x79, 0x6e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, - 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0b, - 0x61, 0x6e, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x79, 0x6e, 0x63, 0x22, 0x6a, 0x0a, 0x12, 0x50, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x61, 0x6e, 0x67, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x02, 0x74, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, - 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x65, - 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x84, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, - 0x61, 0x73, 0x68, 0x12, 0x43, 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x61, 0x6e, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x53, - 0x79, 0x6e, 0x63, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, 0x6e, - 0x63, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, - 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x40, - 0x0a, 0x1a, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, 0x6e, 0x63, 0x52, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x68, 0x65, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, - 0x22, 0x71, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, 0x6e, - 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x06, 0x72, 0x61, - 0x6e, 0x67, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x61, 0x6e, 0x79, - 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x79, 0x6e, 0x63, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x06, 0x72, 0x61, 0x6e, - 0x67, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x15, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, - 0x61, 0x6e, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x79, 0x6e, 0x63, 0x2e, 0x50, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, - 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x72, 0x0a, 0x0f, 0x43, 0x6f, 0x6c, 0x64, - 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x61, 0x6e, - 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x79, 0x6e, 0x63, 0x2e, 0x43, 0x6f, 0x6c, 0x64, 0x53, 0x79, - 0x6e, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x22, 0x9f, 0x01, 0x0a, - 0x10, 0x43, 0x6f, 0x6c, 0x64, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x72, 0x63, 0x33, 0x32, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x05, 0x63, 0x72, 0x63, 0x33, 0x32, 0x12, 0x45, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, - 0x61, 0x6e, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x79, 0x6e, 0x63, 0x2e, 0x43, 0x6f, 0x6c, 0x64, - 0x53, 0x79, 0x6e, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x2a, 0x5d, - 0x0a, 0x08, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x0e, 0x0a, 0x0a, 0x55, 0x6e, - 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x78, - 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x6f, - 0x72, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x55, 0x6e, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, - 0x65, 0x64, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0b, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x10, 0xe8, 0x07, 0x2a, 0x36, 0x0a, - 0x14, 0x43, 0x6f, 0x6c, 0x64, 0x53, 0x79, 0x6e, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x50, 0x6f, 0x67, 0x72, 0x65, 0x62, 0x10, - 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x41, 0x6e, 0x79, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x71, 0x6c, - 0x69, 0x74, 0x65, 0x10, 0x01, 0x32, 0xad, 0x01, 0x0a, 0x08, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x79, - 0x6e, 0x63, 0x12, 0x56, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x79, 0x6e, 0x63, 0x12, 0x21, 0x2e, 0x61, 0x6e, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x79, 0x6e, - 0x63, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, 0x6e, 0x63, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x61, 0x6e, 0x79, 0x4e, 0x6f, 0x64, 0x65, - 0x53, 0x79, 0x6e, 0x63, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, - 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x08, 0x43, 0x6f, - 0x6c, 0x64, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x1c, 0x2e, 0x61, 0x6e, 0x79, 0x4e, 0x6f, 0x64, 0x65, - 0x53, 0x79, 0x6e, 0x63, 0x2e, 0x43, 0x6f, 0x6c, 0x64, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x61, 0x6e, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x79, - 0x6e, 0x63, 0x2e, 0x43, 0x6f, 0x6c, 0x64, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x18, 0x5a, 0x16, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x79, 0x6e, - 0x63, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -}) +const file_nodesync_nodesyncproto_protos_nodesync_proto_rawDesc = "" + + "\n" + + ",nodesync/nodesyncproto/protos/nodesync.proto\x12\vanyNodeSync\"j\n" + + "\x12PartitionSyncRange\x12\x12\n" + + "\x04from\x18\x01 \x01(\x04R\x04from\x12\x0e\n" + + "\x02to\x18\x02 \x01(\x04R\x02to\x12\x14\n" + + "\x05limit\x18\x03 \x01(\rR\x05limit\x12\x1a\n" + + "\belements\x18\x04 \x01(\bR\belements\"\x84\x01\n" + + "\x13PartitionSyncResult\x12\x12\n" + + "\x04hash\x18\x01 \x01(\fR\x04hash\x12C\n" + + "\belements\x18\x02 \x03(\v2'.anyNodeSync.PartitionSyncResultElementR\belements\x12\x14\n" + + "\x05count\x18\x03 \x01(\rR\x05count\"@\n" + + "\x1aPartitionSyncResultElement\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + + "\x04head\x18\x02 \x01(\tR\x04head\"q\n" + + "\x14PartitionSyncRequest\x12 \n" + + "\vpartitionId\x18\x01 \x01(\x04R\vpartitionId\x127\n" + + "\x06ranges\x18\x02 \x03(\v2\x1f.anyNodeSync.PartitionSyncRangeR\x06ranges\"S\n" + + "\x15PartitionSyncResponse\x12:\n" + + "\aresults\x18\x01 \x03(\v2 .anyNodeSync.PartitionSyncResultR\aresults\"r\n" + + "\x0fColdSyncRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12E\n" + + "\fprotocolType\x18\x02 \x01(\x0e2!.anyNodeSync.ColdSyncProtocolTypeR\fprotocolType\"\x9f\x01\n" + + "\x10ColdSyncResponse\x12\x1a\n" + + "\bfilename\x18\x02 \x01(\tR\bfilename\x12\x12\n" + + "\x04data\x18\x03 \x01(\fR\x04data\x12\x14\n" + + "\x05crc32\x18\x04 \x01(\rR\x05crc32\x12E\n" + + "\fprotocolType\x18\x05 \x01(\x0e2!.anyNodeSync.ColdSyncProtocolTypeR\fprotocolType\"\xe5\x01\n" + + "\x13AdoptArchiveRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x16\n" + + "\x06srcKey\x18\x02 \x01(\tR\x06srcKey\x12\x18\n" + + "\aoldHash\x18\x03 \x01(\tR\aoldHash\x12\x18\n" + + "\anewHash\x18\x04 \x01(\tR\anewHash\x12\x14\n" + + "\x05eager\x18\x05 \x01(\bR\x05eager\x12&\n" + + "\x0ecompressedSize\x18\x06 \x01(\x03R\x0ecompressedSize\x12*\n" + + "\x10uncompressedSize\x18\a \x01(\x03R\x10uncompressedSize\"O\n" + + "\x14AdoptArchiveResponse\x127\n" + + "\x06result\x18\x01 \x01(\x0e2\x1f.anyNodeSync.AdoptArchiveResultR\x06result*\xe2\x01\n" + + "\bErrCodes\x12\x0e\n" + + "\n" + + "Unexpected\x10\x00\x12\x17\n" + + "\x13ExpectedCoordinator\x10\x01\x12\x16\n" + + "\x12UnsupportedStorage\x10\x02\x12\x10\n" + + "\fSpaceDeleted\x10\x03\x12\x16\n" + + "\x12ArchiveUnavailable\x10\x04\x12\x18\n" + + "\x14ArchiveObjectMissing\x10\x05\x12\x11\n" + + "\rPeerIsNotNode\x10\x06\x12\x12\n" + + "\x0eNotResponsible\x10\a\x12\x18\n" + + "\x14SpacePendingDeletion\x10\b\x12\x10\n" + + "\vErrorOffset\x10\xe8\a*6\n" + + "\x14ColdSyncProtocolType\x12\n" + + "\n" + + "\x06Pogreb\x10\x00\x12\x12\n" + + "\x0eAnystoreSqlite\x10\x01*n\n" + + "\x12AdoptArchiveResult\x12\x12\n" + + "\x0eAdoptArchiveOk\x10\x00\x12\x1f\n" + + "\x1bAdoptArchiveAlreadyHaveSame\x10\x01\x12#\n" + + "\x1fAdoptArchiveAlreadyHaveDiverged\x10\x022\x82\x02\n" + + "\bNodeSync\x12V\n" + + "\rPartitionSync\x12!.anyNodeSync.PartitionSyncRequest\x1a\".anyNodeSync.PartitionSyncResponse\x12I\n" + + "\bColdSync\x12\x1c.anyNodeSync.ColdSyncRequest\x1a\x1d.anyNodeSync.ColdSyncResponse0\x01\x12S\n" + + "\fAdoptArchive\x12 .anyNodeSync.AdoptArchiveRequest\x1a!.anyNodeSync.AdoptArchiveResponseB\x18Z\x16nodesync/nodesyncprotob\x06proto3" var ( file_nodesync_nodesyncproto_protos_nodesync_proto_rawDescOnce sync.Once @@ -613,34 +808,40 @@ func file_nodesync_nodesyncproto_protos_nodesync_proto_rawDescGZIP() []byte { return file_nodesync_nodesyncproto_protos_nodesync_proto_rawDescData } -var file_nodesync_nodesyncproto_protos_nodesync_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_nodesync_nodesyncproto_protos_nodesync_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_nodesync_nodesyncproto_protos_nodesync_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_nodesync_nodesyncproto_protos_nodesync_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_nodesync_nodesyncproto_protos_nodesync_proto_goTypes = []any{ (ErrCodes)(0), // 0: anyNodeSync.ErrCodes (ColdSyncProtocolType)(0), // 1: anyNodeSync.ColdSyncProtocolType - (*PartitionSyncRange)(nil), // 2: anyNodeSync.PartitionSyncRange - (*PartitionSyncResult)(nil), // 3: anyNodeSync.PartitionSyncResult - (*PartitionSyncResultElement)(nil), // 4: anyNodeSync.PartitionSyncResultElement - (*PartitionSyncRequest)(nil), // 5: anyNodeSync.PartitionSyncRequest - (*PartitionSyncResponse)(nil), // 6: anyNodeSync.PartitionSyncResponse - (*ColdSyncRequest)(nil), // 7: anyNodeSync.ColdSyncRequest - (*ColdSyncResponse)(nil), // 8: anyNodeSync.ColdSyncResponse + (AdoptArchiveResult)(0), // 2: anyNodeSync.AdoptArchiveResult + (*PartitionSyncRange)(nil), // 3: anyNodeSync.PartitionSyncRange + (*PartitionSyncResult)(nil), // 4: anyNodeSync.PartitionSyncResult + (*PartitionSyncResultElement)(nil), // 5: anyNodeSync.PartitionSyncResultElement + (*PartitionSyncRequest)(nil), // 6: anyNodeSync.PartitionSyncRequest + (*PartitionSyncResponse)(nil), // 7: anyNodeSync.PartitionSyncResponse + (*ColdSyncRequest)(nil), // 8: anyNodeSync.ColdSyncRequest + (*ColdSyncResponse)(nil), // 9: anyNodeSync.ColdSyncResponse + (*AdoptArchiveRequest)(nil), // 10: anyNodeSync.AdoptArchiveRequest + (*AdoptArchiveResponse)(nil), // 11: anyNodeSync.AdoptArchiveResponse } var file_nodesync_nodesyncproto_protos_nodesync_proto_depIdxs = []int32{ - 4, // 0: anyNodeSync.PartitionSyncResult.elements:type_name -> anyNodeSync.PartitionSyncResultElement - 2, // 1: anyNodeSync.PartitionSyncRequest.ranges:type_name -> anyNodeSync.PartitionSyncRange - 3, // 2: anyNodeSync.PartitionSyncResponse.results:type_name -> anyNodeSync.PartitionSyncResult - 1, // 3: anyNodeSync.ColdSyncRequest.protocolType:type_name -> anyNodeSync.ColdSyncProtocolType - 1, // 4: anyNodeSync.ColdSyncResponse.protocolType:type_name -> anyNodeSync.ColdSyncProtocolType - 5, // 5: anyNodeSync.NodeSync.PartitionSync:input_type -> anyNodeSync.PartitionSyncRequest - 7, // 6: anyNodeSync.NodeSync.ColdSync:input_type -> anyNodeSync.ColdSyncRequest - 6, // 7: anyNodeSync.NodeSync.PartitionSync:output_type -> anyNodeSync.PartitionSyncResponse - 8, // 8: anyNodeSync.NodeSync.ColdSync:output_type -> anyNodeSync.ColdSyncResponse - 7, // [7:9] is the sub-list for method output_type - 5, // [5:7] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 5, // 0: anyNodeSync.PartitionSyncResult.elements:type_name -> anyNodeSync.PartitionSyncResultElement + 3, // 1: anyNodeSync.PartitionSyncRequest.ranges:type_name -> anyNodeSync.PartitionSyncRange + 4, // 2: anyNodeSync.PartitionSyncResponse.results:type_name -> anyNodeSync.PartitionSyncResult + 1, // 3: anyNodeSync.ColdSyncRequest.protocolType:type_name -> anyNodeSync.ColdSyncProtocolType + 1, // 4: anyNodeSync.ColdSyncResponse.protocolType:type_name -> anyNodeSync.ColdSyncProtocolType + 2, // 5: anyNodeSync.AdoptArchiveResponse.result:type_name -> anyNodeSync.AdoptArchiveResult + 6, // 6: anyNodeSync.NodeSync.PartitionSync:input_type -> anyNodeSync.PartitionSyncRequest + 8, // 7: anyNodeSync.NodeSync.ColdSync:input_type -> anyNodeSync.ColdSyncRequest + 10, // 8: anyNodeSync.NodeSync.AdoptArchive:input_type -> anyNodeSync.AdoptArchiveRequest + 7, // 9: anyNodeSync.NodeSync.PartitionSync:output_type -> anyNodeSync.PartitionSyncResponse + 9, // 10: anyNodeSync.NodeSync.ColdSync:output_type -> anyNodeSync.ColdSyncResponse + 11, // 11: anyNodeSync.NodeSync.AdoptArchive:output_type -> anyNodeSync.AdoptArchiveResponse + 9, // [9:12] is the sub-list for method output_type + 6, // [6:9] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name } func init() { file_nodesync_nodesyncproto_protos_nodesync_proto_init() } @@ -653,8 +854,8 @@ func file_nodesync_nodesyncproto_protos_nodesync_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_nodesync_nodesyncproto_protos_nodesync_proto_rawDesc), len(file_nodesync_nodesyncproto_protos_nodesync_proto_rawDesc)), - NumEnums: 2, - NumMessages: 7, + NumEnums: 3, + NumMessages: 9, NumExtensions: 0, NumServices: 1, }, diff --git a/nodesync/nodesyncproto/nodesync_drpc.pb.go b/nodesync/nodesyncproto/nodesync_drpc.pb.go index e5319032..39eaa6cc 100644 --- a/nodesync/nodesyncproto/nodesync_drpc.pb.go +++ b/nodesync/nodesyncproto/nodesync_drpc.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go-drpc. DO NOT EDIT. -// protoc-gen-go-drpc version: v0.0.34 +// protoc-gen-go-drpc version: v1.0.0 // source: nodesync/nodesyncproto/protos/nodesync.proto package nodesyncproto @@ -35,6 +35,7 @@ type DRPCNodeSyncClient interface { PartitionSync(ctx context.Context, in *PartitionSyncRequest) (*PartitionSyncResponse, error) ColdSync(ctx context.Context, in *ColdSyncRequest) (DRPCNodeSync_ColdSyncClient, error) + AdoptArchive(ctx context.Context, in *AdoptArchiveRequest) (*AdoptArchiveResponse, error) } type drpcNodeSyncClient struct { @@ -96,9 +97,19 @@ func (x *drpcNodeSync_ColdSyncClient) RecvMsg(m *ColdSyncResponse) error { return x.MsgRecv(m, drpcEncoding_File_nodesync_nodesyncproto_protos_nodesync_proto{}) } +func (c *drpcNodeSyncClient) AdoptArchive(ctx context.Context, in *AdoptArchiveRequest) (*AdoptArchiveResponse, error) { + out := new(AdoptArchiveResponse) + err := c.cc.Invoke(ctx, "/anyNodeSync.NodeSync/AdoptArchive", drpcEncoding_File_nodesync_nodesyncproto_protos_nodesync_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + type DRPCNodeSyncServer interface { PartitionSync(context.Context, *PartitionSyncRequest) (*PartitionSyncResponse, error) ColdSync(*ColdSyncRequest, DRPCNodeSync_ColdSyncStream) error + AdoptArchive(context.Context, *AdoptArchiveRequest) (*AdoptArchiveResponse, error) } type DRPCNodeSyncUnimplementedServer struct{} @@ -111,9 +122,13 @@ func (s *DRPCNodeSyncUnimplementedServer) ColdSync(*ColdSyncRequest, DRPCNodeSyn return drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) } +func (s *DRPCNodeSyncUnimplementedServer) AdoptArchive(context.Context, *AdoptArchiveRequest) (*AdoptArchiveResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + type DRPCNodeSyncDescription struct{} -func (DRPCNodeSyncDescription) NumMethods() int { return 2 } +func (DRPCNodeSyncDescription) NumMethods() int { return 3 } func (DRPCNodeSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { switch n { @@ -135,6 +150,15 @@ func (DRPCNodeSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Receiv &drpcNodeSync_ColdSyncStream{in2.(drpc.Stream)}, ) }, DRPCNodeSyncServer.ColdSync, true + case 2: + return "/anyNodeSync.NodeSync/AdoptArchive", drpcEncoding_File_nodesync_nodesyncproto_protos_nodesync_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCNodeSyncServer). + AdoptArchive( + ctx, + in1.(*AdoptArchiveRequest), + ) + }, DRPCNodeSyncServer.AdoptArchive, true default: return "", nil, nil, nil, false } @@ -153,6 +177,10 @@ type drpcNodeSync_PartitionSyncStream struct { drpc.Stream } +func (x *drpcNodeSync_PartitionSyncStream) GetStream() drpc.Stream { + return x.Stream +} + func (x *drpcNodeSync_PartitionSyncStream) SendAndClose(m *PartitionSyncResponse) error { if err := x.MsgSend(m, drpcEncoding_File_nodesync_nodesyncproto_protos_nodesync_proto{}); err != nil { return err @@ -169,6 +197,30 @@ type drpcNodeSync_ColdSyncStream struct { drpc.Stream } +func (x *drpcNodeSync_ColdSyncStream) GetStream() drpc.Stream { + return x.Stream +} + func (x *drpcNodeSync_ColdSyncStream) Send(m *ColdSyncResponse) error { return x.MsgSend(m, drpcEncoding_File_nodesync_nodesyncproto_protos_nodesync_proto{}) } + +type DRPCNodeSync_AdoptArchiveStream interface { + drpc.Stream + SendAndClose(*AdoptArchiveResponse) error +} + +type drpcNodeSync_AdoptArchiveStream struct { + drpc.Stream +} + +func (x *drpcNodeSync_AdoptArchiveStream) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcNodeSync_AdoptArchiveStream) SendAndClose(m *AdoptArchiveResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_nodesync_nodesyncproto_protos_nodesync_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/nodesync/nodesyncproto/nodesync_vtproto.pb.go b/nodesync/nodesyncproto/nodesync_vtproto.pb.go index 102a1518..0b69f3a4 100644 --- a/nodesync/nodesyncproto/nodesync_vtproto.pb.go +++ b/nodesync/nodesyncproto/nodesync_vtproto.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go-vtproto. DO NOT EDIT. -// protoc-gen-go-vtproto version: v0.6.0 +// protoc-gen-go-vtproto version: v0.6.1-0.20240319094008-0393e58bdf10 // source: nodesync/nodesyncproto/protos/nodesync.proto package nodesyncproto @@ -377,6 +377,125 @@ func (m *ColdSyncResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *AdoptArchiveRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AdoptArchiveRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *AdoptArchiveRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.UncompressedSize != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.UncompressedSize)) + i-- + dAtA[i] = 0x38 + } + if m.CompressedSize != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.CompressedSize)) + i-- + dAtA[i] = 0x30 + } + if m.Eager { + i-- + if m.Eager { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x28 + } + if len(m.NewHash) > 0 { + i -= len(m.NewHash) + copy(dAtA[i:], m.NewHash) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.NewHash))) + i-- + dAtA[i] = 0x22 + } + if len(m.OldHash) > 0 { + i -= len(m.OldHash) + copy(dAtA[i:], m.OldHash) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.OldHash))) + i-- + dAtA[i] = 0x1a + } + if len(m.SrcKey) > 0 { + i -= len(m.SrcKey) + copy(dAtA[i:], m.SrcKey) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SrcKey))) + i-- + dAtA[i] = 0x12 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *AdoptArchiveResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AdoptArchiveResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *AdoptArchiveResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Result != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Result)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *PartitionSyncRange) SizeVT() (n int) { if m == nil { return 0 @@ -516,6 +635,54 @@ func (m *ColdSyncResponse) SizeVT() (n int) { return n } +func (m *AdoptArchiveRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.SrcKey) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.OldHash) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.NewHash) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Eager { + n += 2 + } + if m.CompressedSize != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.CompressedSize)) + } + if m.UncompressedSize != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.UncompressedSize)) + } + n += len(m.unknownFields) + return n +} + +func (m *AdoptArchiveResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Result != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Result)) + } + n += len(m.unknownFields) + return n +} + func (m *PartitionSyncRange) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1343,3 +1510,310 @@ func (m *ColdSyncResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *AdoptArchiveRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: AdoptArchiveRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: AdoptArchiveRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SrcKey", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SrcKey = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field OldHash", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.OldHash = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NewHash", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NewHash = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Eager", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Eager = bool(v != 0) + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CompressedSize", wireType) + } + m.CompressedSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CompressedSize |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UncompressedSize", wireType) + } + m.UncompressedSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UncompressedSize |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *AdoptArchiveResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: AdoptArchiveResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: AdoptArchiveResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Result", wireType) + } + m.Result = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Result |= AdoptArchiveResult(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/nodesync/nodesyncproto/protos/nodesync.proto b/nodesync/nodesyncproto/protos/nodesync.proto index 0d0db9ac..f6755c6e 100644 --- a/nodesync/nodesyncproto/protos/nodesync.proto +++ b/nodesync/nodesyncproto/protos/nodesync.proto @@ -7,6 +7,12 @@ enum ErrCodes { Unexpected = 0; ExpectedCoordinator = 1; UnsupportedStorage = 2; + SpaceDeleted = 3; + ArchiveUnavailable = 4; + ArchiveObjectMissing = 5; + PeerIsNotNode = 6; + NotResponsible = 7; + SpacePendingDeletion = 8; ErrorOffset = 1000; } @@ -15,6 +21,10 @@ service NodeSync { rpc PartitionSync(PartitionSyncRequest) returns (PartitionSyncResponse); // ColdSync requests cold sync stream for fast space download rpc ColdSync(ColdSyncRequest) returns (stream ColdSyncResponse); + // AdoptArchive offers the receiver an archived space snapshot parked in the + // shared archive store; the receiver copies the object into its own prefix + // and registers the space as archived without materializing it + rpc AdoptArchive(AdoptArchiveRequest) returns (AdoptArchiveResponse); } // PartitionSyncRange presenting a request for one range @@ -64,4 +74,32 @@ message ColdSyncResponse { enum ColdSyncProtocolType { Pogreb = 0; AnystoreSqlite = 1; +} + +message AdoptArchiveRequest { + string spaceId = 1; + // srcKey is the full key of the gzipped snapshot within the shared bucket (sender's prefix) + string srcKey = 2; + // heads hashes of the space at snapshot time, as stored in the sender's index + string oldHash = 3; + string newHash = 4; + // eager asks the receiver to restore the space in background right away + bool eager = 5; + int64 compressedSize = 6; + int64 uncompressedSize = 7; +} + +message AdoptArchiveResponse { + AdoptArchiveResult result = 1; +} + +enum AdoptArchiveResult { + // the snapshot was copied and registered + AdoptArchiveOk = 0; + // the receiver already holds a copy of the space with the same heads: + // counts as a durable ACK for the sender + AdoptArchiveAlreadyHaveSame = 1; + // the receiver already holds a copy with different heads; the sender must + // converge via tree sync before the handoff can be acknowledged + AdoptArchiveAlreadyHaveDiverged = 2; } \ No newline at end of file diff --git a/nodesync/rpchandler.go b/nodesync/rpchandler.go index c1f52b1f..172d0113 100644 --- a/nodesync/rpchandler.go +++ b/nodesync/rpchandler.go @@ -1,6 +1,8 @@ package nodesync import ( + "context" + "github.com/anyproto/any-sync-node/nodespace" "github.com/anyproto/any-sync-node/nodesync/coldsync" "github.com/anyproto/any-sync-node/nodesync/nodesyncproto" @@ -8,12 +10,26 @@ import ( var _ nodesyncproto.DRPCNodeSyncServer = (*rpcHandler)(nil) +// archiveAdopter is implemented by the archive adopter component +// (archive/adopter); looked up by name to avoid an import cycle. +type archiveAdopter interface { + AdoptArchive(ctx context.Context, req *nodesyncproto.AdoptArchiveRequest) (*nodesyncproto.AdoptArchiveResponse, error) +} + type rpcHandler struct { *nodeRemoteDiffHandler coldSync coldsync.ColdSync nodeSpace nodespace.Service + adopter archiveAdopter } func (r rpcHandler) ColdSync(req *nodesyncproto.ColdSyncRequest, stream nodesyncproto.DRPCNodeSync_ColdSyncStream) error { return r.coldSync.ColdSyncHandle(req, stream) } + +func (r rpcHandler) AdoptArchive(ctx context.Context, req *nodesyncproto.AdoptArchiveRequest) (*nodesyncproto.AdoptArchiveResponse, error) { + if r.adopter == nil { + return nil, nodesyncproto.ErrArchiveUnavailable + } + return r.adopter.AdoptArchive(ctx, req) +} diff --git a/repairer/integration_test.go b/repairer/integration_test.go new file mode 100644 index 00000000..e106f799 --- /dev/null +++ b/repairer/integration_test.go @@ -0,0 +1,212 @@ +package repairer + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/net/rpc" + "github.com/anyproto/any-sync/net/rpc/rpctest" + "github.com/anyproto/any-sync/net/rpc/server" + "github.com/anyproto/any-sync/nodeconf" + "github.com/anyproto/any-sync/nodeconf/mock_nodeconf" + "github.com/anyproto/any-sync/testutil/accounttest" + "github.com/anyproto/any-sync/testutil/anymock" + anystore "github.com/anyproto/any-store" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync-node/nodespace" + "github.com/anyproto/any-sync-node/nodespace/mock_nodespace" + "github.com/anyproto/any-sync-node/nodestorage" + "github.com/anyproto/any-sync-node/nodesync/coldsync" + "github.com/anyproto/any-sync-node/nodesync/nodesyncproto" +) + +// TestIntegration_RepairFlows drives the repairer end to end with two real +// node stacks and the real coldsync protocol over an in-process drpc pair: +// node B holds a real (GenStorage-created) space; node A repairs its broken +// local state by pulling from B. +func TestIntegration_RepairFlows(t *testing.T) { + t.Run("corrupted db: quarantined and replaced from neighbor", func(t *testing.T) { + fx := newIntFixture(t) + spaceId := fx.genSpaceOnB(t) + // node A: an index entry with heads and a corrupted db on disk + fx.corruptSpaceOnA(t, spaceId) + + require.NoError(t, fx.repairerA.repairSpace(spaceId)) + + // the corrupted db is preserved in quarantine + quarantined, err := os.ReadDir(filepath.Join(fx.dirA, ".quarantine")) + require.NoError(t, err) + require.Len(t, quarantined, 1) + assert.Contains(t, quarantined[0].Name(), spaceId) + fx.assertRepairedOnA(t, spaceId) + }) + t.Run("missing db file: pulled from neighbor", func(t *testing.T) { + fx := newIntFixture(t) + spaceId := fx.genSpaceOnB(t) + // node A: an index entry with heads, but no data on disk at all + require.NoError(t, fx.storageA.IndexStorage().UpdateHash(ctx, nodestorage.SpaceUpdate{ + SpaceId: spaceId, OldHash: "lost", NewHash: "lost", + })) + require.NoError(t, fx.storageA.IndexStorage().MarkError(ctx, spaceId, "db file is missing")) + + require.NoError(t, fx.repairerA.repairSpace(spaceId)) + fx.assertRepairedOnA(t, spaceId) + }) + t.Run("space missing everywhere: entry garbage-collected over the wire", func(t *testing.T) { + fx := newIntFixture(t) + const spaceId = "never.pushed" + // an entry without heads: leftover of an uncommitted push + require.NoError(t, fx.storageA.IndexStorage().SetSpaceStatus(ctx, spaceId, nodestorage.SpaceStatusError, "")) + + require.NoError(t, fx.repairerA.repairSpace(spaceId)) + + // the entry is gone entirely: the id is usable again + _, err := fx.storageA.IndexStorage().SpaceStatusEntry(ctx, spaceId) + assert.ErrorIs(t, err, anystore.ErrDocNotFound) + assert.Equal(t, uint32(1), fx.repairerA.stat.droppedEntries.Load()) + }) +} + +type intFixture struct { + appA, appB *app.App + dirA string + storageA nodestorage.NodeStorage + storageB nodestorage.NodeStorage + repairerA *repairer +} + +func (fx *intFixture) genSpaceOnB(t *testing.T) (spaceId string) { + store := nodestorage.GenStorage(t, fx.storageB, 3, 64) + return store.Id() +} + +func (fx *intFixture) corruptSpaceOnA(t *testing.T, spaceId string) { + spaceDir := fx.storageA.StoreDir(spaceId) + require.NoError(t, os.MkdirAll(spaceDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(spaceDir, "store.db"), []byte("this is not a sqlite database"), 0o644)) + require.NoError(t, fx.storageA.IndexStorage().UpdateHash(ctx, nodestorage.SpaceUpdate{ + SpaceId: spaceId, OldHash: "corrupt", NewHash: "corrupt", + })) + require.NoError(t, fx.storageA.IndexStorage().MarkError(ctx, spaceId, "malformed database")) +} + +func (fx *intFixture) assertRepairedOnA(t *testing.T, spaceId string) { + // status cleared, heads registered from the pulled copy + entry, err := fx.storageA.IndexStorage().SpaceStatusEntry(ctx, spaceId) + require.NoError(t, err) + assert.Equal(t, nodestorage.SpaceStatusOk, entry.Status) + assert.NotEqual(t, "corrupt", entry.NewHash) + assert.NotEqual(t, "lost", entry.NewHash) + // the pulled db is a valid space storage + ss, err := fx.storageA.SpaceStorage(ctx, spaceId) + require.NoError(t, err) + state, err := ss.StateStorage().GetState(ctx) + require.NoError(t, err) + assert.Equal(t, spaceId, state.SpaceId) + require.NoError(t, ss.Close(ctx)) +} + +// coldSyncServer exposes only the ColdSync RPC of the NodeSync service. +type coldSyncServer struct { + nodesyncproto.DRPCNodeSyncUnimplementedServer + coldSync coldsync.ColdSync +} + +func (s *coldSyncServer) ColdSync(req *nodesyncproto.ColdSyncRequest, stream nodesyncproto.DRPCNodeSync_ColdSyncStream) error { + return s.coldSync.ColdSyncHandle(req, stream) +} + +func newIntFixture(t *testing.T) *intFixture { + ctrl := gomock.NewController(t) + fx := &intFixture{ + appA: new(app.App), + appB: new(app.App), + dirA: t.TempDir(), + } + + newNode := func(a *app.App, dir string) (nodestorage.NodeStorage, coldsync.ColdSync, *rpctest.TestPool, server.DRPCServer) { + storage := nodestorage.New() + cs := coldsync.New() + tp := rpctest.NewTestPool() + ts := server.New() + nodeSpace := mock_nodespace.NewMockService(ctrl) + anymock.ExpectComp(nodeSpace.EXPECT(), nodespace.CName) + a.Register(intConfig{dir: dir}). + Register(&accounttest.AccountTestService{}). + Register(&archiveStub{}). + Register(storage). + Register(nodeSpace). + Register(tp). + Register(ts). + Register(cs) + return storage, cs, tp, ts + } + + var ( + csA, csB coldsync.ColdSync + tpA *rpctest.TestPool + tsA, tsB server.DRPCServer + ) + fx.storageA, csA, tpA, tsA = newNode(fx.appA, fx.dirA) + fx.storageB, csB, _, tsB = newNode(fx.appB, t.TempDir()) + + // node A's repairer with a nodeconf pointing at node B as the only owner + nodeConf := mock_nodeconf.NewMockService(ctrl) + anymock.ExpectComp(nodeConf.EXPECT(), nodeconf.CName) + nodeConf.EXPECT().IsResponsible(gomock.Any()).AnyTimes().Return(true) + nodeConf.EXPECT().NodeIds(gomock.Any()).AnyTimes().Return([]string{"nodeB"}) + fx.repairerA = &repairer{disableLoop: true} + fx.appA.Register(nodeConf).Register(fx.repairerA) + + require.NoError(t, fx.appA.Start(ctx)) + require.NoError(t, fx.appB.Start(ctx)) + + // serve only ColdSync on node B and wire a live in-process conn pair + require.NoError(t, nodesyncproto.DRPCRegisterNodeSync(tsB, &coldSyncServer{coldSync: csB})) + mcA, mcB := rpctest.MultiConnPair("nodeB", "nodeA") + pB, err := peer.NewPeer(mcA, tsA) + require.NoError(t, err) + require.NoError(t, tpA.AddPeer(ctx, pB)) + _, err = peer.NewPeer(mcB, tsB) + require.NoError(t, err) + + _ = csA + t.Cleanup(func() { + require.NoError(t, fx.appA.Close(ctx)) + require.NoError(t, fx.appB.Close(ctx)) + ctrl.Finish() + }) + return fx +} + +type intConfig struct { + dir string +} + +func (c intConfig) Init(_ *app.App) error { return nil } +func (c intConfig) Name() string { return "config" } + +func (c intConfig) GetStorage() nodestorage.Config { + return nodestorage.Config{Path: c.dir, AnyStorePath: c.dir} +} + +func (c intConfig) GetDrpc() rpc.Config { + return rpc.Config{Stream: rpc.StreamConfig{MaxMsgSizeMb: 10}} +} + +// archiveStub satisfies nodestorage's "node.archive" dependency. +type archiveStub struct{} + +func (a *archiveStub) Init(_ *app.App) error { return nil } +func (a *archiveStub) Name() string { return "node.archive" } + +func (a *archiveStub) Restore(_ context.Context, _ string) error { + return anystore.ErrDocNotFound +} diff --git a/repairer/repairer.go b/repairer/repairer.go new file mode 100644 index 00000000..489241d6 --- /dev/null +++ b/repairer/repairer.go @@ -0,0 +1,240 @@ +package repairer + +import ( + "context" + "errors" + "time" + + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/commonspace/spacesyncproto" + "github.com/anyproto/any-sync/metric" + "github.com/anyproto/any-sync/net/rpc/rpcerr" + "github.com/anyproto/any-sync/nodeconf" + "go.uber.org/zap" + + "github.com/anyproto/any-sync-node/nodestorage" + "github.com/anyproto/any-sync-node/nodesync/coldsync" +) + +const CName = "node.repairer" + +var log = logger.NewNamed(CName) + +const ( + defaultRepairIntervalMinutes = 60 + // spaceRepairTimeout bounds a single space repair (quarantine + cold pull) + spaceRepairTimeout = time.Minute * 10 +) + +func New() Repairer { + return new(repairer) +} + +// Repairer periodically self-heals spaces whose index status is Error +// (failed archive, corrupted db, missing data) while this node is responsible +// for them. It first tries to repair in place (the error may have been +// transient); a genuinely broken db is quarantined under /.quarantine +// (data preserved for the operator) and a valid copy is pulled from another +// responsible node with the regular coldsync — the same mechanic anti-entropy +// uses for missing spaces, so it works on every network, shared bucket or not. +type Repairer interface { + app.ComponentRunnable +} + +type configGetter interface { + GetRepairer() Config +} + +type Config struct { + // RepairIntervalMinutes is the period of the repair cycle, default 60. + RepairIntervalMinutes int `yaml:"repairIntervalMinutes"` +} + +type repairer struct { + nodeConf nodeconf.Service + storage nodestorage.NodeStorage + coldSync coldsync.ColdSync + conf Config + + runCtx context.Context + runCtxCancel context.CancelFunc + done chan struct{} + stat *repairerStat + + // disableLoop keeps the background loop off (unit tests drive cycles directly) + disableLoop bool +} + +func (r *repairer) Init(a *app.App) (err error) { + r.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.Service) + r.storage = a.MustComponent(nodestorage.CName).(nodestorage.NodeStorage) + r.coldSync = a.MustComponent(coldsync.CName).(coldsync.ColdSync) + if g, ok := a.Component("config").(configGetter); ok { + r.conf = g.GetRepairer() + } + if r.conf.RepairIntervalMinutes <= 0 { + r.conf.RepairIntervalMinutes = defaultRepairIntervalMinutes + } + r.runCtx, r.runCtxCancel = context.WithCancel(context.Background()) + r.done = make(chan struct{}) + r.stat = new(repairerStat) + if m := a.Component(metric.CName); m != nil { + registerMetric(r.stat, m.(metric.Metric).Registry()) + } + return +} + +func (r *repairer) Name() (name string) { + return CName +} + +func (r *repairer) Run(_ context.Context) (err error) { + if r.disableLoop { + close(r.done) + return + } + go r.loop() + return +} + +func (r *repairer) loop() { + defer close(r.done) + ticker := time.NewTicker(time.Duration(r.conf.RepairIntervalMinutes) * time.Minute) + defer ticker.Stop() + r.repairCycle() + for { + select { + case <-r.runCtx.Done(): + return + case <-ticker.C: + } + r.repairCycle() + } +} + +func (r *repairer) repairCycle() { + var candidates []string + err := r.storage.IndexStorage().ReadSpacesByStatus(r.runCtx, nodestorage.SpaceStatusError, func(spaceId string) (bool, error) { + // spaces we are not responsible for are not repairable from owners + // that way; they stay visible for the operator + if r.nodeConf.IsResponsible(spaceId) { + candidates = append(candidates, spaceId) + } + return true, nil + }) + if err != nil { + log.Warn("repair cycle: can't read index", zap.Error(err)) + return + } + r.stat.errored.Store(uint32(len(candidates))) + if len(candidates) == 0 { + return + } + log.Info("repair cycle started", zap.Int("spaces", len(candidates))) + var repaired, failed int + for _, spaceId := range candidates { + if r.runCtx.Err() != nil { + return + } + if err := r.repairSpace(spaceId); err != nil { + failed++ + log.Warn("space repair failed", zap.String("spaceId", spaceId), zap.Error(err)) + } else { + repaired++ + } + } + r.stat.errored.Store(uint32(failed)) + log.Info("repair cycle finished", zap.Int("repaired", repaired), zap.Int("failed", failed)) +} + +func (r *repairer) repairSpace(spaceId string) (err error) { + ctx, cancel := context.WithTimeout(r.runCtx, spaceRepairTimeout) + defer cancel() + index := r.storage.IndexStorage() + + entry, err := index.SpaceStatusEntry(ctx, spaceId) + if err != nil { + return + } + // the space must be openable for validation, and Error status blocks + // opening; on any failure below the status flips back to Error + if err = index.SetSpaceStatus(ctx, spaceId, nodestorage.SpaceStatusOk, ""); err != nil { + return + } + defer func() { + if err != nil { + if sErr := index.SetSpaceStatus(ctx, spaceId, nodestorage.SpaceStatusError, ""); sErr != nil { + log.Error("can't restore error status", zap.String("spaceId", spaceId), zap.Error(sErr)) + } + } + }() + + if r.storage.SpaceExists(spaceId) { + // the error may have been transient (e.g. a failed archive upload): + // if the db opens and indexes fine, the space is healthy in place + if _, ipErr := r.storage.IndexSpace(ctx, spaceId, true); ipErr == nil { + log.Info("space repaired in place", zap.String("spaceId", spaceId)) + r.stat.repairedInPlace.Add(1) + return nil + } + // genuinely broken: park the data for the operator and pull a fresh copy + quarantinePath, qErr := r.storage.QuarantineSpace(ctx, spaceId) + if qErr != nil { + return qErr + } + r.stat.quarantined.Add(1) + log.Warn("corrupted space quarantined", zap.String("spaceId", spaceId), zap.String("path", quarantinePath)) + } + + // pull a valid copy from a responsible neighbor + var ( + pulled bool + owners = r.nodeConf.NodeIds(spaceId) + missing int + ) + for _, peerId := range owners { + pErr := r.coldSync.Sync(ctx, spaceId, peerId) + if pErr == nil { + pulled = true + break + } + if errors.Is(rpcerr.Unwrap(pErr), spacesyncproto.ErrSpaceMissing) { + // the owner definitively does not hold the space (as opposed to + // being unreachable or holding a broken copy) + missing++ + } + log.Debug("repair: cold pull failed", zap.String("spaceId", spaceId), zap.String("peerId", peerId), zap.Error(pErr)) + } + if !pulled { + if len(owners) > 0 && missing == len(owners) && entry.NewHash == "" { + // nobody holds the space, we hold nothing, and no heads were ever + // recorded: the entry is a leftover of a space that never durably + // existed (e.g. an uncommitted space push) — garbage-collect it so + // the id is usable again and the error stops alerting + if dErr := index.DeleteSpaceEntry(ctx, spaceId); dErr != nil { + return dErr + } + log.Info("repair: dropped an entry of a space that never existed anywhere", zap.String("spaceId", spaceId)) + r.stat.droppedEntries.Add(1) + return nil + } + return errNoValidCopy + } + // validate and register the pulled copy (index hashes + nodehead) + if _, err = r.storage.IndexSpace(ctx, spaceId, true); err != nil { + return err + } + log.Info("space repaired from a neighbor", zap.String("spaceId", spaceId)) + r.stat.repaired.Add(1) + return nil +} + +func (r *repairer) Close(_ context.Context) (err error) { + r.runCtxCancel() + select { + case <-r.done: + case <-time.After(time.Second * 10): + } + return +} diff --git a/repairer/repairer_test.go b/repairer/repairer_test.go new file mode 100644 index 00000000..1c1acec3 --- /dev/null +++ b/repairer/repairer_test.go @@ -0,0 +1,178 @@ +package repairer + +import ( + "context" + "errors" + "testing" + + "github.com/anyproto/any-sync/commonspace/spacesyncproto" + + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/nodeconf" + "github.com/anyproto/any-sync/nodeconf/mock_nodeconf" + "github.com/anyproto/any-sync/testutil/anymock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync-node/nodestorage" + "github.com/anyproto/any-sync-node/nodestorage/mock_nodestorage" + "github.com/anyproto/any-sync-node/nodesync/coldsync" + "github.com/anyproto/any-sync-node/nodesync/coldsync/mock_coldsync" +) + +var ctx = context.Background() + +const spaceId = "err.space" + +func TestRepairer_RepairSpace(t *testing.T) { + t.Run("transient error repaired in place", func(t *testing.T) { + fx := newFixture(t) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusError, NewHash: "someHash"}, nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusOk, "").Return(nil) + fx.storage.EXPECT().SpaceExists(spaceId).Return(true) + fx.storage.EXPECT().IndexSpace(gomock.Any(), spaceId, true).Return(nil, nil) + + require.NoError(t, fx.repairSpace(spaceId)) + assert.Equal(t, uint32(1), fx.stat.repairedInPlace.Load()) + }) + t.Run("corrupted db quarantined and pulled from a neighbor", func(t *testing.T) { + fx := newFixture(t) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusError, NewHash: "someHash"}, nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusOk, "").Return(nil) + fx.storage.EXPECT().SpaceExists(spaceId).Return(true) + // in-place validation fails: db is corrupted + fx.storage.EXPECT().IndexSpace(gomock.Any(), spaceId, true).Return(nil, errors.New("malformed database")) + fx.storage.EXPECT().QuarantineSpace(gomock.Any(), spaceId).Return("/quarantine/err.space-1", nil) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1", "p2"}) + // first peer fails, second delivers + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p1").Return(errors.New("unreachable")) + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p2").Return(nil) + fx.storage.EXPECT().IndexSpace(gomock.Any(), spaceId, true).Return(nil, nil) + + require.NoError(t, fx.repairSpace(spaceId)) + assert.Equal(t, uint32(1), fx.stat.quarantined.Load()) + assert.Equal(t, uint32(1), fx.stat.repaired.Load()) + }) + t.Run("missing data pulled without quarantine", func(t *testing.T) { + fx := newFixture(t) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusError, NewHash: "someHash"}, nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusOk, "").Return(nil) + fx.storage.EXPECT().SpaceExists(spaceId).Return(false) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1"}) + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p1").Return(nil) + fx.storage.EXPECT().IndexSpace(gomock.Any(), spaceId, true).Return(nil, nil) + + require.NoError(t, fx.repairSpace(spaceId)) + assert.Equal(t, uint32(1), fx.stat.repaired.Load()) + }) + t.Run("no valid copy anywhere: status flips back to Error", func(t *testing.T) { + fx := newFixture(t) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusError, NewHash: "someHash"}, nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusOk, "").Return(nil) + fx.storage.EXPECT().SpaceExists(spaceId).Return(false) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1", "p2"}) + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p1").Return(errors.New("unreachable")) + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p2").Return(errors.New("unreachable")) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusError, "").Return(nil) + + err := fx.repairSpace(spaceId) + assert.ErrorIs(t, err, errNoValidCopy) + }) + t.Run("never existed anywhere: entry garbage-collected", func(t *testing.T) { + fx := newFixture(t) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusError}, nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusOk, "").Return(nil) + fx.storage.EXPECT().SpaceExists(spaceId).Return(false) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1", "p2"}) + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p1").Return(spacesyncproto.ErrSpaceMissing) + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p2").Return(spacesyncproto.ErrSpaceMissing) + fx.indexStorage.EXPECT().DeleteSpaceEntry(gomock.Any(), spaceId).Return(nil) + + require.NoError(t, fx.repairSpace(spaceId)) + assert.Equal(t, uint32(1), fx.stat.droppedEntries.Load()) + }) + t.Run("missing everywhere but heads were recorded: keep Error", func(t *testing.T) { + fx := newFixture(t) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusError, NewHash: "realHeads"}, nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusOk, "").Return(nil) + fx.storage.EXPECT().SpaceExists(spaceId).Return(false) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1", "p2"}) + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p1").Return(spacesyncproto.ErrSpaceMissing) + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p2").Return(spacesyncproto.ErrSpaceMissing) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusError, "").Return(nil) + + assert.ErrorIs(t, fx.repairSpace(spaceId), errNoValidCopy) + }) + t.Run("pulled copy fails validation: status flips back to Error", func(t *testing.T) { + fx := newFixture(t) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusError, NewHash: "someHash"}, nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusOk, "").Return(nil) + fx.storage.EXPECT().SpaceExists(spaceId).Return(false) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1"}) + fx.coldSync.EXPECT().Sync(gomock.Any(), spaceId, "p1").Return(nil) + fx.storage.EXPECT().IndexSpace(gomock.Any(), spaceId, true).Return(nil, errors.New("malformed database")) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusError, "").Return(nil) + + assert.Error(t, fx.repairSpace(spaceId)) + }) +} + +func TestRepairer_RepairCycle(t *testing.T) { + fx := newFixture(t) + // two errored spaces: one responsible (repairable), one not (operator territory) + fx.indexStorage.EXPECT().ReadSpacesByStatus(gomock.Any(), nodestorage.SpaceStatusError, gomock.Any()).DoAndReturn( + func(_ context.Context, _ nodestorage.SpaceStatus, iter func(string) (bool, error)) error { + _, _ = iter(spaceId) + _, _ = iter("foreign.space") + return nil + }) + fx.nodeConf.EXPECT().IsResponsible(spaceId).Return(true) + fx.nodeConf.EXPECT().IsResponsible("foreign.space").Return(false) + // repairSpace path for the responsible one + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(nodestorage.SpaceStatusEntry{Status: nodestorage.SpaceStatusError, NewHash: "someHash"}, nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusOk, "").Return(nil) + fx.storage.EXPECT().SpaceExists(spaceId).Return(true) + fx.storage.EXPECT().IndexSpace(gomock.Any(), spaceId, true).Return(nil, nil) + + fx.repairCycle() + assert.Equal(t, uint32(0), fx.stat.errored.Load()) + assert.Equal(t, uint32(1), fx.stat.repairedInPlace.Load()) +} + +type fixture struct { + *repairer + a *app.App + storage *mock_nodestorage.MockNodeStorage + indexStorage *mock_nodestorage.MockIndexStorage + nodeConf *mock_nodeconf.MockService + coldSync *mock_coldsync.MockColdSync +} + +func newFixture(t *testing.T) *fixture { + ctrl := gomock.NewController(t) + fx := &fixture{ + repairer: &repairer{disableLoop: true}, + a: new(app.App), + storage: mock_nodestorage.NewMockNodeStorage(ctrl), + indexStorage: mock_nodestorage.NewMockIndexStorage(ctrl), + nodeConf: mock_nodeconf.NewMockService(ctrl), + coldSync: mock_coldsync.NewMockColdSync(ctrl), + } + anymock.ExpectComp(fx.storage.EXPECT(), nodestorage.CName) + anymock.ExpectComp(fx.nodeConf.EXPECT(), nodeconf.CName) + anymock.ExpectComp(fx.coldSync.EXPECT(), coldsync.CName) + fx.storage.EXPECT().IndexStorage().AnyTimes().Return(fx.indexStorage) + + fx.a.Register(fx.storage). + Register(fx.nodeConf). + Register(fx.coldSync). + Register(fx.repairer) + + require.NoError(t, fx.a.Start(ctx)) + t.Cleanup(func() { + require.NoError(t, fx.a.Close(ctx)) + ctrl.Finish() + }) + return fx +} diff --git a/repairer/stat.go b/repairer/stat.go new file mode 100644 index 00000000..3141329d --- /dev/null +++ b/repairer/stat.go @@ -0,0 +1,51 @@ +package repairer + +import ( + "errors" + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" +) + +var errNoValidCopy = errors.New("no responsible node could provide a valid copy") + +type repairerStat struct { + // errored is the number of responsible spaces in Error status after the + // last repair cycle (0 = nothing left to repair) + errored atomic.Uint32 + // repairedInPlace: transient errors cleared without touching data + repairedInPlace atomic.Uint32 + // quarantined: corrupted dbs parked under /.quarantine + quarantined atomic.Uint32 + // repaired: valid copies pulled from responsible neighbors + repaired atomic.Uint32 + // droppedEntries: garbage-collected entries of spaces that never existed + // anywhere (definitively missing on all owners, no heads recorded) + droppedEntries atomic.Uint32 +} + +func registerMetric(s *repairerStat, registry *prometheus.Registry) { + gauge := func(name, help string, value func() float64) { + registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "node", + Subsystem: "repairer", + Name: name, + Help: help, + }, value)) + } + gauge("errored", "responsible spaces still in Error status after the last repair cycle", func() float64 { + return float64(s.errored.Load()) + }) + gauge("repaired_in_place", "transient errors cleared without touching data, since start", func() float64 { + return float64(s.repairedInPlace.Load()) + }) + gauge("quarantined", "corrupted dbs parked under .quarantine, since start", func() float64 { + return float64(s.quarantined.Load()) + }) + gauge("repaired", "valid copies pulled from responsible neighbors, since start", func() float64 { + return float64(s.repaired.Load()) + }) + gauge("dropped_entries", "garbage-collected entries of spaces that never existed anywhere, since start", func() float64 { + return float64(s.droppedEntries.Load()) + }) +} diff --git a/resharder/integration_test.go b/resharder/integration_test.go new file mode 100644 index 00000000..ce2e816d --- /dev/null +++ b/resharder/integration_test.go @@ -0,0 +1,350 @@ +package resharder + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + anystore "github.com/anyproto/any-store" + "github.com/anyproto/any-store/anyenc" + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/net/rpc/rpctest" + "github.com/anyproto/any-sync/nodeconf" + "github.com/anyproto/any-sync/nodeconf/mock_nodeconf" + "github.com/anyproto/any-sync/testutil/anymock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync-node/archive" + "github.com/anyproto/any-sync-node/archive/adopter" + "github.com/anyproto/any-sync-node/archive/archivestore" + "github.com/anyproto/any-sync-node/nodehead" + "github.com/anyproto/any-sync-node/nodehead/mock_nodehead" + "github.com/anyproto/any-sync-node/nodestorage" + "github.com/anyproto/any-sync-node/nodesync/hotsync" + "github.com/anyproto/any-sync-node/nodesync/hotsync/mock_hotsync" + "github.com/anyproto/any-sync-node/nodesync/nodesyncproto" +) + +// TestIntegration_DrainAdoptRestore drives the full resharding data plane +// in-process with two real node stacks (real nodestorage + real archive +// component) and a shared in-memory bucket: +// +// node A (drainer): live space -> ForceArchive -> AdoptArchive to node B -> +// verified handoff -> local deletion (status Moved) +// node B (adopter): server-side copy -> index registration -> eager restore +// -> live SQLite db on disk +func TestIntegration_DrainAdoptRestore(t *testing.T) { + bucket := newMemBucket() + nodeA := newTestNode(t, "nodeA", bucket) + nodeB := newTestNode(t, "nodeB", bucket) + + const spaceId = "integration.space" + + // create a "space" db on node A and index it + spaceDir := nodeA.storage.StoreDir(spaceId) + require.NoError(t, os.MkdirAll(spaceDir, 0o755)) + db, err := anystore.Open(ctx, filepath.Join(spaceDir, "store.db"), nil) + require.NoError(t, err) + _, err = db.CreateCollection(ctx, "objects") + require.NoError(t, err) + // the snapshot heads are read from the space state (as spacestorage keeps it) + stateColl, err := db.Collection(ctx, "state") + require.NoError(t, err) + arena := &anyenc.Arena{} + stateDoc := arena.NewObject() + stateDoc.Set("id", arena.NewString(spaceId)) + stateDoc.Set("oh", arena.NewString("h-old")) + stateDoc.Set("nh", arena.NewString("h-new")) + require.NoError(t, stateColl.Insert(ctx, stateDoc)) + require.NoError(t, db.Close()) + require.NoError(t, nodeA.storage.IndexStorage().UpdateHash(ctx, nodestorage.SpaceUpdate{ + SpaceId: spaceId, OldHash: "h-old", NewHash: "h-new", + })) + + // node A is no longer responsible; node B is the only owner (minAcks=1) + nodeA.nodeConf.EXPECT().IsResponsible(spaceId).AnyTimes().Return(false) + nodeA.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"nodeB"}) + // node B accepts adoption from node A (a tree node) and registers heads + nodeB.nodeConf.EXPECT().NodeTypes("nodeA").Return([]nodeconf.NodeType{nodeconf.NodeTypeTree}) + nodeB.nodeConf.EXPECT().IsResponsible(spaceId).Return(true) + nodeB.nodeHead.EXPECT().SetHead(spaceId, "h-old", "h-new").Return(0, nil) + + // wire the drainer's adopt call straight into node B's adopter + nodeA.resharder.adoptFn = func(_ context.Context, peerId string, req *nodesyncproto.AdoptArchiveRequest) (*nodesyncproto.AdoptArchiveResponse, error) { + require.Equal(t, "nodeB", peerId) + return nodeB.adopter.AdoptArchive(peer.CtxWithPeerId(ctx, "nodeA"), req) + } + + ok, err := nodeA.resharder.drainSpace(spaceId) + require.NoError(t, err) + require.True(t, ok) + + // node A: local db gone, index says Moved, own archive object deleted + assert.False(t, nodeA.storage.SpaceExists(spaceId)) + statusA, err := nodeA.storage.IndexStorage().SpaceStatus(ctx, spaceId) + require.NoError(t, err) + assert.Equal(t, nodestorage.SpaceStatusMoved, statusA) + okA, err := nodeA.archiveStore.Exists(ctx, spaceId) + require.NoError(t, err) + assert.False(t, okA) + + // node B: object in its prefix, index entry Archived with the same heads + okB, err := nodeB.archiveStore.Exists(ctx, spaceId) + require.NoError(t, err) + assert.True(t, okB) + entryB, err := nodeB.storage.IndexStorage().SpaceStatusEntry(ctx, spaceId) + require.NoError(t, err) + // the eager restore may already have flipped Archived -> Ok + assert.Contains(t, []nodestorage.SpaceStatus{nodestorage.SpaceStatusArchived, nodestorage.SpaceStatusOk}, entryB.Status) + assert.Equal(t, "h-new", entryB.NewHash) + + // eager restore materializes the db on node B + require.Eventually(t, func() bool { + statusB, sErr := nodeB.storage.IndexStorage().SpaceStatus(ctx, spaceId) + return sErr == nil && statusB == nodestorage.SpaceStatusOk + }, time.Second*10, time.Millisecond*50, "eager restore did not materialize the space on node B") + assert.True(t, nodeB.storage.SpaceExists(spaceId)) + + // the restored db is openable and holds the original collection + restored, err := anystore.Open(ctx, filepath.Join(nodeB.storage.StoreDir(spaceId), "store.db"), nil) + require.NoError(t, err) + defer restored.Close() + colls, err := restored.GetCollectionNames(ctx) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"objects", "state"}, colls) +} + +type testNode struct { + name string + a *app.App + storage nodestorage.NodeStorage + archive archive.Archive + archiveStore archivestore.ArchiveStore + adopter adopter.Adopter + resharder *resharder + nodeConf *mock_nodeconf.MockService + nodeHead *mock_nodehead.MockNodeHead +} + +// realS3RunId makes all nodes of one test run share a prefix namespace when +// the real-S3 backend is enabled. +var realS3RunId = fmt.Sprintf("reshard-int-%d", time.Now().UnixNano()) + +// newIntegrationStore returns the in-memory bucket store, or the real S3 +// store when ARCHIVE_TEST_S3_* env vars are configured (see realstore_test.go +// in archivestore) — the latter runs the whole drain flow against a real +// bucket, e.g. GCS interop. +func newIntegrationStore(bucket *memBucket, nodeName string) (archivestore.ArchiveStore, archivestore.Config) { + accessKey := os.Getenv("ARCHIVE_TEST_S3_ACCESS_KEY") + secretKey := os.Getenv("ARCHIVE_TEST_S3_SECRET_KEY") + s3Bucket := os.Getenv("ARCHIVE_TEST_S3_BUCKET") + if accessKey == "" || secretKey == "" || s3Bucket == "" { + return newMemStore(bucket, nodeName), archivestore.Config{} + } + region := os.Getenv("ARCHIVE_TEST_S3_REGION") + if region == "" { + region = "us-east-1" + } + return archivestore.New(), archivestore.Config{ + Enabled: true, + Region: region, + Bucket: s3Bucket, + Endpoint: os.Getenv("ARCHIVE_TEST_S3_ENDPOINT"), + ForcePathStyle: true, + KeyPrefix: realS3RunId + "/" + nodeName, + Shared: true, + Credentials: archivestore.Credentials{ + AccessKey: accessKey, + SecretKey: secretKey, + }, + } +} + +func newTestNode(t *testing.T, name string, bucket *memBucket) *testNode { + ctrl := gomock.NewController(t) + store, storeConf := newIntegrationStore(bucket, name) + n := &testNode{ + name: name, + a: new(app.App), + storage: nodestorage.New(), + archive: archive.New(), + archiveStore: store, + adopter: adopter.New(), + resharder: New().(*resharder), + nodeConf: mock_nodeconf.NewMockService(ctrl), + nodeHead: mock_nodehead.NewMockNodeHead(ctrl), + } + hotSync := mock_hotsync.NewMockHotSync(ctrl) + anymock.ExpectComp(n.nodeConf.EXPECT(), nodeconf.CName) + anymock.ExpectComp(n.nodeHead.EXPECT(), nodehead.CName) + anymock.ExpectComp(hotSync.EXPECT(), hotsync.CName) + hotSync.EXPECT().SetMetric(gomock.Any(), gomock.Any()).AnyTimes() + hotSync.EXPECT().UpdateQueue(gomock.Any()).AnyTimes() + n.nodeConf.EXPECT().ObserveChanges(gomock.Any()) + n.nodeConf.EXPECT().Configuration().AnyTimes().Return(nodeconf.Configuration{Epoch: 1}) + + n.a.Register(testNodeConfig{dir: t.TempDir(), s3: storeConf}). + Register(&syncWaiterStub{}). + Register(store). + Register(n.archive). + Register(n.storage). + Register(n.nodeHead). + Register(n.nodeConf). + Register(hotSync). + Register(rpctest.NewTestPool()). + Register(n.adopter). + Register(n.resharder) + + require.NoError(t, n.a.Start(ctx)) + t.Cleanup(func() { + require.NoError(t, n.a.Close(ctx)) + }) + return n +} + +type testNodeConfig struct { + dir string + s3 archivestore.Config +} + +func (c testNodeConfig) Init(_ *app.App) error { return nil } +func (c testNodeConfig) Name() string { return "config" } + +func (c testNodeConfig) GetStorage() nodestorage.Config { + return nodestorage.Config{Path: c.dir, AnyStorePath: c.dir} +} + +func (c testNodeConfig) GetArchive() archive.Config { + // periodic archiving off; the restore worker runs regardless + return archive.Config{Enabled: false} +} + +func (c testNodeConfig) GetS3Store() archivestore.Config { + return c.s3 +} + +type syncWaiterStub struct{} + +func (s *syncWaiterStub) Init(_ *app.App) error { return nil } +func (s *syncWaiterStub) Name() string { return "node.nodesync" } + +func (s *syncWaiterStub) WaitSyncOnStart() <-chan struct{} { + ch := make(chan struct{}) + close(ch) + return ch +} + +// memBucket is a shared in-memory "s3 bucket" for the test network. +type memBucket struct { + mu sync.Mutex + objects map[string][]byte + mtimes map[string]time.Time +} + +func newMemBucket() *memBucket { + return &memBucket{objects: map[string][]byte{}, mtimes: map[string]time.Time{}} +} + +func (b *memBucket) has(key string) bool { + b.mu.Lock() + defer b.mu.Unlock() + _, ok := b.objects[key] + return ok +} + +// memStore implements archivestore.ArchiveStore over the shared bucket. +type memStore struct { + bucket *memBucket + prefix string +} + +func newMemStore(bucket *memBucket, nodeName string) *memStore { + return &memStore{bucket: bucket, prefix: nodeName + "/"} +} + +func (m *memStore) Init(_ *app.App) error { return nil } +func (m *memStore) Name() string { return archivestore.CName } + +func (m *memStore) Get(_ context.Context, name string) (io.ReadCloser, error) { + m.bucket.mu.Lock() + defer m.bucket.mu.Unlock() + data, ok := m.bucket.objects[m.prefix+name] + if !ok { + return nil, archivestore.ErrNotFound + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (m *memStore) Put(_ context.Context, name string, data io.ReadSeeker) error { + raw, err := io.ReadAll(data) + if err != nil { + return err + } + m.bucket.mu.Lock() + defer m.bucket.mu.Unlock() + m.bucket.objects[m.prefix+name] = raw + m.bucket.mtimes[m.prefix+name] = time.Now() + return nil +} + +func (m *memStore) Delete(_ context.Context, name string) error { + m.bucket.mu.Lock() + defer m.bucket.mu.Unlock() + delete(m.bucket.objects, m.prefix+name) + delete(m.bucket.mtimes, m.prefix+name) + return nil +} + +func (m *memStore) Exists(_ context.Context, name string) (bool, error) { + return m.bucket.has(m.prefix + name), nil +} + +func (m *memStore) Key(name string) string { + return m.prefix + name +} + +func (m *memStore) CopyFrom(_ context.Context, srcKey, name string) error { + m.bucket.mu.Lock() + defer m.bucket.mu.Unlock() + data, ok := m.bucket.objects[srcKey] + if !ok { + return archivestore.ErrNotFound + } + m.bucket.objects[m.prefix+name] = data + m.bucket.mtimes[m.prefix+name] = time.Now() + return nil +} + +func (m *memStore) Shared() bool { return true } + +func (m *memStore) List(_ context.Context, iter func(name string, lastModified time.Time) (bool, error)) error { + m.bucket.mu.Lock() + type obj struct { + name string + mtime time.Time + } + var objs []obj + for key := range m.bucket.objects { + if strings.HasPrefix(key, m.prefix) { + objs = append(objs, obj{strings.TrimPrefix(key, m.prefix), m.bucket.mtimes[key]}) + } + } + m.bucket.mu.Unlock() + for _, o := range objs { + cont, err := iter(o.name, o.mtime) + if err != nil || !cont { + return err + } + } + return nil +} diff --git a/resharder/resharder.go b/resharder/resharder.go new file mode 100644 index 00000000..64a42824 --- /dev/null +++ b/resharder/resharder.go @@ -0,0 +1,444 @@ +package resharder + +import ( + "context" + "errors" + "time" + + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/metric" + "github.com/anyproto/any-sync/net/pool" + "github.com/anyproto/any-sync/net/rpc/rpcerr" + "github.com/anyproto/any-sync/nodeconf" + "go.uber.org/zap" + "storj.io/drpc" + + "github.com/anyproto/any-sync-node/archive" + "github.com/anyproto/any-sync-node/archive/archivestore" + "github.com/anyproto/any-sync-node/nodehead" + "github.com/anyproto/any-sync-node/nodestorage" + "github.com/anyproto/any-sync-node/nodesync/hotsync" + "github.com/anyproto/any-sync-node/nodesync/nodesyncproto" +) + +const CName = "node.resharder" + +var log = logger.NewNamed(CName) + +const ( + defaultDrainIntervalMinutes = 60 + // spaceOpTimeout bounds a single space handoff (snapshot + up to RF adopt RPCs) + spaceOpTimeout = time.Minute * 10 +) + +func New() Resharder { + return new(resharder) +} + +// Resharder implements the draining side of resharding: when the network +// configuration changes and this node is no longer responsible for a space, +// the resharder hands the space off to the current owners through the shared +// archive store (AdoptArchive) and deletes the local copy only after at least +// two current owners durably hold the same heads. +// +// The machinery is gated on the archive store being shared (s3Store.shared); +// networks without a shared bucket keep today's behavior. +type Resharder interface { + app.ComponentRunnable +} + +type configGetter interface { + GetResharder() Config +} + +type Config struct { + // DrainIntervalMinutes is the period of the background drain cycle + // (it also runs immediately on a network configuration change). + DrainIntervalMinutes int `yaml:"drainIntervalMinutes"` +} + +type resharder struct { + nodeConf nodeconf.Service + storage nodestorage.NodeStorage + nodeHead nodehead.NodeHead + archive archive.Archive + archiveStore archivestore.ArchiveStore + pool pool.Pool + hotSync hotsync.HotSync + conf Config + + trigger chan struct{} + runCtx context.Context + runCtxCancel context.CancelFunc + done chan struct{} + stat *resharderStat + + // adoptFn is the AdoptArchive call to a peer; replaceable in tests + adoptFn func(ctx context.Context, peerId string, req *nodesyncproto.AdoptArchiveRequest) (*nodesyncproto.AdoptArchiveResponse, error) +} + +func (r *resharder) Init(a *app.App) (err error) { + r.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.Service) + r.storage = a.MustComponent(nodestorage.CName).(nodestorage.NodeStorage) + r.nodeHead = a.MustComponent(nodehead.CName).(nodehead.NodeHead) + r.archive = a.MustComponent(archive.CName).(archive.Archive) + r.archiveStore = a.MustComponent(archivestore.CName).(archivestore.ArchiveStore) + r.pool = a.MustComponent(pool.CName).(pool.Pool) + r.hotSync = a.MustComponent(hotsync.CName).(hotsync.HotSync) + if g, ok := a.Component("config").(configGetter); ok { + r.conf = g.GetResharder() + } + if r.conf.DrainIntervalMinutes <= 0 { + r.conf.DrainIntervalMinutes = defaultDrainIntervalMinutes + } + r.trigger = make(chan struct{}, 1) + r.done = make(chan struct{}) + r.runCtx, r.runCtxCancel = context.WithCancel(context.Background()) + r.stat = new(resharderStat) + if r.adoptFn == nil { + r.adoptFn = r.adopt + } + if m := a.Component(metric.CName); m != nil { + registerMetric(r.stat, m.(metric.Metric).Registry()) + } + r.nodeConf.ObserveChanges(func(prev, cur nodeconf.NodeConf) { + if sameTreeMembers(prev.Configuration(), cur.Configuration()) { + // addresses or non-tree nodes changed: the ring is identical, no + // space changes hands — don't waste a full index scan (the + // periodic cycle remains as a backstop) + log.Info("network configuration changed without tree membership changes: drain not scheduled", + zap.Uint64("prevEpoch", prev.Configuration().Epoch), + zap.Uint64("curEpoch", cur.Configuration().Epoch)) + return + } + log.Info("network configuration changed, scheduling drain cycle", + zap.Uint64("prevEpoch", prev.Configuration().Epoch), + zap.Uint64("curEpoch", cur.Configuration().Epoch)) + r.Trigger() + }) + return +} + +// sameTreeMembers reports whether both configurations contain the same set of +// tree-node peer ids. Equal membership means an identical chash ring (member +// capacities are constant today; revisit if capacity weights are introduced). +func sameTreeMembers(a, b nodeconf.Configuration) bool { + treeSet := func(c nodeconf.Configuration) map[string]struct{} { + set := make(map[string]struct{}) + for _, n := range c.Nodes { + if n.HasType(nodeconf.NodeTypeTree) { + set[n.PeerId] = struct{}{} + } + } + return set + } + as, bs := treeSet(a), treeSet(b) + if len(as) != len(bs) { + return false + } + for id := range as { + if _, ok := bs[id]; !ok { + return false + } + } + return true +} + +func (r *resharder) Name() (name string) { + return CName +} + +func (r *resharder) Run(_ context.Context) (err error) { + if !r.archiveStore.Shared() { + log.Info("archive store is not shared: resharding drain is disabled") + r.stat.state.Store(stateDisabled) + close(r.done) + return + } + r.stat.state.Store(stateIdle) + go r.loop() + return +} + +// Trigger schedules an immediate drain cycle. +func (r *resharder) Trigger() { + select { + case r.trigger <- struct{}{}: + default: + } +} + +func (r *resharder) loop() { + defer close(r.done) + ticker := time.NewTicker(time.Duration(r.conf.DrainIntervalMinutes) * time.Minute) + defer ticker.Stop() + // initial cycle picks up drains interrupted by a restart + r.drainCycle() + for { + select { + case <-r.runCtx.Done(): + return + case <-r.trigger: + case <-ticker.C: + } + r.drainCycle() + } +} + +func (r *resharder) drainCycle() { + st := time.Now() + r.stat.epoch.Store(r.nodeConf.Configuration().Epoch) + defer func() { + r.stat.lastCycleUnix.Store(time.Now().Unix()) + }() + r.reconcileUnindexed() + var candidates []string + err := r.storage.IndexStorage().ReadHashes(r.runCtx, func(update nodestorage.SpaceUpdate) (bool, error) { + if !r.nodeConf.IsResponsible(update.SpaceId) { + candidates = append(candidates, update.SpaceId) + } + return true, nil + }) + if err != nil { + log.Warn("drain cycle: can't read index", zap.Error(err)) + return + } + r.stat.draining.Store(uint32(len(candidates))) + if len(candidates) == 0 { + r.stat.state.Store(stateIdle) + return + } + r.stat.state.Store(stateDraining) + log.Info("drain cycle started", zap.Int("spaces", len(candidates))) + var moved, parked int + for _, spaceId := range candidates { + if r.runCtx.Err() != nil { + return + } + ok, err := r.drainSpace(spaceId) + if err != nil { + r.stat.errors.Add(1) + log.Warn("drain space failed", zap.String("spaceId", spaceId), zap.Error(err)) + } + if ok { + moved++ + r.stat.moved.Add(1) + } else { + parked++ + } + } + r.stat.draining.Store(uint32(parked)) + if parked == 0 { + r.stat.state.Store(stateIdle) + } + log.Info("drain cycle finished", zap.Int("moved", moved), zap.Int("parked", parked), zap.Duration("dur", time.Since(st))) +} + +// reconcileUnindexed indexes local space directories that have no index entry +// (legacy leftovers), so they become visible to draining and anti-entropy. +func (r *resharder) reconcileUnindexed() { + ids, err := r.storage.AllSpaceIds() + if err != nil { + log.Warn("drain cycle: can't list space dirs", zap.Error(err)) + return + } + index := r.storage.IndexStorage() + for _, id := range ids { + if r.runCtx.Err() != nil { + return + } + if _, err = index.SpaceStatusEntry(r.runCtx, id); err == nil { + continue + } + ctx, cancel := context.WithTimeout(r.runCtx, time.Minute) + if _, iErr := r.storage.IndexSpace(ctx, id, true); iErr != nil { + log.Warn("drain cycle: can't index unindexed space dir", zap.String("spaceId", id), zap.Error(iErr)) + } else { + log.Info("drain cycle: indexed a space dir without an index entry", zap.String("spaceId", id)) + } + cancel() + } +} + +// drainSpace hands one space off to the current owners. It returns ok=true +// when the local copy was deleted (or the space turned out to be handled +// already); ok=false parks the space for the next cycle. +func (r *resharder) drainSpace(spaceId string) (ok bool, err error) { + ctx, cancel := context.WithTimeout(r.runCtx, spaceOpTimeout) + defer cancel() + + index := r.storage.IndexStorage() + entry, err := index.SpaceStatusEntry(ctx, spaceId) + if err != nil { + return false, err + } + switch entry.Status { + case nodestorage.SpaceStatusOk, nodestorage.SpaceStatusArchived: + default: + // deletion flow, errors and already-moved spaces are not drained + return true, nil + } + // the configuration could have changed since the candidate list was built + if r.nodeConf.IsResponsible(spaceId) { + return true, nil + } + + status := entry.Status + if status == nodestorage.SpaceStatusArchived { + exists, hErr := r.archiveStore.Exists(ctx, spaceId) + if hErr != nil { + return false, hErr + } + if !exists { + if !r.storage.SpaceExists(spaceId) { + // no data anywhere locally: nothing to hand off; leave the entry + // for the operator, the owners sync from the other replicas + return false, index.MarkError(ctx, spaceId, "drain: archived but object and db are missing") + } + // the object is gone but the db is present: repair the status first + // (opening a space whose index says Archived would trigger a restore + // of the very object that is missing) and hand it off as live + if err = index.SetSpaceStatus(ctx, spaceId, nodestorage.SpaceStatusOk, ""); err != nil { + return false, err + } + status = nodestorage.SpaceStatusOk + } + } + + // heads advertised to the owners; for live spaces they are read from the + // snapshot itself so they exactly describe the uploaded object + oldHash, newHash := entry.OldHash, entry.NewHash + compressedSize, uncompressedSize := entry.ArchiveSizeCompressed, entry.ArchiveSizeUncompressed + eager := false + if status == nodestorage.SpaceStatusOk { + if !r.storage.SpaceExists(spaceId) { + // index entry without local data: nothing to hand off + return true, index.SetSpaceStatus(ctx, spaceId, nodestorage.SpaceStatusMoved, "") + } + if oldHash, newHash, compressedSize, uncompressedSize, err = r.archive.ForceArchive(ctx, spaceId); err != nil { + return false, err + } + // writes may have landed while the snapshot was taken/uploaded: park + // and retry next cycle rather than hand off a stale object + cur, cErr := index.SpaceStatusEntry(ctx, spaceId) + if cErr != nil { + return false, cErr + } + if cur.NewHash != newHash { + log.Info("drain: space changed during snapshot, retrying next cycle", zap.String("spaceId", spaceId)) + return false, nil + } + eager = true + } + + req := &nodesyncproto.AdoptArchiveRequest{ + SpaceId: spaceId, + SrcKey: r.archiveStore.Key(spaceId), + OldHash: oldHash, + NewHash: newHash, + Eager: eager, + CompressedSize: compressedSize, + UncompressedSize: uncompressedSize, + } + + owners := r.nodeConf.NodeIds(spaceId) + if len(owners) == 0 { + // no reachable owners in the current ring (e.g. a force-applied broken + // configuration): never delete without a real handoff + r.stat.parked.Add(1) + return false, nil + } + minAcks := 2 + if len(owners) < minAcks { + minAcks = len(owners) + } + var acks, diverged int + for _, peerId := range owners { + resp, aErr := r.adoptFn(ctx, peerId, req) + if aErr != nil { + if errors.Is(aErr, nodesyncproto.ErrSpaceDeleted) { + // the network deleted this space: drop the local copy + log.Info("drain: space is deleted by the network", zap.String("spaceId", spaceId)) + return true, r.deleteLocal(ctx, spaceId, entry) + } + log.Debug("drain: adopt failed", zap.String("spaceId", spaceId), zap.String("peerId", peerId), zap.Error(aErr)) + continue + } + switch resp.Result { + case nodesyncproto.AdoptArchiveResult_AdoptArchiveOk, + nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveSame: + acks++ + case nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveDiverged: + diverged++ + } + } + if acks < minAcks { + if diverged > 0 { + // an owner holds a different version: converge via tree sync + // (loading the space syncs it with the current owners), retry next cycle + r.hotSync.UpdateQueue([]string{spaceId}) + } + r.stat.parked.Add(1) + return false, nil + } + // deletion safety: the heads the owners ACKed must still be the local + // heads — a mismatch means writes landed during the handoff + cur, err := index.SpaceStatusEntry(ctx, spaceId) + if err != nil { + return false, err + } + if cur.NewHash != newHash || cur.OldHash != oldHash { + log.Info("drain: heads changed during handoff, retrying next cycle", zap.String("spaceId", spaceId)) + return false, nil + } + return true, r.deleteLocal(ctx, spaceId, entry) +} + +func (r *resharder) deleteLocal(ctx context.Context, spaceId string, entry nodestorage.SpaceStatusEntry) (err error) { + // delete data first, status marker last: a crash in between leaves the + // space a drain candidate and the next cycle finishes the job + if r.storage.SpaceExists(spaceId) { + if err = r.storage.DeleteSpaceStorage(ctx, spaceId); err != nil { + return + } + } else { + if err = r.nodeHead.DeleteHeads(spaceId); err != nil { + log.Warn("drain: can't delete heads", zap.String("spaceId", spaceId), zap.Error(err)) + err = nil + } + } + if dErr := r.archiveStore.Delete(ctx, spaceId); dErr != nil && !errors.Is(dErr, archivestore.ErrNotFound) { + log.Warn("drain: can't delete archive object", zap.String("spaceId", spaceId), zap.Error(dErr)) + } + if err = r.storage.IndexStorage().SetSpaceStatus(ctx, spaceId, nodestorage.SpaceStatusMoved, ""); err != nil { + return + } + log.Info("space handed off and deleted locally", zap.String("spaceId", spaceId)) + return +} + +func (r *resharder) adopt(ctx context.Context, peerId string, req *nodesyncproto.AdoptArchiveRequest) (resp *nodesyncproto.AdoptArchiveResponse, err error) { + p, err := r.pool.Get(ctx, peerId) + if err != nil { + return + } + err = p.DoDrpc(ctx, func(conn drpc.Conn) error { + var dErr error + resp, dErr = nodesyncproto.NewDRPCNodeSyncClient(conn).AdoptArchive(ctx, req) + return dErr + }) + if err != nil { + return nil, rpcerr.Unwrap(err) + } + return +} + +func (r *resharder) Close(_ context.Context) (err error) { + r.runCtxCancel() + select { + case <-r.done: + case <-time.After(time.Second * 10): + } + return +} diff --git a/resharder/resharder_test.go b/resharder/resharder_test.go new file mode 100644 index 00000000..fbbcd19e --- /dev/null +++ b/resharder/resharder_test.go @@ -0,0 +1,332 @@ +package resharder + +import ( + "context" + "fmt" + "testing" + + anystore "github.com/anyproto/any-store" + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/net/pool" + "github.com/anyproto/any-sync/net/rpc/rpctest" + "github.com/anyproto/any-sync/nodeconf" + "github.com/anyproto/any-sync/nodeconf/mock_nodeconf" + "github.com/anyproto/any-sync/testutil/anymock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync-node/archive" + "github.com/anyproto/any-sync-node/archive/archivestore" + "github.com/anyproto/any-sync-node/archive/archivestore/mock_archivestore" + "github.com/anyproto/any-sync-node/archive/mock_archive" + "github.com/anyproto/any-sync-node/nodehead" + "github.com/anyproto/any-sync-node/nodehead/mock_nodehead" + "github.com/anyproto/any-sync-node/nodestorage" + "github.com/anyproto/any-sync-node/nodestorage/mock_nodestorage" + "github.com/anyproto/any-sync-node/nodesync/hotsync" + "github.com/anyproto/any-sync-node/nodesync/hotsync/mock_hotsync" + "github.com/anyproto/any-sync-node/nodesync/nodesyncproto" +) + +var ctx = context.Background() + +const spaceId = "space.id" + +func okEntry() nodestorage.SpaceStatusEntry { + return nodestorage.SpaceStatusEntry{ + SpaceId: spaceId, + Status: nodestorage.SpaceStatusOk, + OldHash: "oldHash", + NewHash: "newHash", + } +} + +func archivedEntry() nodestorage.SpaceStatusEntry { + e := okEntry() + e.Status = nodestorage.SpaceStatusArchived + e.ArchiveSizeCompressed = 10 + e.ArchiveSizeUncompressed = 20 + return e +} + +func respOk() *nodesyncproto.AdoptArchiveResponse { + return &nodesyncproto.AdoptArchiveResponse{Result: nodesyncproto.AdoptArchiveResult_AdoptArchiveOk} +} + +func respSame() *nodesyncproto.AdoptArchiveResponse { + return &nodesyncproto.AdoptArchiveResponse{Result: nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveSame} +} + +func respDiverged() *nodesyncproto.AdoptArchiveResponse { + return &nodesyncproto.AdoptArchiveResponse{Result: nodesyncproto.AdoptArchiveResult_AdoptArchiveAlreadyHaveDiverged} +} + +func TestResharder_DrainSpace(t *testing.T) { + t.Run("archived space handed off with 2 acks", func(t *testing.T) { + fx := newFixture(t) + fx.nodeConf.EXPECT().IsResponsible(spaceId).Return(false) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(archivedEntry(), nil) + fx.archiveStore.EXPECT().Exists(gomock.Any(), spaceId).Return(true, nil) + fx.archiveStore.EXPECT().Key(spaceId).Return("me/" + spaceId) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1", "p2", "p3"}) + fx.adoptResponses["p1"] = respOk() + fx.adoptResponses["p2"] = respSame() + fx.adoptResponses["p3"] = respDiverged() + // heads recheck before deletion + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(archivedEntry(), nil) + // deletion: no local db -> heads deleted directly, object removed, moved + fx.storage.EXPECT().SpaceExists(spaceId).Return(false) + fx.nodeHead.EXPECT().DeleteHeads(spaceId).Return(nil) + fx.archiveStore.EXPECT().Delete(gomock.Any(), spaceId).Return(nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusMoved, "").Return(nil) + + ok, err := fx.drainSpace(spaceId) + require.NoError(t, err) + assert.True(t, ok) + assert.ElementsMatch(t, []string{"p1", "p2", "p3"}, fx.adopted) + }) + t.Run("live space force-archived and handed off", func(t *testing.T) { + fx := newFixture(t) + fx.nodeConf.EXPECT().IsResponsible(spaceId).Return(false) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(okEntry(), nil) + fx.storage.EXPECT().SpaceExists(spaceId).Return(true) + fx.archive.EXPECT().ForceArchive(gomock.Any(), spaceId).Return("oldHash", "newHash", int64(11), int64(22), nil) + // post-snapshot check: index still matches the snapshot heads + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(okEntry(), nil) + fx.archiveStore.EXPECT().Key(spaceId).Return("me/" + spaceId) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1", "p2"}) + fx.adoptResponses["p1"] = respOk() + fx.adoptResponses["p2"] = respOk() + // heads recheck + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(okEntry(), nil) + // deletion: local db exists + fx.storage.EXPECT().SpaceExists(spaceId).Return(true) + fx.storage.EXPECT().DeleteSpaceStorage(gomock.Any(), spaceId).Return(nil) + fx.archiveStore.EXPECT().Delete(gomock.Any(), spaceId).Return(nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusMoved, "").Return(nil) + + ok, err := fx.drainSpace(spaceId) + require.NoError(t, err) + assert.True(t, ok) + // eager restore requested for live spaces + assert.True(t, fx.lastReq.Eager) + }) + t.Run("not enough acks parks and converges", func(t *testing.T) { + fx := newFixture(t) + fx.nodeConf.EXPECT().IsResponsible(spaceId).Return(false) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(archivedEntry(), nil) + fx.archiveStore.EXPECT().Exists(gomock.Any(), spaceId).Return(true, nil) + fx.archiveStore.EXPECT().Key(spaceId).Return("me/" + spaceId) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1", "p2", "p3"}) + fx.adoptResponses["p1"] = respOk() + fx.adoptResponses["p2"] = respDiverged() + fx.adoptResponses["p3"] = respDiverged() + fx.hotSync.EXPECT().UpdateQueue([]string{spaceId}) + + ok, err := fx.drainSpace(spaceId) + require.NoError(t, err) + assert.False(t, ok) + }) + t.Run("space deleted by network drops local copy", func(t *testing.T) { + fx := newFixture(t) + fx.nodeConf.EXPECT().IsResponsible(spaceId).Return(false) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(archivedEntry(), nil) + fx.archiveStore.EXPECT().Exists(gomock.Any(), spaceId).Return(true, nil) + fx.archiveStore.EXPECT().Key(spaceId).Return("me/" + spaceId) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1", "p2", "p3"}) + fx.adoptErrors["p1"] = nodesyncproto.ErrSpaceDeleted + fx.storage.EXPECT().SpaceExists(spaceId).Return(false) + fx.nodeHead.EXPECT().DeleteHeads(spaceId).Return(nil) + fx.archiveStore.EXPECT().Delete(gomock.Any(), spaceId).Return(nil) + fx.indexStorage.EXPECT().SetSpaceStatus(gomock.Any(), spaceId, nodestorage.SpaceStatusMoved, "").Return(nil) + + ok, err := fx.drainSpace(spaceId) + require.NoError(t, err) + assert.True(t, ok) + }) + t.Run("heads drift during handoff postpones deletion", func(t *testing.T) { + fx := newFixture(t) + fx.nodeConf.EXPECT().IsResponsible(spaceId).Return(false) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(archivedEntry(), nil) + fx.archiveStore.EXPECT().Exists(gomock.Any(), spaceId).Return(true, nil) + fx.archiveStore.EXPECT().Key(spaceId).Return("me/" + spaceId) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{"p1", "p2"}) + fx.adoptResponses["p1"] = respOk() + fx.adoptResponses["p2"] = respOk() + drifted := archivedEntry() + drifted.NewHash = "differentHash" + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(drifted, nil) + + ok, err := fx.drainSpace(spaceId) + require.NoError(t, err) + assert.False(t, ok) + }) + t.Run("responsible again: skip", func(t *testing.T) { + fx := newFixture(t) + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(archivedEntry(), nil) + fx.nodeConf.EXPECT().IsResponsible(spaceId).Return(true) + ok, err := fx.drainSpace(spaceId) + require.NoError(t, err) + assert.True(t, ok) + }) + t.Run("statuses out of scope are skipped", func(t *testing.T) { + fx := newFixture(t) + e := okEntry() + e.Status = nodestorage.SpaceStatusRemovePrepare + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(e, nil) + ok, err := fx.drainSpace(spaceId) + require.NoError(t, err) + assert.True(t, ok) + }) +} + +func TestResharder_DrainCycle(t *testing.T) { + fx := newFixture(t) + // no unindexed dirs + fx.storage.EXPECT().AllSpaceIds().Return(nil, nil) + // two spaces in the index: one responsible, one not + fx.indexStorage.EXPECT().ReadHashes(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, iter func(nodestorage.SpaceUpdate) (bool, error)) error { + _, _ = iter(nodestorage.SpaceUpdate{SpaceId: "keep.id"}) + _, _ = iter(nodestorage.SpaceUpdate{SpaceId: spaceId}) + return nil + }) + fx.nodeConf.EXPECT().IsResponsible("keep.id").Return(true) + fx.nodeConf.EXPECT().IsResponsible(spaceId).Return(false) + // drainSpace path: entry lookup fails with not found -> parked with error + fx.indexStorage.EXPECT().SpaceStatusEntry(gomock.Any(), spaceId).Return(nodestorage.SpaceStatusEntry{}, anystore.ErrDocNotFound) + + fx.drainCycle() + assert.Equal(t, uint32(1), fx.stat.draining.Load()) +} + +type fixture struct { + *resharder + a *app.App + storage *mock_nodestorage.MockNodeStorage + indexStorage *mock_nodestorage.MockIndexStorage + archiveStore *mock_archivestore.MockArchiveStore + archive *mock_archive.MockArchive + nodeHead *mock_nodehead.MockNodeHead + nodeConf *mock_nodeconf.MockService + hotSync *mock_hotsync.MockHotSync + observer nodeconf.ChangeObserver + adopted []string + lastReq *nodesyncproto.AdoptArchiveRequest + adoptResponses map[string]*nodesyncproto.AdoptArchiveResponse + adoptErrors map[string]error +} + +func newFixture(t *testing.T) *fixture { + ctrl := gomock.NewController(t) + fx := &fixture{ + resharder: New().(*resharder), + a: new(app.App), + storage: mock_nodestorage.NewMockNodeStorage(ctrl), + indexStorage: mock_nodestorage.NewMockIndexStorage(ctrl), + archiveStore: mock_archivestore.NewMockArchiveStore(ctrl), + archive: mock_archive.NewMockArchive(ctrl), + nodeHead: mock_nodehead.NewMockNodeHead(ctrl), + nodeConf: mock_nodeconf.NewMockService(ctrl), + hotSync: mock_hotsync.NewMockHotSync(ctrl), + adoptResponses: map[string]*nodesyncproto.AdoptArchiveResponse{}, + adoptErrors: map[string]error{}, + } + fx.resharder.adoptFn = func(_ context.Context, peerId string, req *nodesyncproto.AdoptArchiveRequest) (*nodesyncproto.AdoptArchiveResponse, error) { + fx.adopted = append(fx.adopted, peerId) + fx.lastReq = req + if err, ok := fx.adoptErrors[peerId]; ok { + return nil, err + } + if resp, ok := fx.adoptResponses[peerId]; ok { + return resp, nil + } + return nil, context.DeadlineExceeded + } + + anymock.ExpectComp(fx.storage.EXPECT(), nodestorage.CName) + anymock.ExpectComp(fx.archiveStore.EXPECT(), archivestore.CName) + anymock.ExpectComp(fx.archive.EXPECT(), archive.CName) + anymock.ExpectComp(fx.nodeHead.EXPECT(), nodehead.CName) + anymock.ExpectComp(fx.nodeConf.EXPECT(), nodeconf.CName) + anymock.ExpectComp(fx.hotSync.EXPECT(), hotsync.CName) + fx.hotSync.EXPECT().SetMetric(gomock.Any(), gomock.Any()).AnyTimes() + fx.storage.EXPECT().IndexStorage().AnyTimes().Return(fx.indexStorage) + fx.nodeConf.EXPECT().ObserveChanges(gomock.Any()).Do(func(observer nodeconf.ChangeObserver) { + fx.observer = observer + }) + fx.nodeConf.EXPECT().Configuration().AnyTimes().Return(nodeconf.Configuration{Epoch: 1}) + // Run gates on Shared; keep the background loop off in unit tests + fx.archiveStore.EXPECT().Shared().Return(false) + + tp := rpctest.NewTestPool() + var _ pool.Pool = tp + + fx.a.Register(fx.storage). + Register(fx.archiveStore). + Register(fx.archive). + Register(fx.nodeHead). + Register(fx.nodeConf). + Register(fx.hotSync). + Register(tp). + Register(fx.resharder) + + require.NoError(t, fx.a.Start(ctx)) + t.Cleanup(func() { + require.NoError(t, fx.a.Close(ctx)) + ctrl.Finish() + }) + return fx +} + +func confWithTreeNodes(epoch uint64, addrSuffix string, peerIds ...string) nodeconf.Configuration { + c := nodeconf.Configuration{Id: fmt.Sprintf("cfg-%d", epoch), NetworkId: "net", Epoch: epoch} + for _, id := range peerIds { + c.Nodes = append(c.Nodes, nodeconf.Node{ + PeerId: id, + Addresses: []string{id + addrSuffix}, + Types: []nodeconf.NodeType{nodeconf.NodeTypeTree}, + }) + } + return c +} + +func TestResharder_ObserverSkipsUnchangedTreeSet(t *testing.T) { + fx := newFixture(t) + require.NotNil(t, fx.observer) + + prev := newRingConf("p1") + prev.set(confWithTreeNodes(1, ":1001", "p1", "p2", "p3"), nil) + + t.Run("addresses changed, same tree members: no drain scheduled", func(t *testing.T) { + cur := newRingConf("p1") + cur.set(confWithTreeNodes(2, ":2002", "p1", "p2", "p3"), nil) + fx.observer(prev, cur) + select { + case <-fx.resharder.trigger: + t.Fatal("drain cycle must not be scheduled for an address-only change") + default: + } + }) + t.Run("tree member added: drain scheduled", func(t *testing.T) { + cur := newRingConf("p1") + cur.set(confWithTreeNodes(3, ":1001", "p1", "p2", "p3", "p4"), nil) + fx.observer(prev, cur) + select { + case <-fx.resharder.trigger: + default: + t.Fatal("drain cycle must be scheduled when the tree set changes") + } + }) + t.Run("tree member replaced: drain scheduled", func(t *testing.T) { + cur := newRingConf("p1") + cur.set(confWithTreeNodes(4, ":1001", "p1", "p2", "p9"), nil) + fx.observer(prev, cur) + select { + case <-fx.resharder.trigger: + default: + t.Fatal("drain cycle must be scheduled when a tree member is replaced") + } + }) +} diff --git a/resharder/scenarios_test.go b/resharder/scenarios_test.go new file mode 100644 index 00000000..78866bd1 --- /dev/null +++ b/resharder/scenarios_test.go @@ -0,0 +1,428 @@ +package resharder + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "testing" + + anystore "github.com/anyproto/any-store" + "github.com/anyproto/any-store/anyenc" + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/net/rpc/rpctest" + "github.com/anyproto/any-sync/nodeconf" + "github.com/anyproto/any-sync/testutil/anymock" + "github.com/anyproto/go-chash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync-node/archive" + "github.com/anyproto/any-sync-node/archive/adopter" + "github.com/anyproto/any-sync-node/nodehead" + "github.com/anyproto/any-sync-node/nodehead/mock_nodehead" + "github.com/anyproto/any-sync-node/nodestorage" + "github.com/anyproto/any-sync-node/nodesync/hotsync" + "github.com/anyproto/any-sync-node/nodesync/hotsync/mock_hotsync" + "github.com/anyproto/any-sync-node/nodesync/nodesyncproto" +) + +const scenarioNetworkId = "testnet" + +// TestScenario_AddNode covers the "add new node(s)" operation end to end over +// real chash rings: nodes A,B,C hold every space (3 nodes, RF=3); node D is +// added in epoch 2. Every node that lost a partition to D hands the affected +// spaces off (2 surviving owners ACK by heads, D adopts the snapshot) and +// deletes its copy; spaces whose ownership did not change stay untouched. +func TestScenario_AddNode(t *testing.T) { + net := newScenarioNet(t, "A", "B", "C", "D") + net.setTreeNodes(1, "A", "B", "C") + + spaceIds := scenarioSpaceIds(12) + for _, id := range spaceIds { + // with 3 tree nodes and RF=3 every node owns every space + for _, n := range []string{"A", "B", "C"} { + net.createSpace(t, n, id) + } + } + + // epoch 2: node D joins + net.setTreeNodes(2, "A", "B", "C", "D") + for _, n := range net.order { + net.nodes[n].resharder.drainCycle() + } + + var movedToD int + for _, id := range spaceIds { + owners := net.owners(id) + for _, n := range []string{"A", "B", "C"} { + node := net.nodes[n] + status, err := node.storage.IndexStorage().SpaceStatus(ctx, id) + require.NoError(t, err) + if owners[n] { + assert.Equal(t, nodestorage.SpaceStatusOk, status, "space %s must stay on owner %s", id, n) + assert.True(t, node.storage.SpaceExists(id)) + } else { + assert.Equal(t, nodestorage.SpaceStatusMoved, status, "space %s must be drained from %s", id, n) + assert.False(t, node.storage.SpaceExists(id)) + assert.False(t, net.bucket.has(n+"/"+id), "drained node %s must not keep the archive object of %s", n, id) + } + } + if owners["D"] { + movedToD++ + entry, err := net.nodes["D"].storage.IndexStorage().SpaceStatusEntry(ctx, id) + require.NoError(t, err) + // the eager restore may already have flipped Archived -> Ok + assert.Contains(t, []nodestorage.SpaceStatus{nodestorage.SpaceStatusArchived, nodestorage.SpaceStatusOk}, entry.Status) + assert.Equal(t, "new-"+id, entry.NewHash, "space %s adopted by D with the advertised heads", id) + assert.True(t, net.bucket.has("D/"+id), "space %s must be parked in D's prefix", id) + } + } + require.Greater(t, movedToD, 0, "adding a node must move some partitions to it") +} + +// TestScenario_RemoveNode covers the "remove node(s)" operation: nodes +// A,B,C,D hold spaces per the epoch-1 ring; epoch 2 removes D entirely. +// D hands every space it still holds to the current owners — receivers +// authenticate it through the retained configuration history — and ends up +// holding nothing; the member that was not an owner before adopts the space. +func TestScenario_RemoveNode(t *testing.T) { + net := newScenarioNet(t, "A", "B", "C", "D") + net.setTreeNodes(1, "A", "B", "C", "D") + + spaceIds := scenarioSpaceIds(12) + for _, id := range spaceIds { + for n := range net.owners(id) { + net.createSpace(t, n, id) + } + } + + // epoch 2: node D is removed from the configuration + net.setTreeNodes(2, "A", "B", "C") + for _, n := range net.order { + net.nodes[n].resharder.drainCycle() + } + + nodeD := net.nodes["D"] + for _, id := range spaceIds { + statusD, err := nodeD.storage.IndexStorage().SpaceStatus(ctx, id) + if err == nil && statusD != nodestorage.SpaceStatusOk { + // D held this space in epoch 1: it must be fully drained + assert.Equal(t, nodestorage.SpaceStatusMoved, statusD, "space %s must be drained from removed node D", id) + assert.False(t, nodeD.storage.SpaceExists(id)) + assert.False(t, net.bucket.has("D/"+id)) + } + // every space must now be present on all remaining owners + for _, n := range []string{"A", "B", "C"} { + entry, err := net.nodes[n].storage.IndexStorage().SpaceStatusEntry(ctx, id) + require.NoError(t, err, "space %s must have an index entry on %s", id, n) + assert.Contains(t, + []nodestorage.SpaceStatus{nodestorage.SpaceStatusOk, nodestorage.SpaceStatusArchived}, + entry.Status, "space %s must be live or archived on %s", id, n) + assert.Equal(t, "new-"+id, entry.NewHash) + } + } + // D holds no data at all anymore + dirs, err := nodeD.storage.AllSpaceIds() + require.NoError(t, err) + assert.Empty(t, dirs, "removed node must hold no space dirs") +} + +func scenarioSpaceIds(n int) (ids []string) { + for i := 0; i < n; i++ { + ids = append(ids, fmt.Sprintf("scn%d.space%d", i, i)) + } + return +} + +// scenarioNet is a small in-process network: every node runs the real +// storage/archive/adopter/resharder stack over a shared in-memory bucket; +// topology comes from ring-backed nodeconf fakes sharing real chash rings. +type scenarioNet struct { + t *testing.T + bucket *memBucket + order []string + nodes map[string]*testNode + confs map[string]*ringConf + historyRef *fakeConfHistory + // ring is the current shared ring (same on every node in these tests) + ring chash.CHash +} + +func newScenarioNet(t *testing.T, names ...string) *scenarioNet { + net := &scenarioNet{ + t: t, + bucket: newMemBucket(), + order: names, + nodes: map[string]*testNode{}, + confs: map[string]*ringConf{}, + } + history := newFakeConfHistory() + for _, name := range names { + conf := newRingConf(name) + net.confs[name] = conf + net.nodes[name] = newScenarioNode(t, name, net.bucket, conf, history) + } + // route adopt calls to the target node's adopter with the sender identity + for _, name := range names { + sender := name + net.nodes[sender].resharder.adoptFn = func(_ context.Context, peerId string, req *nodesyncproto.AdoptArchiveRequest) (*nodesyncproto.AdoptArchiveResponse, error) { + target, ok := net.nodes[peerId] + if !ok { + return nil, fmt.Errorf("unknown peer %s", peerId) + } + return target.adopter.AdoptArchive(peer.CtxWithPeerId(ctx, sender), req) + } + } + net.historyRef = history + return net +} + +func (net *scenarioNet) setTreeNodes(epoch uint64, names ...string) { + ring := newTreeRing(net.t, names) + net.ring = ring + conf := nodeconf.Configuration{ + Id: fmt.Sprintf("cfg-%d", epoch), + NetworkId: scenarioNetworkId, + Epoch: epoch, + } + for _, n := range names { + conf.Nodes = append(conf.Nodes, nodeconf.Node{PeerId: n, Types: []nodeconf.NodeType{nodeconf.NodeTypeTree}}) + } + net.historyRef.add(conf) + for _, c := range net.confs { + c.set(conf, ring) + } +} + +// owners returns the current owner set of a space. +func (net *scenarioNet) owners(spaceId string) map[string]bool { + members := net.ring.GetMembers(nodeconf.ReplKey(spaceId)) + res := map[string]bool{} + for _, m := range members { + res[m.Id()] = true + } + return res +} + +func (net *scenarioNet) createSpace(t *testing.T, nodeName, spaceId string) { + node := net.nodes[nodeName] + spaceDir := node.storage.StoreDir(spaceId) + require.NoError(t, os.MkdirAll(spaceDir, 0o755)) + db, err := anystore.Open(ctx, filepath.Join(spaceDir, "store.db"), nil) + require.NoError(t, err) + _, err = db.CreateCollection(ctx, "objects") + require.NoError(t, err) + stateColl, err := db.Collection(ctx, "state") + require.NoError(t, err) + arena := &anyenc.Arena{} + stateDoc := arena.NewObject() + stateDoc.Set("id", arena.NewString(spaceId)) + stateDoc.Set("oh", arena.NewString("old-"+spaceId)) + stateDoc.Set("nh", arena.NewString("new-"+spaceId)) + require.NoError(t, stateColl.Insert(ctx, stateDoc)) + require.NoError(t, db.Close()) + require.NoError(t, node.storage.IndexStorage().UpdateHash(ctx, nodestorage.SpaceUpdate{ + SpaceId: spaceId, OldHash: "old-" + spaceId, NewHash: "new-" + spaceId, + })) +} + +func newScenarioNode(t *testing.T, name string, bucket *memBucket, conf *ringConf, history *fakeConfHistory) *testNode { + ctrl := gomock.NewController(t) + n := &testNode{ + name: name, + a: new(app.App), + storage: nodestorage.New(), + archive: archive.New(), + archiveStore: newMemStore(bucket, name), + adopter: adopter.New(), + resharder: New().(*resharder), + } + nodeHead := mock_nodehead.NewMockNodeHead(ctrl) + hotSync := mock_hotsync.NewMockHotSync(ctrl) + anymock.ExpectComp(nodeHead.EXPECT(), nodehead.CName) + anymock.ExpectComp(hotSync.EXPECT(), hotsync.CName) + nodeHead.EXPECT().SetHead(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(0, nil) + nodeHead.EXPECT().DeleteHeads(gomock.Any()).AnyTimes().Return(nil) + hotSync.EXPECT().SetMetric(gomock.Any(), gomock.Any()).AnyTimes() + hotSync.EXPECT().UpdateQueue(gomock.Any()).AnyTimes() + + n.a.Register(testNodeConfig{dir: t.TempDir()}). + Register(&syncWaiterStub{}). + Register(n.archiveStore). + Register(n.archive). + Register(n.storage). + Register(nodeHead). + Register(conf). + Register(history). + Register(hotSync). + Register(rpctest.NewTestPool()). + Register(n.adopter). + Register(n.resharder) + + require.NoError(t, n.a.Start(ctx)) + t.Cleanup(func() { + require.NoError(t, n.a.Close(ctx)) + }) + return n +} + +// ringConf is a nodeconf.Service fake backed by a real chash ring. +type ringConf struct { + self string + mu sync.RWMutex + conf nodeconf.Configuration + ring chash.CHash + observers []nodeconf.ChangeObserver +} + +func newRingConf(self string) *ringConf { + return &ringConf{self: self} +} + +func (c *ringConf) set(conf nodeconf.Configuration, ring chash.CHash) { + c.mu.Lock() + c.conf = conf + c.ring = ring + c.mu.Unlock() +} + +func (c *ringConf) Init(_ *app.App) error { return nil } +func (c *ringConf) Name() string { return nodeconf.CName } +func (c *ringConf) Run(_ context.Context) error { return nil } +func (c *ringConf) Close(_ context.Context) error { return nil } +func (c *ringConf) Id() string { c.mu.RLock(); defer c.mu.RUnlock(); return c.conf.Id } +func (c *ringConf) Configuration() nodeconf.Configuration { + c.mu.RLock() + defer c.mu.RUnlock() + return c.conf +} + +func (c *ringConf) NodeIds(spaceId string) (res []string) { + c.mu.RLock() + defer c.mu.RUnlock() + for _, m := range c.ring.GetMembers(nodeconf.ReplKey(spaceId)) { + if m.Id() != c.self { + res = append(res, m.Id()) + } + } + return +} + +func (c *ringConf) IsResponsible(spaceId string) bool { + c.mu.RLock() + defer c.mu.RUnlock() + for _, m := range c.ring.GetMembers(nodeconf.ReplKey(spaceId)) { + if m.Id() == c.self { + return true + } + } + return false +} + +func (c *ringConf) NodeTypes(nodeId string) []nodeconf.NodeType { + c.mu.RLock() + defer c.mu.RUnlock() + for _, n := range c.conf.Nodes { + if n.PeerId == nodeId { + return n.Types + } + } + return nil +} + +func (c *ringConf) FilePeers() []string { return nil } +func (c *ringConf) ConsensusPeers() []string { return nil } +func (c *ringConf) CoordinatorPeers() []string { return nil } +func (c *ringConf) NamingNodePeers() []string { return nil } +func (c *ringConf) PaymentProcessingNodePeers() []string { return nil } +func (c *ringConf) PeerAddresses(string) ([]string, bool) { return nil, false } +func (c *ringConf) CHash() chash.CHash { c.mu.RLock(); defer c.mu.RUnlock(); return c.ring } +func (c *ringConf) Partition(spaceId string) int { + c.mu.RLock() + defer c.mu.RUnlock() + return c.ring.GetPartition(nodeconf.ReplKey(spaceId)) +} +func (c *ringConf) ObserveChanges(o nodeconf.ChangeObserver) { + c.mu.Lock() + c.observers = append(c.observers, o) + c.mu.Unlock() +} +func (c *ringConf) NetworkCompatibilityStatus() nodeconf.NetworkCompatibilityStatus { + return nodeconf.NetworkCompatibilityStatusOk +} + +type chMember string + +func (m chMember) Id() string { return string(m) } +func (m chMember) Capacity() float64 { return 1 } + +func newTreeRing(t *testing.T, names []string) chash.CHash { + ring, err := chash.New(chash.Config{ + PartitionCount: nodeconf.PartitionCount, + ReplicationFactor: nodeconf.ReplicationFactor, + }) + require.NoError(t, err) + members := make([]chash.Member, 0, len(names)) + for _, n := range names { + members = append(members, chMember(n)) + } + require.NoError(t, ring.AddMembers(members...)) + return ring +} + +// fakeConfHistory implements nodeconf.HistoryStore for the scenario network. +type fakeConfHistory struct { + mu sync.Mutex + confs map[uint64]nodeconf.Configuration + last nodeconf.Configuration +} + +func newFakeConfHistory() *fakeConfHistory { + return &fakeConfHistory{confs: map[uint64]nodeconf.Configuration{}} +} + +func (h *fakeConfHistory) add(conf nodeconf.Configuration) { + h.mu.Lock() + h.confs[conf.Epoch] = conf + h.last = conf + h.mu.Unlock() +} + +func (h *fakeConfHistory) Init(_ *app.App) error { return nil } +func (h *fakeConfHistory) Name() string { return nodeconf.CNameStore } + +func (h *fakeConfHistory) GetLast(_ context.Context, _ string) (nodeconf.Configuration, error) { + h.mu.Lock() + defer h.mu.Unlock() + return h.last, nil +} + +func (h *fakeConfHistory) SaveLast(_ context.Context, c nodeconf.Configuration) error { + h.add(c) + return nil +} + +func (h *fakeConfHistory) GetByEpoch(_ context.Context, _ string, epoch uint64) (nodeconf.Configuration, error) { + h.mu.Lock() + defer h.mu.Unlock() + c, ok := h.confs[epoch] + if !ok { + return c, nodeconf.ErrConfigurationNotFound + } + return c, nil +} + +func (h *fakeConfHistory) Epochs(_ context.Context, _ string) (epochs []uint64, err error) { + h.mu.Lock() + defer h.mu.Unlock() + for e := range h.confs { + epochs = append(epochs, e) + } + sort.Slice(epochs, func(i, j int) bool { return epochs[i] < epochs[j] }) + return +} diff --git a/resharder/stat.go b/resharder/stat.go new file mode 100644 index 00000000..3e0ec62c --- /dev/null +++ b/resharder/stat.go @@ -0,0 +1,67 @@ +package resharder + +import ( + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" +) + +// resharder states exported via the node_resharder_state gauge. +const ( + // stateDisabled: the archive store is not shared, the machinery is off + stateDisabled uint32 = 0 + // stateIdle: nothing to drain (resharding complete or never needed) + stateIdle uint32 = 1 + // stateDraining: the node holds spaces it is no longer responsible for + stateDraining uint32 = 2 +) + +type resharderStat struct { + // state is one of stateDisabled/stateIdle/stateDraining + state atomic.Uint32 + // epoch is the network configuration epoch of the last drain cycle + epoch atomic.Uint64 + // lastCycleUnix is when the last drain cycle finished + lastCycleUnix atomic.Int64 + // draining is the number of local spaces this node is no longer + // responsible for and still holds (0 when resharding is complete) + draining atomic.Uint32 + // moved is the number of spaces handed off and deleted locally + moved atomic.Uint32 + // parked is the number of handoffs postponed to the next cycle + parked atomic.Uint32 + // errors is the number of failed drain attempts + errors atomic.Uint32 +} + +func registerMetric(s *resharderStat, registry *prometheus.Registry) { + gauge := func(name, help string, value func() float64) { + registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "node", + Subsystem: "resharder", + Name: name, + Help: help, + }, value)) + } + gauge("state", "resharding state: 0 disabled (archive store not shared), 1 idle, 2 draining", func() float64 { + return float64(s.state.Load()) + }) + gauge("epoch", "network configuration epoch of the last drain cycle", func() float64 { + return float64(s.epoch.Load()) + }) + gauge("last_cycle_unix", "unix time the last drain cycle finished, 0 if none ran yet", func() float64 { + return float64(s.lastCycleUnix.Load()) + }) + gauge("draining", "spaces this node is no longer responsible for and still holds", func() float64 { + return float64(s.draining.Load()) + }) + gauge("moved", "spaces handed off and deleted locally since start", func() float64 { + return float64(s.moved.Load()) + }) + gauge("parked", "handoffs postponed to the next cycle since start", func() float64 { + return float64(s.parked.Load()) + }) + gauge("errors", "failed drain attempts since start", func() float64 { + return float64(s.errors.Load()) + }) +}