From 0092b6be7a9593d7164eb1acf74e10f2de38f3ac Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Date: Tue, 9 Jun 2026 18:04:02 +0530 Subject: [PATCH 1/5] change the xattr keys used by GetDocumentWithRaw and GetDocSyncData --- db/crud.go | 4 +-- db/import_test.go | 72 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/db/crud.go b/db/crud.go index a23e8f1bc8..7096b4c6a3 100644 --- a/db/crud.go +++ b/db/crud.go @@ -86,7 +86,7 @@ func (c *DatabaseCollection) GetDocumentWithRaw(ctx context.Context, docid strin // If existing doc wasn't an SG Write, import the doc. if !isSgWrite { // reload to get revseqno for on-demand import - doc, rawBucketDoc, err = c.getDocWithXattrs(ctx, key, append(c.syncGlobalSyncAndUserXattrKeys(), base.VirtualXattrRevSeqNo), unmarshalLevel) + doc, rawBucketDoc, err = c.getDocWithXattrs(ctx, key, c.syncGlobalSyncMouRevSeqNoAndUserXattrKeys(), unmarshalLevel) if err != nil { return nil, nil, err } @@ -170,7 +170,7 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( if c.UseXattrs() { // Retrieve doc and xattr from bucket, unmarshal only xattr. // Triggers on-demand import when document xattr doesn't match cas. - rawDoc, xattrs, cas, getErr := c.dataStore.GetWithXattrs(ctx, key, c.syncGlobalSyncAndUserXattrKeys()) + rawDoc, xattrs, cas, getErr := c.dataStore.GetWithXattrs(ctx, key, c.syncGlobalSyncMouRevSeqNoAndUserXattrKeys()) if getErr != nil { return emptySyncData, getErr } diff --git a/db/import_test.go b/db/import_test.go index 58e554be56..0a53069d3d 100644 --- a/db/import_test.go +++ b/db/import_test.go @@ -253,6 +253,78 @@ func TestOnDemandImport(t *testing.T) { }) } }) + + // Verify that the reload performed before an on-demand import (crud.go GetDocumentWithRaw) + // fetches the _mou xattr, so rawBucketDoc returned to the caller contains it. + t.Run("on-demand get includes mou xattr in reload", func(t *testing.T) { + if !db.UseMou() { + t.Skip("Test requires MOU support") + } + docKey := baseKey + "_mouReload" + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + // SDK write: creates the doc without _sync or _mou + _, err := collection.dataStore.WriteCas(ctx, docKey, 0, 0, []byte(`{"foo":"bar"}`), 0) + require.NoError(t, err) + + // First on-demand import via GetDocument: SG writes _sync and _mou + importedDoc, err := collection.GetDocument(ctx, docKey, DocUnmarshalAll) + require.NoError(t, err) + require.NotNil(t, importedDoc.MetadataOnlyUpdate) + + // External SDK write to the body: changes the CRC32 so IsSGWrite returns false on the + // next read. Rosmar's WriteCas with a non-zero CAS preserves existing xattrs (including _mou). + _, err = collection.dataStore.WriteCas(ctx, docKey, 0, importedDoc.Cas, []byte(`{"foo":"baz"}`), 0) + require.NoError(t, err) + + // GetDocumentWithRaw detects a non-SG write, reloads the doc (crud.go:89), then imports it. + // The reload must include _mou in its xattr list so the returned rawBucketDoc is complete. + _, rawBucketDoc, err := collection.GetDocumentWithRaw(ctx, docKey, DocUnmarshalAll) + require.NoError(t, err) + require.NotNil(t, rawBucketDoc) + + // rawBucketDoc.Xattrs reflects the pre-import state fetched at the reload step. + // Without the fix, _mou was absent from the reload xattr list, so this would be nil + // even though the bucket has a _mou xattr from the first import. + mouBytes := rawBucketDoc.Xattrs[base.MouXattrName] + require.NotNil(t, mouBytes, "_mou must be fetched during the on-demand import reload") + var mou MetadataOnlyUpdate + require.NoError(t, base.JSONUnmarshal(mouBytes, &mou)) + require.NotEmpty(t, mou.HexCAS) + }) + + // Verify that GetDocSyncData fetches _revseqno so the on-demand import it triggers + // records the correct PreviousRevSeqNo in _mou (not 0). + t.Run("on-demand GetDocSyncData sets correct mou pRev", func(t *testing.T) { + if !db.UseMou() { + t.Skip("Test requires MOU support") + } + docKey := baseKey + "_syncDataMou" + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + // SDK write: creates doc without _sync or _mou + _, err := collection.dataStore.WriteCas(ctx, docKey, 0, 0, []byte(`{"foo":"bar"}`), 0) + require.NoError(t, err) + + // Capture revSeqNo before import — it must appear as _mou.pRev after import. + // Without the fix, GetDocSyncData fetches without _revseqno, so doc.RevSeqNo=0 + // and _mou.pRev is written as 0 instead of the correct value. + startingRevSeqNo, _, err := collection.getRevSeqNo(ctx, docKey) + require.NoError(t, err) + + // GetDocSyncData detects a non-SG-write and triggers on-demand import. + _, err = collection.GetDocSyncData(ctx, docKey) + require.NoError(t, err) + + // Read _mou from the bucket to verify PreviousRevSeqNo was set correctly. + xattrs, _, err := collection.dataStore.GetXattrs(ctx, docKey, []string{base.MouXattrName}) + require.NoError(t, err) + var mou MetadataOnlyUpdate + require.NoError(t, base.JSONUnmarshal(xattrs[base.MouXattrName], &mou)) + require.Equal(t, startingRevSeqNo, mou.PreviousRevSeqNo, + "_mou.pRev must equal the pre-import revSeqNo, not 0") + }) + testCases := []struct { name string eccv bool From 7b7a597ecd0df7e8f0c3bac844604cdebbb40a26 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Date: Wed, 10 Jun 2026 15:27:30 +0530 Subject: [PATCH 2/5] fixes based on pr comments - use mou and revSeq only for on demand import --- db/crud.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db/crud.go b/db/crud.go index 7096b4c6a3..7534e5abd5 100644 --- a/db/crud.go +++ b/db/crud.go @@ -170,7 +170,7 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( if c.UseXattrs() { // Retrieve doc and xattr from bucket, unmarshal only xattr. // Triggers on-demand import when document xattr doesn't match cas. - rawDoc, xattrs, cas, getErr := c.dataStore.GetWithXattrs(ctx, key, c.syncGlobalSyncMouRevSeqNoAndUserXattrKeys()) + rawDoc, xattrs, cas, getErr := c.dataStore.GetWithXattrs(ctx, key, c.syncGlobalSyncAndUserXattrKeys()) if getErr != nil { return emptySyncData, getErr } @@ -190,6 +190,11 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( if !isSgWrite { var importErr error + rawDoc, xattrs, cas, getErr := c.dataStore.GetWithXattrs(ctx, key, c.syncGlobalSyncMouRevSeqNoAndUserXattrKeys()) + if getErr != nil { + return emptySyncData, getErr + } + doc, importErr = c.OnDemandImportForGet(ctx, docid, doc, rawDoc, xattrs, cas) if importErr != nil { return emptySyncData, importErr From fb291f8552079d34141e23179506519e4d7abe1f Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Date: Thu, 11 Jun 2026 12:59:35 +0530 Subject: [PATCH 3/5] fix unit tests --- db/crud.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/db/crud.go b/db/crud.go index 7534e5abd5..7376fec817 100644 --- a/db/crud.go +++ b/db/crud.go @@ -190,11 +190,18 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( if !isSgWrite { var importErr error - rawDoc, xattrs, cas, getErr := c.dataStore.GetWithXattrs(ctx, key, c.syncGlobalSyncMouRevSeqNoAndUserXattrKeys()) + rawDoc, xattrs, cas, getErr = c.dataStore.GetWithXattrs(ctx, key, c.syncGlobalSyncMouRevSeqNoAndUserXattrKeys()) if getErr != nil { return emptySyncData, getErr } + // Re-unmarshal with the full xattr set so doc.RevSeqNo and doc.MetadataOnlyUpdate + // are populated for use by OnDemandImportForGet. + doc, unmarshalErr = c.unmarshalDocumentWithXattrs(ctx, docid, nil, xattrs, cas, DocUnmarshalSync) + if unmarshalErr != nil { + return emptySyncData, unmarshalErr + } + doc, importErr = c.OnDemandImportForGet(ctx, docid, doc, rawDoc, xattrs, cas) if importErr != nil { return emptySyncData, importErr From 836541206d88199c7b5d22940af9b59271cf3ba0 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Date: Fri, 12 Jun 2026 11:41:34 +0530 Subject: [PATCH 4/5] fixes based on PR comments - check if document is imported before performing on demand import --- db/crud.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/db/crud.go b/db/crud.go index 7376fec817..8a90d7f282 100644 --- a/db/crud.go +++ b/db/crud.go @@ -198,6 +198,15 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( // Re-unmarshal with the full xattr set so doc.RevSeqNo and doc.MetadataOnlyUpdate // are populated for use by OnDemandImportForGet. doc, unmarshalErr = c.unmarshalDocumentWithXattrs(ctx, docid, nil, xattrs, cas, DocUnmarshalSync) + + isSgWrite, crc32Match, _ := doc.IsSGWrite(ctx, rawDoc) + if crc32Match { + c.dbStats().Database().Crc32MatchCount.Add(1) + } + + if isSgWrite { + return doc.SyncData, nil + } if unmarshalErr != nil { return emptySyncData, unmarshalErr } From ab3f0819119395e0ed677e926dd7af9157d61d02 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Date: Fri, 12 Jun 2026 11:56:20 +0530 Subject: [PATCH 5/5] fixes based on copilot comments --- db/crud.go | 10 +++++----- db/import_test.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/db/crud.go b/db/crud.go index 8a90d7f282..4a060e8d6d 100644 --- a/db/crud.go +++ b/db/crud.go @@ -198,18 +198,18 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( // Re-unmarshal with the full xattr set so doc.RevSeqNo and doc.MetadataOnlyUpdate // are populated for use by OnDemandImportForGet. doc, unmarshalErr = c.unmarshalDocumentWithXattrs(ctx, docid, nil, xattrs, cas, DocUnmarshalSync) + if unmarshalErr != nil { + return emptySyncData, unmarshalErr + } - isSgWrite, crc32Match, _ := doc.IsSGWrite(ctx, rawDoc) + isSgWriteAfterReload, crc32Match, _ := doc.IsSGWrite(ctx, rawDoc) if crc32Match { c.dbStats().Database().Crc32MatchCount.Add(1) } - if isSgWrite { + if isSgWriteAfterReload { return doc.SyncData, nil } - if unmarshalErr != nil { - return emptySyncData, unmarshalErr - } doc, importErr = c.OnDemandImportForGet(ctx, docid, doc, rawDoc, xattrs, cas) if importErr != nil { diff --git a/db/import_test.go b/db/import_test.go index 0a53069d3d..b8cdd9658e 100644 --- a/db/import_test.go +++ b/db/import_test.go @@ -277,7 +277,7 @@ func TestOnDemandImport(t *testing.T) { _, err = collection.dataStore.WriteCas(ctx, docKey, 0, importedDoc.Cas, []byte(`{"foo":"baz"}`), 0) require.NoError(t, err) - // GetDocumentWithRaw detects a non-SG write, reloads the doc (crud.go:89), then imports it. + // GetDocumentWithRaw detects a non-SG write, reloads the doc, then imports it. // The reload must include _mou in its xattr list so the returned rawBucketDoc is complete. _, rawBucketDoc, err := collection.GetDocumentWithRaw(ctx, docKey, DocUnmarshalAll) require.NoError(t, err)