From 3c2034e41cd0474d47a44d8f806d216fff821bdb Mon Sep 17 00:00:00 2001 From: Sarah Johnson Date: Mon, 13 Jul 2026 11:25:00 +0100 Subject: [PATCH 01/25] feat: add related content at edition level --- application/application.go | 25 +++++++++++------- application/application_test.go | 46 +++++++++++++++++++++++++++++++++ models/version.go | 1 + mongo/dataset_store.go | 4 +++ swagger.yaml | 9 ++++--- 5 files changed, 73 insertions(+), 12 deletions(-) diff --git a/application/application.go b/application/application.go index 8887f767..5c5e81e0 100644 --- a/application/application.go +++ b/application/application.go @@ -230,34 +230,41 @@ func populateNewVersionDoc(currentVersion, originalVersion *models.Version) (*mo return nil, err } - var alerts []models.Alert + var related_content []models.GeneralDetails + if currentVersion.RelatedContent != nil { + related_content = append(related_content, *currentVersion.RelatedContent...) + } + if version.RelatedContent != nil { + related_content = append(related_content, *version.RelatedContent...) + } + if related_content != nil { + version.RelatedContent = &related_content + } + var alerts []models.Alert if version.Alerts != nil { alerts = append(alerts, *version.Alerts...) } - if alerts != nil { version.Alerts = &alerts } - if version.CollectionID == "" { - // will be checked later if state:published - version.CollectionID = currentVersion.CollectionID - } - var latestChanges []models.LatestChange if currentVersion.LatestChanges != nil { latestChanges = append(latestChanges, *currentVersion.LatestChanges...) } - if version.LatestChanges != nil { latestChanges = append(latestChanges, *version.LatestChanges...) } - if latestChanges != nil { version.LatestChanges = &latestChanges } + if version.CollectionID == "" { + // will be checked later if state:published + version.CollectionID = currentVersion.CollectionID + } + if version.ReleaseDate == "" { version.ReleaseDate = currentVersion.ReleaseDate } diff --git a/application/application_test.go b/application/application_test.go index 050a8e0c..3be7a92c 100644 --- a/application/application_test.go +++ b/application/application_test.go @@ -3082,6 +3082,52 @@ func TestPopulateNewVersionDocWithEditionChange(t *testing.T) { }) } +func TestPopulateNewVersionDocWithRelatedContent(t *testing.T) { + t.Parallel() + Convey("Given versions with related content", t, func() { + currentVersion := &models.Version{ + State: models.PublishedState, + ReleaseDate: "2024-12-31", + Version: 1, + ID: "789", + } + + originalVersion := &models.Version{ + Type: models.Static.String(), + } + + Convey("When only the current version includes related content", func() { + currentVersion.RelatedContent = &[]models.GeneralDetails{ + {Title: "Related content title", HRef: "https://www.ons.gov.uk/my-related-page", Description: "Related content description"}, + } + version, err := populateNewVersionDoc(currentVersion, originalVersion) + So(err, ShouldBeNil) + So(version, ShouldNotBeNil) + So(*version.RelatedContent, ShouldEqual, *currentVersion.RelatedContent) + }) + Convey("When only the original version includes related content", func() { + originalVersion.RelatedContent = &[]models.GeneralDetails{ + {Title: "Related content title", HRef: "https://www.ons.gov.uk/my-related-page", Description: "Related content description"}, + } + version, err := populateNewVersionDoc(currentVersion, originalVersion) + So(err, ShouldBeNil) + So(version, ShouldNotBeNil) + So(*version.RelatedContent, ShouldEqual, *originalVersion.RelatedContent) + }) + Convey("When both the current version and the original version include related content", func() { + originalVersion.RelatedContent = &[]models.GeneralDetails{ + {Title: "Related content title", HRef: "https://www.ons.gov.uk/my-related-page", Description: "Related content description"}, + } + currentVersion.RelatedContent = &[]models.GeneralDetails{ + {Title: "More Related content title", HRef: "https://www.ons.gov.uk/my-other-related-page", Description: "Additional Related content description"}, + } + version, err := populateNewVersionDoc(currentVersion, originalVersion) + So(err, ShouldBeNil) + So(version, ShouldNotBeNil) + So(len(*version.RelatedContent), ShouldEqual, len(*currentVersion.RelatedContent)+len(*originalVersion.RelatedContent)) + }) + }) +} func TestPopulateVersionInfoEditionValidationNonStatic(t *testing.T) { t.Parallel() diff --git a/models/version.go b/models/version.go index 13215564..efd7f66e 100644 --- a/models/version.go +++ b/models/version.go @@ -52,6 +52,7 @@ type Version struct { Distributions *[]Distribution `bson:"distributions,omitempty" json:"distributions,omitempty"` IsMigration *bool `bson:"is_migration,omitempty" json:"is_migration,omitempty"` PreviousEditionId []string `bson:"previous_edition_id,omitempty" json:"previous_edition_id,omitempty"` + RelatedContent *[]GeneralDetails `bson:"related_content,omitempty" json:"related_content,omitempty"` } // Alert represents an object containing information on an alert diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index 24c15778..8e46d973 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -620,6 +620,10 @@ func createVersionUpdateQuery(version *models.Version, newETag string) bson.M { setUpdates["previous_edition_id"] = version.PreviousEditionId } + if version.RelatedContent != nil { + setUpdates["related_content"] = version.RelatedContent + } + if newETag != "" { setUpdates["e_tag"] = newETag } diff --git a/swagger.yaml b/swagger.yaml index 4034727a..70aef91d 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1564,7 +1564,6 @@ definitions: type: string default: "Open Government Licence v3.0" links: - readOnly: true $ref: "#/definitions/DatasetLinks" methodologies: description: "A list of methodologies for the dataset." @@ -1587,7 +1586,7 @@ definitions: publishers: description: A list of the publishers for the dataset. type: array - minLength: 1 + minItems: 1 items: $ref: "#/definitions/Publisher" default: @@ -2230,7 +2229,6 @@ definitions: - readOnly: true - $ref: "#/definitions/DatasetID" dataset_links: - readOnly: true $ref: "#/definitions/DatasetLinks" description: description: "A description for a dataset" @@ -2709,6 +2707,11 @@ definitions: - official - official-in-development - no-accreditation + related_content: + description: "A list of website content related to this version." + type: array + items: + $ref: "#/definitions/RelatedLink" release_date: description: "The release date of this version of the dataset" type: string From 6c3e409ba5407f04093c519e5e7dc0c6a7804397 Mon Sep 17 00:00:00 2001 From: Hugh Brace Date: Tue, 14 Jul 2026 14:44:05 +0100 Subject: [PATCH 02/25] Store previous series IDs for datasets --- api/dataset.go | 4 ++++ api/dataset_test.go | 6 ++++-- features/datasets_id_web.feature | 18 ++++++++++++++++++ features/static_dataset_put.feature | 5 ++++- models/dataset.go | 1 + 5 files changed, 31 insertions(+), 3 deletions(-) diff --git a/api/dataset.go b/api/dataset.go index 4e476c29..dd6d5bec 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -246,6 +246,8 @@ func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { dataset.Current.Topics = nil } + dataset.Current.PreviousSeriesId = nil + datasetResponse = dataset.Current } else { // User has valid authentication to get raw dataset document @@ -685,6 +687,8 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { return nil, err } } else if dataset.Type == models.Static.String() && dataset.ID != currentDataset.ID { + dataset.PreviousSeriesId = append([]string{}, currentDataset.Next.PreviousSeriesId...) + dataset.PreviousSeriesId = append(dataset.PreviousSeriesId, currentDataset.ID) renamedDatasetUpdate := &models.DatasetUpdate{ ID: dataset.ID, Next: dataset, diff --git a/api/dataset_test.go b/api/dataset_test.go index aeedbac6..30b5fa2c 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -1957,13 +1957,13 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { }) Convey("A successful request to rename an unpublished static dataset returns 200 OK response", t, func() { - b := `{"id":"456","contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"static","keywords":["keyword","keyword 2"],"topics":["topic-0","topic-1"],"license":"Open Government Licence v3.0"}` + b := `{"id":"456","contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"static","keywords":["keyword","keyword 2"],"topics":["topic-0","topic-1"],"license":"Open Government Licence v3.0","previous_series_id":["789"]}` r := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(context.Context, string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{ID: "123", Next: &models.Dataset{Type: models.Static.String(), Title: "CensusEthnicity", State: models.CreatedState, Topics: []string{"topic-0", "topic-1"}}}, nil + return &models.DatasetUpdate{ID: "123", Next: &models.Dataset{Type: models.Static.String(), Title: "CensusEthnicity", State: models.CreatedState, Topics: []string{"topic-0", "topic-1"}, PreviousSeriesId: []string{"789"}}}, nil }, CheckDatasetExistsFunc: func(ctx context.Context, id, state string) error { return errs.ErrDatasetNotFound @@ -2002,6 +2002,8 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { So(mockedDataStore.CheckDatasetExistsCalls(), ShouldHaveLength, 1) So(mockedDataStore.GetVersionsStaticNoLimitCalls(), ShouldHaveLength, 1) So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpsertDatasetCalls()[0].ID, ShouldEqual, "456") + So(mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.PreviousSeriesId, ShouldResemble, []string{"789", "123"}) So(mockedDataStore.DeleteDatasetCalls(), ShouldHaveLength, 1) So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 0) }) diff --git a/features/datasets_id_web.feature b/features/datasets_id_web.feature index ad5d86f3..ceb3ed48 100644 --- a/features/datasets_id_web.feature +++ b/features/datasets_id_web.feature @@ -8,6 +8,12 @@ Feature: GET /datasets/{id} in web mode "id": "published-dataset", "state": "published", "title": "Published Dataset" + }, + { + "id": "published-dataset-with-previous-series", + "state": "published", + "title": "Published Dataset with previous series ID", + "previous_series_id": ["old-dataset-id"] } ] """ @@ -23,6 +29,18 @@ Feature: GET /datasets/{id} in web mode "title": "Published Dataset" } """ + + Scenario: Retrieving a published dataset with previous series returns 200 with the fields redacted + When I GET "/datasets/published-dataset-with-previous-series" + Then I should receive the following JSON response with status "200": + """ + { + "id": "published-dataset-with-previous-series", + "last_updated": "{{DYNAMIC_TIMESTAMP}}", + "state": "published", + "title": "Published Dataset with previous series ID" + } + """ Scenario: Retrieving a non-existing dataset returns 404 When I GET "/datasets/non-existing-dataset" diff --git a/features/static_dataset_put.feature b/features/static_dataset_put.feature index 52c2f739..9a5265bd 100644 --- a/features/static_dataset_put.feature +++ b/features/static_dataset_put.feature @@ -160,7 +160,10 @@ Feature: PUT /datasets/{id} for static datasets "href": "/datasets/new-dataset-id/editions/2025/versions/1", "id": "1" } - } + }, + "previous_series_id": [ + "old-dataset-id" + ] } """ And the static version in the database for id "version-1" should be: diff --git a/models/dataset.go b/models/dataset.go index 0a8e9d75..731899f0 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -120,6 +120,7 @@ type Dataset struct { Survey string `bson:"survey,omitempty" json:"survey,omitempty"` RelatedContent []GeneralDetails `bson:"related_content,omitempty" json:"related_content,omitempty"` Topics []string `bson:"topics,omitempty" json:"topics,omitempty"` + PreviousSeriesId []string `bson:"previous_series_id,omitempty" json:"previous_series_id,omitempty"` } // DatasetLinks represents a list of specific links related to the dataset resource From c8f47df12ae7ce6babfb32a51cd9101f6e7e2dd7 Mon Sep 17 00:00:00 2001 From: Hugh Brace Date: Wed, 15 Jul 2026 11:33:17 +0100 Subject: [PATCH 03/25] Redact previous series ID for web mode when EnableURLRewriting is true --- api/dataset.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/dataset.go b/api/dataset.go index dd6d5bec..f04d80f7 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -224,6 +224,7 @@ func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { } log.Info(ctx, "getDataset endpoint: get dataset with auth", logData) } else { + dataset.Current.PreviousSeriesId = nil datasetResponse, err = utils.RewriteDatasetWithoutAuth(ctx, dataset, datasetLinksBuilder) if err != nil { log.Error(ctx, "getDataset endpoint: failed to rewrite dataset without authorisation", err, logData) From ea2fd62c47dc4327ae81a3ba33037e163a0c7f60 Mon Sep 17 00:00:00 2001 From: Hugh Brace Date: Fri, 17 Jul 2026 10:52:08 +0100 Subject: [PATCH 04/25] Redact previous series IDs when GET /datasets --- api/dataset.go | 1 + api/dataset_test.go | 84 ++++++++++++++++++++++++++++++++ features/public_datasets.feature | 28 ++++++++++- utils/rewriting.go | 1 + 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/api/dataset.go b/api/dataset.go index f04d80f7..8c9b0f06 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -914,6 +914,7 @@ func mapResults(results []*models.DatasetUpdate) []*models.Dataset { continue } item.Current.ID = item.ID + item.Current.PreviousSeriesId = nil items = append(items, item.Current) } return items diff --git a/api/dataset_test.go b/api/dataset_test.go index 30b5fa2c..d56c5547 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -376,6 +376,90 @@ func TestGetDatasetsReturnsOK(t *testing.T) { So(mockedDataStore.GetDatasetsCalls()[0].Offset, ShouldEqual, 12) }) + Convey("A successful web-mode request to get datasets does not expose previous_series_id", t, func() { + r := &http.Request{} + w := httptest.NewRecorder() + address, err := neturl.Parse("localhost:20000/datasets") + So(err, ShouldBeNil) + r.URL = address + + mockedDataStore := &storetest.StorerMock{ + GetDatasetsFunc: func(context.Context, int, int, bool) ([]*models.DatasetUpdate, int, error) { + return []*models.DatasetUpdate{{ + ID: "123-456", + Current: &models.Dataset{ + ID: "123-456", + Type: models.Static.String(), + PreviousSeriesId: []string{"old-series-id"}, + }, + }}, 1, nil + }, + } + + authorisationMock := &authMock.MiddlewareMock{ + RequireFunc: func(permission string, handlerFunc http.HandlerFunc) http.HandlerFunc { + return handlerFunc + }, + ParseFunc: func(token string) (*permissionsAPISDK.EntityData, error) { + return nil, permissionsAPISDK.ErrFailedToParsePermissionsResponse + }, + } + + api := GetAPIWithCMDMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, authorisationMock, application.SearchContentUpdatedProducer{}, &cloudflareMocks.ClienterMock{}, &applicationMocks.AuditServiceMock{}, &applicationMocks.StaticDatasetServiceMock{}, nil, &filesAPISDKMocks.ClienterMock{}) + api.enableURLRewriting = true + + actualResponse, actualTotalCount, err := api.getDatasets(w, r, 11, 12) + So(err, ShouldBeNil) + So(actualTotalCount, ShouldEqual, 1) + + datasets, ok := actualResponse.([]*models.Dataset) + So(ok, ShouldBeTrue) + So(datasets, ShouldHaveLength, 1) + So(datasets[0].PreviousSeriesId, ShouldBeNil) + }) + + Convey("A successful web-mode request to get datasets does not expose previous_series_id when URL rewriting is disabled", t, func() { + r := &http.Request{} + w := httptest.NewRecorder() + address, err := neturl.Parse("localhost:20000/datasets") + So(err, ShouldBeNil) + r.URL = address + + mockedDataStore := &storetest.StorerMock{ + GetDatasetsFunc: func(context.Context, int, int, bool) ([]*models.DatasetUpdate, int, error) { + return []*models.DatasetUpdate{{ + ID: "123-456", + Current: &models.Dataset{ + ID: "123-456", + Type: models.Static.String(), + PreviousSeriesId: []string{"old-series-id"}, + }, + }}, 1, nil + }, + } + + authorisationMock := &authMock.MiddlewareMock{ + RequireFunc: func(permission string, handlerFunc http.HandlerFunc) http.HandlerFunc { + return handlerFunc + }, + ParseFunc: func(token string) (*permissionsAPISDK.EntityData, error) { + return nil, permissionsAPISDK.ErrFailedToParsePermissionsResponse + }, + } + + api := GetAPIWithCMDMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, authorisationMock, application.SearchContentUpdatedProducer{}, &cloudflareMocks.ClienterMock{}, &applicationMocks.AuditServiceMock{}, &applicationMocks.StaticDatasetServiceMock{}, nil, &filesAPISDKMocks.ClienterMock{}) + api.enableURLRewriting = false + + actualResponse, actualTotalCount, err := api.getDatasets(w, r, 11, 12) + So(err, ShouldBeNil) + So(actualTotalCount, ShouldEqual, 1) + + datasets, ok := actualResponse.([]*models.Dataset) + So(ok, ShouldBeTrue) + So(datasets, ShouldHaveLength, 1) + So(datasets[0].PreviousSeriesId, ShouldBeNil) + }) + Convey("A successful request to get datasetwith type query parameter returns 200 OK response, and limit and offset are delegated to the datastore", t, func() { r := &http.Request{} w := httptest.NewRecorder() diff --git a/features/public_datasets.feature b/features/public_datasets.feature index 539fe78b..a7d10dce 100644 --- a/features/public_datasets.feature +++ b/features/public_datasets.feature @@ -86,4 +86,30 @@ Feature: Dataset API } """ Then the HTTP status code should be "405" - \ No newline at end of file + + Scenario: Get /datasets does not return redacted fields + Given I have these datasets: + """ + [ + { + "id": "population-estimates", + "previous_series_id": ["old-dataset-id"] + } + ] + """ + When I GET "/datasets" + Then I should receive the following JSON response with status "200": + """ + { + "count":1, + "items": [ + { + "id": "population-estimates", + "last_updated":"0001-01-01T00:00:00Z" + } + ], + "limit":20, + "offset":0, + "total_count":1 + } + """ diff --git a/utils/rewriting.go b/utils/rewriting.go index 142d1b84..cb557a7b 100644 --- a/utils/rewriting.go +++ b/utils/rewriting.go @@ -51,6 +51,7 @@ func RewriteDatasetsWithoutAuth(ctx context.Context, results []*models.DatasetUp return nil, err } item.Current.ID = item.ID + item.Current.PreviousSeriesId = nil items = append(items, item.Current) } } From 30b88c0b278ace11cd0d659c7af9c9c994f8b82a Mon Sep 17 00:00:00 2001 From: Hugh Brace Date: Fri, 17 Jul 2026 11:07:05 +0100 Subject: [PATCH 05/25] Fix linting issue --- utils/rewriting.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/utils/rewriting.go b/utils/rewriting.go index cb557a7b..f759be51 100644 --- a/utils/rewriting.go +++ b/utils/rewriting.go @@ -44,16 +44,18 @@ func RewriteDatasetsWithoutAuth(ctx context.Context, results []*models.DatasetUp items := []*models.Dataset{} for _, item := range results { - if item.Current != nil { - err := RewriteDatasetLinks(ctx, item.Current.Links, datasetLinksBuilder) - if err != nil { - log.Error(ctx, "failed to rewrite 'current' links", err) - return nil, err - } - item.Current.ID = item.ID - item.Current.PreviousSeriesId = nil - items = append(items, item.Current) + if item.Current == nil { + continue } + + err := RewriteDatasetLinks(ctx, item.Current.Links, datasetLinksBuilder) + if err != nil { + log.Error(ctx, "failed to rewrite 'current' links", err) + return nil, err + } + item.Current.ID = item.ID + item.Current.PreviousSeriesId = nil + items = append(items, item.Current) } return items, nil } From c79498a148398e03280f646353f9f6e17da2b5df Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Mon, 20 Jul 2026 13:54:19 +0100 Subject: [PATCH 06/25] avoid data race on config in parallel tests --- api/versions_test.go | 2 ++ api/webendpoints_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/api/versions_test.go b/api/versions_test.go index 0b9d902c..82fbd716 100644 --- a/api/versions_test.go +++ b/api/versions_test.go @@ -2339,9 +2339,11 @@ func TestPutVersionGenerateDownloadsError(t *testing.T) { r := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123/editions/2017/versions/1", bytes.NewBufferString(versionAssociatedPayload)) w := httptest.NewRecorder() + mu.Lock() cfg, err := config.Get() So(err, ShouldBeNil) cfg.EnablePrivateEndpoints = true + mu.Unlock() authorisationMock := &authMock.MiddlewareMock{ RequireFunc: func(permission string, handlerFunc http.HandlerFunc) http.HandlerFunc { diff --git a/api/webendpoints_test.go b/api/webendpoints_test.go index 7b8b1646..2e002fb8 100644 --- a/api/webendpoints_test.go +++ b/api/webendpoints_test.go @@ -384,6 +384,8 @@ func GetWebAPIWithMocks(ctx context.Context, mockedDataStore store.Storer, mocke DownloadGenerators: mockedMapSMGeneratedDownloads, } + mu.Lock() + defer mu.Unlock() cfg, err := config.Get() So(err, ShouldBeNil) cfg.ServiceAuthToken = authToken From d9ed7bedec252fb9fe52c458608f635b791ed32c Mon Sep 17 00:00:00 2001 From: Hugh Brace Date: Wed, 22 Jul 2026 13:26:17 +0100 Subject: [PATCH 07/25] Fix audit falure for CVE-2026-56852 --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c6aad03f..bdd7dd4d 100644 --- a/go.mod +++ b/go.mod @@ -172,11 +172,11 @@ require ( go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.49.0 // indirect - golang.org/x/sync v0.19.0 // indirect + golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/text v0.40.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260202165425-ce8ad4cf556b // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260202165425-ce8ad4cf556b // indirect google.golang.org/grpc v1.76.0 // indirect diff --git a/go.sum b/go.sum index 82d50442..a1bc8cdf 100644 --- a/go.sum +++ b/go.sum @@ -498,8 +498,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -518,8 +518,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -556,8 +556,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= From c7a4ebfa5b2cd742013757eb9d973f5d7db0468a Mon Sep 17 00:00:00 2001 From: Sarah Johnson Date: Fri, 24 Jul 2026 11:00:20 +0100 Subject: [PATCH 08/25] feat: add related content for versions --- application/application.go | 7 ++----- application/application_test.go | 20 +++++++++----------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/application/application.go b/application/application.go index 9d89c925..9c55e3b1 100644 --- a/application/application.go +++ b/application/application.go @@ -223,17 +223,14 @@ func (smDS *StateMachineDatasetAPI) PopulateVersionInfo(ctx context.Context, ver } //nolint:gocyclo // cyclomatic complexity 21 of func `populateNewVersionDoc` is high (> 20) -func populateNewVersionDoc(currentVersion, originalVersion *models.Version) (*models.Version, error) { +func populateNewVersionDoc(currentVersion, versionUpdate *models.Version) (*models.Version, error) { var version models.Version - err := copier.Copy(&version, originalVersion) // create local copy that escapes to the HEAP at the end of this function + err := copier.Copy(&version, versionUpdate) // create local copy that escapes to the HEAP at the end of this function if err != nil { return nil, err } var related_content []models.GeneralDetails - if currentVersion.RelatedContent != nil { - related_content = append(related_content, *currentVersion.RelatedContent...) - } if version.RelatedContent != nil { related_content = append(related_content, *version.RelatedContent...) } diff --git a/application/application_test.go b/application/application_test.go index 3be7a92c..dc91bfdd 100644 --- a/application/application_test.go +++ b/application/application_test.go @@ -3091,40 +3091,38 @@ func TestPopulateNewVersionDocWithRelatedContent(t *testing.T) { Version: 1, ID: "789", } - - originalVersion := &models.Version{ + versionUpdate := &models.Version{ Type: models.Static.String(), } - Convey("When only the current version includes related content", func() { currentVersion.RelatedContent = &[]models.GeneralDetails{ {Title: "Related content title", HRef: "https://www.ons.gov.uk/my-related-page", Description: "Related content description"}, } - version, err := populateNewVersionDoc(currentVersion, originalVersion) + version, err := populateNewVersionDoc(currentVersion, versionUpdate) So(err, ShouldBeNil) So(version, ShouldNotBeNil) - So(*version.RelatedContent, ShouldEqual, *currentVersion.RelatedContent) + So(version.RelatedContent, ShouldBeNil) }) Convey("When only the original version includes related content", func() { - originalVersion.RelatedContent = &[]models.GeneralDetails{ + versionUpdate.RelatedContent = &[]models.GeneralDetails{ {Title: "Related content title", HRef: "https://www.ons.gov.uk/my-related-page", Description: "Related content description"}, } - version, err := populateNewVersionDoc(currentVersion, originalVersion) + version, err := populateNewVersionDoc(currentVersion, versionUpdate) So(err, ShouldBeNil) So(version, ShouldNotBeNil) - So(*version.RelatedContent, ShouldEqual, *originalVersion.RelatedContent) + So(*version.RelatedContent, ShouldEqual, *versionUpdate.RelatedContent) }) Convey("When both the current version and the original version include related content", func() { - originalVersion.RelatedContent = &[]models.GeneralDetails{ + versionUpdate.RelatedContent = &[]models.GeneralDetails{ {Title: "Related content title", HRef: "https://www.ons.gov.uk/my-related-page", Description: "Related content description"}, } currentVersion.RelatedContent = &[]models.GeneralDetails{ {Title: "More Related content title", HRef: "https://www.ons.gov.uk/my-other-related-page", Description: "Additional Related content description"}, } - version, err := populateNewVersionDoc(currentVersion, originalVersion) + version, err := populateNewVersionDoc(currentVersion, versionUpdate) So(err, ShouldBeNil) So(version, ShouldNotBeNil) - So(len(*version.RelatedContent), ShouldEqual, len(*currentVersion.RelatedContent)+len(*originalVersion.RelatedContent)) + So(*version.RelatedContent, ShouldEqual, *versionUpdate.RelatedContent) }) }) } From a31aa8589502ead53e31456a548e0c0a6b0bcf69 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Mon, 27 Jul 2026 15:07:34 +0100 Subject: [PATCH 09/25] ensure static editions are always first sorted by oldest version release data --- mongo/version_store.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mongo/version_store.go b/mongo/version_store.go index dddab187..6c89c846 100644 --- a/mongo/version_store.go +++ b/mongo/version_store.go @@ -339,7 +339,7 @@ func (m *Mongo) GetAllStaticVersions(ctx context.Context, datasetID, state strin } // GetEditionsStatic retrieves a paginated list of editions for a given dataset. -// Response is ordered by the release date of the oldest version for each edition. +// Editions are ordered by the release date of the oldest version for each edition. // Each Version record is mapped to an EditionUpdate with Current and Next set depending on the state. func (m *Mongo) GetEditionsStatic(ctx context.Context, datasetID, state string, offset, limit int) ([]*models.EditionUpdate, int, error) { selector := bson.M{"links.dataset.id": datasetID} @@ -355,9 +355,9 @@ func (m *Mongo) GetEditionsStatic(ctx context.Context, datasetID, state string, "$min": "$release_date", }, }}, - {"$sort": bson.M{ - "oldest_version_release_date": -1, - "_id": 1, + {"$sort": bson.D{ + {Key: "oldest_version_release_date", Value: -1}, + {Key: "_id", Value: 1}, }}, } From 6caf80cc0db8f5992f19714791b951c2f8112572 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Mon, 27 Jul 2026 15:35:52 +0100 Subject: [PATCH 10/25] fix audit issues with go.opentelemetry.io/otel --- go.mod | 12 ++++++------ go.sum | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index bdd7dd4d..f96fc278 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-avro/avro v0.0.0-20171219232920-444163702c11 // indirect github.com/go-json-experiment/json v0.0.0-20250813233538-9b1f9ea2e11b // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -163,19 +163,19 @@ require ( go.opentelemetry.io/contrib/propagators/b3 v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.36.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.49.0 // indirect golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.40.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260202165425-ce8ad4cf556b // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260202165425-ce8ad4cf556b // indirect diff --git a/go.sum b/go.sum index a1bc8cdf..1c6db92a 100644 --- a/go.sum +++ b/go.sum @@ -190,8 +190,8 @@ github.com/go-avro/avro v0.0.0-20171219232920-444163702c11/go.mod h1:kxj6THYP0dm github.com/go-json-experiment/json v0.0.0-20250813233538-9b1f9ea2e11b h1:6Q4zRHXS/YLOl9Ng1b1OOOBWMidAQZR3Gel0UKPC/KU= github.com/go-json-experiment/json v0.0.0-20250813233538-9b1f9ea2e11b/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -467,22 +467,22 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 h1:SoCgXYF4ISDtNyfLUzsGDa go.opentelemetry.io/contrib/propagators/jaeger v1.36.0/go.mod h1:VHu48l0YTRKSObdPQ+Sb8xMZvdnJlN7yhHuHoPgNqHM= go.opentelemetry.io/contrib/propagators/ot v1.36.0 h1:UBoZjbx483GslNKYK2YpfvePTJV4BHGeFd8+b7dexiM= go.opentelemetry.io/contrib/propagators/ot v1.36.0/go.mod h1:adDDRry19/n9WoA7mSCMjoVJcmzK/bZYzX9SR+g2+W4= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0 h1:SNhVp/9q4Go/XHBkQ1/d5u9P/U+L1yaGPoi0x+mStaI= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0/go.mod h1:tx8OOlGH6R4kLV67YaYO44GFXloEjGPZuMjEkaaqIp4= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -541,8 +541,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 8b97893a75f3d15b9de52f89627f99b20e28db59 Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Tue, 28 Jul 2026 11:23:29 +0100 Subject: [PATCH 11/25] add is_migration series level --- api/dataset.go | 8 ++ api/dataset_test.go | 92 ++++++++++++++++++++- features/datasets_id_web.feature | 16 +++- features/private_datasets.feature | 121 ++++++++++++++++++++++++++++ features/public_datasets.feature | 32 +++++++- features/static_dataset_put.feature | 2 + models/dataset.go | 1 + mongo/dataset_store.go | 4 + mongo/dataset_test.go | 18 +++++ swagger.yaml | 5 +- utils/rewriting.go | 1 + 11 files changed, 296 insertions(+), 4 deletions(-) diff --git a/api/dataset.go b/api/dataset.go index 8c9b0f06..6b1bab73 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -225,6 +225,7 @@ func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { log.Info(ctx, "getDataset endpoint: get dataset with auth", logData) } else { dataset.Current.PreviousSeriesId = nil + dataset.Current.IsMigration = nil datasetResponse, err = utils.RewriteDatasetWithoutAuth(ctx, dataset, datasetLinksBuilder) if err != nil { log.Error(ctx, "getDataset endpoint: failed to rewrite dataset without authorisation", err, logData) @@ -248,6 +249,7 @@ func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { } dataset.Current.PreviousSeriesId = nil + dataset.Current.IsMigration = nil datasetResponse = dataset.Current } else { @@ -690,6 +692,11 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { } else if dataset.Type == models.Static.String() && dataset.ID != currentDataset.ID { dataset.PreviousSeriesId = append([]string{}, currentDataset.Next.PreviousSeriesId...) dataset.PreviousSeriesId = append(dataset.PreviousSeriesId, currentDataset.ID) + + if dataset.IsMigration == nil { + dataset.IsMigration = currentDataset.Next.IsMigration + } + renamedDatasetUpdate := &models.DatasetUpdate{ ID: dataset.ID, Next: dataset, @@ -915,6 +922,7 @@ func mapResults(results []*models.DatasetUpdate) []*models.Dataset { } item.Current.ID = item.ID item.Current.PreviousSeriesId = nil + item.Current.IsMigration = nil items = append(items, item.Current) } return items diff --git a/api/dataset_test.go b/api/dataset_test.go index d56c5547..4a8b48c7 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -48,6 +48,8 @@ const ( authToken = "dataset" ) +func boolPtr(b bool) *bool { return &b } + var ( datasetPayload = `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"filterable"}` datasetPayloadWithID = `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","keywords":["keyword"],"links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","id": "ageing-population-estimates", "next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"filterable"}` @@ -61,6 +63,7 @@ var ( datasetPayloadWithTypeStatic = `{"id":"123","contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"static","keywords":["keyword","keyword 2"],"topics":["topic-0","topic-1"],"license":"Open Government Licence v3.0"}` datasetPayloadWithStatePublished = `{"id":"123","contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"static-published","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"StaticPublished","theme":"population","state":"published","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"static","keywords":["keyword","keyword 2"],"topics":["topic-0","topic-1"],"license":"Open Government Licence v3.0"}` datasetPayloadWithStateAssociated = `{"id":"123","contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"static-associated","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"StaticAssociated","theme":"population","state":"associated","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"static","keywords":["keyword","keyword 2"],"topics":["topic-0","topic-1"],"license":"Open Government Licence v3.0"}` + datasetPayloadWithIDAndIsMigration = `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","keywords":["keyword"],"links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","id": "ageing-population-estimates", "next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"filterable","is_migration":false}` editionPayload = `"{\"edition\":\"2017\",\"state\":\"created\",\"license\":\"ONS\",\"release_date\":\"2017-04-04\",\"version\":\"1\"}"` @@ -391,6 +394,7 @@ func TestGetDatasetsReturnsOK(t *testing.T) { ID: "123-456", Type: models.Static.String(), PreviousSeriesId: []string{"old-series-id"}, + IsMigration: boolPtr(true), }, }}, 1, nil }, @@ -416,6 +420,7 @@ func TestGetDatasetsReturnsOK(t *testing.T) { So(ok, ShouldBeTrue) So(datasets, ShouldHaveLength, 1) So(datasets[0].PreviousSeriesId, ShouldBeNil) + So(datasets[0].IsMigration, ShouldBeNil) }) Convey("A successful web-mode request to get datasets does not expose previous_series_id when URL rewriting is disabled", t, func() { @@ -835,6 +840,48 @@ func TestGetDatasetReturnsOK(t *testing.T) { So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) }) + + Convey("When a web mode request gets a dataset, is_migration is not returned", t, func() { + r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123-456", http.NoBody) + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(context.Context, string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{ + ID: "123", + Current: &models.Dataset{ID: "123", IsMigration: boolPtr(true)}, + Next: &models.Dataset{ID: "123", IsMigration: boolPtr(true)}, + }, nil + }, + } + + api := GetWebAPIWithMocks(context.Background(), mockedDataStore, &mocks.DownloadsGeneratorMock{}, &authMock.MiddlewareMock{}, &authMock.PermissionsCheckerMock{}, &clientsidentity.Client{}, &applicationMocks.AuditServiceMock{}, &applicationMocks.StaticDatasetServiceMock{}) + api.enableURLRewriting = false + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusOK) + So(w.Body.String(), ShouldNotContainSubstring, "is_migration") + }) + + Convey("When a web mode request gets a dataset with URL rewriting enabled, is_migration is not returned", t, func() { + r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123-456", http.NoBody) + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(context.Context, string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{ + ID: "123", + Current: &models.Dataset{ID: "123", IsMigration: boolPtr(true)}, + Next: &models.Dataset{ID: "123", IsMigration: boolPtr(true)}, + }, nil + }, + } + + api := GetWebAPIWithMocks(context.Background(), mockedDataStore, &mocks.DownloadsGeneratorMock{}, &authMock.MiddlewareMock{}, &authMock.PermissionsCheckerMock{}, &clientsidentity.Client{}, &applicationMocks.AuditServiceMock{}, &applicationMocks.StaticDatasetServiceMock{}) + api.enableURLRewriting = true + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusOK) + So(w.Body.String(), ShouldNotContainSubstring, "is_migration") + }) } func TestGetDatasetReturnsError(t *testing.T) { @@ -1802,6 +1849,47 @@ func TestAddDatasetNew(t *testing.T) { So(w.Code, ShouldEqual, http.StatusBadRequest) }) + + Convey("A request to post a dataset with is_migration false stores the value", t, func() { + b := datasetPayloadWithIDAndIsMigration + r := createRequestWithAuth("POST", "http://localhost:22000/datasets", bytes.NewBufferString(b)) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(context.Context, string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + CheckDatasetTitleExistFunc: func(ctx context.Context, title string) (bool, error) { + return false, nil + }, + UpsertDatasetFunc: func(context.Context, string, *models.DatasetUpdate) error { + return nil + }, + } + + authorisationMock := &authMock.MiddlewareMock{ + RequireFunc: func(permission string, handlerFunc http.HandlerFunc) http.HandlerFunc { + return handlerFunc + }, + ParseFunc: func(token string) (*permissionsAPISDK.EntityData, error) { + return testEntityData, nil + }, + } + + auditServiceMock := &applicationMocks.AuditServiceMock{ + RecordDatasetAuditEventFunc: func(ctx context.Context, requestedBy models.RequestedBy, action models.Action, resource string, dataset *models.Dataset) error { + return nil + }, + } + + api := GetAPIWithCMDMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, authorisationMock, application.SearchContentUpdatedProducer{}, &cloudflareMocks.ClienterMock{}, auditServiceMock, &applicationMocks.StaticDatasetServiceMock{}, nil, &filesAPISDKMocks.ClienterMock{}) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusCreated) + So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 1) + So(mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.IsMigration, ShouldNotBeNil) + So(*mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.IsMigration, ShouldBeFalse) + }) } func TestPutDatasetReturnsSuccessfully(t *testing.T) { @@ -2047,7 +2135,7 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(context.Context, string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{ID: "123", Next: &models.Dataset{Type: models.Static.String(), Title: "CensusEthnicity", State: models.CreatedState, Topics: []string{"topic-0", "topic-1"}, PreviousSeriesId: []string{"789"}}}, nil + return &models.DatasetUpdate{ID: "123", Next: &models.Dataset{Type: models.Static.String(), Title: "CensusEthnicity", State: models.CreatedState, Topics: []string{"topic-0", "topic-1"}, PreviousSeriesId: []string{"789"}, IsMigration: boolPtr(true)}}, nil }, CheckDatasetExistsFunc: func(ctx context.Context, id, state string) error { return errs.ErrDatasetNotFound @@ -2088,6 +2176,8 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 1) So(mockedDataStore.UpsertDatasetCalls()[0].ID, ShouldEqual, "456") So(mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.PreviousSeriesId, ShouldResemble, []string{"789", "123"}) + So(mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.IsMigration, ShouldNotBeNil) + So(*mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.IsMigration, ShouldBeTrue) So(mockedDataStore.DeleteDatasetCalls(), ShouldHaveLength, 1) So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 0) }) diff --git a/features/datasets_id_web.feature b/features/datasets_id_web.feature index ceb3ed48..0d9bdecd 100644 --- a/features/datasets_id_web.feature +++ b/features/datasets_id_web.feature @@ -13,7 +13,8 @@ Feature: GET /datasets/{id} in web mode "id": "published-dataset-with-previous-series", "state": "published", "title": "Published Dataset with previous series ID", - "previous_series_id": ["old-dataset-id"] + "previous_series_id": ["old-dataset-id"], + "is_migration": true } ] """ @@ -48,4 +49,17 @@ Feature: GET /datasets/{id} in web mode And I should receive the following response: """ dataset not found + """ + + Scenario: Retrieving a published dataset with URL rewriting enabled returns 200 with the fields redacted + Given URL rewriting is enabled + When I GET "/datasets/published-dataset-with-previous-series" + Then I should receive the following JSON response with status "200": + """ + { + "id": "published-dataset-with-previous-series", + "last_updated": "{{DYNAMIC_TIMESTAMP}}", + "state": "published", + "title": "Published Dataset with previous series ID" + } """ \ No newline at end of file diff --git a/features/private_datasets.feature b/features/private_datasets.feature index 44b7b2ff..f4f77f22 100644 --- a/features/private_datasets.feature +++ b/features/private_datasets.feature @@ -818,4 +818,125 @@ Feature: Private Dataset API And I should receive the following response: """ spaces are not allowed in the ID field + """ + + Scenario: Adding is_migration to a dataset + Given I have these datasets: + """ + [ + { + "id": "population-estimates" + } + ] + """ + When I PUT "/datasets/population-estimates" + """ + { + "is_migration": false + } + """ + Then I should receive the following JSON response with status "200": + """ + { + "is_migration": false, + "last_updated": "0001-01-01T00:00:00Z" + } + """ + And the document in the database for id "population-estimates" should be: + """ + { + "id": "population-estimates", + "is_migration": false + } + """ + + Scenario: GET /datasets returns is_migration when authorised + Given I have these datasets: + """ + [ + { + "id": "population-estimates", + "is_migration": true + } + ] + """ + When I GET "/datasets" + Then I should receive the following JSON response with status "200": + """ + { + "count": 1, + "items": [ + { + "id": "population-estimates", + "next": { + "id": "population-estimates", + "is_migration": true, + "last_updated": "0001-01-01T00:00:00Z" + }, + "current": { + "id": "population-estimates", + "is_migration": true, + "last_updated": "0001-01-01T00:00:00Z" + } + } + ], + "limit": 20, + "offset": 0, + "total_count": 1 + } + """ + + Scenario: Creating a new dataset with is_migration + When I POST "/datasets" + """ + { + "id": "ageing-population-estimates", + "state": "anything", + "title": "CID", + "type": "filterable", + "description": "census", + "keywords": [ + "keyword" + ], + "next_release": "2016-04-04", + "contacts": [ + { + "email": "testing@hotmail.com", + "name": "John Cox", + "telephone": "01623 456789" + } + ], + "is_migration": true + } + """ + Then the HTTP status code should be "201" + And the document in the database for id "ageing-population-estimates" should be: + """ + { + "id": "ageing-population-estimates", + "state": "created", + "title": "CID", + "type": "filterable", + "links": { + "editions": { + "href": "http://localhost:22000/datasets/ageing-population-estimates/editions" + }, + "self": { + "href": "http://localhost:22000/datasets/ageing-population-estimates" + } + }, + "description": "census", + "keywords": [ + "keyword" + ], + "next_release": "2016-04-04", + "contacts": [ + { + "email": "testing@hotmail.com", + "name": "John Cox", + "telephone": "01623 456789" + } + ], + "is_migration": true + } """ \ No newline at end of file diff --git a/features/public_datasets.feature b/features/public_datasets.feature index a7d10dce..3837ac2b 100644 --- a/features/public_datasets.feature +++ b/features/public_datasets.feature @@ -93,7 +93,37 @@ Feature: Dataset API [ { "id": "population-estimates", - "previous_series_id": ["old-dataset-id"] + "previous_series_id": ["old-dataset-id"], + "is_migration": true + } + ] + """ + When I GET "/datasets" + Then I should receive the following JSON response with status "200": + """ + { + "count":1, + "items": [ + { + "id": "population-estimates", + "last_updated":"0001-01-01T00:00:00Z" + } + ], + "limit":20, + "offset":0, + "total_count":1 + } + """ + + Scenario: GET /datasets with URL rewriting enabled does not return redacted fields + Given URL rewriting is enabled + And I have these datasets: + """ + [ + { + "id": "population-estimates", + "previous_series_id": ["old-dataset-id"], + "is_migration": true } ] """ diff --git a/features/static_dataset_put.feature b/features/static_dataset_put.feature index 9a5265bd..7ffd63c4 100644 --- a/features/static_dataset_put.feature +++ b/features/static_dataset_put.feature @@ -13,6 +13,7 @@ Feature: PUT /datasets/{id} for static datasets "title": "Original Title", "description": "A static dataset", "state": "created", + "is_migration": true, "topics": [ "old-topic", "topic-1" @@ -145,6 +146,7 @@ Feature: PUT /datasets/{id} for static datasets "prices" ], "type": "static", + "is_migration": true, "topics": [ "economy-topic-id", "topic-1" diff --git a/models/dataset.go b/models/dataset.go index 731899f0..7b3d54b4 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -121,6 +121,7 @@ type Dataset struct { RelatedContent []GeneralDetails `bson:"related_content,omitempty" json:"related_content,omitempty"` Topics []string `bson:"topics,omitempty" json:"topics,omitempty"` PreviousSeriesId []string `bson:"previous_series_id,omitempty" json:"previous_series_id,omitempty"` + IsMigration *bool `bson:"is_migration,omitempty" json:"is_migration,omitempty"` } // DatasetLinks represents a list of specific links related to the dataset resource diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index 8e46d973..a11f564e 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -502,6 +502,10 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.topics"] = dataset.Topics } + if dataset.IsMigration != nil { + updates["next.is_migration"] = dataset.IsMigration + } + log.Info(ctx, "built update query for dataset resource", log.Data{"datasetID": id, "dataset": dataset, "updates": updates}) return updates } diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index e9ef3c90..826b5a64 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -194,6 +194,7 @@ func TestDatasetUpdateQuery(t *testing.T) { Title: "Related content 2", }} nationalStatistic := true + isMigration := true expectedUpdate := bson.M{ "next.collection_id": "12345678", @@ -222,6 +223,7 @@ func TestDatasetUpdateQuery(t *testing.T) { "next.subtopics": subtopics, "next.survey": survey, "next.related_content": relatedContent, + "next.is_migration": &isMigration, } dataset := &models.Dataset{ @@ -255,6 +257,7 @@ func TestDatasetUpdateQuery(t *testing.T) { Subtopics: subtopics, Survey: survey, RelatedContent: relatedContent, + IsMigration: &isMigration, } selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) @@ -276,6 +279,21 @@ func TestDatasetUpdateQuery(t *testing.T) { So(selector, ShouldNotBeNil) So(selector, ShouldResemble, expectedUpdate) }) + + Convey("When is_migration is set to false", t, func() { + isMigration := false + dataset := &models.Dataset{ + IsMigration: &isMigration, + } + + expectedUpdate := bson.M{ + "next.is_migration": &isMigration, + } + + selector := createDatasetUpdateQuery(testContext, "123", dataset, models.CreatedState) + So(selector, ShouldNotBeNil) + So(selector, ShouldResemble, expectedUpdate) + }) } func TestVersionUpdateQuery(t *testing.T) { diff --git a/swagger.yaml b/swagger.yaml index 70aef91d..dd48c445 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1550,7 +1550,10 @@ definitions: minLength: 50 maxLength: 5000 is_based_on: - $ref: "#/definitions/IsBasedOn" + $ref: "#/definitions/IsBasedOn" + is_migration: + description: "Indicates whether this dataset series was created as part of a dataset migration" + type: boolean keywords: description: "A list of keywords for a dataset" type: array diff --git a/utils/rewriting.go b/utils/rewriting.go index f759be51..3e1b7f6a 100644 --- a/utils/rewriting.go +++ b/utils/rewriting.go @@ -55,6 +55,7 @@ func RewriteDatasetsWithoutAuth(ctx context.Context, results []*models.DatasetUp } item.Current.ID = item.ID item.Current.PreviousSeriesId = nil + item.Current.IsMigration = nil items = append(items, item.Current) } return items, nil From 250ea52181d7205122b06e0a9c76c130b2032f96 Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Tue, 28 Jul 2026 11:25:01 +0100 Subject: [PATCH 12/25] adjust swagger alignment --- swagger.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swagger.yaml b/swagger.yaml index dd48c445..48dddb71 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1550,7 +1550,7 @@ definitions: minLength: 50 maxLength: 5000 is_based_on: - $ref: "#/definitions/IsBasedOn" + $ref: "#/definitions/IsBasedOn" is_migration: description: "Indicates whether this dataset series was created as part of a dataset migration" type: boolean From 4e2291cb602dae4c8c7ba51f0366826165b23364 Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Tue, 28 Jul 2026 11:44:33 +0100 Subject: [PATCH 13/25] upgrade dependencies --- go.mod | 20 ++++++++++---------- go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index bdd7dd4d..38c554f6 100644 --- a/go.mod +++ b/go.mod @@ -163,23 +163,23 @@ require ( go.opentelemetry.io/contrib/propagators/b3 v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.36.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.48.0 // indirect + golang.org/x/crypto v0.50.0 // indirect golang.org/x/mod v0.37.0 // indirect - golang.org/x/net v0.49.0 // indirect + golang.org/x/net v0.53.0 // indirect golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.40.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260202165425-ce8ad4cf556b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260202165425-ce8ad4cf556b // indirect - google.golang.org/grpc v1.76.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a1bc8cdf..12fe8649 100644 --- a/go.sum +++ b/go.sum @@ -467,22 +467,22 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 h1:SoCgXYF4ISDtNyfLUzsGDa go.opentelemetry.io/contrib/propagators/jaeger v1.36.0/go.mod h1:VHu48l0YTRKSObdPQ+Sb8xMZvdnJlN7yhHuHoPgNqHM= go.opentelemetry.io/contrib/propagators/ot v1.36.0 h1:UBoZjbx483GslNKYK2YpfvePTJV4BHGeFd8+b7dexiM= go.opentelemetry.io/contrib/propagators/ot v1.36.0/go.mod h1:adDDRry19/n9WoA7mSCMjoVJcmzK/bZYzX9SR+g2+W4= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0 h1:SNhVp/9q4Go/XHBkQ1/d5u9P/U+L1yaGPoi0x+mStaI= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0/go.mod h1:tx8OOlGH6R4kLV67YaYO44GFXloEjGPZuMjEkaaqIp4= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -495,8 +495,8 @@ golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= @@ -512,8 +512,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -541,14 +541,14 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -567,14 +567,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260202165425-ce8ad4cf556b h1:SGYyueaEovpqmWmtTvwtVgo638V/QFE2zlTCnRrR3jg= -google.golang.org/genproto/googleapis/api v0.0.0-20260202165425-ce8ad4cf556b/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260202165425-ce8ad4cf556b h1:GZxXGdFaHX27ZSMHudWc4FokdD+xl8BC2UJm1OVIEzs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260202165425-ce8ad4cf556b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= -google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 h1:PGIdqvwfpMUyUP+QAlAnKTSWQ671SmYjoou2/5j7HXk= From 26e79d0b29aead420e479c76a0e03c014bae572e Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Tue, 28 Jul 2026 15:05:57 +0100 Subject: [PATCH 14/25] upgrade google.golang.org/grpc to v1.82.1 --- go.mod | 10 +++++----- go.sum | 28 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index f96fc278..7c10e0e5 100644 --- a/go.mod +++ b/go.mod @@ -171,15 +171,15 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.48.0 // indirect + golang.org/x/crypto v0.50.0 // indirect golang.org/x/mod v0.37.0 // indirect - golang.org/x/net v0.49.0 // indirect + golang.org/x/net v0.53.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.40.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260202165425-ce8ad4cf556b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260202165425-ce8ad4cf556b // indirect - google.golang.org/grpc v1.76.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1c6db92a..eb4e2ca7 100644 --- a/go.sum +++ b/go.sum @@ -495,8 +495,8 @@ golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= @@ -512,8 +512,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -547,8 +547,8 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -567,14 +567,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260202165425-ce8ad4cf556b h1:SGYyueaEovpqmWmtTvwtVgo638V/QFE2zlTCnRrR3jg= -google.golang.org/genproto/googleapis/api v0.0.0-20260202165425-ce8ad4cf556b/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260202165425-ce8ad4cf556b h1:GZxXGdFaHX27ZSMHudWc4FokdD+xl8BC2UJm1OVIEzs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260202165425-ce8ad4cf556b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= -google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 h1:PGIdqvwfpMUyUP+QAlAnKTSWQ671SmYjoou2/5j7HXk= From a87dc8f88678785dac6f4d016c7459ae72f67a11 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Tue, 28 Jul 2026 17:30:32 +0100 Subject: [PATCH 15/25] refactor DELETE /datasets/{id} so all associated versions all deleted --- api/dataset.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/api/dataset.go b/api/dataset.go index 8c9b0f06..04c35599 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -820,10 +820,7 @@ func (api *DatasetAPI) deleteDataset(w http.ResponseWriter, r *http.Request) { // Find any editions/versions associated with the dataset based on the type if currentDataset.Next.Type == models.Static.String() { - // Limit is set to DEFAULT_LIMIT (20) to prevent unbounded queries. - // If a dataset has more than DEFAULT_LIMIT unpublished editions/versions, only the first DEFAULT_LIMIT will be deleted. - // Refactoring is required if more than DEFAULT_LIMIT editions/versions per dataset is a possibility. - versionDocs, _, err := api.dataStore.Backend.GetAllStaticVersions(ctx, currentDataset.ID, "", 0, api.defaultLimit) + versions, _, err := api.dataStore.Backend.GetVersionsStaticNoLimit(ctx, currentDataset.ID, "") if err != nil { if err == errs.ErrVersionsNotFound { log.Info(ctx, "deleteDataset endpoint: dataset didn't contain any versions, continuing to delete dataset", logData) @@ -833,9 +830,9 @@ func (api *DatasetAPI) deleteDataset(w http.ResponseWriter, r *http.Request) { } } - for i := range versionDocs { - if versionDocs[i].Distributions != nil { - for _, distribution := range *versionDocs[i].Distributions { + for i := range versions { + if versions[i].Distributions != nil { + for _, distribution := range *versions[i].Distributions { logData["distribution_title"] = distribution.Title logData["distribution_download_url"] = distribution.DownloadURL @@ -848,7 +845,7 @@ func (api *DatasetAPI) deleteDataset(w http.ResponseWriter, r *http.Request) { } } - err := api.dataStore.Backend.DeleteStaticDatasetVersion(ctx, currentDataset.ID, versionDocs[i].Edition, versionDocs[i].Version) + err := api.dataStore.Backend.DeleteStaticDatasetVersion(ctx, currentDataset.ID, versions[i].Edition, versions[i].Version) if err != nil { log.Error(ctx, "deleteDataset endpoint: failed to delete version", err, logData) return err From af13a37390a7a98a89258a05a81577c2193da40e Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Tue, 28 Jul 2026 17:30:59 +0100 Subject: [PATCH 16/25] update DeleteDataset unit tests and mock call assersions --- api/dataset_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/api/dataset_test.go b/api/dataset_test.go index d56c5547..4cf35627 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -3322,7 +3322,7 @@ func TestDeleteDatasetReturnsSuccessfully(t *testing.T) { }, }, nil }, - GetAllStaticVersionsFunc: func(context.Context, string, string, int, int) ([]*models.Version, int, error) { + GetVersionsStaticNoLimitFunc: func(context.Context, string, string) ([]*models.Version, int, error) { versions := []*models.Version{ { ID: "V1", @@ -3388,7 +3388,7 @@ func TestDeleteDatasetReturnsSuccessfully(t *testing.T) { So(w.Code, ShouldEqual, http.StatusNoContent) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetAllStaticVersionsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionsStaticNoLimitCalls()), ShouldEqual, 1) So(len(mockFilesAPIClient.DeleteFileCalls()), ShouldEqual, 2) So(len(mockedDataStore.DeleteStaticDatasetVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 1) @@ -3411,7 +3411,7 @@ func TestDeleteDatasetReturnsSuccessfully(t *testing.T) { }, }, nil }, - GetAllStaticVersionsFunc: func(context.Context, string, string, int, int) ([]*models.Version, int, error) { + GetVersionsStaticNoLimitFunc: func(context.Context, string, string) ([]*models.Version, int, error) { version := []*models.Version{} return version, 0, errs.ErrVersionsNotFound }, @@ -3440,6 +3440,7 @@ func TestDeleteDatasetReturnsSuccessfully(t *testing.T) { So(w.Code, ShouldEqual, http.StatusNoContent) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionsStaticNoLimitCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 1) So(auditServiceMock.RecordDatasetAuditEventCalls(), ShouldHaveLength, 1) So(auditServiceMock.RecordDatasetAuditEventCalls()[0].Action, ShouldEqual, models.ActionDelete) @@ -3695,7 +3696,7 @@ func TestDeleteDatasetReturnsError(t *testing.T) { }, }, nil }, - GetAllStaticVersionsFunc: func(context.Context, string, string, int, int) ([]*models.Version, int, error) { + GetVersionsStaticNoLimitFunc: func(context.Context, string, string) ([]*models.Version, int, error) { versions := []*models.Version{ { ID: "V1", @@ -3714,7 +3715,7 @@ func TestDeleteDatasetReturnsError(t *testing.T) { }, }, } - return versions, 1, nil + return versions, len(versions), nil }, DeleteStaticDatasetVersionFunc: func(ctx context.Context, datasetID, editionID string, version int) error { return errs.ErrInternalServer @@ -3735,6 +3736,7 @@ func TestDeleteDatasetReturnsError(t *testing.T) { assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionsStaticNoLimitCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteStaticDatasetVersionCalls()), ShouldEqual, 1) }) @@ -3752,7 +3754,7 @@ func TestDeleteDatasetReturnsError(t *testing.T) { }, }, nil }, - GetAllStaticVersionsFunc: func(context.Context, string, string, int, int) ([]*models.Version, int, error) { + GetVersionsStaticNoLimitFunc: func(context.Context, string, string) ([]*models.Version, int, error) { versions := []*models.Version{ { ID: "1", @@ -3769,7 +3771,7 @@ func TestDeleteDatasetReturnsError(t *testing.T) { }, }, } - return versions, 1, nil + return versions, len(versions), nil }, } @@ -3794,6 +3796,7 @@ func TestDeleteDatasetReturnsError(t *testing.T) { assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionsStaticNoLimitCalls()), ShouldEqual, 1) So(len(mockFilesAPIClient.DeleteFileCalls()), ShouldEqual, 1) }) } From f32cc8df06dad1a3db3e9090057da2c5834d4918 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Tue, 28 Jul 2026 17:31:36 +0100 Subject: [PATCH 17/25] remove defaultLimit from DatasetAPI struct as it is no longer used --- api/api.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/api.go b/api/api.go index ab3a5551..6475003e 100644 --- a/api/api.go +++ b/api/api.go @@ -80,7 +80,6 @@ type DatasetAPI struct { instancePublishedChecker *instance.PublishCheck versionPublishedChecker *PublishCheck MaxRequestOptions int - defaultLimit int smDatasetAPI *application.StateMachineDatasetAPI auditService application.AuditService staticDatasetService application.StaticDatasetService @@ -108,7 +107,6 @@ func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, d versionPublishedChecker: nil, instancePublishedChecker: nil, MaxRequestOptions: cfg.MaxRequestOptions, - defaultLimit: cfg.DefaultLimit, smDatasetAPI: smDatasetAPI, permissionsChecker: permissionsChecker, auditService: auditService, From 23f7b54dc51fd80e5c931f416dde59e53a0dfd69 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Wed, 29 Jul 2026 15:53:22 +0100 Subject: [PATCH 18/25] add 409 errors for id changes on migrated datasets --- apierrors/errors.go | 116 +++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 56 deletions(-) diff --git a/apierrors/errors.go b/apierrors/errors.go index dec0aea7..34ba6a33 100644 --- a/apierrors/errors.go +++ b/apierrors/errors.go @@ -15,55 +15,57 @@ func (e ErrInvalidPatch) Error() string { // A list of error messages for Dataset API var ( - ErrAddDatasetAlreadyExists = errors.New("dataset already exists") - ErrAddDatasetTitleAlreadyExists = errors.New("dataset title already exists") - ErrDatasetTypeInvalid = errors.New("invalid dataset type") - ErrTypeMismatch = errors.New("type mismatch") - ErrAddUpdateDatasetBadRequest = errors.New("failed to parse json body") - ErrConflictUpdatingInstance = errors.New("conflict updating instance resource") - ErrDatasetNotFound = errors.New("dataset not found") - ErrDeletePublishedDatasetForbidden = errors.New("a published dataset cannot be deleted") - ErrDeletePublishedVersionForbidden = errors.New("a published version cannot be deleted") - ErrDimensionNodeNotFound = errors.New("dimension node not found") - ErrDimensionNotFound = errors.New("dimension not found") - ErrDimensionOptionNotFound = errors.New("dimension option not found") - ErrDimensionsNotFound = errors.New("dimensions not found") - ErrEditionNotFound = errors.New("edition not found") - ErrEditionsNotFound = errors.New("no editions were found") - ErrIncorrectStateToDetach = errors.New("only versions with a state of edition-confirmed or associated can be detached") - ErrInstanceNotFound = errors.New("instance not found") - ErrInstanceConflict = errors.New("instance does not match the expected eTag") - ErrInternalServer = errors.New("internal error") - ErrInsertedObservationsInvalidSyntax = errors.New("inserted observation request parameter not an integer") - ErrInvalidQueryParameter = errors.New("invalid query parameter") - ErrInvalidBody = errors.New("invalid request body") - ErrTooManyQueryParameters = errors.New("too many query parameters have been provided") - ErrMetadataVersionNotFound = errors.New("version not found") - ErrMissingJobProperties = errors.New("missing job properties") - ErrMissingParameters = errors.New("missing properties in JSON") - ErrResourcePublished = errors.New("unable to update resource as it has been published") - ErrResourceState = errors.New("incorrect resource state") - ErrUnableToParseJSON = errors.New("failed to parse json body") - ErrUnableToReadMessage = errors.New("failed to read message body") - ErrUnauthorised = errors.New("unauthorised access to API") - ErrVersionMissingState = errors.New("missing state from version") - ErrVersionNotFound = errors.New("version not found") - ErrVersionsNotFound = errors.New("no versions were found") - ErrInvalidVersion = errors.New("invalid version requested") - ErrVersionAlreadyExists = errors.New("an unpublished version of this dataset already exists") - ErrNotFound = errors.New("not found") - ErrMissingDatasetID = errors.New("invalid fields: missing dataset id in request body") - ErrEditionAlreadyExists = errors.New("the edition already exists") - ErrEditionTitleAlreadyExists = errors.New("the edition-title already exists") - ErrInvalidDatasetTypeForEditionUpdate = errors.New("unable to update edition-id, invalid dataset type") - ErrSpacesNotAllowedInID = errors.New("spaces are not allowed in the ID field") - ErrCannotChangeIDForPublishedDataset = errors.New("cannot change the dataset ID for a published dataset") - ErrFileMetadataNotFound = errors.New("file metadata not found") - ErrFileNotInCorrectState = errors.New("file not in correct state") - ErrInvalidParamCombination = errors.New("cannot request state and published parameters at the same time") - ErrMethodNotAllowed = errors.New("method not allowed") - ErrPublishedDatasetTopicChange = errors.New("canonical topic can't be changed once a series is published") - ErrStateNotFound = errors.New("incorrect state, can be one of the following: edition-confirmed, associated, approved or published") + ErrAddDatasetAlreadyExists = errors.New("dataset already exists") + ErrAddDatasetTitleAlreadyExists = errors.New("dataset title already exists") + ErrDatasetTypeInvalid = errors.New("invalid dataset type") + ErrTypeMismatch = errors.New("type mismatch") + ErrAddUpdateDatasetBadRequest = errors.New("failed to parse json body") + ErrConflictUpdatingInstance = errors.New("conflict updating instance resource") + ErrDatasetNotFound = errors.New("dataset not found") + ErrDeletePublishedDatasetForbidden = errors.New("a published dataset cannot be deleted") + ErrDeletePublishedVersionForbidden = errors.New("a published version cannot be deleted") + ErrDimensionNodeNotFound = errors.New("dimension node not found") + ErrDimensionNotFound = errors.New("dimension not found") + ErrDimensionOptionNotFound = errors.New("dimension option not found") + ErrDimensionsNotFound = errors.New("dimensions not found") + ErrEditionNotFound = errors.New("edition not found") + ErrEditionsNotFound = errors.New("no editions were found") + ErrIncorrectStateToDetach = errors.New("only versions with a state of edition-confirmed or associated can be detached") + ErrInstanceNotFound = errors.New("instance not found") + ErrInstanceConflict = errors.New("instance does not match the expected eTag") + ErrInternalServer = errors.New("internal error") + ErrInsertedObservationsInvalidSyntax = errors.New("inserted observation request parameter not an integer") + ErrInvalidQueryParameter = errors.New("invalid query parameter") + ErrInvalidBody = errors.New("invalid request body") + ErrTooManyQueryParameters = errors.New("too many query parameters have been provided") + ErrMetadataVersionNotFound = errors.New("version not found") + ErrMissingJobProperties = errors.New("missing job properties") + ErrMissingParameters = errors.New("missing properties in JSON") + ErrResourcePublished = errors.New("unable to update resource as it has been published") + ErrResourceState = errors.New("incorrect resource state") + ErrUnableToParseJSON = errors.New("failed to parse json body") + ErrUnableToReadMessage = errors.New("failed to read message body") + ErrUnauthorised = errors.New("unauthorised access to API") + ErrVersionMissingState = errors.New("missing state from version") + ErrVersionNotFound = errors.New("version not found") + ErrVersionsNotFound = errors.New("no versions were found") + ErrInvalidVersion = errors.New("invalid version requested") + ErrVersionAlreadyExists = errors.New("an unpublished version of this dataset already exists") + ErrNotFound = errors.New("not found") + ErrMissingDatasetID = errors.New("invalid fields: missing dataset id in request body") + ErrEditionAlreadyExists = errors.New("the edition already exists") + ErrEditionTitleAlreadyExists = errors.New("the edition-title already exists") + ErrInvalidDatasetTypeForEditionUpdate = errors.New("unable to update edition-id, invalid dataset type") + ErrSpacesNotAllowedInID = errors.New("spaces are not allowed in the ID field") + ErrCannotChangeIDForPublishedDataset = errors.New("cannot change the dataset ID for a published dataset") + ErrCannotChangeDatasetIDForMigratedDataset = errors.New("cannot change the dataset ID for a migrated dataset") + ErrCannotChangeEditionIDForMigratedEdition = errors.New("cannot change the edition ID for a migrated edition") + ErrFileMetadataNotFound = errors.New("file metadata not found") + ErrFileNotInCorrectState = errors.New("file not in correct state") + ErrInvalidParamCombination = errors.New("cannot request state and published parameters at the same time") + ErrMethodNotAllowed = errors.New("method not allowed") + ErrPublishedDatasetTopicChange = errors.New("canonical topic can't be changed once a series is published") + ErrStateNotFound = errors.New("incorrect state, can be one of the following: edition-confirmed, associated, approved or published") ErrExpectedResourceStateOfCreated = errors.New("unable to update resource, expected resource to have a state of created") ErrExpectedResourceStateOfSubmitted = errors.New("unable to update resource, expected resource to have a state of submitted") @@ -101,13 +103,15 @@ var ( } ConflictRequestMap = map[error]bool{ - ErrAddDatasetAlreadyExists: true, - ErrCannotChangeIDForPublishedDataset: true, - ErrConflictUpdatingInstance: true, - ErrInstanceConflict: true, - ErrEditionAlreadyExists: true, - ErrEditionTitleAlreadyExists: true, - ErrFileNotInCorrectState: true, + ErrAddDatasetAlreadyExists: true, + ErrCannotChangeIDForPublishedDataset: true, + ErrCannotChangeDatasetIDForMigratedDataset: true, + ErrCannotChangeEditionIDForMigratedEdition: true, + ErrConflictUpdatingInstance: true, + ErrInstanceConflict: true, + ErrEditionAlreadyExists: true, + ErrEditionTitleAlreadyExists: true, + ErrFileNotInCorrectState: true, } ForbiddenMap = map[error]bool{ From c9864a5997dad8261cb2843958f6da428f6a7ab7 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Wed, 29 Jul 2026 15:53:47 +0100 Subject: [PATCH 19/25] prevent id being changed on migrated datasets --- api/dataset.go | 14 ++++++++++---- api/dataset_test.go | 39 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/api/dataset.go b/api/dataset.go index 7db75074..8cc08d02 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -44,10 +44,11 @@ var ( // errors that should return a 409 status datasetsConflict = map[error]bool{ - errs.ErrCannotChangeIDForPublishedDataset: true, - errs.ErrAddDatasetAlreadyExists: true, - errs.ErrAddDatasetTitleAlreadyExists: true, - errs.ErrPublishedDatasetTopicChange: true, + errs.ErrCannotChangeIDForPublishedDataset: true, + errs.ErrCannotChangeDatasetIDForMigratedDataset: true, + errs.ErrAddDatasetAlreadyExists: true, + errs.ErrAddDatasetTitleAlreadyExists: true, + errs.ErrPublishedDatasetTopicChange: true, } ) @@ -614,6 +615,11 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { return nil, errs.ErrCannotChangeIDForPublishedDataset } + if isIDChanged && currentDataset.Next.IsMigration != nil && *currentDataset.Next.IsMigration { + log.Error(ctx, "putDataset endpoint: cannot change dataset ID for migrated dataset", errs.ErrCannotChangeDatasetIDForMigratedDataset, data) + return nil, errs.ErrCannotChangeDatasetIDForMigratedDataset + } + if !isPublished && isIDChanged { err := api.dataStore.Backend.CheckDatasetExists(ctx, dataset.ID, "") if err == nil { diff --git a/api/dataset_test.go b/api/dataset_test.go index dff5db76..b1072d2a 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -2135,7 +2135,7 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(context.Context, string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{ID: "123", Next: &models.Dataset{Type: models.Static.String(), Title: "CensusEthnicity", State: models.CreatedState, Topics: []string{"topic-0", "topic-1"}, PreviousSeriesId: []string{"789"}, IsMigration: boolPtr(true)}}, nil + return &models.DatasetUpdate{ID: "123", Next: &models.Dataset{Type: models.Static.String(), Title: "CensusEthnicity", State: models.CreatedState, Topics: []string{"topic-0", "topic-1"}, PreviousSeriesId: []string{"789"}}}, nil }, CheckDatasetExistsFunc: func(ctx context.Context, id, state string) error { return errs.ErrDatasetNotFound @@ -2176,8 +2176,6 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 1) So(mockedDataStore.UpsertDatasetCalls()[0].ID, ShouldEqual, "456") So(mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.PreviousSeriesId, ShouldResemble, []string{"789", "123"}) - So(mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.IsMigration, ShouldNotBeNil) - So(*mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.IsMigration, ShouldBeTrue) So(mockedDataStore.DeleteDatasetCalls(), ShouldHaveLength, 1) So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 0) }) @@ -3272,6 +3270,41 @@ func TestPutDatasetReturnsError(t *testing.T) { So(mockedDataStore.DeleteDatasetCalls(), ShouldHaveLength, 0) }) + Convey("When a request is made to change the dataset id of a migrated dataset", t, func() { + b := `{"id":"456","contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"static-published","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"StaticPublished","theme":"population","state":"published","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"static","keywords":["keyword","keyword 2"],"topics":["topic-0","topic-1"],"license":"Open Government Licence v3.0"}` + r := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(context.Context, string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{ + ID: "123", + Current: &models.Dataset{Type: models.Static.String(), State: models.PublishedState, Topics: []string{"topic-0", "topic-1"}, IsMigration: new(true)}, + Next: &models.Dataset{Type: models.Static.String(), Title: "StaticPublished", Topics: []string{"topic-0", "topic-1"}, IsMigration: new(true)}, + }, nil + }, + } + + authorisationMock := &authMock.MiddlewareMock{ + RequireFunc: func(permission string, handlerFunc http.HandlerFunc) http.HandlerFunc { + return handlerFunc + }, + ParseFunc: func(token string) (*permissionsAPISDK.EntityData, error) { + return testEntityData, nil + }, + } + + api := GetAPIWithCMDMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, authorisationMock, application.SearchContentUpdatedProducer{}, &cloudflareMocks.ClienterMock{}, &applicationMocks.AuditServiceMock{}, &applicationMocks.StaticDatasetServiceMock{}, nil, &filesAPISDKMocks.ClienterMock{}) + api.Router.ServeHTTP(w, r) + + Convey("Then it should return a 409 conflict response", func() { + So(w.Code, ShouldEqual, http.StatusConflict) + So(w.Body.String(), ShouldContainSubstring, errs.ErrCannotChangeDatasetIDForMigratedDataset.Error()) + + So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1) + }) + }) + Convey("When PUT static unpublished dataset calls trying to change id to an existing one returns 409 response", t, func() { b := `{"id":"456","contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"static","keywords":["keyword","keyword 2"],"topics":["topic-0","topic-1"],"license":"Open Government Licence v3.0"}` r := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) From 46a2a275b699edac78654908cce74e0b9f3f99b7 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Wed, 29 Jul 2026 15:54:35 +0100 Subject: [PATCH 20/25] prevent id being changed on migrated edition --- api/versions.go | 8 +++++++- api/versions_test.go | 43 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/api/versions.go b/api/versions.go index 07308d04..27ae510f 100644 --- a/api/versions.go +++ b/api/versions.go @@ -428,7 +428,7 @@ func (api *DatasetAPI) putVersion(w http.ResponseWriter, r *http.Request) { if version.DatasetID != "" { if err := utils.ValidateIDNoSpaces(version.DatasetID); err != nil { - log.Error(ctx, "putVersion endpoint: edition ID in request body contains spaces", err, data) + log.Error(ctx, "putVersion endpoint: dataset ID in request body contains spaces", err, data) handleVersionAPIErr(ctx, err, w, data) return } @@ -457,6 +457,12 @@ func (api *DatasetAPI) putVersion(w http.ResponseWriter, r *http.Request) { // Only validate uniqueness IF edition or title is changing if editionChanged { + if existingVersion.IsMigration != nil && *existingVersion.IsMigration { + log.Error(ctx, "cannot change edition ID for migrated edition", errs.ErrCannotChangeEditionIDForMigratedEdition, data) + handleVersionAPIErr(ctx, errs.ErrCannotChangeEditionIDForMigratedEdition, w, data) + return + } + checkErr := api.dataStore.Backend.CheckEditionExistsStatic(ctx, version.DatasetID, version.Edition, "") if checkErr == nil { log.Error(ctx, "edition ID already exists for this dataset", errs.ErrEditionAlreadyExists, data) diff --git a/api/versions_test.go b/api/versions_test.go index 82fbd716..5f4d11db 100644 --- a/api/versions_test.go +++ b/api/versions_test.go @@ -5303,7 +5303,7 @@ func TestPutVersionIsMigration(t *testing.T) { Convey("When is_migration is included in a PUT version request body", t, func() { trueVal := true - b := `{"edition_title":"Updated Edition Title","release_date":"2017-04-04","is_migration":true,"type":"static"}` + b := `{"edition":"2017","edition_title":"Original Title","release_date":"2017-04-04","is_migration":true,"type":"static"}` r := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123/editions/2017/versions/1", bytes.NewBufferString(b)) w := httptest.NewRecorder() @@ -5377,6 +5377,47 @@ func TestPutVersionIsMigration(t *testing.T) { }) } +func TestPutVersion_EditionIDCannotBeChangedForMigratedEdition(t *testing.T) { + t.Parallel() + + Convey("When a request is made to the change the edition ID of a migrated edition", t, func() { + b := `{"edition":"changed-id","edition_title":"Original Title","release_date":"2017-04-04","type":"static"}` + r := createRequestWithAuth(http.MethodPut, "http://localhost:22000/datasets/123/editions/2017/versions/1", bytes.NewBufferString(b)) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetVersionFunc: func(context.Context, string, string, int, string) (*models.Version, error) { + return nil, errs.ErrVersionNotFound + }, + GetVersionStaticFunc: func(ctx context.Context, datasetID, editionID string, version int, state string) (*models.Version, error) { + return &models.Version{ + Edition: "2017", + IsMigration: new(true), + }, nil + }, + } + + authorisationMock := &authMock.MiddlewareMock{ + RequireFunc: func(permission string, handlerFunc http.HandlerFunc) http.HandlerFunc { + return handlerFunc + }, + ParseFunc: func(token string) (*permissionsAPISDK.EntityData, error) { + return testEntityData, nil + }, + } + + api := GetAPIWithCMDMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, authorisationMock, application.SearchContentUpdatedProducer{}, &cloudflareMocks.ClienterMock{}, &applicationMocks.AuditServiceMock{}, &applicationMocks.StaticDatasetServiceMock{}, nil, &filesAPISDKMocks.ClienterMock{}) + api.Router.ServeHTTP(w, r) + + Convey("Then it returns a 409 Conflict", func() { + So(w.Code, ShouldEqual, http.StatusConflict) + So(w.Body.String(), ShouldContainSubstring, errs.ErrCannotChangeEditionIDForMigratedEdition.Error()) + + So(len(mockedDataStore.GetVersionStaticCalls()), ShouldEqual, 1) + }) + }) +} + func TestPutStateApproveDistributionFilesCheck(t *testing.T) { buildAPIWithApproval := func( mockedDataStore store.Storer, From a3c6c81887bb14d5bc2f22ec866d38714246a019 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Wed, 29 Jul 2026 15:55:15 +0100 Subject: [PATCH 21/25] add component tests for id changes on migrated datasets --- api/dataset_test.go | 7 ++-- features/static_dataset_put.feature | 47 +++++++++++++++++++++- features/static_versions_put.feature | 59 ++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 6 deletions(-) diff --git a/api/dataset_test.go b/api/dataset_test.go index b1072d2a..44d03ea4 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -3271,16 +3271,15 @@ func TestPutDatasetReturnsError(t *testing.T) { }) Convey("When a request is made to change the dataset id of a migrated dataset", t, func() { - b := `{"id":"456","contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"static-published","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"StaticPublished","theme":"population","state":"published","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"static","keywords":["keyword","keyword 2"],"topics":["topic-0","topic-1"],"license":"Open Government Licence v3.0"}` + b := `{"id":"456","contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"migrated-dataset","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"Migrated Dataset","theme":"population","state":"created","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","href":"https://www.ons.gov.uk/"},"type":"static","keywords":["keyword","keyword 2"],"topics":["topic-0","topic-1"],"license":"Open Government Licence v3.0"}` r := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(context.Context, string) (*models.DatasetUpdate, error) { return &models.DatasetUpdate{ - ID: "123", - Current: &models.Dataset{Type: models.Static.String(), State: models.PublishedState, Topics: []string{"topic-0", "topic-1"}, IsMigration: new(true)}, - Next: &models.Dataset{Type: models.Static.String(), Title: "StaticPublished", Topics: []string{"topic-0", "topic-1"}, IsMigration: new(true)}, + ID: "123", + Next: &models.Dataset{Type: models.Static.String(), Title: "Migrated Dataset", Topics: []string{"topic-0", "topic-1"}, IsMigration: new(true)}, }, nil }, } diff --git a/features/static_dataset_put.feature b/features/static_dataset_put.feature index 7ffd63c4..163ff009 100644 --- a/features/static_dataset_put.feature +++ b/features/static_dataset_put.feature @@ -13,7 +13,6 @@ Feature: PUT /datasets/{id} for static datasets "title": "Original Title", "description": "A static dataset", "state": "created", - "is_migration": true, "topics": [ "old-topic", "topic-1" @@ -66,6 +65,19 @@ Feature: PUT /datasets/{id} for static datasets "topic-1" ] } + }, + { + "next": { + "id": "migrated-dataset-id", + "state": "created", + "type": "static", + "title": "Migrated Dataset", + "topics": [ + "old-topic", + "topic-1" + ], + "is_migration": true + } } ] """ @@ -146,7 +158,6 @@ Feature: PUT /datasets/{id} for static datasets "prices" ], "type": "static", - "is_migration": true, "topics": [ "economy-topic-id", "topic-1" @@ -265,6 +276,38 @@ Feature: PUT /datasets/{id} for static datasets cannot change the dataset ID for a published dataset """ + Scenario: Cannot change id of a migrated dataset + When I PUT "/datasets/migrated-dataset-id" + """ + { + "id": "new-dataset-id", + "type": "static", + "title": "Migrated Dataset", + "description": "Migrated static dataset", + "next_release": "2026-01-01T00:00:00Z", + "contacts": [ + { + "name": "John Doe", + "email": "john@example.com" + } + ], + "license": "Open Government Licence v3.0", + "keywords": [ + "economy", + "prices" + ], + "topics": [ + "old-topic", + "topic-1" + ] + } + """ + Then the HTTP status code should be "409" + And I should receive the following response: + """ + cannot change the dataset ID for a migrated dataset + """ + Scenario: Cannot change canonical topic of published static dataset When I PUT "/datasets/published-dataset-id" """ diff --git a/features/static_versions_put.feature b/features/static_versions_put.feature index e8d902f3..0d4258fd 100644 --- a/features/static_versions_put.feature +++ b/features/static_versions_put.feature @@ -1100,6 +1100,65 @@ Feature: Static Dataset Versions PUT API } """ + Scenario: PUT fails when attempting to change edition ID for a migrated version + Given I have a static dataset with version: + """ + { + "dataset": { + "id": "migrated-dataset", + "title": "Migrated Dataset", + "state": "associated", + "type": "static" + }, + "version": { + "id": "migrated-version", + "edition": "original-edition", + "edition_title": "Original Edition", + "links": { + "dataset": { + "id": "migrated-dataset" + }, + "edition": { + "href": "/datasets/migrated-dataset/editions/original-edition", + "id": "original-edition" + }, + "self": { + "href": "/datasets/migrated-dataset/editions/original-edition/versions/1" + } + }, + "version": 1, + "release_date": "2025-01-01T09:00:00.000Z", + "state": "associated", + "type": "static", + "is_migration": true, + "distributions": [ + { + "title": "csv", + "format": "csv", + "media_type": "text/csv", + "download_url": "/uuid/filename.csv", + "byte_size": 125000 + } + ] + } + } + """ + And private endpoints are enabled + And I am an admin user + When I PUT "/datasets/migrated-dataset/editions/original-edition/versions/1" + """ + { + "edition": "changed-edition", + "edition_title": "Changed Edition", + "type": "static" + } + """ + Then the HTTP status code should be "409" + And I should receive the following response: + """ + cannot change the edition ID for a migrated edition + """ + Scenario: PUT successfully updates edition ID and all associated links Given I have a static dataset with version: """ From f3931a8de50838cf70af0749373d251080e9090b Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Thu, 30 Jul 2026 07:48:44 +0100 Subject: [PATCH 22/25] wait for async Kafka message and Cloudflare purge in unit tests --- api/dataset_test.go | 11 ----------- api/versions_test.go | 22 +++++++++++++++++++++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/api/dataset_test.go b/api/dataset_test.go index 44d03ea4..dacab2cc 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -33,7 +33,6 @@ import ( "github.com/ONSdigital/dp-dataset-api/url" filesAPISDK "github.com/ONSdigital/dp-files-api/sdk" filesAPISDKMocks "github.com/ONSdigital/dp-files-api/sdk/mocks" - kafka "github.com/ONSdigital/dp-kafka/v4" dprequest "github.com/ONSdigital/dp-net/v3/request" "github.com/gorilla/mux" @@ -79,16 +78,6 @@ var ( mu sync.Mutex ) -func getSearchContentUpdatedMock() *mocks.KafkaProducerMock { - producerMock := &mocks.KafkaProducerMock{ - OutputFunc: func() chan kafka.BytesMessage { - return make(chan kafka.BytesMessage, 1) - }, - } - - return producerMock -} - // GetAPIWithCMDMocks also used in other tests, so exported func GetAPIWithCMDMocks(mockedDataStore store.Storer, mockedGeneratedDownloads DownloadsGenerator, authorisationMock *authMock.MiddlewareMock, searchContentUpdated application.SearchContentUpdatedProducer, cloudflareMock *cloudflareMocks.ClienterMock, auditServiceMock *applicationMocks.AuditServiceMock, staticDatasetServiceMock *applicationMocks.StaticDatasetServiceMock, topicAPISDKMock *topicAPISDKMocks.ClienterMock, filesAPISDKMock *filesAPISDKMocks.ClienterMock) *DatasetAPI { mu.Lock() diff --git a/api/versions_test.go b/api/versions_test.go index 5f4d11db..2df69ecb 100644 --- a/api/versions_test.go +++ b/api/versions_test.go @@ -29,6 +29,7 @@ import ( filesAPISDK "github.com/ONSdigital/dp-files-api/sdk" filesAPISDKMocks "github.com/ONSdigital/dp-files-api/sdk/mocks" filesAPIErrors "github.com/ONSdigital/dp-files-api/store" + kafka "github.com/ONSdigital/dp-kafka/v4" permissionsAPISDK "github.com/ONSdigital/dp-permissions-api/sdk" topicAPISDKMocks "github.com/ONSdigital/dp-topic-api/sdk/mocks" "github.com/ONSdigital/log.go/v2/log" @@ -4970,11 +4971,19 @@ func TestPutStateReturnsOk(t *testing.T) { }, } - scuProducerMock := getSearchContentUpdatedMock() + outputCalled := make(chan bool, 1) + scuProducerMock := &mocks.KafkaProducerMock{ + OutputFunc: func() chan kafka.BytesMessage { + outputCalled <- true + return make(chan kafka.BytesMessage, 1) + }, + } searchContentUpdated := application.SearchContentUpdatedProducer{Producer: scuProducerMock} + purgeByPrefixesCalled := make(chan bool, 1) cloudflareMock := &cloudflareMocks.ClienterMock{ PurgeByPrefixesFunc: func(ctx context.Context, prefixes []string) error { + purgeByPrefixesCalled <- true return nil }, GetTimeoutFunc: func() time.Duration { @@ -4991,6 +5000,17 @@ func TestPutStateReturnsOk(t *testing.T) { api := GetAPIWithCMDMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, authorisationMock, searchContentUpdated, cloudflareMock, auditServiceMock, &applicationMocks.StaticDatasetServiceMock{}, &topicAPISDKMocks.ClienterMock{}, &mockFilesAPIClient) api.Router.ServeHTTP(w, r) + select { + case <-outputCalled: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for kafka message to be sent") + } + select { + case <-purgeByPrefixesCalled: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for Cloudflare purge") + } + So(w.Code, ShouldEqual, http.StatusOK) So(mockedDataStore.GetVersionStaticCalls(), ShouldHaveLength, 2) So(mockedDataStore.UpdateStateStaticCalls(), ShouldHaveLength, 1) From 5785265ee300f17de4222770dc84e1405767cefe Mon Sep 17 00:00:00 2001 From: Sarah Johnson Date: Thu, 30 Jul 2026 14:46:17 +0100 Subject: [PATCH 23/25] fix: related content now deleted from database --- mongo/dataset_store.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index a11f564e..5fe49ca0 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -626,6 +626,8 @@ func createVersionUpdateQuery(version *models.Version, newETag string) bson.M { if version.RelatedContent != nil { setUpdates["related_content"] = version.RelatedContent + } else { + setUpdates["related_content"] = nil } if newETag != "" { From 19b0d1d423608947610f92c867a4b1b366310b15 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Mon, 3 Aug 2026 11:27:26 +0100 Subject: [PATCH 24/25] resolve merge conflicts From 5a96a9f00ab4c66295f93f2b838a886301087188 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Mon, 3 Aug 2026 13:37:19 +0100 Subject: [PATCH 25/25] remove duplicate assersion --- api/dataset_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/dataset_test.go b/api/dataset_test.go index f07376d7..dacab2cc 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -2165,8 +2165,6 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 1) So(mockedDataStore.UpsertDatasetCalls()[0].ID, ShouldEqual, "456") So(mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.PreviousSeriesId, ShouldResemble, []string{"789", "123"}) - So(mockedDataStore.UpsertDatasetCalls()[0].ID, ShouldEqual, "456") - So(mockedDataStore.UpsertDatasetCalls()[0].DatasetDoc.Next.PreviousSeriesId, ShouldResemble, []string{"789", "123"}) So(mockedDataStore.DeleteDatasetCalls(), ShouldHaveLength, 1) So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 0) })