Skip to content

Commit 85f283f

Browse files
lpusoklpusok
authored
[SSW-2210] Paging workarounds and improvements (#263)
Added fallbacks for paging API error (invalid cursor). In case the existing paging logic fails, we use workarounds such as fetch400Profiles. Removed API calls for devices included in profiles, this should help with rate limits in the long term. To do this we use the UDIDs to check for profile validity, instead of the opaque ID from the API.
1 parent 8403574 commit 85f283f

File tree

17 files changed

+453
-81
lines changed

17 files changed

+453
-81
lines changed

‎autocodesign/autocodesign.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ type Profile interface {
2222
ID() string
2323
Attributes() appstoreconnect.ProfileAttributes
2424
CertificateIDs() ([]string, error)
25-
DeviceIDs() ([]string, error)
25+
DeviceUDIDs() ([]string, error)
2626
BundleID() (appstoreconnect.BundleID, error)
2727
Entitlements() (Entitlements, error)
2828
}
@@ -67,6 +67,12 @@ type Certificate struct {
6767
ID string
6868
}
6969

70+
// DeviceIDs is the opaque ID of a device (from used in the Developer Portal API)
71+
type DeviceIDs []string
72+
73+
// DeviceUDIDs are used in the provisioning profiles, as unique device identifiers
74+
type DeviceUDIDs []string
75+
7076
// DevPortalClient abstract away the Apple Developer Portal API
7177
type DevPortalClient interface {
7278
Login() error
@@ -170,8 +176,8 @@ func (m codesignAssetManager) EnsureCodesignAssets(appLayout AppLayout, opts Cod
170176
return nil, err
171177
}
172178

173-
var devPortalDeviceIDs []string
174-
var devPortalDeviceUDIDs []string
179+
var devPortalDeviceIDs DeviceIDs
180+
var devPortalDeviceUDIDs DeviceUDIDs
175181
if DistributionTypeRequiresDeviceList(distrTypes) {
176182
devPortalDevices, err := EnsureTestDevices(m.devPortalClient, opts.BitriseTestDevices, appLayout.Platform)
177183
if err != nil {
@@ -208,7 +214,7 @@ func (m codesignAssetManager) EnsureCodesignAssets(appLayout AppLayout, opts Cod
208214
printMissingCodeSignAssets(missingAppLayout)
209215

210216
// Ensure Profiles
211-
newCodesignAssets, err := ensureProfiles(m.devPortalClient, distrType, certsByType, *missingAppLayout, devPortalDeviceIDs, opts.MinProfileValidityDays)
217+
newCodesignAssets, err := ensureProfiles(m.devPortalClient, distrType, certsByType, *missingAppLayout, devPortalDeviceIDs, devPortalDeviceUDIDs, opts.MinProfileValidityDays)
212218
if err != nil {
213219
switch {
214220
case errors.As(err, &ErrAppClipAppID{}):

‎autocodesign/autocodesign_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func newMockProfile(m profileArgs) Profile {
4949
profile.On("BundleID").Return(func() appstoreconnect.BundleID {
5050
return m.appID
5151
}, nil)
52-
profile.On("DeviceIDs").Return(func() []string {
52+
profile.On("DeviceUDIDs").Return(func() []string {
5353
return m.devices
5454
}, nil)
5555
profile.On("CertificateIDs").Return(func() []string {

‎autocodesign/devportalclient/appstoreconnect/bundleids.go‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,49 @@ import (
77
// BundleIDsEndpoint ...
88
const BundleIDsEndpoint = "bundleIds"
99

10+
// ListBundleIDsSortOption ...
11+
type ListBundleIDsSortOption string
12+
13+
// ListBundleIDsSortOptions ...
14+
const (
15+
ListBundleIDsSortOptionName ListBundleIDsSortOption = "name"
16+
ListBundleIDsSortOptionNameDesc ListBundleIDsSortOption = "-name"
17+
ListBundleIDsSortOptionPlatform ListBundleIDsSortOption = "platform"
18+
ListBundleIDsSortOptionPlatformDesc ListBundleIDsSortOption = "-platform"
19+
ListBundleIDsSortOptionIdentifier ListBundleIDsSortOption = "identifier"
20+
ListBundleIDsSortOptionIdentifierDesc ListBundleIDsSortOption = "-identifier"
21+
ListBundleIDsSortOptionSeedId ListBundleIDsSortOption = "seedId"
22+
ListBundleIDsSortOptionSeedIdDesc ListBundleIDsSortOption = "-seedId"
23+
ListBundleIDsSortOptionID ListBundleIDsSortOption = "id"
24+
ListBundleIDsSortOptionIDDesc ListBundleIDsSortOption = "-id"
25+
)
26+
1027
// ListBundleIDsOptions ...
1128
type ListBundleIDsOptions struct {
1229
PagingOptions
13-
FilterIdentifier string `url:"filter[identifier],omitempty"`
14-
FilterName string `url:"filter[name],omitempty"`
15-
FilterPlatform BundleIDPlatform `url:"filter[platform],omitempty"`
16-
Include string `url:"include,omitempty"`
30+
FilterIdentifier string `url:"filter[identifier],omitempty"`
31+
FilterName string `url:"filter[name],omitempty"`
32+
FilterPlatform BundleIDPlatform `url:"filter[platform],omitempty"`
33+
Include string `url:"include,omitempty"`
34+
Sort ListBundleIDsSortOption `url:"sort,omitempty"`
35+
}
36+
37+
// PagingInformationPaging ...
38+
type PagingInformationPaging struct {
39+
Total int `json:"total,omitempty"`
40+
Limit int `json:"limit,omitempty"`
41+
}
42+
43+
// PagingInformation ...
44+
type PagingInformation struct {
45+
Paging PagingInformationPaging `json:"paging,omitempty"`
1746
}
1847

1948
// PagedDocumentLinks ...
2049
type PagedDocumentLinks struct {
21-
Next string `json:"next,omitempty"`
50+
First string `json:"first,omitempty"`
51+
Next string `json:"next,omitempty"`
52+
Self string `json:"self,omitempty"`
2253
}
2354

2455
// BundleIDAttributes ...
@@ -58,6 +89,7 @@ type BundleID struct {
5889
type BundleIdsResponse struct {
5990
Data []BundleID `json:"data,omitempty"`
6091
Links PagedDocumentLinks `json:"links,omitempty"`
92+
Meta PagingInformation `json:"meta,omitempty"`
6193
}
6294

6395
// ListBundleIDs ...

‎autocodesign/devportalclient/appstoreconnect/certificates.go‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,27 @@ import (
88
// CertificatesEndpoint ...
99
const CertificatesEndpoint = "certificates"
1010

11+
// ListCertificatesSortOption ...
12+
type ListCertificatesSortOption string
13+
14+
// ListCertificatesSortOptions ...
15+
const (
16+
ListCertificatesSortOptionDisplayName ListCertificatesSortOption = "displayName"
17+
ListCertificatesSortOptionDisplayNameDesc ListCertificatesSortOption = "-displayName"
18+
ListCertificatesSortOptionCertificateType ListCertificatesSortOption = "certificateType"
19+
ListCertificatesSortOptionCertificateTypeDesc ListCertificatesSortOption = "-certificateType"
20+
ListCertificatesSortOptionSerialNumber ListCertificatesSortOption = "serialNumber"
21+
ListCertificatesSortOptionSerialNumberDesc ListCertificatesSortOption = "-serialNumber"
22+
ListCertificatesSortOptionID ListCertificatesSortOption = "id"
23+
ListCertificatesSortOptionIDDesc ListCertificatesSortOption = "-id"
24+
)
25+
1126
// ListCertificatesOptions ...
1227
type ListCertificatesOptions struct {
1328
PagingOptions
14-
FilterSerialNumber string `url:"filter[serialNumber],omitempty"`
15-
FilterCertificateType CertificateType `url:"filter[certificateType],omitempty"`
29+
FilterSerialNumber string `url:"filter[serialNumber],omitempty"`
30+
FilterCertificateType CertificateType `url:"filter[certificateType],omitempty"`
31+
Sort ListCertificatesSortOption `url:"sort,omitempty"`
1632
}
1733

1834
// CertificateType ...
@@ -53,6 +69,7 @@ type Certificate struct {
5369
type CertificatesResponse struct {
5470
Data []Certificate `json:"data"`
5571
Links PagedDocumentLinks `json:"links,omitempty"`
72+
Meta PagingInformation `json:"meta,omitempty"`
5673
}
5774

5875
// ListCertificates ...

‎autocodesign/devportalclient/appstoreconnect/devices.go‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,30 @@ import (
77
// DevicesEndpoint ...
88
const DevicesEndpoint = "devices"
99

10+
// ListDevicesSortOption ...
11+
type ListDevicesSortOption string
12+
13+
// ListDevicesSortOptions ...
14+
const (
15+
ListDevicesSortOptionName ListDevicesSortOption = "name"
16+
ListDevicesSortOptionNameDesc ListDevicesSortOption = "-name"
17+
ListDevicesSortOptionPlatform ListDevicesSortOption = "platform"
18+
ListDevicesSortOptionPlatformDesc ListDevicesSortOption = "-platform"
19+
ListDevicesSortOptionUDID ListDevicesSortOption = "udid"
20+
ListDevicesSortOptionUDIDDesc ListDevicesSortOption = "-udid"
21+
ListDevicesSortOptionStatus ListDevicesSortOption = "status"
22+
ListDevicesSortOptionStatusDesc ListDevicesSortOption = "-status"
23+
ListDevicesSortOptionID ListDevicesSortOption = "id"
24+
ListDevicesSortOptionIDDesc ListDevicesSortOption = "-id"
25+
)
26+
1027
// ListDevicesOptions ...
1128
type ListDevicesOptions struct {
1229
PagingOptions
13-
FilterUDID string `url:"filter[udid],omitempty"`
14-
FilterPlatform DevicePlatform `url:"filter[platform],omitempty"`
15-
FilterStatus Status `url:"filter[status],omitempty"`
30+
FilterUDID string `url:"filter[udid],omitempty"`
31+
FilterPlatform DevicePlatform `url:"filter[platform],omitempty"`
32+
FilterStatus Status `url:"filter[status],omitempty"`
33+
Sort ListDevicesSortOption `url:"sort,omitempty"`
1634
}
1735

1836
// DeviceClass ...
@@ -68,6 +86,7 @@ type Device struct {
6886
type DevicesResponse struct {
6987
Data []Device `json:"data"`
7088
Links PagedDocumentLinks `json:"links,omitempty"`
89+
Meta PagingInformation `json:"meta,omitempty"`
7190
}
7291

7392
// DeviceResponse ...

‎autocodesign/devportalclient/appstoreconnect/error.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package appstoreconnect
33
import (
44
"fmt"
55
"net/http"
6+
"strings"
67
)
78

89
// ErrorResponseError ...
@@ -37,6 +38,17 @@ func (r ErrorResponse) Error() string {
3738
return m
3839
}
3940

41+
// IsCursorInvalid ...
42+
func (r ErrorResponse) IsCursorInvalid() bool {
43+
// {"errors"=>[{"id"=>"[ ... ]", "status"=>"400", "code"=>"PARAMETER_ERROR.INVALID", "title"=>"A parameter has an invalid value", "detail"=>"'eyJvZmZzZXQiOiIyMCJ9' is not a valid cursor for this request", "source"=>{"parameter"=>"cursor"}}]}
44+
for _, err := range r.Errors {
45+
if err.Code == "PARAMETER_ERROR.INVALID" && strings.Contains(err.Detail, "is not a valid cursor for this request") {
46+
return true
47+
}
48+
}
49+
return false
50+
}
51+
4052
// DeviceRegistrationError ...
4153
type DeviceRegistrationError struct {
4254
Reason string

‎autocodesign/devportalclient/appstoreconnect/profiles.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ type ProfilesResponse struct {
127127
Attributes serialized.Object `json:"attributes"`
128128
} `json:"included"`
129129
Links PagedDocumentLinks `json:"links,omitempty"`
130+
Meta PagingInformation `json:"meta,omitempty"`
130131
}
131132

132133
// ListProfiles ...

‎autocodesign/devportalclient/appstoreconnectclient/certificates.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@ package appstoreconnectclient
22

33
import (
44
"crypto/x509"
5+
"errors"
56
"fmt"
67
"math/big"
78

9+
"github.com/bitrise-io/go-utils/log"
810
"github.com/bitrise-io/go-xcode/certificateutil"
911
"github.com/bitrise-io/go-xcode/v2/autocodesign"
1012
"github.com/bitrise-io/go-xcode/v2/autocodesign/devportalclient/appstoreconnect"
@@ -85,13 +87,61 @@ func queryCertificatesByType(client *appstoreconnect.Client, certificateType app
8587
FilterCertificateType: certificateType,
8688
})
8789
if err != nil {
90+
var apiError *appstoreconnect.ErrorResponse
91+
if ok := errors.As(err, &apiError); ok {
92+
if apiError.IsCursorInvalid() {
93+
log.Warnf("Cursor is invalid, falling back to listing certificates with 400 limit")
94+
return list400Certificates(client, certificateType)
95+
}
96+
}
8897
return nil, err
8998
}
99+
90100
certificates = append(certificates, response.Data...)
91101

92102
nextPageURL = response.Links.Next
93103
if nextPageURL == "" {
94104
return parseCertificatesResponse(certificates)
95105
}
106+
if len(certificates) >= response.Meta.Paging.Total {
107+
log.Warnf("All certificates fetched, but next page URL is not empty")
108+
return parseCertificatesResponse(certificates)
109+
}
96110
}
97111
}
112+
113+
func list400Certificates(client *appstoreconnect.Client, certificateType appstoreconnect.CertificateType) ([]autocodesign.Certificate, error) {
114+
certificatesByID := map[string]appstoreconnect.Certificate{}
115+
var totalCount int
116+
for _, sort := range []appstoreconnect.ListCertificatesSortOption{appstoreconnect.ListCertificatesSortOptionID, appstoreconnect.ListCertificatesSortOptionIDDesc} {
117+
response, err := client.Provisioning.ListCertificates(&appstoreconnect.ListCertificatesOptions{
118+
PagingOptions: appstoreconnect.PagingOptions{
119+
Limit: 200,
120+
},
121+
FilterCertificateType: certificateType,
122+
Sort: sort,
123+
})
124+
if err != nil {
125+
return nil, err
126+
}
127+
128+
for _, responseCertificate := range response.Data {
129+
certificatesByID[responseCertificate.ID] = responseCertificate
130+
}
131+
132+
if totalCount == 0 {
133+
totalCount = response.Meta.Paging.Total
134+
}
135+
}
136+
137+
if totalCount > 400 {
138+
log.Warnf("Unable to retrieve all certificates: more than 400 certificates available (%s)", totalCount)
139+
}
140+
141+
var certificates []appstoreconnect.Certificate
142+
for _, certificate := range certificatesByID {
143+
certificates = append(certificates, certificate)
144+
}
145+
146+
return parseCertificatesResponse(certificates)
147+
}

‎autocodesign/devportalclient/appstoreconnectclient/devices.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"fmt"
66
"net/http"
77

8+
"github.com/bitrise-io/go-utils/log"
9+
810
"github.com/bitrise-io/go-xcode/v2/autocodesign/devportalclient/appstoreconnect"
911
"github.com/bitrise-io/go-xcode/v2/devportalservice"
1012
)
@@ -34,6 +36,13 @@ func (d *DeviceClient) ListDevices(udid string, platform appstoreconnect.DeviceP
3436
FilterStatus: appstoreconnect.Enabled,
3537
})
3638
if err != nil {
39+
var apiError *appstoreconnect.ErrorResponse
40+
if ok := errors.As(err, &apiError); ok {
41+
if apiError.IsCursorInvalid() {
42+
log.Warnf("Cursor is invalid, falling back to listing devices with 400 limit")
43+
return d.list400Devices(udid, platform)
44+
}
45+
}
3746
return nil, err
3847
}
3948

@@ -43,7 +52,50 @@ func (d *DeviceClient) ListDevices(udid string, platform appstoreconnect.DeviceP
4352
if nextPageURL == "" {
4453
return devices, nil
4554
}
55+
if len(devices) >= response.Meta.Paging.Total {
56+
log.Warnf("All devices fetched, but next page URL is not empty")
57+
return devices, nil
58+
}
59+
}
60+
}
61+
62+
// list400Devices is used to work around a specific App Store Connect API bug
63+
func (d *DeviceClient) list400Devices(udid string, platform appstoreconnect.DevicePlatform) ([]appstoreconnect.Device, error) {
64+
devicesByID := map[string]appstoreconnect.Device{}
65+
var totalCount int
66+
for _, sort := range []appstoreconnect.ListDevicesSortOption{appstoreconnect.ListDevicesSortOptionID, appstoreconnect.ListDevicesSortOptionIDDesc} {
67+
response, err := d.client.Provisioning.ListDevices(&appstoreconnect.ListDevicesOptions{
68+
PagingOptions: appstoreconnect.PagingOptions{
69+
Limit: 200,
70+
},
71+
FilterUDID: udid,
72+
FilterPlatform: platform,
73+
FilterStatus: appstoreconnect.Enabled,
74+
Sort: sort,
75+
})
76+
if err != nil {
77+
return nil, err
78+
}
79+
80+
for _, responseDevice := range response.Data {
81+
devicesByID[responseDevice.ID] = responseDevice
82+
}
83+
84+
if totalCount == 0 {
85+
totalCount = response.Meta.Paging.Total
86+
}
87+
}
88+
89+
if totalCount > 400 {
90+
log.Warnf("Unable to retrieve all devices: more than 400 devices available (%s)", totalCount)
4691
}
92+
93+
var devices []appstoreconnect.Device
94+
for _, device := range devicesByID {
95+
devices = append(devices, device)
96+
}
97+
98+
return devices, nil
4799
}
48100

49101
// RegisterDevice ...

0 commit comments

Comments
 (0)