Skip to content

Commit cc2701f

Browse files
NikitaSmalljeroiraz
authored andcommitted
feat: add s3 (aws) role based auth as an option
feat: use aws role to provide credentials when the role is provided, the storage gets the credentials based on the security metadata. This change also includes the refresh of the credentials. feat: update readme so there is an example of role-based config feat: fix typo feat: use generic parameter to enable role lookup feat: use logging within the credentials refresh routine feat: update readme feat: use named constant for period fix: correct immudb naming fix: address style-related comments fix: correct minor comments feat: use instance metadata url as a flag with the default
1 parent 2dc527f commit cc2701f

File tree

8 files changed

+340
-29
lines changed

8 files changed

+340
-29
lines changed

README.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,29 @@ such as with AWS ECS Fargate, the identifier can be taken from S3. To enable tha
181181
export IMMUDB_S3_EXTERNAL_IDENTIFIER=true
182182
```
183183

184+
You can also use the role-based credentials for more flexible and secure configuration.
185+
This allows the service to be used with instance role configuration without a user entity.
186+
The following example shows how to run immudb with the S3 storage enabled using AWS Roles:
187+
188+
```bash
189+
export IMMUDB_S3_STORAGE=true
190+
export IMMUDB_S3_ROLE_ENABLED=true
191+
export IMMUDB_S3_BUCKET_NAME=<BUCKET NAME>
192+
export IMMUDB_S3_LOCATION=<AWS S3 REGION>
193+
export IMMUDB_S3_PATH_PREFIX=testing-001
194+
export IMMUDB_S3_ENDPOINT="https://${IMMUDB_S3_BUCKET_NAME}.s3.${IMMUDB_S3_LOCATION}.amazonaws.com"
195+
196+
./immudb
197+
```
198+
199+
Optionally, you can specify the exact role immudb should be using with:
200+
201+
```bash
202+
export IMMUDB_S3_ROLE=<AWS S3 ACCESS ROLE NAME>
203+
```
204+
205+
Remember, the `IMMUDB_S3_ROLE_ENABLED` parameter still should be on.
206+
184207
You can also easily use immudb with compatible s3 alternatives
185208
such as the [minio](https://github.com/minio/minio) server:
186209

cmd/immudb/command/init.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,13 +72,16 @@ func (cl *Commandline) setupFlags(cmd *cobra.Command, options *server.Options) {
7272
cmd.Flags().Int("pgsql-server-port", 5432, "pgsql server port")
7373
cmd.Flags().Bool("pprof", false, "add pprof profiling endpoint on the metrics server")
7474
cmd.Flags().Bool("s3-storage", false, "enable or disable s3 storage")
75+
cmd.Flags().Bool("s3-role-enabled", false, "enable role-based authentication for s3 storage")
7576
cmd.Flags().String("s3-endpoint", "", "s3 endpoint")
77+
cmd.Flags().String("s3-role", "", "role name for role-based authentication attempt for s3 storage")
7678
cmd.Flags().String("s3-access-key-id", "", "s3 access key id")
7779
cmd.Flags().String("s3-secret-key", "", "s3 secret access key")
7880
cmd.Flags().String("s3-bucket-name", "", "s3 bucket name")
7981
cmd.Flags().String("s3-location", "", "s3 location (region)")
8082
cmd.Flags().String("s3-path-prefix", "", "s3 path prefix (multiple immudb instances can share the same bucket if they have different prefixes)")
8183
cmd.Flags().Bool("s3-external-identifier", false, "use the remote identifier if there is no local identifier")
84+
cmd.Flags().String("s3-instance-metadata-url", "http://169.254.169.254", "s3 instance metadata url")
8285
cmd.Flags().Int("max-sessions", 100, "maximum number of simultaneously opened sessions")
8386
cmd.Flags().Duration("max-session-inactivity-time", 3*time.Minute, "max session inactivity time is a duration after which an active session is declared inactive by the server. A session is kept active if server is still receiving requests from client (keep-alive or other methods)")
8487
cmd.Flags().Duration("max-session-age-time", 0, "the current default value is infinity. max session age time is a duration after which session will be forcibly closed")
@@ -135,12 +138,15 @@ func setupDefaults(options *server.Options) {
135138
viper.SetDefault("pprof", false)
136139
viper.SetDefault("s3-storage", false)
137140
viper.SetDefault("s3-endpoint", "")
141+
viper.SetDefault("s3-role-enabled", false)
142+
viper.SetDefault("s3-role", "")
138143
viper.SetDefault("s3-access-key-id", "")
139144
viper.SetDefault("s3-secret-key", "")
140145
viper.SetDefault("s3-bucket-name", "")
141146
viper.SetDefault("s3-location", "")
142147
viper.SetDefault("s3-path-prefix", "")
143148
viper.SetDefault("s3-external-identifier", false)
149+
viper.SetDefault("s3-instance-metadata-url", "http://169.254.169.254")
144150
viper.SetDefault("max-sessions", 100)
145151
viper.SetDefault("max-session-inactivity-time", 3*time.Minute)
146152
viper.SetDefault("max-session-age-time", 0)

cmd/immudb/command/parse_options.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,23 +86,29 @@ func parseOptions() (options *server.Options, err error) {
8686
swaggerUIEnabled := viper.GetBool("swaggerui")
8787

8888
s3Storage := viper.GetBool("s3-storage")
89+
s3RoleEnabled := viper.GetBool("s3-role-enabled")
90+
s3Role := viper.GetString("s3-role")
8991
s3Endpoint := viper.GetString("s3-endpoint")
9092
s3AccessKeyID := viper.GetString("s3-access-key-id")
9193
s3SecretKey := viper.GetString("s3-secret-key")
9294
s3BucketName := viper.GetString("s3-bucket-name")
9395
s3Location := viper.GetString("s3-location")
9496
s3PathPrefix := viper.GetString("s3-path-prefix")
9597
s3ExternalIdentifier := viper.GetBool("s3-external-identifier")
98+
s3MetadataURL := viper.GetString("s3-instance-metadata-url")
9699

97100
remoteStorageOptions := server.DefaultRemoteStorageOptions().
98101
WithS3Storage(s3Storage).
102+
WithS3RoleEnabled(s3RoleEnabled).
103+
WithS3Role(s3Role).
99104
WithS3Endpoint(s3Endpoint).
100105
WithS3AccessKeyID(s3AccessKeyID).
101106
WithS3SecretKey(s3SecretKey).
102107
WithS3BucketName(s3BucketName).
103108
WithS3Location(s3Location).
104109
WithS3PathPrefix(s3PathPrefix).
105-
WithS3ExternalIdentifier(s3ExternalIdentifier)
110+
WithS3ExternalIdentifier(s3ExternalIdentifier).
111+
WithS3InstanceMetadataURL(s3MetadataURL)
106112

107113
sessionOptions := sessions.DefaultOptions().
108114
WithMaxSessions(viper.GetInt("max-sessions")).

embedded/remotestorage/s3/s3.go

Lines changed: 160 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"crypto/sha256"
2424
"encoding/base64"
2525
"encoding/hex"
26+
"encoding/json"
2627
"encoding/xml"
2728
"errors"
2829
"fmt"
@@ -32,6 +33,7 @@ import (
3233
"net/http"
3334
"net/url"
3435
"os"
36+
"regexp"
3537
"sort"
3638
"strconv"
3739
"strings"
@@ -41,17 +43,24 @@ import (
4143
)
4244

4345
type Storage struct {
44-
endpoint string
45-
accessKeyID string
46-
secretKey string
47-
bucket string
48-
prefix string
49-
location string
50-
httpClient *http.Client
46+
endpoint string
47+
S3RoleEnabled bool
48+
s3Role string
49+
accessKeyID string
50+
secretKey string
51+
bucket string
52+
prefix string
53+
location string
54+
httpClient *http.Client
55+
56+
awsInstanceMetadataURL string
57+
awsCredsRefreshPeriod time.Duration
5158
}
5259

5360
var (
5461
ErrInvalidArguments = errors.New("invalid arguments")
62+
ErrKeyCredentialsProvided = errors.New("remote storage configuration already includes access key and/or secret key")
63+
ErrCredentialsCannotBeFound = errors.New("cannot find credentials based on instance role remote storage")
5564
ErrInvalidArgumentsOffsSize = fmt.Errorf("%w: negative offset or zero size", ErrInvalidArguments)
5665
ErrInvalidArgumentsNameStartSlash = fmt.Errorf("%w: name can not start with /", ErrInvalidArguments)
5766
ErrInvalidArgumentsNameEndSlash = fmt.Errorf("%w: name can not end with /", ErrInvalidArguments)
@@ -73,17 +82,22 @@ var (
7382
ErrInvalidResponseSubPathUnescape = fmt.Errorf("%w: error un-escaping object name", ErrInvalidResponse)
7483

7584
ErrTooManyRedirects = errors.New("too many redirects")
85+
86+
arnRoleRegex = regexp.MustCompile(`arn:.*\/(.*)`)
7687
)
7788

7889
const maxRedirects = 5
7990

8091
func Open(
8192
endpoint string,
93+
S3RoleEnabled bool,
94+
s3Role string,
8295
accessKeyID string,
8396
secretKey string,
8497
bucket string,
8598
location string,
8699
prefix string,
100+
awsInstanceMetadataURL string,
87101
) (remotestorage.Storage, error) {
88102

89103
// Endpoint must always end with '/'
@@ -106,19 +120,30 @@ func Open(
106120
prefix = prefix + "/"
107121
}
108122

109-
return &Storage{
110-
endpoint: endpoint,
111-
accessKeyID: accessKeyID,
112-
secretKey: secretKey,
113-
bucket: bucket,
114-
location: location,
115-
prefix: prefix,
123+
s3storage := &Storage{
124+
endpoint: endpoint,
125+
S3RoleEnabled: S3RoleEnabled,
126+
s3Role: s3Role,
127+
accessKeyID: accessKeyID,
128+
secretKey: secretKey,
129+
bucket: bucket,
130+
location: location,
131+
prefix: prefix,
116132
httpClient: &http.Client{
117133
CheckRedirect: func(req *http.Request, via []*http.Request) error {
118134
return http.ErrUseLastResponse
119135
},
120136
},
121-
}, nil
137+
awsInstanceMetadataURL: awsInstanceMetadataURL,
138+
awsCredsRefreshPeriod: time.Minute,
139+
}
140+
141+
err := s3storage.getRoleCredentials()
142+
if err != nil {
143+
return nil, err
144+
}
145+
146+
return s3storage, nil
122147
}
123148

124149
func (s *Storage) Kind() string {
@@ -691,4 +716,124 @@ func (s *Storage) scanObjectNames(ctx context.Context, prefix string, limit int)
691716
return entries, subPaths, nil
692717
}
693718

719+
func (s *Storage) getRoleCredentials() error {
720+
if !s.S3RoleEnabled {
721+
return nil
722+
}
723+
724+
var err error
725+
s.accessKeyID, s.secretKey, err = s.requestCredentials()
726+
if err != nil {
727+
return err
728+
}
729+
730+
s3CredentialsRefreshTicker := time.NewTicker(s.awsCredsRefreshPeriod)
731+
go func() {
732+
for {
733+
select {
734+
case _ = <-s3CredentialsRefreshTicker.C:
735+
accessKeyID, secretKey, err := s.requestCredentials()
736+
if err != nil {
737+
log.Printf("S3 role credentials lookup failed with an error: %v", err)
738+
continue
739+
}
740+
s.accessKeyID, s.secretKey = accessKeyID, secretKey
741+
}
742+
}
743+
}()
744+
745+
return nil
746+
}
747+
748+
func (s *Storage) requestCredentials() (string, string, error) {
749+
tokenReq, err := http.NewRequest("PUT", fmt.Sprintf("%s%s",
750+
s.awsInstanceMetadataURL,
751+
"/latest/api/token",
752+
), nil)
753+
if err != nil {
754+
return "", "", errors.New("cannot form metadata token request")
755+
}
756+
757+
tokenReq.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "21600")
758+
759+
tokenResp, err := http.DefaultClient.Do(tokenReq)
760+
if err != nil {
761+
return "", "", errors.New("cannot get metadata token")
762+
}
763+
defer tokenResp.Body.Close()
764+
765+
token, err := ioutil.ReadAll(tokenResp.Body)
766+
if err != nil {
767+
return "", "", errors.New("cannot read metadata token")
768+
}
769+
770+
role := s.s3Role
771+
if s.s3Role == "" {
772+
roleReq, err := http.NewRequest("GET", fmt.Sprintf("%s%s",
773+
s.awsInstanceMetadataURL,
774+
"/latest/meta-data/iam/info",
775+
), nil)
776+
if err != nil {
777+
return "", "", errors.New("cannot form role name request")
778+
}
779+
780+
roleReq.Header.Set("X-aws-ec2-metadata-token", string(token))
781+
roleResp, err := http.DefaultClient.Do(roleReq)
782+
if err != nil {
783+
return "", "", errors.New("cannot get role name")
784+
}
785+
defer roleResp.Body.Close()
786+
787+
creds, err := ioutil.ReadAll(roleResp.Body)
788+
if err != nil {
789+
return "", "", errors.New("cannot read role name")
790+
}
791+
792+
var metadata struct {
793+
InstanceProfileArn string `json:"InstanceProfileArn"`
794+
}
795+
if err := json.Unmarshal(creds, &metadata); err != nil {
796+
return "", "", errors.New("cannot parse role name")
797+
}
798+
799+
match := arnRoleRegex.FindStringSubmatch(metadata.InstanceProfileArn)
800+
if len(match) < 2 {
801+
return "", "", ErrCredentialsCannotBeFound
802+
}
803+
804+
role = match[1]
805+
}
806+
807+
credsReq, err := http.NewRequest("GET", fmt.Sprintf("%s%s/%s",
808+
s.awsInstanceMetadataURL,
809+
"/latest/meta-data/iam/security-credentials",
810+
role,
811+
), nil)
812+
if err != nil {
813+
return "", "", errors.New("cannot form role credentials request")
814+
}
815+
816+
credsReq.Header.Set("X-aws-ec2-metadata-token", string(token))
817+
credsResp, err := http.DefaultClient.Do(credsReq)
818+
if err != nil {
819+
return "", "", errors.New("cannot get role credentials")
820+
}
821+
defer credsResp.Body.Close()
822+
823+
creds, err := ioutil.ReadAll(credsReq.Body)
824+
if err != nil {
825+
return "", "", errors.New("cannot read role credentials")
826+
}
827+
828+
var credentials struct {
829+
AccessKeyID string `json:"AccessKeyId"`
830+
SecretAccessKey string `json:"SecretAccessKey"`
831+
}
832+
if err := json.Unmarshal(creds, &credentials); err != nil {
833+
return "", "", errors.New("cannot parse role credentials")
834+
}
835+
836+
return credentials.AccessKeyID, credentials.SecretAccessKey, nil
837+
}
838+
694839
var _ remotestorage.Storage = (*Storage)(nil)

0 commit comments

Comments
 (0)