Larger file can be sent to credhub

Author: ArthurHlt

README.md

@@ -2,6 +2,8 @@
 
 An [http backend](https://www.terraform.io/docs/backends/types/http.html) which stores and retrieves tfstates files in a secure and encrypted way through [credhub](https://github.com/cloudfoundry-incubator/credhub).
 
+When file is too large for database use in credhub, file will be split in part and gzipped to be sent to credhub and rebuild final file when retrieving from credhub. 
+
 This backend supports [state locking](https://www.terraform.io/docs/state/locking.html).
 
 ## Boshrelease
@@ -67,7 +69,8 @@ There is two different ways to run the server:
 ```yaml
 host: 0.0.0.0 # an be 127.0.0.1 too
 port: 8080 # port to listen
-name: terraform-secure # this name inside credhub to create an unique path for your tfstate
+chunk_size: ~ # Chunk size in number of bytes to split your tfstate inside credhub to leverage database limit (Default: 60000)
+base_path: /terraform-secure-backend/tfstate/pouet #  Create an unique path for your tfstate on credhub
 cert: ~ # Set a path or pem cert string certificate to run your senver in tls (ignored if lets_encrypt_domains is set)
 key: ~ # Set a path or pem key string certificate to run your senver in tls (ignored if lets_encrypt_domains is set)
 log_level: ~ # Verbosity, can be info, debug, warning, error

go.mod

@@ -32,5 +32,8 @@ require (
 	github.com/urfave/cli v1.20.0
 	github.com/zclconf/go-cty v0.0.0-20190201220620-4ca19710f056 // indirect
 	golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613
+	golang.org/x/net v0.0.0-20190225153610-fe579d43d832 // indirect
+	golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect
+	golang.org/x/sys v0.0.0-20190225065934-cc5685c2db12 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -174,16 +174,22 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181129055619-fae4c4e3ad76 h1:xx5MUFyRQRbPk6VjWjIE1epE/K5AoDD8QUN116NCy8k=
 golang.org/x/net v0.0.0-20181129055619-fae4c4e3ad76/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190225153610-fe579d43d832 h1:2IdId8zoI92l1bUzjAOygcAOkmCe13HY1j0rqPPPzB8=
+golang.org/x/net v0.0.0-20190225153610-fe579d43d832/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 h1:0oC8rFnE+74kEmuHZ46F6KHsMr5Gx2gUQPuNz28iQZM=
 golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190225065934-cc5685c2db12 h1:Zw7eRv6INHGfu15LVRN1vrrwusJbnfJjAZn3D1VkQIE=
+golang.org/x/sys v0.0.0-20190225065934-cc5685c2db12/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

server/api.go

@@ -1,25 +1,28 @@
 package server
 
 import (
-	"code.cloudfoundry.org/credhub-cli/credhub/credentials/values"
 	"encoding/json"
 	"fmt"
 	"github.com/gorilla/mux"
 	"github.com/hashicorp/terraform/state"
+	"github.com/orange-cloudfoundry/terraform-secure-backend/server/credhub"
+	"github.com/orange-cloudfoundry/terraform-secure-backend/server/storer"
 	"github.com/sirupsen/logrus"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"strings"
 )
 
 type ApiController struct {
-	name          string
-	credhubClient CredhubClient
+	basePath      string
+	storer        storer.Storer
 	store         *LockStore
+	credhubClient credhub.CredhubClient
 }
 
-func NewApiController(name string, credhubClient CredhubClient, store *LockStore) *ApiController {
-	return &ApiController{name, credhubClient, store}
+func NewApiController(basePath string, credhubClient credhub.CredhubClient, storer storer.Storer, store *LockStore) *ApiController {
+	return &ApiController{basePath, storer, store, credhubClient}
 }
 
 type CredModel struct {
@@ -34,18 +37,7 @@ func (c ApiController) Store(w http.ResponseWriter, req *http.Request) {
 	defer req.Body.Close()
 	entry := logrus.WithField("action", "store").WithField("name", c.RequestName(req))
 	entry.Debug("Storing tfstate")
-	var dataJson map[string]interface{}
-	b, err := ioutil.ReadAll(req.Body)
-	if err != nil {
-		entry.Error(err)
-		panic(err)
-	}
-	err = json.Unmarshal(b, &dataJson)
-	if err != nil {
-		entry.Error(err)
-		panic(err)
-	}
-	_, err = c.credhubClient.SetJSON(c.CredhubName(req), values.JSON(dataJson))
+	err := c.storer.Store(c.CredhubName(req), req.Body)
 	if err != nil {
 		entry.Error(err)
 		panic(err)
@@ -56,7 +48,7 @@ func (c ApiController) Retrieve(w http.ResponseWriter, req *http.Request) {
 	defer req.Body.Close()
 	entry := logrus.WithField("action", "retrieve").WithField("name", c.RequestName(req))
 	entry.Debug("Retrieving tfstate")
-	cred, err := c.credhubClient.GetLatestJSON(c.CredhubName(req))
+	r, err := c.storer.Retrieve(c.CredhubName(req))
 	if err != nil && strings.Contains(err.Error(), "does not exist") {
 		w.WriteHeader(http.StatusNoContent)
 		return
@@ -65,22 +57,22 @@ func (c ApiController) Retrieve(w http.ResponseWriter, req *http.Request) {
 		entry.Error(err)
 		panic(err)
 	}
+	defer r.Close()
 	w.Header().Set("Content-Type", "application/json")
-	b, _ := json.Marshal(cred.Value)
-	w.Write(b)
+	io.Copy(w, r)
 }
 
 func (c ApiController) Delete(w http.ResponseWriter, req *http.Request) {
 	defer req.Body.Close()
-	name := c.CredhubName(req)
+	path := c.CredhubName(req)
 	entry := logrus.WithField("action", "delete").WithField("name", c.RequestName(req))
 	entry.Debug("Deleting tfstate")
-	err := c.credhubClient.Delete(name)
+	err := c.storer.Delete(path)
 	if err != nil {
 		entry.Error(err)
 		panic(err)
 	}
-	err = c.store.DeleteLock(name)
+	err = c.store.DeleteLock(path)
 	if err != nil {
 		entry.Error(err)
 		panic(err)
@@ -119,12 +111,8 @@ func (c ApiController) Lock(w http.ResponseWriter, req *http.Request) {
 	}
 }
 
-func (c ApiController) Path() string {
-	return fmt.Sprintf("%s/%s", CREDHUB_PREFIX, c.name)
-}
-
 func (c ApiController) CredhubName(req *http.Request) string {
-	return fmt.Sprintf("%s/%s", c.Path(), c.RequestName(req))
+	return fmt.Sprintf("%s/%s", c.basePath, c.RequestName(req))
 }
 
 func (c ApiController) RequestName(req *http.Request) string {
@@ -165,7 +153,7 @@ func (c ApiController) UnLock(w http.ResponseWriter, req *http.Request) {
 
 func (c ApiController) List(w http.ResponseWriter, req *http.Request) {
 	entry := logrus.WithField("action", "list")
-	result, err := c.credhubClient.FindByPath(c.Path())
+	result, err := c.credhubClient.FindByPath(c.basePath)
 	if err != nil {
 		entry.Error(err)
 		panic(err)

server/api_test.go

@@ -1,8 +1,6 @@
 package server_test
 
 import (
-	. "github.com/orange-cloudfoundry/terraform-secure-backend/server"
-
 	"bytes"
 	"code.cloudfoundry.org/credhub-cli/credhub/credentials"
 	"code.cloudfoundry.org/credhub-cli/credhub/credentials/values"
@@ -11,7 +9,9 @@ import (
 	"github.com/hashicorp/terraform/state"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/orange-cloudfoundry/terraform-secure-backend/server/serverfakes"
+	. "github.com/orange-cloudfoundry/terraform-secure-backend/server"
+	"github.com/orange-cloudfoundry/terraform-secure-backend/server/credhub/credhubfakes"
+	"github.com/orange-cloudfoundry/terraform-secure-backend/server/storer"
 	log "github.com/sirupsen/logrus"
 	"io/ioutil"
 	"net/http"
@@ -20,15 +20,17 @@ import (
 
 var _ = Describe("Api", func() {
 	log.SetOutput(ioutil.Discard)
-	var fakeClient *serverfakes.FakeCredhubClient
+	var fakeClient *credhubfakes.FakeCredhubClient
+	var cStorer *storer.Credhub
 	var apiController *ApiController
 	var lockStore *LockStore
 	var responseRecorder *httptest.ResponseRecorder
 	BeforeEach(func() {
 		responseRecorder = httptest.NewRecorder()
-		fakeClient = new(serverfakes.FakeCredhubClient)
+		fakeClient = new(credhubfakes.FakeCredhubClient)
+		cStorer = storer.NewCredhub(fakeClient)
 		lockStore = NewLockStore(fakeClient)
-		apiController = NewApiController("test", fakeClient, lockStore)
+		apiController = NewApiController("test", fakeClient, cStorer, lockStore)
 	})
 	Context("Store", func() {
 		It("should store data when giving state", func() {