Merge pull request #24 from nimbolus/fix-state-migration

Fix state migration

pkg/server/handler.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -130,7 +131,11 @@ func Get(w http.ResponseWriter, r *http.Request, state *terraform.State, store s
 	log.Debugf("get state with id %s", state.ID)
 	stateID := state.ID
 	state, err := store.GetState(state.ID)
-	if err != nil {
+	if errors.Is(err, storage.ErrStateNotFound) {
+		log.Debugf("state with id %s does not exist", stateID)
+		HTTPResponse(w, r, http.StatusNotFound, err.Error())
+		return
+	} else if err != nil {
 		log.Warnf("failed to get state with id %s: %v", stateID, err)
 		HTTPResponse(w, r, http.StatusBadRequest, err.Error())
 		return

pkg/server/handler_test.go

@@ -23,8 +23,10 @@ import (
 	tf "github.com/nimbolus/terraform-backend/pkg/terraform"
 )
 
-func NewStateHandler(t *testing.T) http.Handler {
-	store, err := filesystem.NewFileSystemStorage(filepath.Join("./handler_test", "storage"))
+var terraformBinary = flag.String("tf", "terraform", "terraform binary")
+
+func NewStateHandler(t *testing.T, baseDir string) http.Handler {
+	store, err := filesystem.NewFileSystemStorage(filepath.Join(baseDir, "storage"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -40,11 +42,9 @@ func NewStateHandler(t *testing.T) http.Handler {
 	return r
 }
 
-var terraformBinary = flag.String("tf", "terraform", "terraform binary")
-
-func terraformOptions(t *testing.T, addr string) *terraform.Options {
+func terraformOptions(t *testing.T, baseDir, addr string) *terraform.Options {
 	return terraform.WithDefaultRetryableErrors(t, &terraform.Options{
-		TerraformDir:    "./handler_test",
+		TerraformDir:    baseDir,
 		TerraformBinary: *terraformBinary,
 		Vars:            map[string]interface{}{},
 		Reconfigure:     true,
@@ -61,7 +61,7 @@ func terraformOptions(t *testing.T, addr string) *terraform.Options {
 }
 
 func TestServerHandler_VerifyLockOnPush(t *testing.T) {
-	s := httptest.NewServer(NewStateHandler(t))
+	s := httptest.NewServer(NewStateHandler(t, "./handler_test"))
 	defer s.Close()
 
 	address, err := url.JoinPath(s.URL, "/state/project1/example")
@@ -72,7 +72,7 @@ func TestServerHandler_VerifyLockOnPush(t *testing.T) {
 	simulateLock(t, address, true)
 
 	for _, doLock := range []bool{true, false} {
-		terraformOptions := terraformOptions(t, address)
+		terraformOptions := terraformOptions(t, "./handler_test", address)
 		terraformOptions.Lock = doLock
 
 		_, err = terraform.InitAndApplyE(t, terraformOptions)
@@ -85,15 +85,15 @@ func TestServerHandler_VerifyLockOnPush(t *testing.T) {
 }
 
 func TestServerHandler(t *testing.T) {
-	s := httptest.NewServer(NewStateHandler(t))
+	s := httptest.NewServer(NewStateHandler(t, "./handler_test"))
 	defer s.Close()
 
 	address, err := url.JoinPath(s.URL, "/state/project1/example")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	terraformOptions := terraformOptions(t, address)
+	terraformOptions := terraformOptions(t, "./handler_test", address)
 
 	// Clean up resources with "terraform destroy" at the end of the test.
 	defer terraform.Destroy(t, terraformOptions)

pkg/server/state_migration_test.go

@@ -0,0 +1,94 @@
+//go:build integration || handler
+// +build integration handler
+
+package server
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+)
+
+func TestStateMigration(t *testing.T) {
+	baseDir := "./state_migration_test"
+
+	s := httptest.NewServer(NewStateHandler(t, baseDir))
+	defer s.Close()
+
+	address, err := url.JoinPath(s.URL, "/state/project1/example")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	terraformOptions := terraformOptions(t, baseDir, address)
+	terraformOptions.Reconfigure = false
+	terraformOptions.Lock = true
+	backendConf := terraformOptions.BackendConfig
+
+	// init with http backend
+	if err := os.WriteFile(filepath.Join(baseDir, "backend.tf"), []byte("terraform {\n  backend \"http\" {}\n}"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := terraform.InitAndApplyE(t, terraformOptions); err != nil {
+		t.Fatal(err)
+	}
+
+	// init -migrate-state to local backend
+	terraformOptions.MigrateState = true
+	terraformOptions.BackendConfig = map[string]interface{}{}
+
+	if err := os.WriteFile(filepath.Join(baseDir, "backend.tf"), []byte("terraform {\n  backend \"local\" {}\n}"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := terraform.InitAndApplyE(t, terraformOptions); err != nil {
+		t.Fatal(err)
+	}
+
+	// init -migrate-state to http backend
+	terraformOptions.BackendConfig = backendConf
+
+	if err := os.WriteFile(filepath.Join(baseDir, "backend.tf"), []byte("terraform {\n  backend \"http\" {}\n}"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := terraform.InitAndApplyE(t, terraformOptions); err != nil {
+		t.Fatal(err)
+	}
+
+	// destroy
+	if _, err := terraform.DestroyE(t, terraformOptions); err != nil {
+		t.Fatal(err)
+	}
+
+	req, err := http.NewRequest(http.MethodDelete, address, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	req.Header.Set("Authorization", fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte("basic:some-random-secret"))))
+
+	if _, err := http.DefaultClient.Do(req); err != nil {
+		t.Fatal(err)
+	}
+
+	// verify state file is deleted
+	if _, err := os.Stat(filepath.Join(baseDir, "storage/d82238e1158b32f0b445c5da058608a8c1d83551f890b19b7e90d78cce1a808d.tfstate")); !os.IsNotExist(err) {
+		t.Fatal("state file should be deleted")
+	}
+
+	// cleanup
+	for _, f := range []string{"terraform.tfstate", ".terraform/terraform.tfstate"} {
+		if err := os.Remove(filepath.Join(baseDir, f)); err != nil {
+			t.Fatal(err)
+		}
+	}
+}

pkg/server/state_migration_test/.gitignore

@@ -0,0 +1,5 @@
+.terraform/
+.terraform.lock.hcl
+terraform.tfstate
+terraform.tfstate.backup
+storage/

pkg/server/state_migration_test/backend.tf

@@ -0,0 +1,3 @@
+terraform {
+  backend "http" {}
+}

pkg/server/state_migration_test/main.tf

@@ -0,0 +1,4 @@
+resource "local_file" "dummy" {
+  filename = "dummy.txt"
+  content  = "dummy"
+}

pkg/storage/filesystem/filesystem.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/nimbolus/terraform-backend/pkg/storage"
 	"github.com/nimbolus/terraform-backend/pkg/terraform"
 )
 
@@ -35,13 +36,7 @@ func (f *FileSystemStorage) SaveState(s *terraform.State) error {
 
 func (f *FileSystemStorage) GetState(id string) (*terraform.State, error) {
 	if _, err := os.Stat(f.getFileName(id)); errors.Is(err, os.ErrNotExist) {
-		f, err := os.Create(f.getFileName(id))
-		if err != nil {
-			return nil, err
-		}
-		defer f.Close()
-
-		return &terraform.State{}, nil
+		return nil, storage.ErrStateNotFound
 	}
 
 	d, err := os.ReadFile(f.getFileName(id))

pkg/storage/postgres/postgres.go

@@ -3,10 +3,12 @@ package postgres
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 	"time"
 
 	pgclient "github.com/nimbolus/terraform-backend/pkg/client/postgres"
+	"github.com/nimbolus/terraform-backend/pkg/storage"
 	"github.com/nimbolus/terraform-backend/pkg/terraform"
 )
 
@@ -70,7 +72,9 @@ func (p *PostgresStorage) GetState(id string) (*terraform.State, error) {
 	s := &terraform.State{}
 
 	err := p.db.QueryRow(`SELECT state_data FROM `+p.table+` WHERE state_id = $1`, id).Scan(&s.Data)
-	if err != nil {
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, storage.ErrStateNotFound
+	} else if err != nil {
 		return nil, err
 	}
 

pkg/storage/s3/s3.go

@@ -8,6 +8,7 @@ import (
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 
+	"github.com/nimbolus/terraform-backend/pkg/storage"
 	"github.com/nimbolus/terraform-backend/pkg/terraform"
 )
 
@@ -67,7 +68,7 @@ func (s *S3Storage) GetState(id string) (*terraform.State, error) {
 	buf := new(bytes.Buffer)
 	if _, err := buf.ReadFrom(obj); err != nil {
 		if minio.ToErrorResponse(err).Code == "NoSuchKey" {
-			return state, nil
+			return nil, storage.ErrStateNotFound
 		}
 		return state, err
 	}

pkg/storage/storage.go

@@ -1,9 +1,15 @@
 package storage
 
 import (
+	"errors"
+
 	"github.com/nimbolus/terraform-backend/pkg/terraform"
 )
 
+var (
+	ErrStateNotFound = errors.New("state does not exist")
+)
+
 type Storage interface {
 	GetName() string
 	SaveState(s *terraform.State) error

pkg/storage/util/storagetest.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"errors"
 	"testing"
 
 	"github.com/spf13/viper"
@@ -21,6 +22,11 @@ func StorageTest(t *testing.T, s storage.Storage) {
 		Data:    []byte("test"),
 	}
 
+	nonExisting, err := s.GetState(state.ID)
+	if nonExisting != nil || !errors.Is(err, storage.ErrStateNotFound) {
+		t.Error("non existing state should return ErrStateNotFound")
+	}
+
 	if err := s.SaveState(state); err != nil {
 		t.Error(err)
 	}