Soak test for kubernetes backend

This commit is contained in:
John Houston 2020-06-05 13:32:59 -04:00
parent 23fb8f6d21
commit e5f7237582
3 changed files with 2085 additions and 8 deletions

View File

@ -1,10 +1,16 @@
package kubernetes
import (
"fmt"
"math/rand"
"os"
"sync"
"testing"
"time"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/states/statemgr"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@ -30,7 +36,7 @@ func testACC(t *testing.T) {
namespace = "default"
}
cleanupK8sSecrets(t)
cleanupK8sResources(t)
}
func TestBackend_impl(t *testing.T) {
@ -39,7 +45,7 @@ func TestBackend_impl(t *testing.T) {
func TestBackend(t *testing.T) {
testACC(t)
defer cleanupK8sSecrets(t)
defer cleanupK8sResources(t)
b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
"secret_suffix": secretSuffix,
@ -51,7 +57,7 @@ func TestBackend(t *testing.T) {
func TestBackendLocks(t *testing.T) {
testACC(t)
defer cleanupK8sSecrets(t)
defer cleanupK8sResources(t)
// Get the backend. We need two to test locking.
b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
@ -67,7 +73,57 @@ func TestBackendLocks(t *testing.T) {
backend.TestBackendStateForceUnlock(t, b1, b2)
}
func cleanupK8sSecrets(t *testing.T) {
func TestBackendLocksSoak(t *testing.T) {
testACC(t)
defer cleanupK8sResources(t)
clientCount := 1000
lockCount := 0
lockers := []statemgr.Locker{}
for i := 0; i < clientCount; i++ {
b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
"secret_suffix": secretSuffix,
}))
s, err := b.StateMgr(backend.DefaultStateName)
if err != nil {
t.Fatalf("Error creating state manager: %v", err)
}
lockers = append(lockers, s.(statemgr.Locker))
}
wg := sync.WaitGroup{}
for i, l := range lockers {
wg.Add(1)
go func(locker statemgr.Locker, i int) {
r := rand.Intn(10)
time.Sleep(time.Duration(r) * time.Microsecond)
li := state.NewLockInfo()
li.Operation = "test"
li.Who = fmt.Sprintf("client-%v", i)
_, err := locker.Lock(li)
if err == nil {
t.Logf("[INFO] Client %v got the lock\r\n", i)
lockCount++
}
wg.Done()
}(l, i)
}
wg.Wait()
if lockCount > 1 {
t.Fatalf("multiple backend clients were able to acquire a lock, count: %v", lockCount)
}
if lockCount == 0 {
t.Fatal("no clients were able to acquire a lock")
}
}
func cleanupK8sResources(t *testing.T) {
// Get a backend to use the k8s client
b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
"secret_suffix": secretSuffix,
@ -80,7 +136,7 @@ func cleanupK8sSecrets(t *testing.T) {
t.Fatal(err)
}
// Get state secrets based off the tfstateKey label
// Delete secrets
opts := metav1.ListOptions{LabelSelector: tfstateKey + "=true"}
secrets, err := sClient.List(opts)
if err != nil {
@ -106,6 +162,32 @@ func cleanupK8sSecrets(t *testing.T) {
}
}
leaseClient, err := b.KubernetesLeaseClient()
if err != nil {
t.Fatal(err)
}
// Delete leases
leases, err := leaseClient.List(opts)
if err != nil {
t.Fatal(err)
}
for _, lease := range leases.Items {
labels := lease.GetLabels()
key, ok := labels[tfstateSecretSuffixKey]
if !ok {
continue
}
if key == secretSuffix {
err = leaseClient.Delete(lease.GetName(), delOps)
if err != nil {
errs = append(errs, err)
}
}
}
if len(errs) > 0 {
t.Fatal(errs)
}

View File

@ -15,7 +15,7 @@ func TestRemoteClient_impl(t *testing.T) {
func TestRemoteClient(t *testing.T) {
testACC(t)
defer cleanupK8sSecrets(t)
defer cleanupK8sResources(t)
b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
"secret_suffix": secretSuffix,
@ -31,7 +31,7 @@ func TestRemoteClient(t *testing.T) {
func TestRemoteClientLocks(t *testing.T) {
testACC(t)
defer cleanupK8sSecrets(t)
defer cleanupK8sResources(t)
b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
"secret_suffix": secretSuffix,
@ -56,7 +56,7 @@ func TestRemoteClientLocks(t *testing.T) {
func TestForceUnlock(t *testing.T) {
testACC(t)
defer cleanupK8sSecrets(t)
defer cleanupK8sResources(t)
b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
"secret_suffix": secretSuffix,

File diff suppressed because it is too large Load Diff