From de2ef5e836206ca9796945029b6b6f097fd11b26 Mon Sep 17 00:00:00 2001 From: Alvin Lin Date: Mon, 3 Oct 2022 19:14:50 -0700 Subject: [PATCH] Cheery pick fix for issue #4888 (#4891) * Cheery pick fix for issue #4888 Signed-off-by: Alvin Lin --- CHANGELOG.md | 4 +- VERSION | 2 +- .../running-chunks-storage-with-cassandra.md | 4 +- pkg/alertmanager/alertstore/local/store.go | 36 +++++++++- .../alertstore/local/store_test.go | 19 +++++- pkg/alertmanager/multitenant_test.go | 66 +++++++++++++++++++ 6 files changed, 123 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfb60cc3b6..f03fe8fdb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog -## master / unreleased - +## 1.13.1 2022-10-03 +[BUGFIX] AlertManager: fixed issue introduced by #4495 where templates files were being deleted when using alertmanager local store. #4890 ## 1.13.0 2022-07-14 * [CHANGE] Changed default for `-ingester.min-ready-duration` from 1 minute to 15 seconds. #4539 diff --git a/VERSION b/VERSION index feaae22bac..b50dd27dd9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.0 +1.13.1 diff --git a/docs/chunks-storage/running-chunks-storage-with-cassandra.md b/docs/chunks-storage/running-chunks-storage-with-cassandra.md index 20762169aa..9cc6acbf6f 100644 --- a/docs/chunks-storage/running-chunks-storage-with-cassandra.md +++ b/docs/chunks-storage/running-chunks-storage-with-cassandra.md @@ -111,12 +111,12 @@ storage: ``` The latest tag is not published for the Cortex docker image. Visit quay.io/repository/cortexproject/cortex -to find the latest stable version tag and use it in the command below (currently it is `v1.11.0`). +to find the latest stable version tag and use it in the command below (currently it is `v1.13.1`). Run Cortex using the latest stable version: ``` -docker run -d --name=cortex -v $(pwd)/single-process-config.yaml:/etc/single-process-config.yaml -p 9009:9009 quay.io/cortexproject/cortex:v1.11.0 -config.file=/etc/single-process-config.yaml +docker run -d --name=cortex -v $(pwd)/single-process-config.yaml:/etc/single-process-config.yaml -p 9009:9009 quay.io/cortexproject/cortex:v1.13.1 -config.file=/etc/single-process-config.yaml ``` In case you prefer to run the master version, please follow this [documentation](./chunks-storage-getting-started.md) on how to build Cortex from source. diff --git a/pkg/alertmanager/alertstore/local/store.go b/pkg/alertmanager/alertstore/local/store.go index 53ffa7db3e..9970df1e27 100644 --- a/pkg/alertmanager/alertstore/local/store.go +++ b/pkg/alertmanager/alertstore/local/store.go @@ -15,7 +15,8 @@ import ( ) const ( - Name = "local" + Name = "local" + templatesDir = "templates" ) var ( @@ -149,9 +150,42 @@ func (f *Store) reloadConfigs() (map[string]alertspb.AlertConfigDesc, error) { // The file name must correspond to the user tenant ID user := strings.TrimSuffix(info.Name(), ext) + // Load template files + userTemplateDir := filepath.Join(f.cfg.Path, user, templatesDir) + var templates []*alertspb.TemplateDesc + + if _, e := os.Stat(userTemplateDir); e == nil { + err = filepath.Walk(userTemplateDir, func(templatePath string, info os.FileInfo, err error) error { + if err != nil { + return errors.Wrapf(err, "unable to walk file path at %s", templatePath) + } + // Ignore files that are directories + if info.IsDir() { + return nil + } + content, err := os.ReadFile(templatePath) + if err != nil { + return errors.Wrapf(err, "unable to read alertmanager templates %s", templatePath) + } + + templates = append(templates, &alertspb.TemplateDesc{ + Body: string(content), + Filename: info.Name(), + }) + return nil + }) + + if err != nil { + return errors.Wrapf(err, "unable to list alertmanager templates: %s", userTemplateDir) + } + } else if !os.IsNotExist(e) { + return errors.Wrapf(e, "unable to read alertmanager templates %s", path) + } + configs[user] = alertspb.AlertConfigDesc{ User: user, RawConfig: string(content), + Templates: templates, } return nil }) diff --git a/pkg/alertmanager/alertstore/local/store_test.go b/pkg/alertmanager/alertstore/local/store_test.go index 9baace4313..4072bf9c30 100644 --- a/pkg/alertmanager/alertstore/local/store_test.go +++ b/pkg/alertmanager/alertstore/local/store_test.go @@ -82,17 +82,22 @@ func TestStore_GetAlertConfigs(t *testing.T) { // The storage contains some configs. { user1Cfg := prepareAlertmanagerConfig("user-1") - require.NoError(t, ioutil.WriteFile(filepath.Join(storeDir, "user-1.yaml"), []byte(user1Cfg), os.ModePerm)) + user1Dir, user1TemplateDir := prepareUserDir(t, storeDir, true, "user-1") + require.NoError(t, os.WriteFile(filepath.Join(user1Dir, "user-1.yaml"), []byte(user1Cfg), os.ModePerm)) + + require.NoError(t, os.WriteFile(filepath.Join(user1TemplateDir, "template.tpl"), []byte("testTemplate"), os.ModePerm)) configs, err := store.GetAlertConfigs(ctx, []string{"user-1", "user-2"}) require.NoError(t, err) assert.Contains(t, configs, "user-1") assert.NotContains(t, configs, "user-2") assert.Equal(t, user1Cfg, configs["user-1"].RawConfig) + assert.Equal(t, "testTemplate", configs["user-1"].Templates[0].Body) // Add another user config. user2Cfg := prepareAlertmanagerConfig("user-2") - require.NoError(t, ioutil.WriteFile(filepath.Join(storeDir, "user-2.yaml"), []byte(user2Cfg), os.ModePerm)) + user2Dir, _ := prepareUserDir(t, storeDir, false, "user-2") + require.NoError(t, os.WriteFile(filepath.Join(user2Dir, "user-2.yaml"), []byte(user2Cfg), os.ModePerm)) configs, err = store.GetAlertConfigs(ctx, []string{"user-1", "user-2"}) require.NoError(t, err) @@ -103,6 +108,16 @@ func TestStore_GetAlertConfigs(t *testing.T) { } } +func prepareUserDir(t *testing.T, storeDir string, createTemplateDir bool, user string) (userDir string, templateDir string) { + userDir = filepath.Join(storeDir, user) + templateDir = filepath.Join(userDir, templatesDir) + require.NoError(t, os.MkdirAll(userDir, os.ModePerm)) + if createTemplateDir { + require.NoError(t, os.MkdirAll(templateDir, os.ModePerm)) + } + return +} + func prepareLocalStore(t *testing.T) (store *Store, storeDir string) { var err error diff --git a/pkg/alertmanager/multitenant_test.go b/pkg/alertmanager/multitenant_test.go index 237ce44884..c35a2624d8 100644 --- a/pkg/alertmanager/multitenant_test.go +++ b/pkg/alertmanager/multitenant_test.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "sync" "testing" @@ -40,6 +41,7 @@ import ( "github.com/cortexproject/cortex/pkg/alertmanager/alertspb" "github.com/cortexproject/cortex/pkg/alertmanager/alertstore" "github.com/cortexproject/cortex/pkg/alertmanager/alertstore/bucketclient" + "github.com/cortexproject/cortex/pkg/alertmanager/alertstore/local" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv/consul" "github.com/cortexproject/cortex/pkg/storage/bucket" @@ -166,6 +168,49 @@ func TestMultitenantAlertmanagerConfig_Validate(t *testing.T) { } } +func TestMultitenantAlertmanager_loadAndSyncConfigsLocalStorage(t *testing.T) { + storeDir := t.TempDir() + store, _ := local.NewStore(local.StoreConfig{Path: storeDir}) + config := `global: + resolve_timeout: 1m + smtp_require_tls: false + +route: + receiver: 'email' + +receivers: +- name: 'email' + email_configs: + - to: test@example.com + from: test@example.com + smarthost: smtp:2525 +` + user1Dir, user1TemplateDir := prepareUserDir(t, storeDir, "user-1") + user2Dir, _ := prepareUserDir(t, storeDir, "user-2") + require.NoError(t, os.WriteFile(filepath.Join(user1Dir, "user-1.yaml"), []byte(config), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(user2Dir, "user-2.yaml"), []byte(config), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(user1TemplateDir, "template.tpl"), []byte("testTemplate"), os.ModePerm)) + + originalFiles, err := listFiles(storeDir) + require.NoError(t, err) + require.Equal(t, 3, len(originalFiles)) + + cfg := mockAlertmanagerConfig(t) + cfg.DataDir = storeDir + reg := prometheus.NewPedanticRegistry() + am, err := createMultitenantAlertmanager(cfg, nil, nil, store, nil, nil, log.NewNopLogger(), reg) + require.NoError(t, err) + for i := 0; i < 5; i++ { + err = am.loadAndSyncConfigs(context.Background(), reasonPeriodic) + require.NoError(t, err) + require.Len(t, am.alertmanagers, 2) + files, err := listFiles(storeDir) + require.NoError(t, err) + // Verify if the files were not deleted + require.Equal(t, originalFiles, files) + } +} + func TestMultitenantAlertmanager_loadAndSyncConfigs(t *testing.T) { ctx := context.Background() @@ -1892,6 +1937,27 @@ func prepareInMemoryAlertStore() alertstore.AlertStore { return bucketclient.NewBucketAlertStore(objstore.NewInMemBucket(), nil, log.NewNopLogger()) } +func prepareUserDir(t *testing.T, storeDir string, user string) (userDir string, templateDir string) { + userDir = filepath.Join(storeDir, user) + templateDir = filepath.Join(userDir, templatesDir) + require.NoError(t, os.MkdirAll(userDir, os.ModePerm)) + require.NoError(t, os.MkdirAll(templateDir, os.ModePerm)) + return +} + +func listFiles(dir string) ([]string, error) { + var r []string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + r = append(r, path) + } + return nil + }) + sort.Strings(r) + + return r, err +} + func TestSafeTemplateFilepath(t *testing.T) { tests := map[string]struct { dir string