feat(plugin): make kv plugin builtin. (#75)

* feat(plugin): make kv plugin builtin.

* doc(changelog): update.

* chore(ci): raise go runtime to 1.16.9 and 1.17.2

.wwhrd.yml

@@ -49,3 +49,7 @@ exceptions:
   - github.com/hashicorp/go-immutable-radix
   # MPL-2.0 - https://github.com/hashicorp/go-uuid/blob/master/LICENSE
   - github.com/hashicorp/go-uuid
+  # MPL-2.0 - https://github.com/hashicorp/serf/blob/master/LICENSE
+  - github.com/hashicorp/serf/coordinate
+  # MPL-2.0 - https://github.com/hashicorp/consul/blob/main/LICENSE
+  - github.com/hashicorp/consul/api

CHANGELOG.md

@@ -19,6 +19,7 @@ FEATURES:
 * template/engine: `jsonEscape` / `jsonUnescape` is added to handle string escaping using JSON character escaping strategy [#70](https://github.com/elastic/harp/pull/70)
 * template/engine: `unquote` is added to unquote a `quote` escaped string. [#70](https://github.com/elastic/harp/pull/70)
 * bundle/prefixer: Globally add a prefix to all secret package. [#74](https://github.com/elastic/harp/pull/74)
+* plugin/kv: Promote harp-kv as builtin. [#75](https://github.com/elastic/harp/pull/75)
 
 ## 0.1.24
 

build/mage/golang/init.go

@@ -27,8 +27,8 @@ import (
 
 // Keep only last 2 versions
 var goVersions = []string{
-	"~1.17.1",
-	"~1.16.8",
+	"~1.17.2",
+	"~1.16.9",
 }
 
 func init() {

cmd/harp/internal/cmd/from.go

@@ -36,6 +36,9 @@ var fromCmd = func() *cobra.Command {
 	cmd.AddCommand(fromDumpCmd())
 	cmd.AddCommand(fromOPLogCmd())
 	cmd.AddCommand(fromObjectCmd())
+	cmd.AddCommand(fromConsulCmd())
+	cmd.AddCommand(fromEtcd3Cmd())
+	cmd.AddCommand(fromZookeeperCmd())
 
 	return cmd
 }

cmd/harp/internal/cmd/from_consul.go

@@ -0,0 +1,88 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+	"context"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/elastic/harp/pkg/kv/consul"
+	"github.com/elastic/harp/pkg/sdk/cmdutil"
+	"github.com/elastic/harp/pkg/sdk/log"
+	"github.com/elastic/harp/pkg/tasks/from"
+)
+
+// -----------------------------------------------------------------------------
+
+type fromConsulParams struct {
+	outputPath           string
+	basePaths            []string
+	lastPathItemAsSecret bool
+}
+
+var fromConsulCmd = func() *cobra.Command {
+	var params fromConsulParams
+
+	cmd := &cobra.Command{
+		Use:   "consul",
+		Short: "Extract KV pairs from Hashicorp Consul KV Store",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Initialize logger and context
+			ctx, cancel := cmdutil.Context(cmd.Context(), "harp-kv-from-consul", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
+			defer cancel()
+
+			runFromConsul(ctx, &params)
+		},
+	}
+
+	// Add parameters
+	cmd.Flags().StringVar(&params.outputPath, "out", "-", "Container output path ('-' for stdout)")
+	cmd.Flags().StringSliceVar(&params.basePaths, "paths", []string{}, "Exported base paths")
+	cmd.Flags().BoolVarP(&params.lastPathItemAsSecret, "last-path-item-as-secret-key", "k", false, "Use the last path element as secret key")
+
+	return cmd
+}
+
+func runFromConsul(ctx context.Context, params *fromConsulParams) {
+	// Create Consul client config from environment.
+	config := api.DefaultConfig()
+
+	// Creates a new client
+	client, err := api.NewClient(config)
+	if err != nil {
+		log.For(ctx).Fatal("unable to connect to consul cluster", zap.Error(err))
+		return
+	}
+
+	// Delegate to task
+	t := &from.ExtractKVTask{
+		Store:                   consul.Store(client),
+		ContainerWriter:         cmdutil.FileWriter(params.outputPath),
+		BasePaths:               params.basePaths,
+		LastPathItemAsSecretKey: params.lastPathItemAsSecret,
+	}
+
+	// Run the task
+	if err := t.Run(ctx); err != nil {
+		log.For(ctx).Fatal("unable to execute kv extract task", zap.Error(err))
+		return
+	}
+}

cmd/harp/internal/cmd/from_etcd3.go

@@ -0,0 +1,137 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+	"context"
+	"time"
+
+	"github.com/spf13/cobra"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.uber.org/zap"
+
+	"github.com/elastic/harp/pkg/kv/etcd3"
+	"github.com/elastic/harp/pkg/sdk/cmdutil"
+	"github.com/elastic/harp/pkg/sdk/log"
+	"github.com/elastic/harp/pkg/sdk/tlsconfig"
+	"github.com/elastic/harp/pkg/tasks/from"
+)
+
+// -----------------------------------------------------------------------------
+
+type fromEtcd3Params struct {
+	outputPath           string
+	basePaths            []string
+	lastPathItemAsSecret bool
+
+	endpoints   []string
+	dialTimeout time.Duration
+	username    string
+	password    string
+
+	useTLS             bool
+	caFile             string
+	certFile           string
+	keyFile            string
+	passphrase         string
+	insecureSkipVerify bool
+}
+
+var fromEtcd3Cmd = func() *cobra.Command {
+	var params fromEtcd3Params
+
+	cmd := &cobra.Command{
+		Use:   "etcd3",
+		Short: "Extract KV pairs from CoreOS Etcdv3 KV Store",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Initialize logger and context
+			ctx, cancel := cmdutil.Context(cmd.Context(), "harp-kv-from-etcdv3", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
+			defer cancel()
+
+			runFromEtcd3(ctx, &params)
+		},
+	}
+
+	// Add parameters
+	cmd.Flags().StringVar(&params.outputPath, "out", "-", "Container output path ('-' for stdout)")
+	cmd.Flags().StringSliceVar(&params.basePaths, "paths", []string{}, "Exported base paths")
+	cmd.Flags().BoolVarP(&params.lastPathItemAsSecret, "last-path-item-as-secret-key", "k", false, "Use the last path element as secret key")
+
+	cmd.Flags().StringArrayVar(&params.endpoints, "endpoints", []string{"http://localhost:2379"}, "Etcd cluster endpoints")
+	cmd.Flags().DurationVar(&params.dialTimeout, "dial-timeout", 15*time.Second, "Etcd cluster dial timeout")
+	cmd.Flags().StringVar(&params.username, "username", "", "Etcd cluster connection username")
+	cmd.Flags().StringVar(&params.password, "password", "", "Etcd cluster connection password")
+
+	cmd.Flags().BoolVar(&params.useTLS, "tls", false, "Enable TLS")
+	cmd.Flags().StringVar(&params.caFile, "ca-file", "", "TLS CA Certificate file path")
+	cmd.Flags().StringVar(&params.certFile, "cert-file", "", "TLS Client certificate file path")
+	cmd.Flags().StringVar(&params.keyFile, "key-file", "", "TLS Client private key file path")
+	cmd.Flags().StringVar(&params.passphrase, "key-passphrase", "", "TLS Client private key passphrase")
+	cmd.Flags().BoolVar(&params.insecureSkipVerify, "insecure-skip-verify", false, "Disable TLS certificate verification")
+
+	return cmd
+}
+
+func runFromEtcd3(ctx context.Context, params *fromEtcd3Params) {
+	// Create config
+	config := clientv3.Config{
+		Context:     ctx,
+		Endpoints:   params.endpoints,
+		DialTimeout: params.dialTimeout,
+		Username:    params.username,
+		Password:    params.password,
+	}
+
+	if params.useTLS {
+		tlsConfig, err := tlsconfig.Client(&tlsconfig.Options{
+			InsecureSkipVerify: params.insecureSkipVerify,
+			CAFile:             params.caFile,
+			CertFile:           params.certFile,
+			KeyFile:            params.keyFile,
+			Passphrase:         params.passphrase,
+		})
+		if err != nil {
+			log.For(ctx).Fatal("unable to initialize TLS settings", zap.Error(err))
+			return
+		}
+
+		// Assign TLS settings
+		config.TLS = tlsConfig
+	}
+
+	// Creates a new client
+	client, err := clientv3.New(config)
+	if err != nil {
+		log.For(ctx).Fatal("unable to connect to etcdv3 cluster", zap.Error(err))
+		return
+	}
+
+	// Delegate to task
+	t := &from.ExtractKVTask{
+		Store:                   etcd3.Store(client),
+		ContainerWriter:         cmdutil.FileWriter(params.outputPath),
+		BasePaths:               params.basePaths,
+		LastPathItemAsSecretKey: params.lastPathItemAsSecret,
+	}
+
+	// Run the task
+	if err := t.Run(ctx); err != nil {
+		log.For(ctx).Fatal("unable to execute kv extract task", zap.Error(err))
+		return
+	}
+}

cmd/harp/internal/cmd/from_zookeeper.go

@@ -0,0 +1,89 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+	"context"
+	"time"
+
+	zk "github.com/samuel/go-zookeeper/zk"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/elastic/harp/pkg/kv/zookeeper"
+	"github.com/elastic/harp/pkg/sdk/cmdutil"
+	"github.com/elastic/harp/pkg/sdk/log"
+	"github.com/elastic/harp/pkg/tasks/from"
+)
+
+// -----------------------------------------------------------------------------
+
+type fromZookeeperParams struct {
+	outputPath           string
+	basePaths            []string
+	lastPathItemAsSecret bool
+
+	endpoints   []string
+	dialTimeout time.Duration
+}
+
+var fromZookeeperCmd = func() *cobra.Command {
+	var params fromZookeeperParams
+	cmd := &cobra.Command{
+		Use:     "zookeeper",
+		Aliases: []string{"zk"},
+		Short:   "Extract KV pairs from Apache Zookeeper KV Store",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Initialize logger and context
+			ctx, cancel := cmdutil.Context(cmd.Context(), "harp-kv-from-zookeeper", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
+			defer cancel()
+
+			runFromZookeeper(ctx, &params)
+		},
+	}
+
+	// Add parameters
+	cmd.Flags().StringArrayVar(&params.endpoints, "endpoints", []string{"127.0.0.1:2181"}, "Zookeeper client endpoints")
+	cmd.Flags().DurationVar(&params.dialTimeout, "dial-timeout", 15*time.Second, "Zookeeper client dial timeout")
+	cmd.Flags().BoolVarP(&params.lastPathItemAsSecret, "last-path-item-as-secret-key", "k", false, "Use the last path element as secret key")
+
+	return cmd
+}
+
+func runFromZookeeper(ctx context.Context, params *fromZookeeperParams) {
+	// Create config
+	client, _, err := zk.Connect(params.endpoints, params.dialTimeout)
+	if err != nil {
+		log.For(ctx).Fatal("unable to connect to zookeeper cluster", zap.Error(err))
+		return
+	}
+
+	// Delegate to task
+	t := &from.ExtractKVTask{
+		Store:                   zookeeper.Store(client),
+		ContainerWriter:         cmdutil.FileWriter(params.outputPath),
+		BasePaths:               params.basePaths,
+		LastPathItemAsSecretKey: params.lastPathItemAsSecret,
+	}
+
+	// Run the task
+	if err := t.Run(ctx); err != nil {
+		log.For(ctx).Fatal("unable to execute kv extract task", zap.Error(err))
+		return
+	}
+}

docs/cmd/harp_from.md

@@ -12,9 +12,12 @@ Secret container generation commands
 
 * [harp](harp.md)	 - Extensible secret management tool
 * [harp from bundle-template](harp_from_bundle-template.md)	 - Generate a secret container from a BundleTemplate manifest
+* [harp from consul](harp_from_consul.md)	 - Extract KV pairs from Hashicorp Consul KV Store
 * [harp from dump](harp_from_dump.md)	 - Import from bundle dump output as a secret container
+* [harp from etcd3](harp_from_etcd3.md)	 - Extract KV pairs from CoreOS Etcdv3 KV Store
 * [harp from jsonmap](harp_from_jsonmap.md)	 - Convert a JSON map to a secret-container
 * [harp from object](harp_from_object.md)	 - Convert an object to a secret-container
 * [harp from oplog](harp_from_oplog.md)	 - Convert a JSON oplog to a secret-container
 * [harp from vault](harp_from_vault.md)	 - Pull a list of Vault K/V paths as a secret container
+* [harp from zookeeper](harp_from_zookeeper.md)	 - Extract KV pairs from Apache Zookeeper KV Store
 

docs/cmd/harp_from_consul.md

@@ -0,0 +1,21 @@
+## harp from consul
+
+Extract KV pairs from Hashicorp Consul KV Store
+
+```
+harp from consul [flags]
+```
+
+### Options
+
+```
+  -h, --help                           help for consul
+  -k, --last-path-item-as-secret-key   Use the last path element as secret key
+      --out string                     Container output path ('-' for stdout) (default "-")
+      --paths strings                  Exported base paths
+```
+
+### SEE ALSO
+
+* [harp from](harp_from.md)	 - Secret container generation commands
+