add 'expire' batch job type (#213)

.github/workflows/vulncheck.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: [ 1.19.x, 1.20.x ]
+        go-version: [ 1.19.12 ]
     steps:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3

batch-job.go

@@ -36,12 +36,14 @@ type BatchJobType string
 const (
 	BatchJobReplicate BatchJobType = "replicate"
 	BatchJobKeyRotate BatchJobType = "keyrotate"
+	BatchJobExpire    BatchJobType = "expire"
 )
 
 // SupportedJobTypes supported job types
 var SupportedJobTypes = []BatchJobType{
 	BatchJobReplicate,
 	BatchJobKeyRotate,
+	BatchJobExpire,
 	// add new job types
 }
 
@@ -142,6 +144,51 @@ const BatchJobKeyRotateTemplate = `keyrotate:
       delay: "500ms" # least amount of delay between each retry
 `
 
+// BatchJobExpireTemplate provides a sample template
+// for batch expiring objects
+const BatchJobExpireTemplate = `expire:
+  apiVersion: v1
+  bucket: BUCKET
+  prefix: PREFIX
+  # optional flags
+  flags:
+    filter:
+      olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+      createdBefore: "date" # match objects created before "date"
+
+      tags:
+        - key: "name"
+          value: "pick*" # match objects with tag 'name', all values starting with 'pick'
+
+      metadata:
+        - key: "content-type"
+          value: "image/*" # match objects with 'content-type', all values starting with 'image/'
+
+      size:
+        lesserThan:  "" # match objects with size lesser than this value (e.g. 10KiB)
+        greaterThan: "" # match objects with size greater than this value (e.g. 10KiB)
+
+      name:
+        endsWith:   "" # match objects ending with this string
+        contains:   "" # match objects that contain this string
+        startsWith: "" # match objects starting with this string
+
+      # number of versions to retain
+      # e.g.
+      # maxVersions: 0  # expire all versions
+      # maxVersions: 1  # expire all versions except latest
+      # maxVersions: 10 # expire all versions higher than 10
+      maxVersions: 0 # default
+
+    notify:
+      endpoint: "https://notify.endpoint" # notification endpoint to receive job completion status
+      token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+
+    retry:
+      attempts: 10 # number of retries for the job before giving up
+      delay: "500ms" # least amount of delay between each retry
+`
+
 // BatchJobResult returned by StartBatchJob
 type BatchJobResult struct {
 	ID      string        `json:"id"`
@@ -211,14 +258,16 @@ type GenerateBatchJobOpts struct {
 // GenerateBatchJob creates a new job template from standard template
 // TODO: allow configuring yaml values
 func (adm *AdminClient) GenerateBatchJob(_ context.Context, opts GenerateBatchJobOpts) (string, error) {
+	// TODO: allow configuring the template to fill values from GenerateBatchJobOpts
 	switch opts.Type {
 	case BatchJobReplicate:
-		// TODO: allow configuring the template to fill values from GenerateBatchJobOpts
 		return BatchJobReplicateTemplate, nil
 	case BatchJobKeyRotate:
 		return BatchJobKeyRotateTemplate, nil
+	case BatchJobExpire:
+		return BatchJobExpireTemplate, nil
 	}
-	return "", fmt.Errorf("unsupported batch type requested: %s", opts.Type)
+	return "", fmt.Errorf("unknown batch job requested: %s", opts.Type)
 }
 
 // ListBatchJobsResult contains entries for all current jobs.

service-commands.go

@@ -105,6 +105,8 @@ type ServiceTraceOpts struct {
 	Healing           bool
 	BatchReplication  bool
 	BatchKeyRotation  bool
+	BatchExpire       bool
+	BatchAll          bool
 	Rebalance         bool
 	ReplicationResync bool
 	Bootstrap         bool
@@ -124,8 +126,16 @@ func (t ServiceTraceOpts) TraceTypes() TraceType {
 	tt.SetIf(t.Scanner, TraceScanner)
 	tt.SetIf(t.Decommission, TraceDecommission)
 	tt.SetIf(t.Healing, TraceHealing)
-	tt.SetIf(t.BatchReplication, TraceBatchReplication)
-	tt.SetIf(t.BatchKeyRotation, TraceBatchKeyRotation)
+	if t.BatchAll {
+		tt.SetIf(true, TraceBatchReplication)
+		tt.SetIf(true, TraceBatchKeyRotation)
+		tt.SetIf(true, TraceBatchExpire)
+	} else {
+		tt.SetIf(t.BatchReplication, TraceBatchReplication)
+		tt.SetIf(t.BatchKeyRotation, TraceBatchKeyRotation)
+		tt.SetIf(t.BatchExpire, TraceBatchExpire)
+	}
+
 	tt.SetIf(t.Rebalance, TraceRebalance)
 	tt.SetIf(t.ReplicationResync, TraceReplicationResync)
 	tt.SetIf(t.Bootstrap, TraceBootstrap)
@@ -149,6 +159,12 @@ func (t ServiceTraceOpts) AddParams(u url.Values) {
 	u.Set("healing", strconv.FormatBool(t.Healing))
 	u.Set("batch-replication", strconv.FormatBool(t.BatchReplication))
 	u.Set("batch-keyrotation", strconv.FormatBool(t.BatchKeyRotation))
+	u.Set("batch-expire", strconv.FormatBool(t.BatchExpire))
+	if t.BatchAll {
+		u.Set("batch-replication", "true")
+		u.Set("batch-keyrotation", "true")
+		u.Set("batch-expire", "true")
+	}
 	u.Set("rebalance", strconv.FormatBool(t.Rebalance))
 	u.Set("replication-resync", strconv.FormatBool(t.ReplicationResync))
 	u.Set("bootstrap", strconv.FormatBool(t.Bootstrap))
@@ -165,6 +181,7 @@ func (t *ServiceTraceOpts) ParseParams(r *http.Request) (err error) {
 	t.Healing = r.Form.Get("healing") == "true"
 	t.BatchReplication = r.Form.Get("batch-replication") == "true"
 	t.BatchKeyRotation = r.Form.Get("batch-keyrotation") == "true"
+	t.BatchExpire = r.Form.Get("batch-expire") == "true"
 	t.Rebalance = r.Form.Get("rebalance") == "true"
 	t.Storage = r.Form.Get("storage") == "true"
 	t.Internal = r.Form.Get("internal") == "true"

trace.go

@@ -49,6 +49,8 @@ const (
 	TraceBatchReplication
 	// TraceBatchKeyRotation will trace batch keyrotation operations.
 	TraceBatchKeyRotation
+	// TraceBatchExpire will trace batch expiration operations.
+	TraceBatchExpire
 	// TraceRebalance will trace rebalance operations
 	TraceRebalance
 	// TraceReplicationResync will trace replication resync operations.
@@ -66,6 +68,11 @@ const (
 	TraceAll TraceType = (1 << iota) - 1
 )
 
+const (
+	// TraceBatch will trace all batch operations.
+	TraceBatch = TraceBatchReplication | TraceBatchKeyRotation | TraceBatchExpire // |TraceBatch<NextFeature>
+)
+
 // Contains returns whether all flags in other is present in t.
 func (t TraceType) Contains(other TraceType) bool {
 	return t&other == other