From b5a1ce79a3e25dd3d346b905e2e758f9dbcabb94 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 5 Aug 2023 01:53:07 -0700 Subject: [PATCH] add 'expire' batch job type (#213) --- .github/workflows/vulncheck.yml | 2 +- batch-job.go | 53 +++++++++++++++++++++++++++++++-- service-commands.go | 21 +++++++++++-- trace.go | 7 +++++ tracetype_string.go | 28 +++++++++-------- 5 files changed, 93 insertions(+), 18 deletions(-) diff --git a/.github/workflows/vulncheck.yml b/.github/workflows/vulncheck.yml index ff423bb..9c0a700 100644 --- a/.github/workflows/vulncheck.yml +++ b/.github/workflows/vulncheck.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: [ 1.19.x, 1.20.x ] + go-version: [ 1.19.12 ] steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 diff --git a/batch-job.go b/batch-job.go index 9bf631a..3ee6f93 100644 --- a/batch-job.go +++ b/batch-job.go @@ -36,12 +36,14 @@ type BatchJobType string const ( BatchJobReplicate BatchJobType = "replicate" BatchJobKeyRotate BatchJobType = "keyrotate" + BatchJobExpire BatchJobType = "expire" ) // SupportedJobTypes supported job types var SupportedJobTypes = []BatchJobType{ BatchJobReplicate, BatchJobKeyRotate, + BatchJobExpire, // add new job types } @@ -142,6 +144,51 @@ const BatchJobKeyRotateTemplate = `keyrotate: delay: "500ms" # least amount of delay between each retry ` +// BatchJobExpireTemplate provides a sample template +// for batch expiring objects +const BatchJobExpireTemplate = `expire: + apiVersion: v1 + bucket: BUCKET + prefix: PREFIX + # optional flags + flags: + filter: + olderThan: "7d" # match objects older than this value (e.g. 7d10h31s) + createdBefore: "date" # match objects created before "date" + + tags: + - key: "name" + value: "pick*" # match objects with tag 'name', all values starting with 'pick' + + metadata: + - key: "content-type" + value: "image/*" # match objects with 'content-type', all values starting with 'image/' + + size: + lesserThan: "" # match objects with size lesser than this value (e.g. 10KiB) + greaterThan: "" # match objects with size greater than this value (e.g. 10KiB) + + name: + endsWith: "" # match objects ending with this string + contains: "" # match objects that contain this string + startsWith: "" # match objects starting with this string + + # number of versions to retain + # e.g. + # maxVersions: 0 # expire all versions + # maxVersions: 1 # expire all versions except latest + # maxVersions: 10 # expire all versions higher than 10 + maxVersions: 0 # default + + notify: + endpoint: "https://notify.endpoint" # notification endpoint to receive job completion status + token: "Bearer xxxxx" # optional authentication token for the notification endpoint + + retry: + attempts: 10 # number of retries for the job before giving up + delay: "500ms" # least amount of delay between each retry +` + // BatchJobResult returned by StartBatchJob type BatchJobResult struct { ID string `json:"id"` @@ -211,14 +258,16 @@ type GenerateBatchJobOpts struct { // GenerateBatchJob creates a new job template from standard template // TODO: allow configuring yaml values func (adm *AdminClient) GenerateBatchJob(_ context.Context, opts GenerateBatchJobOpts) (string, error) { + // TODO: allow configuring the template to fill values from GenerateBatchJobOpts switch opts.Type { case BatchJobReplicate: - // TODO: allow configuring the template to fill values from GenerateBatchJobOpts return BatchJobReplicateTemplate, nil case BatchJobKeyRotate: return BatchJobKeyRotateTemplate, nil + case BatchJobExpire: + return BatchJobExpireTemplate, nil } - return "", fmt.Errorf("unsupported batch type requested: %s", opts.Type) + return "", fmt.Errorf("unknown batch job requested: %s", opts.Type) } // ListBatchJobsResult contains entries for all current jobs. diff --git a/service-commands.go b/service-commands.go index aec9e48..1069fba 100644 --- a/service-commands.go +++ b/service-commands.go @@ -105,6 +105,8 @@ type ServiceTraceOpts struct { Healing bool BatchReplication bool BatchKeyRotation bool + BatchExpire bool + BatchAll bool Rebalance bool ReplicationResync bool Bootstrap bool @@ -124,8 +126,16 @@ func (t ServiceTraceOpts) TraceTypes() TraceType { tt.SetIf(t.Scanner, TraceScanner) tt.SetIf(t.Decommission, TraceDecommission) tt.SetIf(t.Healing, TraceHealing) - tt.SetIf(t.BatchReplication, TraceBatchReplication) - tt.SetIf(t.BatchKeyRotation, TraceBatchKeyRotation) + if t.BatchAll { + tt.SetIf(true, TraceBatchReplication) + tt.SetIf(true, TraceBatchKeyRotation) + tt.SetIf(true, TraceBatchExpire) + } else { + tt.SetIf(t.BatchReplication, TraceBatchReplication) + tt.SetIf(t.BatchKeyRotation, TraceBatchKeyRotation) + tt.SetIf(t.BatchExpire, TraceBatchExpire) + } + tt.SetIf(t.Rebalance, TraceRebalance) tt.SetIf(t.ReplicationResync, TraceReplicationResync) tt.SetIf(t.Bootstrap, TraceBootstrap) @@ -149,6 +159,12 @@ func (t ServiceTraceOpts) AddParams(u url.Values) { u.Set("healing", strconv.FormatBool(t.Healing)) u.Set("batch-replication", strconv.FormatBool(t.BatchReplication)) u.Set("batch-keyrotation", strconv.FormatBool(t.BatchKeyRotation)) + u.Set("batch-expire", strconv.FormatBool(t.BatchExpire)) + if t.BatchAll { + u.Set("batch-replication", "true") + u.Set("batch-keyrotation", "true") + u.Set("batch-expire", "true") + } u.Set("rebalance", strconv.FormatBool(t.Rebalance)) u.Set("replication-resync", strconv.FormatBool(t.ReplicationResync)) u.Set("bootstrap", strconv.FormatBool(t.Bootstrap)) @@ -165,6 +181,7 @@ func (t *ServiceTraceOpts) ParseParams(r *http.Request) (err error) { t.Healing = r.Form.Get("healing") == "true" t.BatchReplication = r.Form.Get("batch-replication") == "true" t.BatchKeyRotation = r.Form.Get("batch-keyrotation") == "true" + t.BatchExpire = r.Form.Get("batch-expire") == "true" t.Rebalance = r.Form.Get("rebalance") == "true" t.Storage = r.Form.Get("storage") == "true" t.Internal = r.Form.Get("internal") == "true" diff --git a/trace.go b/trace.go index d3ee8dc..eeca31f 100644 --- a/trace.go +++ b/trace.go @@ -49,6 +49,8 @@ const ( TraceBatchReplication // TraceBatchKeyRotation will trace batch keyrotation operations. TraceBatchKeyRotation + // TraceBatchExpire will trace batch expiration operations. + TraceBatchExpire // TraceRebalance will trace rebalance operations TraceRebalance // TraceReplicationResync will trace replication resync operations. @@ -66,6 +68,11 @@ const ( TraceAll TraceType = (1 << iota) - 1 ) +const ( + // TraceBatch will trace all batch operations. + TraceBatch = TraceBatchReplication | TraceBatchKeyRotation | TraceBatchExpire // |TraceBatch +) + // Contains returns whether all flags in other is present in t. func (t TraceType) Contains(other TraceType) bool { return t&other == other diff --git a/tracetype_string.go b/tracetype_string.go index a1852e8..621bb6a 100644 --- a/tracetype_string.go +++ b/tracetype_string.go @@ -17,15 +17,16 @@ func _() { _ = x[TraceHealing-64] _ = x[TraceBatchReplication-128] _ = x[TraceBatchKeyRotation-256] - _ = x[TraceRebalance-512] - _ = x[TraceReplicationResync-1024] - _ = x[TraceBootstrap-2048] - _ = x[TraceFTP-4096] - _ = x[TraceILM-8192] - _ = x[TraceAll-16383] + _ = x[TraceBatchExpire-512] + _ = x[TraceRebalance-1024] + _ = x[TraceReplicationResync-2048] + _ = x[TraceBootstrap-4096] + _ = x[TraceFTP-8192] + _ = x[TraceILM-16384] + _ = x[TraceAll-32767] } -const _TraceType_name = "OSStorageS3InternalScannerDecommissionHealingBatchReplicationBatchKeyRotationRebalanceReplicationResyncBootstrapFTPILMAll" +const _TraceType_name = "OSStorageS3InternalScannerDecommissionHealingBatchReplicationBatchKeyRotationBatchExpireRebalanceReplicationResyncBootstrapFTPILMAll" var _TraceType_map = map[TraceType]string{ 1: _TraceType_name[0:2], @@ -37,12 +38,13 @@ var _TraceType_map = map[TraceType]string{ 64: _TraceType_name[38:45], 128: _TraceType_name[45:61], 256: _TraceType_name[61:77], - 512: _TraceType_name[77:86], - 1024: _TraceType_name[86:103], - 2048: _TraceType_name[103:112], - 4096: _TraceType_name[112:115], - 8192: _TraceType_name[115:118], - 16383: _TraceType_name[118:121], + 512: _TraceType_name[77:88], + 1024: _TraceType_name[88:97], + 2048: _TraceType_name[97:114], + 4096: _TraceType_name[114:123], + 8192: _TraceType_name[123:126], + 16384: _TraceType_name[126:129], + 32767: _TraceType_name[129:132], } func (i TraceType) String() string {