diff --git a/go.mod b/go.mod index e36a1b6..6eeac46 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/minio/minio-go/v7 v7.0.76 + github.com/minio/pkg/v3 v3.0.20 github.com/prometheus/common v0.59.1 github.com/prometheus/procfs v0.15.1 github.com/prometheus/prom2json v1.4.0 @@ -23,7 +24,10 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/kr/text v0.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect @@ -39,5 +43,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 323a96c..def3d49 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -18,9 +19,17 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -29,6 +38,8 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.76 h1:9nxHH2XDai61cT/EFhyIw/wW4vJfpPNvl7lSFpRt+Ng= github.com/minio/minio-go/v7 v7.0.76/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg= +github.com/minio/pkg/v3 v3.0.20 h1:iZSWNIXOpbVXkLhxbVQofIIJgDwh62FJf4O+ttMobLE= +github.com/minio/pkg/v3 v3.0.20/go.mod h1:LwYm9J2+ZMgnrVvnxxFKss1QCuY86X85ec6CQNv2a4k= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4= @@ -90,7 +101,12 @@ golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jobs/batch.go b/jobs/batch.go new file mode 100644 index 0000000..10ca1ac --- /dev/null +++ b/jobs/batch.go @@ -0,0 +1,214 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// # This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package jobs constists of all structs related to batch job requests and related functionality. +package jobs + +import ( + "context" + "time" + + miniogo "github.com/minio/minio-go/v7" + "github.com/minio/pkg/v3/xtime" +) + +// BatchJobRequest to start batch job +type BatchJobRequest struct { + ID string `yaml:"-" json:"name"` + User string `yaml:"-" json:"user"` + Started time.Time `yaml:"-" json:"started"` + Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"` + KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"` + Expire *BatchJobExpire `yaml:"expire" json:"expire"` + Ctx context.Context `msg:"-"` +} + +// BatchJobReplicateV1 v1 of batch job replication +type BatchJobReplicateV1 struct { + APIVersion string `yaml:"apiVersion" json:"apiVersion"` + Flags BatchJobReplicateFlags `yaml:"flags" json:"flags"` + Target BatchJobReplicateTarget `yaml:"target" json:"target"` + Source BatchJobReplicateSource `yaml:"source" json:"source"` + + Clnt *miniogo.Core `msg:"-"` +} + +// BatchJobReplicateFlags various configurations for replication job definition currently includes +type BatchJobReplicateFlags struct { + Filter BatchReplicateFilter `yaml:"filter" json:"filter"` + Notify BatchJobNotification `yaml:"notify" json:"notify"` + Retry BatchJobRetry `yaml:"retry" json:"retry"` +} + +// BatchReplicateFilter holds all the filters currently supported for batch replication +type BatchReplicateFilter struct { + NewerThan xtime.Duration `yaml:"newerThan,omitempty" json:"newerThan"` + OlderThan xtime.Duration `yaml:"olderThan,omitempty" json:"olderThan"` + CreatedAfter time.Time `yaml:"createdAfter,omitempty" json:"createdAfter"` + CreatedBefore time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"` + Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"` + Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"` +} + +// BatchJobKV is a key-value data type which supports wildcard matching +type BatchJobKV struct { + Line, Col int + Key string `yaml:"key" json:"key"` + Value string `yaml:"value" json:"value"` +} + +// BatchJobNotification stores notification endpoint and token information. +// Used by batch jobs to notify of their status. +type BatchJobNotification struct { + Line, Col int + Endpoint string `yaml:"endpoint" json:"endpoint"` + Token string `yaml:"token" json:"token"` +} + +// BatchJobRetry stores retry configuration used in the event of failures. +type BatchJobRetry struct { + Line, Col int + Attempts int `yaml:"attempts" json:"attempts"` // number of retry attempts + Delay time.Duration `yaml:"delay" json:"delay"` // delay between each retries +} + +// BatchJobReplicateTarget describes target element of the replication job that receives +// the filtered data from source +type BatchJobReplicateTarget struct { + Type BatchJobReplicateResourceType `yaml:"type" json:"type"` + Bucket string `yaml:"bucket" json:"bucket"` + Prefix string `yaml:"prefix" json:"prefix"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + Path string `yaml:"path" json:"path"` + Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"` +} + +// BatchJobReplicateResourceType defines the type of batch jobs +type BatchJobReplicateResourceType string + +// BatchJobReplicateCredentials access credentials for batch replication it may +// be either for target or source. +type BatchJobReplicateCredentials struct { + AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"` + SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"` + SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"` +} + +// BatchJobReplicateSource describes source element of the replication job that is +// the source of the data for the target +type BatchJobReplicateSource struct { + Type BatchJobReplicateResourceType `yaml:"type" json:"type"` + Bucket string `yaml:"bucket" json:"bucket"` + Prefix BatchJobPrefix `yaml:"prefix" json:"prefix"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + Path string `yaml:"path" json:"path"` + Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"` + Snowball BatchJobSnowball `yaml:"snowball" json:"snowball"` +} + +// BatchJobPrefix - to support prefix field yaml unmarshalling with string or slice of strings +type BatchJobPrefix []string + +// BatchJobSnowball describes the snowball feature when replicating objects from a local source to a remote target +type BatchJobSnowball struct { + Line, Col int + Disable *bool `yaml:"disable" json:"disable"` + Batch *int `yaml:"batch" json:"batch"` + InMemory *bool `yaml:"inmemory" json:"inmemory"` + Compress *bool `yaml:"compress" json:"compress"` + SmallerThan *string `yaml:"smallerThan" json:"smallerThan"` + SkipErrs *bool `yaml:"skipErrs" json:"skipErrs"` +} + +// BatchJobKeyRotateV1 v1 of batch key rotation job +type BatchJobKeyRotateV1 struct { + APIVersion string `yaml:"apiVersion" json:"apiVersion"` + Flags BatchJobKeyRotateFlags `yaml:"flags" json:"flags"` + Bucket string `yaml:"bucket" json:"bucket"` + Prefix string `yaml:"prefix" json:"prefix"` + Encryption BatchJobKeyRotateEncryption `yaml:"encryption" json:"encryption"` +} + +// BatchJobKeyRotateFlags various configurations for replication job definition currently includes +type BatchJobKeyRotateFlags struct { + Filter BatchKeyRotateFilter `yaml:"filter" json:"filter"` + Notify BatchJobNotification `yaml:"notify" json:"notify"` + Retry BatchJobRetry `yaml:"retry" json:"retry"` +} + +// BatchKeyRotateFilter holds all the filters currently supported for batch replication +type BatchKeyRotateFilter struct { + NewerThan time.Duration `yaml:"newerThan,omitempty" json:"newerThan"` + OlderThan time.Duration `yaml:"olderThan,omitempty" json:"olderThan"` + CreatedAfter time.Time `yaml:"createdAfter,omitempty" json:"createdAfter"` + CreatedBefore time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"` + Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"` + Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"` + KMSKeyID string `yaml:"kmskeyid" json:"kmskey"` +} + +// BatchJobKeyRotateEncryption defines key rotation encryption options passed +type BatchJobKeyRotateEncryption struct { + Type BatchKeyRotationType `yaml:"type" json:"type"` + Key string `yaml:"key" json:"key"` + Context string `yaml:"context" json:"context"` + KmsContext map[string]string `msg:"-"` +} + +// BatchKeyRotationType defines key rotation type +type BatchKeyRotationType string + +// BatchJobExpire represents configuration parameters for a batch expiration +// job typically supplied in yaml form +type BatchJobExpire struct { + Line, Col int + APIVersion string `yaml:"apiVersion" json:"apiVersion"` + Bucket string `yaml:"bucket" json:"bucket"` + Prefix BatchJobPrefix `yaml:"prefix" json:"prefix"` + NotificationCfg BatchJobNotification `yaml:"notify" json:"notify"` + Retry BatchJobRetry `yaml:"retry" json:"retry"` + Rules []BatchJobExpireFilter `yaml:"rules" json:"rules"` +} + +// BatchJobExpireFilter holds all the filters currently supported for batch replication +type BatchJobExpireFilter struct { + Line, Col int + OlderThan xtime.Duration `yaml:"olderThan,omitempty" json:"olderThan"` + CreatedBefore *time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"` + Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"` + Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"` + Size BatchJobSizeFilter `yaml:"size" json:"size"` + Type string `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` + Purge BatchJobExpirePurge `yaml:"purge" json:"purge"` +} + +// BatchJobSizeFilter supports size based filters - LesserThan and GreaterThan +type BatchJobSizeFilter struct { + Line, Col int + UpperBound BatchJobSize `yaml:"lessThan" json:"lessThan"` + LowerBound BatchJobSize `yaml:"greaterThan" json:"greaterThan"` +} + +// BatchJobExpirePurge type accepts non-negative versions to be retained +type BatchJobExpirePurge struct { + Line, Col int + RetainVersions int `yaml:"retainVersions" json:"retainVersions"` +} + +// BatchJobSize supports humanized byte values in yaml files type BatchJobSize uint64 +type BatchJobSize int64