Skip to content

Commit

Permalink
add support for updating CRDs in operator startup
Browse files Browse the repository at this point in the history
  • Loading branch information
matheusfm committed Apr 16, 2024
1 parent 2bbc4eb commit c42dd7d
Show file tree
Hide file tree
Showing 12 changed files with 439 additions and 15 deletions.
1 change: 1 addition & 0 deletions charts/zora/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ The following table lists the configurable parameters of the Zora chart and thei
| customChecksConfigMap | string | `"zora-custom-checks"` | Custom checks ConfigMap name |
| httpsProxy | string | `""` | HTTPS proxy URL |
| noProxy | string | `"kubernetes.default.svc.*,127.0.0.1,localhost"` | Comma-separated list of URL patterns to be excluded from going through the proxy |
| updateCRDs | bool | `true` for upgrades | Specifies whether CRDs should be updated by operator at startup |

Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,

Expand Down
1 change: 1 addition & 0 deletions charts/zora/templates/operator/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ spec:
- --checks-configmap-namespace={{ .Release.Namespace }}
- --checks-configmap-name={{ .Values.customChecksConfigMap }}
- --kubexns-image={{ printf "%s:%s" .Values.kubexnsImage.repository .Values.kubexnsImage.tag }}
- --update-crds={{ .Values.updateCRDs | default .Release.IsUpgrade }}
image: "{{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.operator.image.pullPolicy }}
ports:
Expand Down
10 changes: 10 additions & 0 deletions charts/zora/templates/operator/rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,16 @@ rules:
- serviceaccounts/status
verbs:
- get
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- get
- list
- patch
- update
- watch
- apiGroups:
- batch
resources:
Expand Down
4 changes: 4 additions & 0 deletions charts/zora/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -285,3 +285,7 @@ customChecksConfigMap: zora-custom-checks
httpsProxy: ""
# -- Comma-separated list of URL patterns to be excluded from going through the proxy
noProxy: kubernetes.default.svc.*,127.0.0.1,localhost

# -- (bool) Specifies whether CRDs should be updated by operator at startup
# @default -- `true` for upgrades
updateCRDs:
1 change: 1 addition & 0 deletions cmd/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ COPY cmd/main.go cmd/main.go
COPY api/ api/
COPY internal/ internal/
COPY pkg/ pkg/
COPY config/crd/bases/ config/crd/bases/

RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go

Expand Down
38 changes: 23 additions & 15 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,17 @@ import (
"strings"
"time"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"

"go.uber.org/zap/zapcore"
apiextensionsv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrllog "sigs.k8s.io/controller-runtime/pkg/log"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
Expand All @@ -41,6 +43,7 @@ import (
zorav1alpha2 "github.com/undistro/zora/api/zora/v1alpha2"
zoracontroller "github.com/undistro/zora/internal/controller/zora"
"github.com/undistro/zora/internal/saas"
"github.com/undistro/zora/pkg/crds"
//+kubebuilder:scaffold:imports
)

Expand All @@ -63,6 +66,7 @@ func main() {
var probeAddr string
var secureMetrics bool
var enableHTTP2 bool
var updateCRDs bool
var defaultPluginsNamespace string
var defaultPluginsNames string
var workerImage string
Expand All @@ -86,6 +90,8 @@ func main() {
"If set the metrics endpoint is served securely")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
flag.BoolVar(&updateCRDs, "update-crds", false,
"If set, operator will update Zora CRDs if needed")
flag.StringVar(&defaultPluginsNamespace, "default-plugins-namespace", "zora-system", "The namespace of default plugins")
flag.StringVar(&defaultPluginsNames, "default-plugins-names", "marvin,popeye", "Comma separated list of default plugins")
flag.StringVar(&workerImage, "worker-image", "ghcr.io/undistro/zora/worker:latest", "Docker image name of Worker container")
Expand Down Expand Up @@ -124,8 +130,8 @@ func main() {
if !enableHTTP2 {
tlsOpts = append(tlsOpts, disableHTTP2)
}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
restConfig := ctrl.GetConfigOrDie()
mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{
BindAddress: metricsAddr,
Expand Down Expand Up @@ -161,28 +167,22 @@ func main() {
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("cluster-controller"),
Config: mgr.GetConfig(),
Config: restConfig,
OnUpdate: onClusterUpdate,
OnDelete: onClusterDelete,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Cluster")
os.Exit(1)
}

kcli, err := kubernetes.NewForConfig(mgr.GetConfig())
if err != nil {
setupLog.Error(err, "unable to create Kubernetes clientset", "controller", "Cluster")
os.Exit(1)
}

annotations, err := annotations(cronJobAnnotations)
if err != nil {
setupLog.Error(err, "unable to parse annotations")
os.Exit(1)
}
if err = (&zoracontroller.ClusterScanReconciler{
Client: mgr.GetClient(),
K8sClient: kcli,
K8sClient: kubernetes.NewForConfigOrDie(restConfig),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("clusterscan-controller"),
DefaultPluginsNamespace: defaultPluginsNamespace,
Expand Down Expand Up @@ -219,9 +219,17 @@ func main() {
setupLog.Error(err, "unable to set up ready check")
os.Exit(1)
}
ctx := ctrl.SetupSignalHandler()

if updateCRDs {
if err := crds.Update(ctrllog.IntoContext(ctx, setupLog), apiextensionsv1client.NewForConfigOrDie(restConfig)); err != nil {
setupLog.Error(err, "unable to update CRDs")
os.Exit(1)
}
}

setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
if err := mgr.Start(ctx); err != nil {
setupLog.Error(err, "problem running manager")
os.Exit(1)
}
Expand Down
6 changes: 6 additions & 0 deletions config/crd/bases/embed.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package bases

import "embed"

//go:embed zora.undistro.io*.yaml
var CRDsFS embed.FS
13 changes: 13 additions & 0 deletions config/crd/bases/embed_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package bases

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestEmbedCRDs(t *testing.T) {
entries, err := CRDsFS.ReadDir(".")
assert.NoError(t, err)
assert.Equal(t, 6, len(entries))
}
10 changes: 10 additions & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,16 @@ rules:
- serviceaccounts/status
verbs:
- get
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- get
- list
- patch
- update
- watch
- apiGroups:
- batch
resources:
Expand Down
1 change: 1 addition & 0 deletions internal/controller/zora/customcheck_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ type CustomCheckReconciler struct {
//+kubebuilder:rbac:groups=zora.undistro.io,resources=customchecks/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=zora.undistro.io,resources=customchecks/finalizers,verbs=update
//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch;update;patch

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
Expand Down
152 changes: 152 additions & 0 deletions pkg/crds/crds.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
// Copyright 2024 Undistro Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package crds

import (
"context"
"fmt"
"path/filepath"
"sort"

"github.com/undistro/zora/config/crd/bases"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/yaml"
)

var CRDs []apiextensionsv1.CustomResourceDefinition

// Update updates Zora CRDs if needed
func Update(ctx context.Context, client *apiextensionsv1client.ApiextensionsV1Client) error {
log := ctrllog.FromContext(ctx)
for _, crd := range CRDs {
existing, err := client.CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
log.Info("CRD not found", "name", crd.Name)
continue
}
return err
}
obj, updatedFields := merge(*existing, crd)
if len(updatedFields) == 0 {
log.Info("Unchanged CRD", "name", crd.Name)
continue
}
if _, err := client.CustomResourceDefinitions().Update(ctx, obj, metav1.UpdateOptions{}); err != nil {
return err
}
log.Info("CRD updated", "name", crd.Name, "changes", updatedFields)
}
return nil
}

func merge(existing, desired apiextensionsv1.CustomResourceDefinition) (*apiextensionsv1.CustomResourceDefinition, []string) {
existingVersions := make(map[string]apiextensionsv1.CustomResourceDefinitionVersion, len(existing.Spec.Versions))
for _, v := range existing.Spec.Versions {
existingVersions[v.Name] = v
}
result := existing.DeepCopy()
var updatedFields []string

if result.Spec.PreserveUnknownFields != desired.Spec.PreserveUnknownFields {
result.Spec.PreserveUnknownFields = desired.Spec.PreserveUnknownFields
updatedFields = append(updatedFields, "spec.preserveUnknownFields")
}

if !equality.Semantic.DeepEqual(conversionOrNone(result.Spec.Conversion), conversionOrNone(desired.Spec.Conversion)) {
result.Spec.Conversion = desired.Spec.Conversion
updatedFields = append(updatedFields, "spec.conversion")
}

sort.Strings(result.Spec.Names.ShortNames)
sort.Strings(desired.Spec.Names.ShortNames)
if !equality.Semantic.DeepEqual(result.Spec.Names.ShortNames, desired.Spec.Names.ShortNames) {
result.Spec.Names.ShortNames = desired.Spec.Names.ShortNames
updatedFields = append(updatedFields, "spec.names.shortNames")
}

for i, desiredVersion := range desired.Spec.Versions {
existingVersion, exists := existingVersions[desiredVersion.Name]
if !exists {
// desired version doesn't exist in the existing CRD
result.Spec.Versions = append(result.Spec.Versions, desiredVersion)
updatedFields = append(updatedFields, fmt.Sprintf(`spec.versions[?(@.name==%q)]`, desiredVersion.Name))
continue
}

if !equality.Semantic.DeepEqual(existingVersion.AdditionalPrinterColumns, desiredVersion.AdditionalPrinterColumns) {
result.Spec.Versions[i].AdditionalPrinterColumns = desiredVersion.AdditionalPrinterColumns
updatedFields = append(updatedFields, fmt.Sprintf(`spec.versions[?(@.name==%q)].additionalPrinterColumns`, desiredVersion.Name))
}
desiredSchemaStatus := desiredVersion.Schema.OpenAPIV3Schema.Properties["status"]
if !equality.Semantic.DeepEqual(existingVersion.Schema.OpenAPIV3Schema.Properties["status"], desiredSchemaStatus) {
result.Spec.Versions[i].Schema.OpenAPIV3Schema.Properties["status"] = desiredSchemaStatus
updatedFields = append(updatedFields, fmt.Sprintf(`spec.versions[?(@.name==%q)].schema.openAPIV3Schema.properties.status`, desiredVersion.Name))
}
if existingVersion.Served != desiredVersion.Served {
result.Spec.Versions[i].Served = desiredVersion.Served
updatedFields = append(updatedFields, fmt.Sprintf(`spec.versions[?(@.name==%q)].served`, desiredVersion.Name))
}
if existingVersion.Storage != desiredVersion.Storage {
result.Spec.Versions[i].Storage = desiredVersion.Storage
updatedFields = append(updatedFields, fmt.Sprintf(`spec.versions[?(@.name==%q)].storage`, desiredVersion.Name))
}
if existingVersion.Deprecated != desiredVersion.Deprecated {
result.Spec.Versions[i].Deprecated = desiredVersion.Deprecated
updatedFields = append(updatedFields, fmt.Sprintf(`spec.versions[?(@.name==%q)].deprecated`, desiredVersion.Name))
}
if existingVersion.DeprecationWarning != desiredVersion.DeprecationWarning {
result.Spec.Versions[i].DeprecationWarning = desiredVersion.DeprecationWarning
updatedFields = append(updatedFields, fmt.Sprintf(`spec.versions[?(@.name==%q)].deprecationWarning`, desiredVersion.Name))
}
}
return result, updatedFields
}

func conversionOrNone(c *apiextensionsv1.CustomResourceConversion) *apiextensionsv1.CustomResourceConversion {
if c != nil {
return c
}
return &apiextensionsv1.CustomResourceConversion{Strategy: apiextensionsv1.NoneConverter}
}

func init() {
entries, err := bases.CRDsFS.ReadDir(".")
if err != nil {
panic(err)
}
crds := make([]apiextensionsv1.CustomResourceDefinition, 0, len(entries))
for _, entry := range entries {
name := entry.Name()
if filepath.Ext(name) != ".yaml" {
continue
}
bs, err := bases.CRDsFS.ReadFile(name)
if err != nil {
panic(err)
}
crd := &apiextensionsv1.CustomResourceDefinition{}
if err := yaml.Unmarshal(bs, crd); err != nil {
panic(err)
}
crds = append(crds, *crd)
}
CRDs = crds
}
Loading

0 comments on commit c42dd7d

Please sign in to comment.