WIP - e2e

.github/actions/kind/action.yml

@@ -0,0 +1,58 @@
+name: "Set up KinD"
+description: "Step to start and configure KinD cluster"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Init directories
+      shell: bash
+      run: |
+        TEMP_DIR="$(pwd)/tmp"
+        mkdir -p "${TEMP_DIR}"
+        echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV
+
+        mkdir -p "$(pwd)/bin"
+        echo "$(pwd)/bin" >> $GITHUB_PATH
+
+    - name: Container image registry
+      shell: bash
+      run: |
+        podman run -d -p 5000:5000 --name registry registry:2.8.1
+
+        export REGISTRY_ADDRESS=$(hostname -i):5000
+        echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV
+        echo "Container image registry started at ${REGISTRY_ADDRESS}"
+
+        KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml
+        echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV
+        envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE}
+
+        sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF
+        [[registry]]
+        prefix = "$REGISTRY_ADDRESS"
+        insecure = true
+        location = "$REGISTRY_ADDRESS"
+        EOF'
+
+    - name: Setup KinD cluster
+      uses: helm/kind-action@v1.8.0
+      with:
+        cluster_name: cluster
+        version: v0.17.0
+        config: ${{ env.KIND_CONFIG_FILE }}
+
+    - name: Print cluster info
+      shell: bash
+      run: |
+        echo "KinD cluster:"
+        kubectl cluster-info
+        kubectl describe nodes
+
+    - name: Install Ingress controller
+      shell: bash
+      run: |
+        VERSION=controller-v1.6.4
+        echo "Deploying Ingress controller into KinD cluster"
+        curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n        - --status-update-interval=10/g" | kubectl apply -f -
+        kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true"
+        kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all

.github/resources-kind/kind.yaml

@@ -0,0 +1,31 @@
+# ---------------------------------------------------------------------------
+#  Copyright 2023.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# ---------------------------------------------------------------------------
+
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+containerdConfigPatches:
+  - |-
+    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${REGISTRY_ADDRESS}"]
+      endpoint = ["http://${REGISTRY_ADDRESS}"]

.github/workflows/e2e_tests.yaml

@@ -0,0 +1,129 @@
+name: e2e
+
+on:
+  pull_request:
+    branches:
+      - main
+      - 'release-*'
+    paths-ignore:
+      - 'docs/**'
+      - '**.adoc'
+      - '**.md'
+      - 'LICENSE'
+  push:
+    branches:
+      - main
+      - 'release-*'
+    paths-ignore:
+      - 'docs/**'
+      - '**.adoc'
+      - '**.md'
+      - 'LICENSE'
+
+concurrency:
+  group: ${{ github.head_ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  kubernetes:
+
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Cleanup
+        run: |
+          ls -lart
+          echo "Initial status:"
+          df -h
+
+          echo "Cleaning up resources:"
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt clean
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker rmi $(docker image ls -aq)
+
+          echo "Final status:"
+          df -h
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Checkout CodeFlare operator repository
+        uses: actions/checkout@v3
+        with:
+          repository: project-codeflare/codeflare-operator
+          path: codeflare-operator
+
+      - name: Set Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: v1.19
+
+      - name: Set up gotestfmt
+        uses: gotesttools/gotestfmt-action@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup and start KinD cluster
+        uses: ./.github/actions/kind
+
+      - name: Deploy CodeFlare stack
+        id: deploy
+        run: |
+          cd codeflare-operator
+          echo Deploying CodeFlare operator
+          IMG="${REGISTRY_ADDRESS}"/codeflare-operator
+          make image-push -e IMG="${IMG}"
+          make deploy -e IMG="${IMG}"
+          kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager
+
+          echo Setting up CodeFlare stack
+          make setup-e2e
+          make defaults manifests generate fmt vet
+
+          cd ..
+
+      - name: Run e2e tests
+        run: |
+          export CODEFLARE_TEST_TIMEOUT_SHORT=1m
+          export CODEFLARE_TEST_TIMEOUT_MEDIUM=5m
+          export CODEFLARE_TEST_TIMEOUT_LONG=10m
+
+          export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
+          echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
+
+          set -euo pipefail
+          go test -timeout 30m -v ./tests/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+
+      - name: Print CodeFlare operator logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing CodeFlare operator logs"
+          kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
+
+      - name: Print MCAD controller logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing MCAD controller logs"
+          kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log
+
+      - name: Print KubeRay operator logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing KubeRay operator logs"
+          kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: always() && steps.deploy.outcome == 'success'
+        with:
+          name: logs
+          retention-days: 10
+          path: |
+            ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log

Dockerfile

@@ -0,0 +1,25 @@
+# Build the manager binary
+FROM registry.access.redhat.com/ubi8/go-toolset:1.19.10-10 as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+# Copy the go source
+COPY main.go main.go
+COPY api/ api/
+COPY controllers/ controllers/
+
+# Build
+USER root
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
+
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
+WORKDIR /
+COPY --from=builder /workspace/manager .
+COPY config/internal config/internal
+
+USER 65532:65532
+ENTRYPOINT ["/manager"]