cloudbuild.yaml

# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Pre-requisites to running with cloud build
# gcloud services enable \
#     compute.googleapis.com \
#     container.googleapis.com \
#     cloudbuild.googleapis.com

timeout: 3600s
substitutions:
  _NAME: broker
  _REGION:
  # Set this to your broker endpoint, if set to 'default', will use cannonical "broker.endpoints.PROJECT_ID.cloud.goog"
  _TURN_REALM: default

steps:
  ###
  # Detect type of Selkies cluster, public or private GKE.
  ###
  - name: "gcr.io/cloud-builders/gcloud"
    id: "selkies-gke-detect"
    entrypoint: bash
    args:
      - -exc
      - |
        PRIVATE_NODES=$(gcloud -q container clusters describe ${_NAME}-${_REGION} --region ${_REGION} --format='value(privateClusterConfig.enablePrivateNodes)')
        if [[ "${PRIVATE_NODES,,}" == true ]]; then
          touch /workspace/PRIVATE_GKE
        fi
  ###
  # Deploy TURN infrastructure based on GKE cluster type.
  ###
  - name: "gcr.io/cloud-builders/gcloud"
    id: "deploy-turn"
    dir: infra/turn
    entrypoint: bash
    args:
      - -exc
      - |
        if [[ -e /workspace/PRIVATE_GKE ]]; then
          # Deploy TURN to private GKE cluster.
          echo "INFO: Deploying TURN to VMs and manifests to private GKE cluster"

          # Set the TURN_REALM
          export TURN_REALM=""
          if [[ "${_TURN_REALM}" == "default" ]]; then
            TURN_REALM="broker.endpoints.${PROJECT_ID}.cloud.goog"
          else
            TURN_REALM="${_TURN_REALM}"
          fi

          # Deploy TURN using Cloud Build
          gcloud -q builds submit --config cloudbuild.private-gke.yaml --substitutions=_REGION=${_REGION},_TURN_REALM=$$TURN_REALM,_ACTION=apply
        else
          # Deploy TURN to public GKE cluster
          echo "INFO: Deploying TURN to public GKE cluster"

          # Get the current TURN shared secret, if any.
          export TURN_SHARED_SECRET=""
          if [[ -n "$(gcloud -q secrets list --filter=name~selkies-turn-shared-secret --format='value(name)')" ]]; then
            TURN_SHARED_SECRET=$(gcloud -q secrets versions access latest --secret selkies-turn-shared-secret)
          fi
          if [[ -z "$${TURN_SHARED_SECRET}" ]]; then
            # Generate new TURN shared secret
            TURN_SHARED_SECRET=$(openssl rand -base64 15)
            gcloud -q secrets create selkies-turn-shared-secret --replication-policy=automatic --data-file <(echo -n ${TURN_SHARED_SECRET?})
          fi

          # Set the TURN_REALM
          export TURN_REALM=""
          if [[ "${_TURN_REALM}" == "default" ]]; then
            TURN_REALM="broker.endpoints.${PROJECT_ID}.cloud.goog"
          else
            TURN_REALM="${_TURN_REALM}"
          fi

          # Deploy TURN using Cloud Build
          gcloud -q builds submit --config cloudbuild.public-gke.yaml --substitutions=_REGION=${_REGION},_TURN_REALM=$$TURN_REALM,_ACTION=apply
        fi

  ###
  # Build images
  ###
  - name: "gcr.io/cloud-builders/gcloud"
    id: "build-images"
    dir: "images"
    args:
      - "builds"
      - "submit"
    waitFor: ["-"]
  ###
  # Deploy autoscale gpu trigger job to ensure at least 1 node is active for the next hour
  ###
  - name: "gcr.io/${PROJECT_ID}/broker-installer"
    id: ensure-gpu-node
    entrypoint: bash
    args:
      - -exc
      - |
        gcloud container clusters get-credentials $${CLOUDSDK_CONTAINER_CLUSTER} --region=$${CLOUDSDK_COMPUTE_REGION}

        kubectl apply -f manifests/node/gpu-node-pool-autoscaler-job.yaml

        kubectl -n kube-system wait pod \
          -l job-name=gpu-node-pool-autoscaler \
          --for=condition=Ready \
          --timeout=900s
    env:
      - "PROJECT_ID=${PROJECT_ID}"
      - "CLOUDSDK_CORE_PROJECT=${PROJECT_ID}"
      - "CLOUDSDK_COMPUTE_REGION=${_REGION}"
      - "CLOUDSDK_CONTAINER_CLUSTER=${_NAME}-${_REGION}"
    waitFor: ["-"]
  ###
  # Deploy manifests
  ###
  - name: "gcr.io/cloud-builders/gcloud"
    id: "deploy-manifests"
    dir: "manifests"
    args:
      - "builds"
      - "submit"
      - "--substitutions=_NAME=${_NAME},_REGION=${_REGION}"
    waitFor: ["build-images"]
  ###
  # Wait for deployment
  ###
  - name: "gcr.io/${PROJECT_ID}/broker-installer"
    id: wait-for-deployment
    entrypoint: "bash"
    args:
      - "-exc"
      - |
        gcloud container clusters get-credentials $${CLOUDSDK_CONTAINER_CLUSTER} --region=$${CLOUDSDK_COMPUTE_REGION}

        kubectl -n kube-system wait pod \
          -l k8s-app=nvidia-driver-installer \
          --for=condition=Ready \
          --timeout=600s

        kubectl -n kube-system wait pod \
          -l app=gpu-node-init \
          --for=condition=Ready \
          --timeout=600s

        kubectl -n kube-system wait pod \
          -l app=gpu-sharing \
          --for=condition=Ready \
          --timeout=600s

        kubectl describe node -l \
          cloud.google.com/gke-accelerator-initialized=true \
          | grep nvidia.com/gpu
    env:
      - "PROJECT_ID=${PROJECT_ID}"
      - "CLOUDSDK_CORE_PROJECT=${PROJECT_ID}"
      - "CLOUDSDK_COMPUTE_REGION=${_REGION}"
      - "CLOUDSDK_CONTAINER_CLUSTER=${_NAME}-${_REGION}"
    waitFor:
      - ensure-gpu-node
      - deploy-manifests