dev to main merge 29 dec

.github/workflows/cd-debezium-jdbc.yaml

@@ -0,0 +1,71 @@
+name: debezium-jdbc
+
+on:
+  workflow_dispatch:
+    inputs:
+      START_BUILD:
+        required: true
+        type: boolean
+        description: This value is ignored, just to trigger dispatch.
+  push:
+    branches:
+      - dev
+      - main
+      - feat/srs-25-etl-deploy
+    paths:
+      - 'etl/**'
+      - '.github/workflows/**'
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./etl/debezium-jdbc
+env:
+  APP_NAME: "debezium-jdbc"
+
+jobs:
+  etl-debezium-jdbc-cd-by-push:
+    runs-on: ubuntu-20.04
+
+    if: github.repository == 'bcgov/nr-epd-digital-services'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set ENV variables
+        id: set-variable
+        # make sure to update the TOOLS_NAME once all the environment for forms-flow-ai-web is ready
+        run: |
+          if [ '${{ github.ref_name }}' == 'dev' ]; then
+            echo "For ${{ github.ref_name }} branch"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="dev"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
+            echo "ENV_NAME="dev"" >> $GITHUB_ENV
+          elif [ '${{ github.ref_name }}' == 'main' ]; then
+            echo "For ${{ github.ref_name }} branch"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="test"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="main"" >> $GITHUB_ENV
+            echo "ENV_NAME="test"" >> $GITHUB_ENV
+          else
+            echo "For ${{ github.ref_name }} branch, defaulting to dev values"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="dev"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
+          fi
+        shell: bash
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OC_SERVER}} --token=${{secrets.OC_TOKEN}}
+
+      - name: Tools project
+        shell: bash
+        run: |
+          oc project ${{ secrets.OC_ETL_NAMESPACE_NAMEPLATE }}-tools
+
+      - name: Start Build Openshift
+        shell: bash
+        run: |
+          oc start-build ${{ env.APP_NAME }}-${{ env.TAG_NAME }}-build --wait
+

.github/workflows/cd-debezium-postgis.yaml

@@ -0,0 +1,83 @@
+name: debezium-postgis Custom
+
+on:
+  workflow_dispatch:
+    inputs:
+      START_BUILD:
+        required: true
+        type: boolean
+        description: This value is ignored, just to trigger dispatch.
+  push:
+    branches:
+      - dev
+      - main
+      - feat/srs-25-etl-deploy
+    paths:
+      - 'etl/**'
+      - '.github/workflows/**'
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./etl/debezium-postgis
+env:
+  APP_NAME: "debezium-postgis"
+
+jobs:
+  etl-debezium-postgis-cd-by-push:
+    runs-on: ubuntu-20.04
+
+    if: github.repository == 'bcgov/nr-epd-digital-services'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set ENV variables
+        id: set-variable
+        # make sure to update the TOOLS_NAME once all the environment for forms-flow-ai-web is ready
+        run: |
+          if [ '${{ github.ref_name }}' == 'dev' ]; then
+            echo "For ${{ github.ref_name }} branch"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="dev"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
+            echo "ENV_NAME="dev"" >> $GITHUB_ENV
+          elif [ '${{ github.ref_name }}' == 'main' ]; then
+            echo "For ${{ github.ref_name }} branch"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="test"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="main"" >> $GITHUB_ENV
+            echo "ENV_NAME="test"" >> $GITHUB_ENV
+          else
+            echo "For ${{ github.ref_name }} branch, defaulting to dev values"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="dev"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
+          fi
+        shell: bash
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OC_SERVER}} --token=${{secrets.OC_TOKEN}}
+
+      - name: Tools project
+        shell: bash
+        run: |
+          oc project ${{ secrets.OC_ETL_NAMESPACE_NAMEPLATE }}-tools
+
+      # Build from either dev or main branch as appropriate
+      - name: Build from ${{ env.BRANCH_NAME }} branch
+        shell: bash
+        run: |
+          oc patch bc/${{ env.APP_NAME }}-build -p '{"spec":{"source":{"git":{"ref":"${{ env.BRANCH_NAME }}"}}}}'
+
+      - name: Start Build Openshift
+        shell: bash
+        run: |
+          oc start-build ${{ env.APP_NAME }}-build --wait
+
+
+      - name: Tag+Deploy for ${{ env.TAG_NAME }}
+        shell: bash
+        run: |
+          oc tag ${{ env.APP_NAME }}:latest ${{ env.APP_NAME }}:${{ env.TAG_NAME }}
+

.github/workflows/ci-cd-openshift-tools.yaml

@@ -9,7 +9,7 @@ on:
     paths:
       - 'backend/**'
       - 'frontend/**'
-      - '.github/workflows/ci-cd-openshift-tools.yaml'
+      - '.github/workflows/ci-cd-openshift-tools.yaml'      
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.gitignore

@@ -114,12 +114,12 @@ snyk.sarif
 
 # Docker/Podman volumes
 .volumes
-backend/etl/debezium-jdbc/.settings/org.eclipse.m2e.core.prefs
-backend/etl/debezium-jdbc/.settings/org.eclipse.jdt.core.prefs
-backend/etl/debezium-jdbc/.settings/org.eclipse.core.resources.prefs
-backend/etl/debezium-jdbc/.project
-backend/etl/debezium-jdbc/.classpath
-backend/etl/debezium-jdbc/target/
+etl/debezium-jdbc/.settings/org.eclipse.m2e.core.prefs
+etl/debezium-jdbc/.settings/org.eclipse.jdt.core.prefs
+etl/debezium-jdbc/.settings/org.eclipse.core.resources.prefs
+etl/debezium-jdbc/.project
+etl/debezium-jdbc/.classpath
+etl/debezium-jdbc/target/
 
 
 notes/

etl/ReadMe.md

@@ -37,18 +37,60 @@ jdbc:oracle:thin:@(description=(address=(protocol=tcps)(host=172.18.215.225.nip.
 
 # Create sample data in postgresql using init.sql
 
+# OpenShift Build and Deploy process
+
+## 1. Build custom postgis
+
+```
+     oc process -f debezium-postgis.build.yaml |oc apply -f - 
+```
+
+## 2. Build kafka jdbc connect component
+
+```
+     oc process -f debezium-jdbc.build.yaml --param-file=.env |oc apply -f - 
+```
+
+## 3. Build kafka broker.
+```
+     oc process -f debezium-kafka.build.yaml |oc apply -f - 
+```
+
+## 4. Deploy postgis (switch to correct project for env before this step)
+```
+     oc process -f debezium-postgis.deploy.yaml |oc apply -f - 
+```
+## 5. Deploy zookeeper
+```
+     oc process -f debezium-zookeeper.deploy.yaml |oc apply -f - 
+```
+## 6. Deploy kafka
+```
+     oc process -f debezium-kafka.deploy.yaml |oc apply -f - 
+```
+## 7. Deploy kafka-jdbc connect.
+```
+     oc process -f debezium-jdbc.deploy.yaml |oc apply -f - 
+```
 
 # Register postgresql connector
 ```
 curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json http://localhost:8083/connectors/ 
 
+curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/
+
+
+
 ```
 
 # Register Oracle jdbc sink connector
 
 ```
 curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json http://localhost:8083/connectors/
 
+curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/
+
+
 ```
 
 # Test with a kafka console consumer.
@@ -67,6 +109,16 @@ curl -X DELETE localhost:8083/connectors/<connector-name>
 
 curl -X DELETE localhost:8083/connectors/oracle-jdbc-sink-connector
 curl -X DELETE localhost:8083/connectors/postgres-source-connector
-```      
+
+curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/oracle-jdbc-sink-connector
+curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/postgres-source-connector
+
+```    
+
+# Get all connectors registered.
+```
+curl localhost:8083/connectors/
+curl https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors
+```
 
 

etl/debezium-jdbc/.classpath

@@ -31,6 +31,19 @@
 			<attribute name="m2e-apt" value="true"/>
 		</attributes>
 	</classpathentry>
+	<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="optional" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="test" value="true"/>
+			<attribute name="optional" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="src" output="target/test-classes" path="target/generated-test-sources/test-annotations">
 		<attributes>
 			<attribute name="optional" value="true"/>

etl/debezium-kafka/Dockerfile

@@ -0,0 +1,6 @@
+
+FROM debezium/kafka:1.9
+
+
+COPY connect-log4j.properties /kafka/config/
+COPY log4j.properties /kafka/config/