diff --git a/etl/ReadMe.md b/etl/ReadMe.md index 6808de55..7bc95c96 100644 --- a/etl/ReadMe.md +++ b/etl/ReadMe.md @@ -37,11 +37,50 @@ jdbc:oracle:thin:@(description=(address=(protocol=tcps)(host=172.18.215.225.nip. # Create sample data in postgresql using init.sql +# OpenShift Build and Deploy process + +## 1. Build custom postgis + +``` + oc process -f debezium-postgis.build.yaml |oc apply -f - +``` + +## 2. Build kafka jdbc connect component + +``` + oc process -f debezium-jdbc.build.yaml --param-file=.env |oc apply -f - +``` + +## 3. Build kafka broker. +``` + oc process -f debezium-kafka.build.yaml |oc apply -f - +``` + +## 4. Deploy postgis (switch to correct project for env before this step) +``` + oc process -f debezium-postgis.deploy.yaml |oc apply -f - +``` +## 5. Deploy zookeeper +``` + oc process -f debezium-zookeeper.deploy.yaml |oc apply -f - +``` +## 6. Deploy kafka +``` + oc process -f debezium-kafka.deploy.yaml |oc apply -f - +``` +## 7. Deploy kafka-jdbc connect. +``` + oc process -f debezium-jdbc.deploy.yaml |oc apply -f - +``` # Register postgresql connector ``` curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json http://localhost:8083/connectors/ +curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/ + + + ``` # Register Oracle jdbc sink connector @@ -49,6 +88,9 @@ curl -H "Content-Type: application/json" -d @register-postgres-source-connector. ``` curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json http://localhost:8083/connectors/ +curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/ + + ``` # Test with a kafka console consumer. @@ -67,6 +109,16 @@ curl -X DELETE localhost:8083/connectors/ curl -X DELETE localhost:8083/connectors/oracle-jdbc-sink-connector curl -X DELETE localhost:8083/connectors/postgres-source-connector -``` + +curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/oracle-jdbc-sink-connector +curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/postgres-source-connector + +``` + +# Get all connectors registered. +``` +curl localhost:8083/connectors/ +curl https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors +``` diff --git a/etl/debezium-kafka/Dockerfile b/etl/debezium-kafka/Dockerfile new file mode 100644 index 00000000..29d7ae26 --- /dev/null +++ b/etl/debezium-kafka/Dockerfile @@ -0,0 +1,6 @@ + +FROM debezium/kafka:1.9 + + +COPY connect-log4j.properties /kafka/config/ +COPY log4j.properties /kafka/config/ \ No newline at end of file diff --git a/etl/debezium-kafka/connect-log4j.properties b/etl/debezium-kafka/connect-log4j.properties new file mode 100644 index 00000000..52bd0b19 --- /dev/null +++ b/etl/debezium-kafka/connect-log4j.properties @@ -0,0 +1,67 @@ +kafka.logs.dir=logs + +log4j.rootLogger=ERROR, stdout + +# Disable excessive reflection warnings - KAFKA-5229 +log4j.logger.org.reflections=ERROR + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.threshold=ERROR +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.cleanerAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +# Turn on all our debugging info +#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender +#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender +#log4j.logger.kafka.perf=DEBUG, kafkaAppender +#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender +#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG +log4j.logger.kafka=ERROR, kafkaAppender + +log4j.logger.kafka.network.RequestChannel$=ERROR, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.request.logger=ERROR, requestAppender +log4j.additivity.kafka.request.logger=false + +log4j.logger.kafka.controller=ERROR, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.kafka.log.LogCleaner=ERROR, cleanerAppender +log4j.additivity.kafka.log.LogCleaner=false + +log4j.logger.state.change.logger=ERROR, stateChangeAppender +log4j.additivity.state.change.logger=false diff --git a/etl/debezium-kafka/log4j.properties b/etl/debezium-kafka/log4j.properties new file mode 100644 index 00000000..52bd0b19 --- /dev/null +++ b/etl/debezium-kafka/log4j.properties @@ -0,0 +1,67 @@ +kafka.logs.dir=logs + +log4j.rootLogger=ERROR, stdout + +# Disable excessive reflection warnings - KAFKA-5229 +log4j.logger.org.reflections=ERROR + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.threshold=ERROR +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.cleanerAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +# Turn on all our debugging info +#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender +#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender +#log4j.logger.kafka.perf=DEBUG, kafkaAppender +#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender +#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG +log4j.logger.kafka=ERROR, kafkaAppender + +log4j.logger.kafka.network.RequestChannel$=ERROR, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.request.logger=ERROR, requestAppender +log4j.additivity.kafka.request.logger=false + +log4j.logger.kafka.controller=ERROR, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.kafka.log.LogCleaner=ERROR, cleanerAppender +log4j.additivity.kafka.log.LogCleaner=false + +log4j.logger.state.change.logger=ERROR, stateChangeAppender +log4j.additivity.state.change.logger=false diff --git a/etl/docker-compose.yaml b/etl/docker-compose.yaml index 579f5c39..e378896a 100644 --- a/etl/docker-compose.yaml +++ b/etl/docker-compose.yaml @@ -8,6 +8,8 @@ services: - 3888:3888 kafka: image: debezium/kafka:${DEBEZIUM_VERSION} + build: + context: debezium-kafka ports: - 9092:9092 links: diff --git a/etl/register-oracle-jdbc-sink-connector.json b/etl/register-oracle-jdbc-sink-connector.json index b72a9285..0addcae8 100644 --- a/etl/register-oracle-jdbc-sink-connector.json +++ b/etl/register-oracle-jdbc-sink-connector.json @@ -6,7 +6,7 @@ "topics.regex": "dbserver1.public.(.*)", "connection.url": "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCPS)(PORT=1543)(HOST=nrcdb01.bcgov))(CONNECT_DATA=(SERVICE_NAME=SD57387.NRS.BCGOV))(SECURITY=(ssl_server_cert_dn=\"CN=nrcdb01.bcgov\")))", "connection.user": "JAITHOMA", - "connection.password": "jaithoma#1020", + "connection.password": "xxxxx", "dialect.name": "OracleDatabaseDialect", "security.protocol":"SSL", "ssl.enabled.protocols": "TLSv1.2,TLSv1.1", @@ -16,7 +16,9 @@ "table.name.format" : "SIS.${topic}", "delete.enabled": "true", "pk.mode": "record_key", - "insert.mode": "upsert", + "insert.mode": "upsert", + "max.retries": "30", + "retry.backoff.ms": "10000", "quote.sql.identifiers": "never", "transforms":"route,ConvertCreatedTimestamp,ConvertUpdatedTimestamp,topicCase", "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", diff --git a/openshift/templates/etl/debezium-kafka.build.yaml b/openshift/templates/etl/debezium-kafka.build.yaml new file mode 100644 index 00000000..79fafc3e --- /dev/null +++ b/openshift/templates/etl/debezium-kafka.build.yaml @@ -0,0 +1,59 @@ +--- +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: "${API_NAME}-build-template" + creationTimestamp: +objects: +- kind: ImageStream + apiVersion: v1 + metadata: + name: "${API_NAME}" +- kind: BuildConfig + apiVersion: v1 + metadata: + name: "${API_NAME}-build" + labels: + app: "${API_NAME}-build" + spec: + runPolicy: Serial + source: + type: Git + git: + uri: "${GIT_REPO_URL}" + ref: "${GIT_REF}" + contextDir: "${SOURCE_CONTEXT_DIR}" + strategy: + type: Docker + dockerStrategy: + dockerfilePath: "${DOCKER_FILE_PATH}" + output: + to: + kind: ImageStreamTag + name: "${API_NAME}:latest" +parameters: +- name: API_NAME + displayName: Name + description: The name assigned to all of the resources defined in this template. + required: true + value: debezium-kafka +- name: GIT_REPO_URL + displayName: Git Repo URL + description: The URL to your GIT repo. + required: true + value: https://github.com/bcgov/nr-epd-digital-services +- name: GIT_REF + displayName: Git Reference + description: The git reference or branch. + required: true + value: dev +- name: SOURCE_CONTEXT_DIR + displayName: Source Context Directory + description: The source context directory. + required: false + value: etl/debezium-kafka +- name: DOCKER_FILE_PATH + displayName: Docker File Path + description: The path to the docker file defining the build. + required: false + value: "Dockerfile" \ No newline at end of file diff --git a/openshift/templates/etl/debezium-kafka.deploy.yaml b/openshift/templates/etl/debezium-kafka.deploy.yaml index 5fc8c37b..4d2717c2 100644 --- a/openshift/templates/etl/debezium-kafka.deploy.yaml +++ b/openshift/templates/etl/debezium-kafka.deploy.yaml @@ -26,6 +26,15 @@ objects: maxSurge: 25% triggers: - type: ConfigChange + - type: ImageChange + imageChangeParams: + automatic: true + containerNames: + - "${API_NAME}" + from: + kind: ImageStreamTag + namespace: "${IMAGE_NAMESPACE}" + name: "${API_NAME}:${TAG_NAME}" replicas: 1 test: false selector: @@ -42,7 +51,7 @@ objects: spec: containers: - name: "${API_NAME}" - image: debezium/kafka:1.9 + image: "${API_NAME}" imagePullPolicy: Always ports: - containerPort: 9092 @@ -108,4 +117,4 @@ parameters: displayName: Environment TAG name description: The TAG name for this environment, e.g., dev, test, prod required: true - value: dev \ No newline at end of file + value: latest \ No newline at end of file