Merge pull request #6 from dqops/develop

Version 0.2.0-beta4 with a fast docker build script.

.dockerignore

@@ -6,7 +6,7 @@ docs
 .idea
 .vscode
 dqoai/target
-distribution/target
+dqops/target
 lib/target
 home/venv
 dqoai/src/main/frontend/build

.github/workflows/main-publish-release.yml

@@ -31,6 +31,21 @@ jobs:
         with:
           path: ./VERSION
 
+      - uses: docker-practice/actions-setup-docker@master
+
+      - uses: mr-smithers-excellent/docker-build-push@v6
+        name: Build and Push Docker Image
+        with:
+          image: dqops/dqo
+          tags: $(date +%s),${{ steps.read_version.outputs.content }},latest
+          dockerfile: Dockerfile-fast
+          enableBuildKit: true
+          multiPlatform: true
+          platform: linux/amd64,linux/arm64
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
       - name: Create GitHub release
         uses: actions/create-release@v1
         id: create_release

.run/dqo run.run.xml

@@ -1,6 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="dqo run" type="JarApplication">
-    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-0.2.0-beta3.jar" />
+    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-0.2.0-beta4.jar" />
     <option name="VM_PARAMETERS" value="-Xmx1024m --add-opens java.base/java.nio=ALL-UNNAMED" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
     <option name="ALTERNATIVE_JRE_PATH" value="temurin-17" />

Dockerfile

@@ -8,6 +8,7 @@ COPY pom.xml .
 COPY dqops/pom.xml dqops/
 COPY distribution/pom.xml distribution/
 COPY lib lib
+COPY VERSION VERSION
 
 # resolve dependencies
 ENV USER_HOME="/user/mvn"
@@ -26,6 +27,7 @@ WORKDIR /workspace/app
 COPY dqops/src dqops/src
 RUN ./mvnw.sh install -DskipTests -pl !distribution
 RUN mkdir -p dqops/target/dependency && (cd dqops/target/dependency; jar -xf ../*.jar)
+RUN mv lib/target/output/dqo-lib-$(cat /workspace/app/VERSION) lib/target/output/dqo-lib
 
 FROM python:3.11.3-slim-bullseye AS dqo-home
 WORKDIR /dqo
@@ -63,6 +65,7 @@ COPY --from=dqo-home /dqo/home home
 
 # copy spring dependencies
 ARG DEPENDENCY=/workspace/app/dqops/target/dependency
+COPY --from=dqo-libs /workspace/app/lib/target/output/dqo-lib/jars /dqo/lib
 COPY --from=dqo-libs ${DEPENDENCY}/BOOT-INF/lib /dqo/lib
 COPY --from=dqo-libs ${DEPENDENCY}/META-INF /dqo/META-INF
 COPY --from=dqo-libs ${DEPENDENCY}/BOOT-INF/classes /dqo

Dockerfile-fast

@@ -0,0 +1,52 @@
+FROM openjdk:17-jdk-slim-buster AS dqo-libs
+RUN apt-get update && apt-get install -y unzip
+
+WORKDIR /workspace/app
+COPY ./distribution/target/ /workspace/app/temp/
+COPY VERSION /workspace/app/temp/
+RUN unzip /workspace/app/temp/dqo-distribution-$(cat /workspace/app/temp/VERSION)-bin.zip -d /workspace/app/home
+WORKDIR /workspace/app/home
+RUN mkdir -p /workspace/app/home/expanded && (cd expanded; jar -xf ../jars/dqo-dqops-$(cat /workspace/app/temp/VERSION).jar) && rm /workspace/app/home/jars/dqo-dqops-$(cat /workspace/app/temp/VERSION).jar
+
+FROM python:3.11.3-slim-bullseye AS dqo-home
+WORKDIR /dqo
+
+# copy dqo home
+COPY home home
+WORKDIR /dqo/home
+RUN rm -rf venv/
+
+# recreate venv
+ENV VIRTUAL_ENV=/dqo/home/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN cp lib/requirements.txt $VIRTUAL_ENV/home_requirements.txt
+RUN pip3 install -r $VIRTUAL_ENV/home_requirements.txt
+
+RUN python3 -m compileall ./
+
+FROM python:3.11.3-slim-bullseye AS dqo-main
+EXPOSE 8888
+WORKDIR /dqo
+
+# install java
+RUN apt-get update && apt-get install -y ca-certificates-java openjdk-17-jre && apt-get clean
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-$TARGETARCH
+ENV PATH=$PATH:$JAVA_HOME/bin
+
+# dqo launch setup
+ENV DQO_HOME=/dqo/home
+ENV DQO_USER_HOME=/dqo/userhome
+ENV DQO_USER_INITIALIZE_USER_HOME=true
+RUN mkdir $DQO_USER_HOME
+RUN touch $DQO_USER_HOME/.DQO_USER_HOME_NOT_MOUNTED
+COPY --from=dqo-home /dqo/home home
+
+# copy spring dependencies
+ARG DEPENDENCY=/workspace/app/home
+COPY --from=dqo-libs ${DEPENDENCY}/jars /dqo/lib
+COPY --from=dqo-libs ${DEPENDENCY}/expanded/BOOT-INF/lib /dqo/lib
+COPY --from=dqo-libs ${DEPENDENCY}/expanded/META-INF /dqo/META-INF
+COPY --from=dqo-libs ${DEPENDENCY}/expanded/BOOT-INF/classes /dqo
+ENV JAVA_TOOL_OPTIONS="-Xmx1024m"
+ENTRYPOINT ["java", "-cp", "/dqo:/dqo/lib/*", "com.dqops.cli.CliApplication"]

VERSION

@@ -1 +1 @@
-0.2.0-beta3
+0.2.0-beta4

distribution/pom.xml

@@ -11,7 +11,7 @@
 
 	<groupId>com.dqops</groupId>
 	<artifactId>dqo-distribution</artifactId>
-    <version>0.2.0-beta3</version>  <!-- DQO Version, do not touch (changed automatically) -->
+    <version>0.2.0-beta4</version>  <!-- DQO Version, do not touch (changed automatically) -->
 	<name>dqo-distribution</name>
 	<description>DQO Data Quality Operations Center final assembly</description>
 	<packaging>pom</packaging>

distribution/python/dqops/version.py

@@ -15,8 +15,8 @@
 # limit
 
 # WARNING: the next two lines with the version numbers (VERSION =, PIP_VERSION =) should not be modified manually. They are changed by a maven profile at compile time.
-VERSION = "0.2.0-beta3"
-PIP_VERSION = "0.2.0.3"
+VERSION = "0.2.0-beta4"
+PIP_VERSION = "0.2.0.4"
 GITHUB_RELEASE = "v" + VERSION + ""
 JAVA_VERSION = "17"
 

dqo

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-export DQO_VERSION=0.2.0-beta3
+export DQO_VERSION=0.2.0-beta4
 
 # Configure local development environment overrides
 if [ -f $(dirname $0)/set-dqo-envs.sh ]; then

dqo.cmd

@@ -15,7 +15,7 @@
 @REM limitations under the License.
 @REM
 
-set DQO_VERSION=0.2.0-beta3
+set DQO_VERSION=0.2.0-beta4
 
 rem Configure local development environment overrides
 if exist "%~dp0set-dqo-envs.cmd" (

dqops/pom.xml

@@ -27,7 +27,7 @@
 	</parent>
 	<groupId>com.dqops</groupId>
 	<artifactId>dqo-dqops</artifactId>
-    <version>0.2.0-beta3</version>  <!-- DQO Version, do not touch (changed automatically) -->
+    <version>0.2.0-beta4</version>  <!-- DQO Version, do not touch (changed automatically) -->
 	<packaging>jar</packaging>
 	<name>dqo-dqops</name>
 	<description>DQO Data Quality Operations Center</description>

dqops/src/main/frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "DQO",
-  "version": "0.2.0-beta3",
+  "version": "0.2.0-beta4",
   "private": true,
   "dependencies": {
     "@codemirror/lang-python": "^6.1.2",

dqops/src/main/java/com/dqops/cli/CliApplication.java

@@ -114,6 +114,12 @@ public static void main(String[] args) {
 			// calls CliMainCommandRunner and calls commands in io.dqo.cli.command, find the right command there if you want to know what happens now
 		}
 	    catch (Throwable t) {
+			if (t instanceof IllegalStateException && t.getCause() instanceof org.jline.reader.EndOfFileException) {
+				System.err.println("DQO cannot open the terminal.");
+				System.err.println("If you have started DQO from docker and want to use the DQO shell, please run the container with docker's \"-it\" parameter.");
+				System.err.println("Alternatively, start DQO from docker in a server headless mode (without the DQO shell) using \"docker run dqops/dqo run\".");
+				System.exit(-1);
+			}
 			LOG.error("Error at starting the application: " + t.getMessage(), t);
 			t.printStackTrace();
 		}

dqops/src/main/resources/banner.txt

@@ -3,4 +3,4 @@
  | |) | | (_) | | (_) | | '_ \ (_-<
  |___/   \__\_\  \___/  | .__/ /__/
                         |_|
- :: DQOps Data Quality Operations Center ::    (v0.2.0-beta3)
+ :: DQOps Data Quality Operations Center ::    (v0.2.0-beta4)

lib/pom.xml

@@ -11,7 +11,7 @@
 
 	<groupId>com.dqops</groupId>
 	<artifactId>dqo-lib</artifactId>
-    <version>0.2.0-beta3</version>  <!-- DQO Version, do not touch (changed automatically) -->
+    <version>0.2.0-beta4</version>  <!-- DQO Version, do not touch (changed automatically) -->
 	<name>lib</name>
 	<description>POM for a list of dependencies to libraries that should be distributed in the "lib" folder, especially all JDBC drivers.</description>
 	<packaging>jar</packaging>

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.dqops</groupId>
     <artifactId>dqo-data-quality-observer</artifactId>
-    <version>0.2.0-beta3</version>  <!-- DQO Version, do not touch (changed automatically) -->
+    <version>0.2.0-beta4</version>  <!-- DQO Version, do not touch (changed automatically) -->
     <packaging>pom</packaging>
     <description>DQO Data Quality Operations Center</description>
 

publish_docs.cmd

@@ -1,4 +1,4 @@
 @echo off
-python3 -m pip install --upgrade --user -r ./requirements.txt
-python3 -m mkdocs build
+python -m pip install --upgrade --user -r ./requirements.txt
+python -m mkdocs build
 gsutil -m rsync -j html,txt,xml,png,js,css,json,svg,gif -r site gs://docs-dqo-ai/docs/