From aa306b089ad0f59e639877d24be0050efec67367 Mon Sep 17 00:00:00 2001 From: Piotr Czarnas Date: Fri, 26 Apr 2024 10:41:25 +0200 Subject: [PATCH 01/88] Avoid circular dependency in creating the terminal. --- .../check/CheckActivateCliCommand.java | 18 +++---- .../commands/check/CheckRunCliCommand.java | 29 ++++++----- .../commands/column/ColumnListCliCommand.java | 23 ++++----- .../connection/ConnectionListCliCommand.java | 23 ++++----- .../ConnectionSchemaListCliCommand.java | 21 ++++---- .../table/ConnectionTableListCliCommand.java | 23 ++++----- .../table/ConnectionTableShowCliCommand.java | 23 ++++----- .../commands/data/DataDeleteCliCommand.java | 21 ++++---- .../commands/table/TableImportCliCommand.java | 22 ++++----- .../commands/table/TableListCliCommand.java | 23 ++++----- .../table/impl/TableCliServiceImpl.java | 14 +++--- .../CommandExecutionErrorHandler.java | 6 +-- .../terminal/TerminalTableWritterImpl.java | 48 ++++++++----------- .../ConnectionAddCliCommandTest.java | 35 ++++++++++++++ .../impl/ConnectionCliServiceImplTests.java | 35 ++++++++++++++ .../cli/terminal/TerminalFactoryStub.java | 38 +++++++++++++++ .../TerminalTableWritterImplTest.java | 2 +- 17 files changed, 234 insertions(+), 170 deletions(-) create mode 100644 dqops/src/test/java/com/dqops/cli/commands/connection/ConnectionAddCliCommandTest.java create mode 100644 dqops/src/test/java/com/dqops/cli/commands/connection/impl/ConnectionCliServiceImplTests.java create mode 100644 dqops/src/test/java/com/dqops/cli/terminal/TerminalFactoryStub.java diff --git a/dqops/src/main/java/com/dqops/cli/commands/check/CheckActivateCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/check/CheckActivateCliCommand.java index 7a3b221923..c9e7ae6664 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/check/CheckActivateCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/check/CheckActivateCliCommand.java @@ -24,10 +24,7 @@ import com.dqops.cli.completion.completedcommands.ITableNameCommand; import com.dqops.cli.completion.completers.*; import com.dqops.cli.output.OutputFormatService; -import com.dqops.cli.terminal.FileWriter; -import com.dqops.cli.terminal.TerminalReader; -import com.dqops.cli.terminal.TerminalTableWritter; -import com.dqops.cli.terminal.TerminalWriter; +import com.dqops.cli.terminal.*; import com.dqops.metadata.search.CheckSearchFilters; import com.dqops.utils.serialization.JsonSerializer; import com.google.common.base.Strings; @@ -48,8 +45,7 @@ @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) @CommandLine.Command(name = "activate", description = "Activates data quality checks matching specified filters") public class CheckActivateCliCommand extends BaseCommand implements ICommand, ITableNameCommand { - private TerminalReader terminalReader; - private TerminalWriter terminalWriter; + private TerminalFactory terminalFactory; private TerminalTableWritter terminalTableWritter; private CheckCliService checkService; private JsonSerializer jsonSerializer; @@ -60,15 +56,13 @@ public CheckActivateCliCommand() { } @Autowired - public CheckActivateCliCommand(TerminalReader terminalReader, - TerminalWriter terminalWriter, + public CheckActivateCliCommand(TerminalFactory terminalFactory, TerminalTableWritter terminalTableWritter, CheckCliService checkService, JsonSerializer jsonSerializer, OutputFormatService outputFormatService, FileWriter fileWriter) { - this.terminalReader = terminalReader; - this.terminalWriter = terminalWriter; + this.terminalFactory = terminalFactory; this.terminalTableWritter = terminalTableWritter; this.checkService = checkService; this.jsonSerializer = jsonSerializer; @@ -338,11 +332,11 @@ public Map getFatalLevelOptions() { public Integer call() throws Exception { if (Strings.isNullOrEmpty(this.connection)) { throwRequiredParameterMissingIfHeadless("--connection"); - this.connection = this.terminalReader.prompt("Connection name (--connection)", null, false); + this.connection = this.terminalFactory.getReader().prompt("Connection name (--connection)", null, false); } if (Strings.isNullOrEmpty(this.check)) { throwRequiredParameterMissingIfHeadless("--check"); - this.check = this.terminalReader.prompt("Data quality check name (--check)", null, false); + this.check = this.terminalFactory.getReader().prompt("Data quality check name (--check)", null, false); } CheckSearchFilters filters = new CheckSearchFilters(); diff --git a/dqops/src/main/java/com/dqops/cli/commands/check/CheckRunCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/check/CheckRunCliCommand.java index 6fae7920b3..4ad1722b41 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/check/CheckRunCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/check/CheckRunCliCommand.java @@ -26,8 +26,8 @@ import com.dqops.cli.output.OutputFormatService; import com.dqops.cli.terminal.FileWriter; import com.dqops.cli.terminal.TablesawDatasetTableModel; +import com.dqops.cli.terminal.TerminalFactory; import com.dqops.cli.terminal.TerminalTableWritter; -import com.dqops.cli.terminal.TerminalWriter; import com.dqops.execution.checks.CheckExecutionErrorSummary; import com.dqops.execution.checks.CheckExecutionSummary; import com.dqops.execution.checks.progress.CheckExecutionProgressListener; @@ -51,7 +51,7 @@ @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) @CommandLine.Command(name = "run", header = "Run data quality checks that match a given condition", description = "Run data quality checks on your dataset that match a given condition. The command output is a table with the results that provides insight into the data quality.") public class CheckRunCliCommand extends BaseCommand implements ICommand, ITableNameCommand { - private TerminalWriter terminalWriter; + private TerminalFactory terminalFactory; private TerminalTableWritter terminalTableWritter; private CheckCliService checkService; private CheckExecutionProgressListenerProvider checkExecutionProgressListenerProvider; @@ -64,19 +64,18 @@ public CheckRunCliCommand() { /** * Dependency injection constructor. - * @param terminalWriter Terminal writer. * @param checkService Check implementation service. * @param jsonSerializer Json serializer. */ @Autowired - public CheckRunCliCommand(TerminalWriter terminalWriter, + public CheckRunCliCommand(TerminalFactory terminalFactory, TerminalTableWritter terminalTableWritter, CheckCliService checkService, CheckExecutionProgressListenerProvider checkExecutionProgressListenerProvider, JsonSerializer jsonSerializer, OutputFormatService outputFormatService, FileWriter fileWriter) { - this.terminalWriter = terminalWriter; + this.terminalFactory = terminalFactory; this.terminalTableWritter = terminalTableWritter; this.checkService = checkService; this.checkExecutionProgressListenerProvider = checkExecutionProgressListenerProvider; @@ -388,22 +387,22 @@ public Integer call() throws Exception { CheckExecutionSummary checkExecutionSummary = this.checkService.runChecks(filters, this.timeWindowFilterParameters, progressListener, this.dummyRun); if (checkExecutionSummary.getTotalChecksExecutedCount() == 0) { - this.terminalWriter.writeLine("No checks with these filters were found."); + this.terminalFactory.getWriter().writeLine("No checks with these filters were found."); return 0; } if (this.mode != CheckRunReportingMode.silent) { TablesawDatasetTableModel tablesawDatasetTableModel = new TablesawDatasetTableModel(checkExecutionSummary.getSummaryTable()); - this.terminalWriter.writeLine("Check evaluation summary per table:"); + this.terminalFactory.getWriter().writeLine("Check evaluation summary per table:"); switch(this.getOutputFormat()) { case CSV: { String csvContent = this.outputFormatService.tableToCsv(tablesawDatasetTableModel); if (this.isWriteToFile()) { CliOperationStatus cliOperationStatus = this.fileWriter.writeStringToFile(csvContent); - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); } else { - this.terminalWriter.write(csvContent); + this.terminalFactory.getWriter().write(csvContent); } break; } @@ -411,10 +410,10 @@ public Integer call() throws Exception { String jsonContent = this.outputFormatService.tableToJson(tablesawDatasetTableModel); if (this.isWriteToFile()) { CliOperationStatus cliOperationStatus = this.fileWriter.writeStringToFile(jsonContent); - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); } else { - this.terminalWriter.write(jsonContent); + this.terminalFactory.getWriter().write(jsonContent); } break; } @@ -423,9 +422,9 @@ public Integer call() throws Exception { TableBuilder tableBuilder = new TableBuilder(tablesawDatasetTableModel); tableBuilder.addInnerBorder(BorderStyle.oldschool); tableBuilder.addHeaderBorder(BorderStyle.oldschool); - String renderedTable = tableBuilder.build().render(this.terminalWriter.getTerminalWidth() - 1); + String renderedTable = tableBuilder.build().render(this.terminalFactory.getWriter().getTerminalWidth() - 1); CliOperationStatus cliOperationStatus = this.fileWriter.writeStringToFile(renderedTable); - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); } else { this.terminalTableWritter.writeTable(tablesawDatasetTableModel, true); @@ -437,9 +436,9 @@ public Integer call() throws Exception { CheckExecutionErrorSummary checkExecutionErrorSummary = checkExecutionSummary.getCheckExecutionErrorSummary(); if (checkExecutionErrorSummary != null) { if (this.mode == CheckRunReportingMode.debug) { - this.terminalWriter.writeLine(checkExecutionErrorSummary.getDebugMessage()); + this.terminalFactory.getWriter().writeLine(checkExecutionErrorSummary.getDebugMessage()); } else { - this.terminalWriter.writeLine(checkExecutionErrorSummary.getSummaryMessage()); + this.terminalFactory.getWriter().writeLine(checkExecutionErrorSummary.getSummaryMessage()); } } } diff --git a/dqops/src/main/java/com/dqops/cli/commands/column/ColumnListCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/column/ColumnListCliCommand.java index 05d2ea852c..b915013f4e 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/column/ColumnListCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/column/ColumnListCliCommand.java @@ -25,10 +25,7 @@ import com.dqops.cli.completion.completers.ColumnNameCompleter; import com.dqops.cli.completion.completers.ConnectionNameCompleter; import com.dqops.cli.completion.completers.FullTableNameCompleter; -import com.dqops.cli.terminal.FileWriter; -import com.dqops.cli.terminal.TablesawDatasetTableModel; -import com.dqops.cli.terminal.TerminalTableWritter; -import com.dqops.cli.terminal.TerminalWriter; +import com.dqops.cli.terminal.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @@ -44,8 +41,8 @@ @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) @CommandLine.Command(name = "list", header = "List the columns that match a given condition", description = "List all the columns in a table or filter them based on a specified condition.") public class ColumnListCliCommand extends BaseCommand implements ICommand, IConnectionNameCommand, ITableNameCommand { + private TerminalFactory terminalFactory; private ColumnCliService columnCliService; - private TerminalWriter terminalWriter; private TerminalTableWritter terminalTableWritter; private FileWriter fileWriter; @@ -53,11 +50,11 @@ public ColumnListCliCommand() { } @Autowired - public ColumnListCliCommand(TerminalWriter terminalWriter, - ColumnCliService columnCliService, + public ColumnListCliCommand(TerminalFactory terminalFactory, + ColumnCliService columnCliService, TerminalTableWritter terminalTableWritter, FileWriter fileWriter) { - this.terminalWriter = terminalWriter; + this.terminalFactory = terminalFactory; this.columnCliService = columnCliService; this.terminalTableWritter = terminalTableWritter; this.fileWriter = fileWriter; @@ -164,24 +161,24 @@ public Integer call() throws Exception { TableBuilder tableBuilder = new TableBuilder(new TablesawDatasetTableModel(cliOperationStatus.getTable())); tableBuilder.addInnerBorder(BorderStyle.oldschool); tableBuilder.addHeaderBorder(BorderStyle.oldschool); - String renderedTable = tableBuilder.build().render(this.terminalWriter.getTerminalWidth() - 1); + String renderedTable = tableBuilder.build().render(this.terminalFactory.getWriter().getTerminalWidth() - 1); CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(renderedTable); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { this.terminalTableWritter.writeTable(cliOperationStatus.getTable(), true); } } else { if (this.isWriteToFile()) { CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(cliOperationStatus.getMessage()); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { - this.terminalWriter.write(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().write(cliOperationStatus.getMessage()); } } return 0; } else { - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); return -1; } } diff --git a/dqops/src/main/java/com/dqops/cli/commands/connection/ConnectionListCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/connection/ConnectionListCliCommand.java index a5e91b88fe..4c458cd5bd 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/connection/ConnectionListCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/connection/ConnectionListCliCommand.java @@ -21,10 +21,7 @@ import com.dqops.cli.commands.connection.impl.ConnectionCliService; import com.dqops.cli.commands.connection.impl.models.ConnectionListModel; import com.dqops.cli.output.OutputFormatService; -import com.dqops.cli.terminal.FileWriter; -import com.dqops.cli.terminal.FormattedTableDto; -import com.dqops.cli.terminal.TerminalTableWritter; -import com.dqops.cli.terminal.TerminalWriter; +import com.dqops.cli.terminal.*; import com.google.api.client.util.Strings; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.ConfigurableBeanFactory; @@ -44,7 +41,7 @@ @CommandLine.Command(name = "list", header = "List connections that match a given condition", description = "Lists all the created connections for the logged-in user that match the conditions specified in the options. It allows the user to filter connections based on various parameters.") public class ConnectionListCliCommand extends BaseCommand implements ICommand { private ConnectionCliService connectionCliService; - private TerminalWriter terminalWriter; + private TerminalFactory terminalFactory; private TerminalTableWritter terminalTableWritter; private OutputFormatService outputFormatService; private FileWriter fileWriter; @@ -54,12 +51,12 @@ public ConnectionListCliCommand() { @Autowired public ConnectionListCliCommand(ConnectionCliService connectionCliService, - TerminalWriter terminalWriter, + TerminalFactory terminalFactory, TerminalTableWritter terminalTableWritter, OutputFormatService outputFormatService, FileWriter fileWriter) { this.connectionCliService = connectionCliService; - this.terminalWriter = terminalWriter; + this.terminalFactory = terminalFactory; this.terminalTableWritter = terminalTableWritter; this.outputFormatService = outputFormatService; this.fileWriter = fileWriter; @@ -107,10 +104,10 @@ public Integer call() throws Exception { String csvContent = this.outputFormatService.tableToCsv(formattedTable); if (this.isWriteToFile()) { CliOperationStatus cliOperationStatus = this.fileWriter.writeStringToFile(csvContent); - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); } else { - this.terminalWriter.write(csvContent); + this.terminalFactory.getWriter().write(csvContent); } break; } @@ -118,10 +115,10 @@ public Integer call() throws Exception { String jsonContent = this.outputFormatService.tableToJson(formattedTable); if (this.isWriteToFile()) { CliOperationStatus cliOperationStatus = this.fileWriter.writeStringToFile(jsonContent); - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); } else { - this.terminalWriter.write(jsonContent); + this.terminalFactory.getWriter().write(jsonContent); } break; } @@ -131,9 +128,9 @@ public Integer call() throws Exception { TableBuilder tableBuilder = new TableBuilder(model); tableBuilder.addInnerBorder(BorderStyle.oldschool); tableBuilder.addHeaderBorder(BorderStyle.oldschool); - String renderedTable = tableBuilder.build().render(this.terminalWriter.getTerminalWidth() - 1); + String renderedTable = tableBuilder.build().render(this.terminalFactory.getWriter().getTerminalWidth() - 1); CliOperationStatus cliOperationStatus = this.fileWriter.writeStringToFile(renderedTable); - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); } else { this.terminalTableWritter.writeTable(formattedTable, true); diff --git a/dqops/src/main/java/com/dqops/cli/commands/connection/schema/ConnectionSchemaListCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/connection/schema/ConnectionSchemaListCliCommand.java index 51c6d701ce..38dd8cbd01 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/connection/schema/ConnectionSchemaListCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/connection/schema/ConnectionSchemaListCliCommand.java @@ -38,8 +38,7 @@ @CommandLine.Command(name = "list", header = "List schemas in the specified connection", description = "It allows the user to view the summary of all schemas in a selected connection.") public class ConnectionSchemaListCliCommand extends BaseCommand implements ICommand { private ConnectionCliService connectionCliService; - private TerminalReader terminalReader; - private TerminalWriter terminalWriter; + private TerminalFactory terminalFactory; private TerminalTableWritter terminalTableWritter; private FileWriter fileWriter; @@ -48,13 +47,11 @@ public ConnectionSchemaListCliCommand() { @Autowired public ConnectionSchemaListCliCommand(ConnectionCliService connectionCliService, - TerminalReader terminalReader, - TerminalWriter terminalWriter, + TerminalFactory terminalFactory, TerminalTableWritter terminalTableWritter, FileWriter fileWriter) { this.connectionCliService = connectionCliService; - this.terminalWriter = terminalWriter; - this.terminalReader = terminalReader; + this.terminalFactory = terminalFactory; this.terminalTableWritter = terminalTableWritter; this.fileWriter = fileWriter; } @@ -78,7 +75,7 @@ public ConnectionSchemaListCliCommand(ConnectionCliService connectionCliService, public Integer call() throws Exception { if (Strings.isNullOrEmpty(this.name)) { throwRequiredParameterMissingIfHeadless("--name"); - this.name = this.terminalReader.prompt("Connection name (--name)", null, false); + this.name = this.terminalFactory.getReader().prompt("Connection name (--name)", null, false); } CliOperationStatus cliOperationStatus= this.connectionCliService.loadSchemaList(this.name, this.getOutputFormat(), dimensions, labels); @@ -88,24 +85,24 @@ public Integer call() throws Exception { TableBuilder tableBuilder = new TableBuilder(new TablesawDatasetTableModel(cliOperationStatus.getTable())); tableBuilder.addInnerBorder(BorderStyle.oldschool); tableBuilder.addHeaderBorder(BorderStyle.oldschool); - String renderedTable = tableBuilder.build().render(this.terminalWriter.getTerminalWidth() - 1); + String renderedTable = tableBuilder.build().render(this.terminalFactory.getWriter().getTerminalWidth() - 1); CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(renderedTable); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { this.terminalTableWritter.writeTable(cliOperationStatus.getTable(), true); } } else { if (this.isWriteToFile()) { CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(cliOperationStatus.getMessage()); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { - this.terminalWriter.write(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().write(cliOperationStatus.getMessage()); } } return 0; } else { - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); return -1; } } diff --git a/dqops/src/main/java/com/dqops/cli/commands/connection/table/ConnectionTableListCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/connection/table/ConnectionTableListCliCommand.java index 3e34a129ca..6e1e6757bb 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/connection/table/ConnectionTableListCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/connection/table/ConnectionTableListCliCommand.java @@ -39,8 +39,7 @@ @CommandLine.Command(name = "list", header = "List tables for the specified connection and schema name.", description = "List all the tables available in the database for the specified connection and schema. It allows the user to view all the tables in the database.") public class ConnectionTableListCliCommand extends BaseCommand implements ICommand { private ConnectionCliService connectionCliService; - private TerminalReader terminalReader; - private TerminalWriter terminalWriter; + private TerminalFactory terminalFactory; private TerminalTableWritter terminalTableWritter; private FileWriter fileWriter; @@ -49,13 +48,11 @@ public ConnectionTableListCliCommand() { @Autowired public ConnectionTableListCliCommand(ConnectionCliService connectionCliService, - TerminalReader terminalReader, - TerminalWriter terminalWriter, + TerminalFactory terminalFactory, TerminalTableWritter terminalTableWritter, FileWriter fileWriter) { this.connectionCliService = connectionCliService; - this.terminalWriter = terminalWriter; - this.terminalReader = terminalReader; + this.terminalFactory = terminalFactory; this.terminalTableWritter = terminalTableWritter; this.fileWriter = fileWriter; } @@ -85,12 +82,12 @@ public ConnectionTableListCliCommand(ConnectionCliService connectionCliService, public Integer call() throws Exception { if (Strings.isNullOrEmpty(this.connection)) { throwRequiredParameterMissingIfHeadless("--connection"); - this.connection = this.terminalReader.prompt("Connection name (--connection)", null, false); + this.connection = this.terminalFactory.getReader().prompt("Connection name (--connection)", null, false); } if (Strings.isNullOrEmpty(this.schema)) { throwRequiredParameterMissingIfHeadless("--schema"); - this.schema = this.terminalReader.prompt("Schema name (--schema)", null, false); + this.schema = this.terminalFactory.getReader().prompt("Schema name (--schema)", null, false); } CliOperationStatus cliOperationStatus = this.connectionCliService.loadTableList(connection, schema, table, this.getOutputFormat(), dimensions, labels); @@ -100,24 +97,24 @@ public Integer call() throws Exception { TableBuilder tableBuilder = new TableBuilder(new TablesawDatasetTableModel(cliOperationStatus.getTable())); tableBuilder.addInnerBorder(BorderStyle.oldschool); tableBuilder.addHeaderBorder(BorderStyle.oldschool); - String renderedTable = tableBuilder.build().render(this.terminalWriter.getTerminalWidth() - 1); + String renderedTable = tableBuilder.build().render(this.terminalFactory.getWriter().getTerminalWidth() - 1); CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(renderedTable); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { this.terminalTableWritter.writeTable(cliOperationStatus.getTable(), true); } } else { if (this.isWriteToFile()) { CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(cliOperationStatus.getMessage()); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { - this.terminalWriter.write(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().write(cliOperationStatus.getMessage()); } } return 0; } else { - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); return -1; } } diff --git a/dqops/src/main/java/com/dqops/cli/commands/connection/table/ConnectionTableShowCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/connection/table/ConnectionTableShowCliCommand.java index 4629979e35..e33a2eba18 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/connection/table/ConnectionTableShowCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/connection/table/ConnectionTableShowCliCommand.java @@ -41,8 +41,7 @@ @CommandLine.Command(name = "show", header = "Show table for connection", description = "Show the details of the specified table in the database for the specified connection. It allows the user to view the details of a specific table in the database.") public class ConnectionTableShowCliCommand extends BaseCommand implements ICommand, IConnectionNameCommand { private ConnectionCliService connectionCliService; - private TerminalReader terminalReader; - private TerminalWriter terminalWriter; + private TerminalFactory terminalFactory; private TerminalTableWritter terminalTableWritter; private FileWriter fileWriter; @@ -51,13 +50,11 @@ public ConnectionTableShowCliCommand() { @Autowired public ConnectionTableShowCliCommand(ConnectionCliService connectionCliService, - TerminalReader terminalReader, - TerminalWriter terminalWriter, + TerminalFactory terminalFactory, TerminalTableWritter terminalTableWritter, FileWriter fileWriter) { this.connectionCliService = connectionCliService; - this.terminalWriter = terminalWriter; - this.terminalReader = terminalReader; + this.terminalFactory = terminalFactory; this.terminalTableWritter = terminalTableWritter; this.fileWriter = fileWriter; } @@ -112,12 +109,12 @@ public void setTableName(String tableName) { public Integer call() throws Exception { if (Strings.isNullOrEmpty(this.connection)) { throwRequiredParameterMissingIfHeadless("--connection"); - this.connection = this.terminalReader.prompt("Connection name (--connection)", null, false); + this.connection = this.terminalFactory.getReader().prompt("Connection name (--connection)", null, false); } if (Strings.isNullOrEmpty(this.table)) { throwRequiredParameterMissingIfHeadless("--table"); - this.table = this.terminalReader.prompt("Full table name (schema.table), supports wildcard patterns 'sch*.tab*'", null, false); + this.table = this.terminalFactory.getReader().prompt("Full table name (schema.table), supports wildcard patterns 'sch*.tab*'", null, false); } CliOperationStatus cliOperationStatus = this.connectionCliService.showTableForConnection(connection, table, this.getOutputFormat()); @@ -127,24 +124,24 @@ public Integer call() throws Exception { TableBuilder tableBuilder = new TableBuilder(new TablesawDatasetTableModel(cliOperationStatus.getTable())); tableBuilder.addInnerBorder(BorderStyle.oldschool); tableBuilder.addHeaderBorder(BorderStyle.oldschool); - String renderedTable = tableBuilder.build().render(this.terminalWriter.getTerminalWidth() - 1); + String renderedTable = tableBuilder.build().render(this.terminalFactory.getWriter().getTerminalWidth() - 1); CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(renderedTable); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { this.terminalTableWritter.writeTable(cliOperationStatus.getTable(), true); } } else { if (this.isWriteToFile()) { CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(cliOperationStatus.getMessage()); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { - this.terminalWriter.write(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().write(cliOperationStatus.getMessage()); } } return 0; } else { - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); return -1; } } diff --git a/dqops/src/main/java/com/dqops/cli/commands/data/DataDeleteCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/data/DataDeleteCliCommand.java index c165395ad1..da8053eeb4 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/data/DataDeleteCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/data/DataDeleteCliCommand.java @@ -27,10 +27,7 @@ import com.dqops.cli.completion.completers.FullTableNameCompleter; import com.dqops.cli.converters.StringToLocalDateCliConverterMonthEnd; import com.dqops.cli.converters.StringToLocalDateCliConverterMonthStart; -import com.dqops.cli.terminal.TablesawDatasetTableModel; -import com.dqops.cli.terminal.TerminalReader; -import com.dqops.cli.terminal.TerminalTableWritter; -import com.dqops.cli.terminal.TerminalWriter; +import com.dqops.cli.terminal.*; import com.dqops.core.jobqueue.jobs.data.DeleteStoredDataQueueJobParameters; import com.dqops.data.statistics.factory.StatisticsCollectorTarget; import org.apache.parquet.Strings; @@ -53,8 +50,7 @@ @CommandLine.Command(name = "delete", header = "Deletes stored data that matches the specified conditions", description = "Deletes stored data that matches specified conditions. Be careful when using this command, as it permanently deletes the selected data and cannot be undone.") public class DataDeleteCliCommand extends BaseCommand implements ICommand, IConnectionNameCommand, ITableNameCommand { private DataCliService dataCliService; - private TerminalReader terminalReader; - private TerminalWriter terminalWriter; + private TerminalFactory terminalFactory; private TerminalTableWritter terminalTableWritter; @@ -65,16 +61,15 @@ public DataDeleteCliCommand() { * Dependency injection constructor. * * @param dataCliService Data CLI service. - * @param terminalReader Terminal reader. - * @param terminalWriter Terminal writer. + * @param terminalFactory Terminal factory. * @param terminalTableWritter Terminal table writer. */ @Autowired public DataDeleteCliCommand(DataCliService dataCliService, - TerminalReader terminalReader, TerminalWriter terminalWriter, TerminalTableWritter terminalTableWritter) { + TerminalFactory terminalFactory, + TerminalTableWritter terminalTableWritter) { this.dataCliService = dataCliService; - this.terminalReader = terminalReader; - this.terminalWriter = terminalWriter; + this.terminalFactory = terminalFactory; this.terminalTableWritter = terminalTableWritter; } @@ -260,12 +255,12 @@ protected DeleteStoredDataQueueJobParameters createDeletionParameters() { public Integer call() throws Exception { if (Strings.isNullOrEmpty(this.connection)) { throwRequiredParameterMissingIfHeadless("--connection"); - this.connection = this.terminalReader.prompt("Connection name (--connection)", null, false); + this.connection = this.terminalFactory.getReader().prompt("Connection name (--connection)", null, false); } DeleteStoredDataQueueJobParameters deletionParameters = this.createDeletionParameters(); CliOperationStatus cliOperationStatus = this.dataCliService.deleteStoredData(deletionParameters); - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); Table cliOperationStatusTable = cliOperationStatus.getTable(); if (cliOperationStatusTable != null) { diff --git a/dqops/src/main/java/com/dqops/cli/commands/table/TableImportCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/table/TableImportCliCommand.java index a0f051f2b4..8a4dcb70a1 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/table/TableImportCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/table/TableImportCliCommand.java @@ -23,6 +23,7 @@ import com.dqops.cli.completion.completedcommands.IConnectionNameCommand; import com.dqops.cli.completion.completers.ConnectionNameCompleter; import com.dqops.cli.completion.completers.SchemaNameCompleter; +import com.dqops.cli.terminal.TerminalFactory; import com.dqops.cli.terminal.TerminalReader; import com.dqops.cli.terminal.TerminalTableWritter; import com.dqops.cli.terminal.TerminalWriter; @@ -42,8 +43,7 @@ @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) @CommandLine.Command(name = "import", header = "Import tables from a specified database", description = "Import the tables from the specified database into the application. It allows the user to import the tables from the database into the application for performing various database operations.") public class TableImportCliCommand extends BaseCommand implements ICommand, IConnectionNameCommand { - private TerminalReader terminalReader; - private TerminalWriter terminalWriter; + private TerminalFactory terminalFactory; private TerminalTableWritter terminalTableWriter; private TableCliService tableImportService; @@ -51,12 +51,10 @@ public TableImportCliCommand() { } @Autowired - public TableImportCliCommand(TerminalReader terminalReader, - TerminalWriter terminalWriter, + public TableImportCliCommand(TerminalFactory terminalFactory, TerminalTableWritter terminalTableWriter, TableCliService tableImportService) { - this.terminalReader = terminalReader; - this.terminalWriter = terminalWriter; + this.terminalFactory = terminalFactory; this.terminalTableWriter = terminalTableWriter; this.tableImportService = tableImportService; } @@ -142,7 +140,7 @@ public Integer call() throws Exception { if (Strings.isNullOrEmpty(this.connection)) { throwRequiredParameterMissingIfHeadless("--connection"); - this.connection = this.terminalReader.prompt("Connection name (--connection)", null, false); + this.connection = this.terminalFactory.getReader().prompt("Connection name (--connection)", null, false); } if (Strings.isNullOrEmpty(this.schema) || StringPatternComparer.isSearchPattern(this.schema)) { @@ -151,14 +149,14 @@ public Integer call() throws Exception { try { schemaTable = this.tableImportService.loadSchemaList(this.connection, this.schema); } catch (TableImportFailedException ex) { - this.terminalWriter.writeLine(String.format("Cannot read schemas from the connection %s, error: %s", this.connection, ex.getMessage())); + this.terminalFactory.getWriter().writeLine(String.format("Cannot read schemas from the connection %s, error: %s", this.connection, ex.getMessage())); return -1; } Integer schemaIndex = this.terminalTableWriter.pickTableRowWithPaging("Select the schema (database, etc.) from which tables will be imported:", schemaTable); if (schemaIndex == null) { - this.terminalWriter.writeLine("No schema was selected."); + this.terminalFactory.getWriter().writeLine("No schema was selected."); return -1; } @@ -167,11 +165,11 @@ public Integer call() throws Exception { CliOperationStatus cliOperationStatus = this.tableImportService.importTables(this.getConnection(), this.getSchema(), this.getTable()); if (cliOperationStatus.isSuccess()) { - this.terminalWriter.writeLine("\nThe following tables were imported:"); - this.terminalWriter.writeTable(cliOperationStatus.getTable(), true); + this.terminalFactory.getWriter().writeLine("\nThe following tables were imported:"); + this.terminalFactory.getWriter().writeTable(cliOperationStatus.getTable(), true); return 0; } else { - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); return -1; } } diff --git a/dqops/src/main/java/com/dqops/cli/commands/table/TableListCliCommand.java b/dqops/src/main/java/com/dqops/cli/commands/table/TableListCliCommand.java index bfe5457167..237e691d65 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/table/TableListCliCommand.java +++ b/dqops/src/main/java/com/dqops/cli/commands/table/TableListCliCommand.java @@ -23,10 +23,7 @@ import com.dqops.cli.completion.completedcommands.IConnectionNameCommand; import com.dqops.cli.completion.completers.ConnectionNameCompleter; import com.dqops.cli.completion.completers.TableNameCompleter; -import com.dqops.cli.terminal.FileWriter; -import com.dqops.cli.terminal.TablesawDatasetTableModel; -import com.dqops.cli.terminal.TerminalTableWritter; -import com.dqops.cli.terminal.TerminalWriter; +import com.dqops.cli.terminal.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @@ -43,8 +40,8 @@ @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) @CommandLine.Command(name = "list", header = "List tables filtered by the given conditions", description = "List all the tables that match a given condition. It allows the user to use various filters, such as table name or schema names to list filtered tables.") public class TableListCliCommand extends BaseCommand implements ICommand, IConnectionNameCommand { + private TerminalFactory terminalFactory; private TableCliService tableImportService; - private TerminalWriter terminalWriter; private TerminalTableWritter terminalTableWritter; private FileWriter fileWriter; @@ -52,12 +49,12 @@ public TableListCliCommand() { } @Autowired - public TableListCliCommand(TerminalWriter terminalWriter, - TableCliService tableImportService, + public TableListCliCommand(TerminalFactory terminalFactory, + TableCliService tableImportService, TerminalTableWritter terminalTableWritter, FileWriter fileWriter) { + this.terminalFactory = terminalFactory; this.tableImportService = tableImportService; - this.terminalWriter = terminalWriter; this.terminalTableWritter = terminalTableWritter; this.fileWriter = fileWriter; } @@ -142,24 +139,24 @@ public Integer call() throws Exception { TableBuilder tableBuilder = new TableBuilder(new TablesawDatasetTableModel(cliOperationStatus.getTable())); tableBuilder.addInnerBorder(BorderStyle.oldschool); tableBuilder.addHeaderBorder(BorderStyle.oldschool); - String renderedTable = tableBuilder.build().render(this.terminalWriter.getTerminalWidth() - 1); + String renderedTable = tableBuilder.build().render(this.terminalFactory.getWriter().getTerminalWidth() - 1); CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(renderedTable); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { this.terminalTableWritter.writeTable(cliOperationStatus.getTable(), true); } } else { if (this.isWriteToFile()) { CliOperationStatus cliOperationStatus2 = this.fileWriter.writeStringToFile(cliOperationStatus.getMessage()); - this.terminalWriter.writeLine(cliOperationStatus2.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus2.getMessage()); } else { - this.terminalWriter.write(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().write(cliOperationStatus.getMessage()); } } return 0; } else { - this.terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); return -1; } } diff --git a/dqops/src/main/java/com/dqops/cli/commands/table/impl/TableCliServiceImpl.java b/dqops/src/main/java/com/dqops/cli/commands/table/impl/TableCliServiceImpl.java index c1311da021..bd33a49a65 100644 --- a/dqops/src/main/java/com/dqops/cli/commands/table/impl/TableCliServiceImpl.java +++ b/dqops/src/main/java/com/dqops/cli/commands/table/impl/TableCliServiceImpl.java @@ -19,6 +19,7 @@ import com.dqops.cli.commands.CliOperationStatus; import com.dqops.cli.commands.TabularOutputFormat; import com.dqops.cli.output.OutputFormatService; +import com.dqops.cli.terminal.TerminalFactory; import com.dqops.cli.terminal.TerminalReader; import com.dqops.cli.terminal.TerminalTableWritter; import com.dqops.cli.terminal.TerminalWriter; @@ -62,10 +63,9 @@ public class TableCliServiceImpl implements TableCliService { private final TableService tableService; private final UserHomeContextFactory userHomeContextFactory; - private final TerminalReader terminalReader; - private final TerminalWriter terminalWriter; private final SecretValueProvider secretValueProvider; private final ConnectionProviderRegistry connectionProviderRegistry; + private final TerminalFactory terminalFactory; private final TerminalTableWritter terminalTableWritter; private final OutputFormatService outputFormatService; private final DqoJobQueue dqoJobQueue; @@ -76,8 +76,7 @@ public class TableCliServiceImpl implements TableCliService { public TableCliServiceImpl(TableService tableService, UserHomeContextFactory userHomeContextFactory, ConnectionProviderRegistry connectionProviderRegistry, - TerminalReader terminalReader, - TerminalWriter terminalWriter, + TerminalFactory terminalFactory, SecretValueProvider secretValueProvider, TerminalTableWritter terminalTableWritter, OutputFormatService outputFormatService, @@ -87,8 +86,7 @@ public TableCliServiceImpl(TableService tableService, this.tableService = tableService; this.userHomeContextFactory = userHomeContextFactory; this.connectionProviderRegistry = connectionProviderRegistry; - this.terminalReader = terminalReader; - this.terminalWriter = terminalWriter; + this.terminalFactory = terminalFactory; this.secretValueProvider = secretValueProvider; this.terminalTableWritter = terminalTableWritter; this.outputFormatService = outputFormatService; @@ -322,8 +320,8 @@ public CliOperationStatus removeTable(String connectionName, String fullTableNam CliOperationStatus listingStatus = listTables(connectionName, fullTableName, TabularOutputFormat.TABLE, null, null); this.terminalTableWritter.writeTable(listingStatus.getTable(), true); - this.terminalWriter.writeLine("Do you want to remove these " + tableWrappers.size() + " tables?"); - boolean response = this.terminalReader.promptBoolean("Yes or No", false); + this.terminalFactory.getWriter().writeLine("Do you want to remove these " + tableWrappers.size() + " tables?"); + boolean response = this.terminalFactory.getReader().promptBoolean("Yes or No", false); if (!response) { cliOperationStatus.setFailedMessage("Delete operation cancelled."); return cliOperationStatus; diff --git a/dqops/src/main/java/com/dqops/cli/exceptions/CommandExecutionErrorHandler.java b/dqops/src/main/java/com/dqops/cli/exceptions/CommandExecutionErrorHandler.java index 25addf614b..37ef2efe52 100644 --- a/dqops/src/main/java/com/dqops/cli/exceptions/CommandExecutionErrorHandler.java +++ b/dqops/src/main/java/com/dqops/cli/exceptions/CommandExecutionErrorHandler.java @@ -18,6 +18,7 @@ import com.dqops.cli.terminal.TerminalFactory; import com.dqops.core.configuration.DqoCoreConfigurationProperties; import com.dqops.utils.exceptions.DqoErrorUserMessage; +import lombok.extern.slf4j.Slf4j; import org.apache.parquet.Strings; import org.jline.reader.UserInterruptException; import org.slf4j.Logger; @@ -28,9 +29,8 @@ /** * Command error handler. */ +@Slf4j public class CommandExecutionErrorHandler implements CommandLine.IExecutionExceptionHandler { - private static final Logger LOG = LoggerFactory.getLogger(CommandExecutionErrorHandler.class); - private final TerminalFactory terminalFactory; private final DqoCoreConfigurationProperties coreConfigurationProperties; @@ -88,7 +88,7 @@ public int handleExecutionException(Exception e, CommandLine commandLine, Comman } } - LOG.debug("Command " + parseResult.toString() + " failed", e); + log.error("Command " + parseResult.toString() + " failed", e); return -1; } } diff --git a/dqops/src/main/java/com/dqops/cli/terminal/TerminalTableWritterImpl.java b/dqops/src/main/java/com/dqops/cli/terminal/TerminalTableWritterImpl.java index c338fda0a9..f5316c62ec 100644 --- a/dqops/src/main/java/com/dqops/cli/terminal/TerminalTableWritterImpl.java +++ b/dqops/src/main/java/com/dqops/cli/terminal/TerminalTableWritterImpl.java @@ -32,24 +32,14 @@ */ @Component public class TerminalTableWritterImpl implements TerminalTableWritter { + private final TerminalFactory terminalFactory; private final FileWriter fileWriter; - private final TerminalReader terminalReader; - private final TerminalWriter terminalWriter; @Autowired - TerminalTableWritterImpl(TerminalFactory terminalFactory, - FileWriter fileWriter) { + public TerminalTableWritterImpl(TerminalFactory terminalFactory, + FileWriter fileWriter) { + this.terminalFactory = terminalFactory; this.fileWriter = fileWriter; - this.terminalReader = terminalFactory.getReader(); - this.terminalWriter = terminalFactory.getWriter(); - } - - TerminalTableWritterImpl(TerminalWriter terminalWriter, - TerminalReader terminalReader, - FileWriter fileWriter) { - this.fileWriter = fileWriter; - this.terminalReader = terminalReader; - this.terminalWriter = terminalWriter; } /** @@ -61,23 +51,23 @@ public class TerminalTableWritterImpl implements TerminalTableWritter { public Integer pickTableRowWithPaging(String question, Table table) { RowSelectionTableModel tableModel = new RowSelectionTableModel(table); - terminalWriter.writeLine(question); + this.terminalFactory.getWriter().writeLine(question); this.writeTable(tableModel, false, true); while (true) { - String line = terminalReader.prompt("Please enter one of the [] values: ", "", false); + String line = this.terminalFactory.getReader().prompt("Please enter one of the [] values: ", "", false); int rowCount = table.rowCount(); try { int pickedNumber = Integer.parseInt(line.trim()); if (pickedNumber <= 0 || pickedNumber > rowCount) { - terminalWriter.write(String.format("Please enter a number between 1 .. %d", rowCount)); + this.terminalFactory.getWriter().write(String.format("Please enter a number between 1 .. %d", rowCount)); } else { return pickedNumber - 1; // WATCH OUT: the user picks a 1-based row index, so the user picks [1] to get the very first row, but rows are 0-based indexed and we will return 0 as the selected row index } } catch (NumberFormatException nfe) { - terminalWriter.write(String.format("Please enter a number between 1 .. %d", rowCount)); + this.terminalFactory.getWriter().write(String.format("Please enter a number between 1 .. %d", rowCount)); } } } @@ -123,7 +113,7 @@ public void writeTable(FormattedTableDto tableData, boolean addBorder) { TableModel model = new BeanListTableModel<>(tableData.getRows().subList(pageStartTableIndex, pageEndTableIndex), tableData.getHeaders()); String renderedTable = renderTable(model, addBorder); - terminalWriter.write(renderedTable); + this.terminalFactory.getWriter().write(renderedTable); if (rowsLeft >= pageHeight) { boolean shouldPrintNextPage = pagingPrompt(tableData, addBorder); @@ -145,7 +135,7 @@ public void writeTable(FormattedTableDto tableData, boolean addBorder) { * @return Page heights in lines. */ private int getPageHeight(boolean addBorder){ - int pageHeight = (addBorder ? terminalWriter.getTerminalHeight() / 3 : terminalWriter.getTerminalHeight() - 2); + int pageHeight = (addBorder ? this.terminalFactory.getWriter().getTerminalHeight() / 3 : this.terminalFactory.getWriter().getTerminalHeight() - 2); if( pageHeight == 0) { pageHeight = 1; } @@ -188,7 +178,7 @@ public void writeTable(Table table, boolean addBorder, boolean noHeader) { int secondLineIndex = renderedTable.indexOf("\n") + 1; renderedTable = renderedTable.substring(secondLineIndex); } - terminalWriter.write(renderedTable); + this.terminalFactory.getWriter().write(renderedTable); if (rowsLeft >= pageHeight) { boolean shouldPrintNextPage = pagingPrompt(table, addBorder); @@ -248,7 +238,7 @@ public void writeTable(TableModel tableModel, boolean addBorder) { public void writeWholeTable(FormattedTableDto tableData, boolean addBorder) { TableModel model = new BeanListTableModel<>(tableData.getRows(), tableData.getHeaders()); String renderedTable = renderTable(model, addBorder); - terminalWriter.write(renderedTable); + this.terminalFactory.getWriter().write(renderedTable); } /** @@ -260,7 +250,7 @@ public void writeWholeTable(FormattedTableDto tableData, boolean addBorder) { public void writeWholeTable(Table table, boolean addBorder) { TablesawDatasetTableModel tableModel = new TablesawDatasetTableModel(table); String renderedTable = renderTable(tableModel, addBorder); - terminalWriter.write(renderedTable); + this.terminalFactory.getWriter().write(renderedTable); } /** @@ -271,7 +261,7 @@ public void writeWholeTable(Table table, boolean addBorder) { @Override public void writeWholeTable(TableModel tableModel, boolean addBorder) { String renderedTable = renderTable(tableModel, addBorder); - terminalWriter.write(renderedTable); + this.terminalFactory.getWriter().write(renderedTable); } /** @@ -282,7 +272,7 @@ public void writeWholeTable(TableModel tableModel, boolean addBorder) { */ private boolean pagingPrompt(FormattedTableDto tableData, boolean addBorder){ try { - int response = terminalReader.promptChar("Show next page? [Y]es / [n]o / [a]ll / [s]ave to file: ", 'y', false); + int response = this.terminalFactory.getReader().promptChar("Show next page? [Y]es / [n]o / [a]ll / [s]ave to file: ", 'y', false); if (response == 'N' || response == 'n') { return false; } @@ -292,7 +282,7 @@ else if (response == 'a' || response == 'A') { } else if (response == 's' || response == 'S') { CliOperationStatus cliOperationStatus = this.writeTableToFile(tableData); - terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); return false; } else if (response == 'y' || response == 'Y') { @@ -312,7 +302,7 @@ else if (response == 'y' || response == 'Y') { */ private boolean pagingPrompt(Table table, boolean addBorder){ try { - int response = terminalReader.promptChar("Show next page? [Y]es / [n]o / [a]ll / [s]ave to file: ", 'y', false); + int response = this.terminalFactory.getReader().promptChar("Show next page? [Y]es / [n]o / [a]ll / [s]ave to file: ", 'y', false); if (response == 'N' || response == 'n') { return false; } @@ -322,7 +312,7 @@ else if (response == 'a' || response == 'A') { } else if (response == 's' || response == 'S') { CliOperationStatus cliOperationStatus = this.writeTableToFile(table); - terminalWriter.writeLine(cliOperationStatus.getMessage()); + this.terminalFactory.getWriter().writeLine(cliOperationStatus.getMessage()); return false; } else if (response == 'y' || response == 'Y') { @@ -369,7 +359,7 @@ private String renderTable(TableModel tableModel, boolean addBorder, int totalAv * @return Rendered table. */ private String renderTable(TableModel tableModel, boolean addBorder){ - return renderTable(tableModel, addBorder, terminalWriter.getTerminalWidth() - 1); + return renderTable(tableModel, addBorder, this.terminalFactory.getWriter().getTerminalWidth() - 1); } } diff --git a/dqops/src/test/java/com/dqops/cli/commands/connection/ConnectionAddCliCommandTest.java b/dqops/src/test/java/com/dqops/cli/commands/connection/ConnectionAddCliCommandTest.java new file mode 100644 index 0000000000..f79723dc90 --- /dev/null +++ b/dqops/src/test/java/com/dqops/cli/commands/connection/ConnectionAddCliCommandTest.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dqops.cli.commands.connection; + +import com.dqops.BaseTest; +import com.dqops.utils.BeanFactoryObjectMother; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +/** + * Unit tests for ConnectionAddCliCommand + */ +@SpringBootTest +public class ConnectionAddCliCommandTest extends BaseTest { + @Test + void ConnectionAddCliCommand_whenCreatedFromIoC_instanceIsCreated() { + ConnectionAddCliCommand bean = BeanFactoryObjectMother.getBeanFactory().getBean(ConnectionAddCliCommand.class); + Assertions.assertNotNull(bean); + } +} diff --git a/dqops/src/test/java/com/dqops/cli/commands/connection/impl/ConnectionCliServiceImplTests.java b/dqops/src/test/java/com/dqops/cli/commands/connection/impl/ConnectionCliServiceImplTests.java new file mode 100644 index 0000000000..779d73df1a --- /dev/null +++ b/dqops/src/test/java/com/dqops/cli/commands/connection/impl/ConnectionCliServiceImplTests.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dqops.cli.commands.connection.impl; + +import com.dqops.BaseTest; +import com.dqops.utils.BeanFactoryObjectMother; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +/** + * Unit tests for ConnectionCliServiceImpl + */ +@SpringBootTest +public class ConnectionCliServiceImplTests extends BaseTest { + @Test + void ConnectionCliService_whenCreatedFromIoC_isCreated() { + ConnectionCliService instance = BeanFactoryObjectMother.getBeanFactory().getBean(ConnectionCliService.class); + Assertions.assertNotNull(instance); + } +} diff --git a/dqops/src/test/java/com/dqops/cli/terminal/TerminalFactoryStub.java b/dqops/src/test/java/com/dqops/cli/terminal/TerminalFactoryStub.java new file mode 100644 index 0000000000..5f0406be39 --- /dev/null +++ b/dqops/src/test/java/com/dqops/cli/terminal/TerminalFactoryStub.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dqops.cli.terminal; + +/** + * Terminal factory test stub. + */ +public class TerminalFactoryStub implements TerminalFactory { + private final TerminalReader reader; + private final TerminalWriter writer; + + public TerminalFactoryStub(TerminalReader terminalReader, TerminalWriter terminalWriter) { + this.reader = terminalReader; + this.writer = terminalWriter; + } + + public TerminalReader getReader() { + return reader; + } + + public TerminalWriter getWriter() { + return writer; + } +} diff --git a/dqops/src/test/java/com/dqops/cli/terminal/TerminalTableWritterImplTest.java b/dqops/src/test/java/com/dqops/cli/terminal/TerminalTableWritterImplTest.java index ca6678047a..2d41c0353f 100644 --- a/dqops/src/test/java/com/dqops/cli/terminal/TerminalTableWritterImplTest.java +++ b/dqops/src/test/java/com/dqops/cli/terminal/TerminalTableWritterImplTest.java @@ -28,7 +28,7 @@ public void setUp() { lineReader = new LineReaderWrapper(); TerminalReaderImpl terminalReader = new TerminalReaderImpl(terminalFactory.getWriter(), lineReader); FileWriter fileWriter = BeanFactoryObjectMother.getBeanFactory().getBean(FileWriter.class); - sut = new TerminalTableWritterImpl(terminalWriter, terminalReader, fileWriter); + sut = new TerminalTableWritterImpl(new TerminalFactoryStub(terminalReader, terminalWriter), fileWriter); terminalWriter.setTerminalWidth(50); terminalWriter.setTerminalHeight(10); } From e026477b11ec676141923bac604d4681420d00cb Mon Sep 17 00:00:00 2001 From: Piotr Czarnas Date: Fri, 26 Apr 2024 11:19:42 +0200 Subject: [PATCH 02/88] Default bean for an empty map, in case that a map is used as parameter for a cli command. --- .../main/java/com/dqops/cli/CliConfiguration.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dqops/src/main/java/com/dqops/cli/CliConfiguration.java b/dqops/src/main/java/com/dqops/cli/CliConfiguration.java index 50e210eb23..025f272e5b 100644 --- a/dqops/src/main/java/com/dqops/cli/CliConfiguration.java +++ b/dqops/src/main/java/com/dqops/cli/CliConfiguration.java @@ -44,12 +44,14 @@ import java.io.PushbackInputStream; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.function.Supplier; /** * Configuration class to configure the terminal and line reader beans. */ -@Lazy +@Lazy(false) @Configuration public class CliConfiguration { /** @@ -187,4 +189,13 @@ public TerminalReader terminalReader(TerminalWriter terminalWriter) { LineReader cliLineReader = (LineReader) StaticBeanFactory.getBeanFactory().getBean("cliLineReader"); return new TerminalReaderImpl(terminalWriter, cliLineReader); } + + /** + * Creates an empty hashmap. + * @return Empty hashmap. + */ + @Bean(name = "java.util.Map") + public Map createEmptyMap() { + return new LinkedHashMap(); + } } From d77f005a8e969d56cac91ea6a9b2370e06a09194 Mon Sep 17 00:00:00 2001 From: Patryk Jatczak Date: Fri, 26 Apr 2024 11:34:55 +0200 Subject: [PATCH 03/88] docs: fixed highlighting --- docs/data-sources/athena.md | 2 +- docs/data-sources/csv.md | 2 +- docs/data-sources/json.md | 2 +- docs/data-sources/parquet.md | 2 +- docs/data-sources/redshift.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/data-sources/athena.md b/docs/data-sources/athena.md index ed0bf37670..7505ae905c 100644 --- a/docs/data-sources/athena.md +++ b/docs/data-sources/athena.md @@ -214,7 +214,7 @@ With DQOps, you can configure credentials to access AWS S3 directly in the platf Please note, that any credentials and secrets shared with the DQOps Cloud or DQOps SaaS instances are stored in the .credentials folder. This folder also contains the default credentials files pair for AWS named **AWS_default_config** and **AWS_default_credentials**. -``` { .asc .annotate hl_lines="4" } +``` { .asc .annotate hl_lines="4-5" } $DQO_USER_HOME ├───... └───.credentials diff --git a/docs/data-sources/csv.md b/docs/data-sources/csv.md index c223c2b6af..86060870b4 100644 --- a/docs/data-sources/csv.md +++ b/docs/data-sources/csv.md @@ -311,7 +311,7 @@ With DQOps, you can configure credentials to access AWS S3 directly in the platf Please note, that any credentials and secrets shared with the DQOps Cloud or DQOps SaaS instances are stored in the .credentials folder. This folder also contains the default credentials files pair for AWS named **AWS_default_config** and **AWS_default_credentials**. -``` { .asc .annotate hl_lines="4" } +``` { .asc .annotate hl_lines="4-5" } $DQO_USER_HOME ├───... └───.credentials diff --git a/docs/data-sources/json.md b/docs/data-sources/json.md index 072efd79c3..15241882a9 100644 --- a/docs/data-sources/json.md +++ b/docs/data-sources/json.md @@ -310,7 +310,7 @@ With DQOps, you can configure credentials to access AWS S3 directly in the platf Please note, that any credentials and secrets shared with the DQOps Cloud or DQOps SaaS instances are stored in the .credentials folder. This folder also contains the default credentials files pair for AWS named **AWS_default_config** and **AWS_default_credentials**. -``` { .asc .annotate hl_lines="4" } +``` { .asc .annotate hl_lines="4-5" } $DQO_USER_HOME ├───... └───.credentials diff --git a/docs/data-sources/parquet.md b/docs/data-sources/parquet.md index d9720a9a91..0120de4a78 100644 --- a/docs/data-sources/parquet.md +++ b/docs/data-sources/parquet.md @@ -298,7 +298,7 @@ With DQOps, you can configure credentials to access AWS S3 directly in the platf Please note, that any credentials and secrets shared with the DQOps Cloud or DQOps SaaS instances are stored in the .credentials folder. This folder also contains the default credentials files pair for AWS named **AWS_default_config** and **AWS_default_credentials**. -``` { .asc .annotate hl_lines="4" } +``` { .asc .annotate hl_lines="4-5" } $DQO_USER_HOME ├───... └───.credentials diff --git a/docs/data-sources/redshift.md b/docs/data-sources/redshift.md index e952acfd8f..6e898a86c4 100644 --- a/docs/data-sources/redshift.md +++ b/docs/data-sources/redshift.md @@ -199,7 +199,7 @@ With DQOps, you can configure credentials to access AWS S3 directly in the platf Please note, that any credentials and secrets shared with the DQOps Cloud or DQOps SaaS instances are stored in the .credentials folder. This folder also contains the default credentials files pair for AWS named **AWS_default_config** and **AWS_default_credentials**. -``` { .asc .annotate hl_lines="4" } +``` { .asc .annotate hl_lines="4-5" } $DQO_USER_HOME ├───... └───.credentials From ac63ae5efe4d2942c37037b9ff24fd8136feaa0b Mon Sep 17 00:00:00 2001 From: Patryk Jatczak Date: Fri, 26 Apr 2024 13:09:54 +0200 Subject: [PATCH 04/88] duckdb directories has a special string for directory mapping --- .../duckdb/DuckdbConnectionProvider.java | 45 +++++++++-------- .../duckdb/DuckdbParametersSpec.java | 22 ++++++++- .../DuckdbConnectionProviderObjectMother.java | 34 +++++++++++++ .../duckdb/DuckdbConnectionProviderTest.java | 48 +++++++++++++++++++ 4 files changed, 128 insertions(+), 21 deletions(-) create mode 100644 dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderObjectMother.java create mode 100644 dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderTest.java diff --git a/dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbConnectionProvider.java b/dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbConnectionProvider.java index a07068cea3..63f18cb8a5 100644 --- a/dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbConnectionProvider.java +++ b/dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbConnectionProvider.java @@ -32,9 +32,7 @@ import tech.tablesaw.api.ColumnType; import tech.tablesaw.columns.Column; -import java.util.Arrays; -import java.util.List; -import java.util.NoSuchElementException; +import java.util.*; import java.util.stream.Collectors; /** @@ -139,30 +137,37 @@ public void promptForConnectionParameters(ConnectionSpec connectionSpec, boolean duckdbSpec.setFilesFormatType(terminalReader.promptEnum("Type of source files for DuckDB", DuckdbFilesFormatType.class, null, false)); } - if (duckdbSpec.getDirectories().isEmpty()) { + if (duckdbSpec.getDirectoriesString().isEmpty()) { if (isHeadless) { throw new CliRequiredParameterMissingException("--duckdb-directories"); } + duckdbSpec.setDirectoriesString(terminalReader.prompt("Virtual schema name and path in a pattern: schema=path. For multiple use pattern: schema=path1,schema=path2", null, false)); + } - String directoriesRaw = terminalReader.prompt("Virtual schema names and paths (in a pattern schema=path)", null, false); - if(!directoriesRaw.contains("/") && !directoriesRaw.contains("\\")){ - throw new RuntimeException("The provided path is invalid. The path should be an absolute path to the directory with folders or files. On Windows use double backslash (\\\\)."); - } - List directories = Arrays.stream(directoriesRaw.split(",")).collect(Collectors.toList()); - - for (String directory : directories) { - List schemaDirectory = Arrays.stream(directory.split("=")) - .map(String::trim) - .collect(Collectors.toList()); - if(schemaDirectory.size() != 2){ - throw new RuntimeException("Unbalanced values for " + directory + "." + - "Ensure you provide directories in a schema=path pattern."); - } - duckdbSpec.getDirectories().put(schemaDirectory.get(0), schemaDirectory.get(1)); - } + Map directories = parseDirectoriesString(duckdbSpec.getDirectoriesString()); + duckdbSpec.setDirectories(directories); + } + Map parseDirectoriesString(String directoriesRaw){ + Map directories = new HashMap<>(); + + if(!directoriesRaw.contains("/") && !directoriesRaw.contains("\\")){ + throw new RuntimeException("The provided path is invalid. The path should be an absolute path to the directory with folders or files. On Windows use double backslash (\\\\)."); + } + List singleMapping = Arrays.stream(directoriesRaw.split(",")).collect(Collectors.toList()); + + for (String mapping : singleMapping) { + List schemaDirectory = Arrays.stream(mapping.split("=")) + .map(String::trim) + .collect(Collectors.toList()); + if(schemaDirectory.size() != 2){ + throw new RuntimeException("Unbalanced values for " + mapping + "." + + "Ensure you provide directories in a schema=path pattern."); + } + directories.put(schemaDirectory.get(0), schemaDirectory.get(1)); } + return directories; } /** diff --git a/dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbParametersSpec.java b/dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbParametersSpec.java index da25af4c4d..65db0e2f3d 100644 --- a/dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbParametersSpec.java +++ b/dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbParametersSpec.java @@ -90,11 +90,14 @@ public class DuckdbParametersSpec extends BaseProviderParametersSpec @JsonSerialize(using = IgnoreEmptyYamlSerializer.class) private ParquetFileFormatSpec parquet; - @CommandLine.Option(names = "--duckdb-directories", split = ",") @JsonPropertyDescription("Virtual schema name to directory mappings. The path must be an absolute path.") @JsonInclude(JsonInclude.Include.NON_EMPTY) private Map directories = new HashMap<>(); + @CommandLine.Option(names = {"--duckdb-directories"}, description = "Virtual schema name to directory mappings. The path must be an absolute path.") + @JsonIgnore + private String directoriesString; + @CommandLine.Option(names = {"--duckdb-storage-type"}, description = "The storage type.") @JsonPropertyDescription("The storage type.") private DuckdbStorageType storageType; @@ -254,6 +257,23 @@ public void setDirectories(Map directories) { this.directories = directories != null ? Collections.unmodifiableMap(directories) : null; } + /** + * Returns a raw directories string containing mapping between schemas and directories + * @return Raw directories string containing mapping between schemas and directories + */ + public String getDirectoriesString() { + return directoriesString; + } + + /** + * Sets a raw directories string containing mapping between schemas and directories + * @param directoriesString Raw directories string containing mapping between schemas and directories + */ + public void setDirectoriesString(String directoriesString) { + setDirtyIf(!Objects.equals(this.directoriesString, directoriesString)); + this.directoriesString = directoriesString; + } + /** * Returns the storage type. * @return the storage type. diff --git a/dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderObjectMother.java b/dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderObjectMother.java new file mode 100644 index 0000000000..a41e53b286 --- /dev/null +++ b/dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderObjectMother.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.connectors.duckdb; + +import com.dqops.utils.BeanFactoryObjectMother; +import org.springframework.beans.factory.BeanFactory; + +/** + * DuckdbConnectionProvider object mother. + */ +public class DuckdbConnectionProviderObjectMother { + /** + * Returns the DuckdbConnectionProvider. + * @return DuckdbConnectionProvider. + */ + public static DuckdbConnectionProvider getProvider() { + BeanFactory beanFactory = BeanFactoryObjectMother.getBeanFactory(); + return beanFactory.getBean(DuckdbConnectionProvider.class); + } + +} diff --git a/dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderTest.java b/dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderTest.java new file mode 100644 index 0000000000..01153df449 --- /dev/null +++ b/dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderTest.java @@ -0,0 +1,48 @@ +package com.dqops.connectors.duckdb; + +import com.dqops.BaseTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.Map; + +@SpringBootTest +class DuckdbConnectionProviderTest extends BaseTest { + + private DuckdbConnectionProvider sut; + + + @BeforeEach + void setUp() { + sut = DuckdbConnectionProviderObjectMother.getProvider(); + } + + @Test + void parseDirectoriesString_whenOneValidMapping_parsesIt() { + String inputString = "schema=/directory"; + + Map mapping = sut.parseDirectoriesString(inputString); + + Assertions.assertEquals(1, mapping.size()); + Assertions.assertTrue(mapping.containsKey("schema")); + Assertions.assertEquals("/directory", mapping.get("schema")); + } + + @Test + void parseDirectoriesString_whenMultiMapping_parsesIt() { + String inputString = "schema1=/directory1,schema2=/directory2"; + + Map mapping = sut.parseDirectoriesString(inputString); + + Assertions.assertEquals(2, mapping.size()); + + Assertions.assertTrue(mapping.containsKey("schema1")); + Assertions.assertEquals("/directory1", mapping.get("schema1")); + + Assertions.assertTrue(mapping.containsKey("schema2")); + Assertions.assertEquals("/directory2", mapping.get("schema2")); + } + +} \ No newline at end of file From 34c9d824bf9a25dc534e3f2f2b75b6951fbdf1c2 Mon Sep 17 00:00:00 2001 From: Aleksy Date: Sat, 27 Apr 2024 17:24:01 +0200 Subject: [PATCH 05/88] dots dimensions --- .../src/pages/Schema/SchemaTables.tsx | 190 +++++++++++++++++- 1 file changed, 179 insertions(+), 11 deletions(-) diff --git a/dqops/src/main/frontend/src/pages/Schema/SchemaTables.tsx b/dqops/src/main/frontend/src/pages/Schema/SchemaTables.tsx index 297ea15f51..d1a6b32389 100644 --- a/dqops/src/main/frontend/src/pages/Schema/SchemaTables.tsx +++ b/dqops/src/main/frontend/src/pages/Schema/SchemaTables.tsx @@ -1,7 +1,15 @@ +import { Tooltip } from '@material-tailwind/react'; +import clsx from 'clsx'; +import moment from 'moment'; import React, { useEffect, useMemo, useState } from 'react'; import { useDispatch } from 'react-redux'; -import { TableListModel } from '../../api'; +import { + DimensionCurrentDataQualityStatusModel, + DimensionCurrentDataQualityStatusModelCurrentSeverityEnum, + TableListModel +} from '../../api'; import Button from '../../components/Button'; +import { getColor } from '../../components/Connection/TableView/TableQualityStatus/TableQualityStatusUtils'; import SvgIcon from '../../components/SvgIcon'; import { addFirstLevelTab } from '../../redux/actions/source.actions'; import { TableApiClient } from '../../services/apiClient'; @@ -157,10 +165,43 @@ export const SchemaTables = () => { ); }; + const getDimensionKey = () => { + const uniqueDimensions: string[] = []; + tables.forEach((table) => { + Object.keys(table.data_quality_status?.dimensions ?? {}).forEach((x) => { + if (!uniqueDimensions.includes(x) && !basicDimensionTypes.includes(x)) { + uniqueDimensions.push(x); + } + }); + }); + + return uniqueDimensions; + }; + const getBasicDimmensionsKeys = (dimentions: any, type: string) => { + const basicDimensions = Object.keys(dimentions ?? {})?.find( + (x) => x === type + ); + return basicDimensions; + }; + const basicDimensionTypes = ['Completeness', 'Validity', 'Consistency']; + + const getAdditionalDimentionsKeys = (dimentions: any) => { + return ( + Object.keys(dimentions ?? {})?.filter( + (x) => !basicDimensionTypes.includes(x) + ) ?? [] + ); + }; return ( - {headeritems.map((item) => renderItem(item.label, item.value))} + + {[ + ...headeritems, + ...basicDimensionTypes.map((x) => ({ label: x, value: x })), + ...getDimensionKey().map((x) => ({ label: x, value: x })) + ].map((item) => renderItem(item.label, item.value))} + {tables.map((item, index) => ( @@ -182,16 +223,105 @@ export const SchemaTables = () => { - {buttonTabs.map((button) => { + {/* {Object.entries(item.data_quality_status?.dimensions ?? {}).map( + ([key, value]) => { + return ( + + ); + } + )} */} + + {basicDimensionTypes.map((dimType) => { + const dimensionKey = getBasicDimmensionsKeys( + item.data_quality_status?.dimensions, + dimType + ); + const currentSeverity = (item.data_quality_status?.dimensions ?? + {})[dimensionKey as any]?.current_severity; + const lastCheckExecutedAt = (item.data_quality_status + ?.dimensions ?? {})?.[dimensionKey as any] + ?.last_check_executed_at; + const severityColor = getColor(currentSeverity as any); + const hasNoSeverity = severityColor.length === 0; + + const dimensionsClassNames = clsx('w-3 h-3', { + 'bg-gray-150': hasNoSeverity && lastCheckExecutedAt, + [severityColor]: !hasNoSeverity, + 'border border-gray-150': hasNoSeverity + }); return ( - + ); + })} + {getAdditionalDimentionsKeys( + item.data_quality_status?.dimensions ?? {} + ).map((dimensionKey: string, dimIndex) => { + return ( + ); })} @@ -201,3 +331,41 @@ export const SchemaTables = () => {
{item?.stage} {item?.filter} + +
+ +
- + +
+ +
+ +
+
); }; + +const renderSecondLevelTooltip = ( + data: DimensionCurrentDataQualityStatusModel | undefined +) => { + if (data && data.last_check_executed_at) { + return ( +
+
+
Last executed at:
+
+ {moment(data?.last_check_executed_at).format('YYYY-MM-DD HH:mm:ss')} +
+
+
+
Current severity level:
+
{data?.current_severity}
+
+
+
Highest historical severity level:
+
{data?.highest_historical_severity}
+
+
+
Quality Dimension:
+
{data?.dimension}
+
+
+ ); + } + return ( +
+
+
Quality Dimension:
+
{data?.dimension}
+
+
No data quality checks configured
+
+ ); +}; From fd2ed85e2309faa95225626dd8a2679839174ba9 Mon Sep 17 00:00:00 2001 From: Piotr Czarnas Date: Sun, 28 Apr 2024 16:22:48 +0200 Subject: [PATCH 06/88] Report the connection name for labels reported to bigquery. --- .../dqops/connectors/bigquery/BigQueryInternalConnection.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dqops/src/main/java/com/dqops/connectors/bigquery/BigQueryInternalConnection.java b/dqops/src/main/java/com/dqops/connectors/bigquery/BigQueryInternalConnection.java index a1b4f5bc70..050448e052 100644 --- a/dqops/src/main/java/com/dqops/connectors/bigquery/BigQueryInternalConnection.java +++ b/dqops/src/main/java/com/dqops/connectors/bigquery/BigQueryInternalConnection.java @@ -34,6 +34,7 @@ public class BigQueryInternalConnection { * @param quotaProjectId GCP quota project id. */ public BigQueryInternalConnection(String connectionName, BigQuery bigQueryClient, String billingProjectId, String quotaProjectId) { + this.connectionName = connectionName; this.bigQueryClient = bigQueryClient; this.billingProjectId = billingProjectId; this.quotaProjectId = quotaProjectId; From ecd3c82372ab70b1ecd0ba5ef605f602b922b337 Mon Sep 17 00:00:00 2001 From: Aleksy Date: Mon, 29 Apr 2024 11:49:15 +0200 Subject: [PATCH 07/88] moved dimensions to separate components --- .../src/pages/Schema/SchemaTableItem.tsx | 107 ++++++ .../Schema/SchemaTableItemDimensions.tsx | 150 +++++++++ .../src/pages/Schema/SchemaTables.tsx | 312 +++--------------- 3 files changed, 310 insertions(+), 259 deletions(-) create mode 100644 dqops/src/main/frontend/src/pages/Schema/SchemaTableItem.tsx create mode 100644 dqops/src/main/frontend/src/pages/Schema/SchemaTableItemDimensions.tsx diff --git a/dqops/src/main/frontend/src/pages/Schema/SchemaTableItem.tsx b/dqops/src/main/frontend/src/pages/Schema/SchemaTableItem.tsx new file mode 100644 index 0000000000..0603eb291e --- /dev/null +++ b/dqops/src/main/frontend/src/pages/Schema/SchemaTableItem.tsx @@ -0,0 +1,107 @@ +import React, { useMemo } from 'react'; +import { useDispatch } from 'react-redux'; +import { TableListModel } from '../../api'; +import Button from '../../components/Button'; +import SvgIcon from '../../components/SvgIcon'; +import { addFirstLevelTab } from '../../redux/actions/source.actions'; +import { CheckTypes, ROUTES } from '../../shared/routes'; +import { useDecodedParams } from '../../utils'; +import SchemaTableItemDimensions from './SchemaTableItemDimensions'; + +type TButtonTabs = { + label: string; + value: string; +}; + +export default function SchemaTableItem({ item }: { item: TableListModel }) { + const { + checkTypes + }: { + checkTypes: CheckTypes; + } = useDecodedParams(); + const dispatch = useDispatch(); + + const goToTable = (item: TableListModel, tab: string) => { + dispatch( + addFirstLevelTab(checkTypes, { + url: ROUTES.TABLE_LEVEL_PAGE( + checkTypes, + item.connection_name ?? '', + item.target?.schema_name ?? '', + item.target?.table_name ?? '', + tab + ), + value: ROUTES.TABLE_LEVEL_VALUE( + checkTypes, + item.connection_name ?? '', + item.target?.schema_name ?? '', + item.target?.table_name ?? '' + ), + state: {}, + label: item.target?.table_name ?? '' + }) + ); + return; + }; + + const buttonTabs: TButtonTabs[] = useMemo(() => { + switch (checkTypes) { + case CheckTypes.PROFILING: + return [ + { + label: 'Basic statistics', + value: 'statistics' + }, + { label: 'Profiling checks', value: 'advanced' }, + { label: 'Profiling table status', value: 'table-quality-status' } + ]; + + case CheckTypes.SOURCES: + return [ + { + label: 'Table configuration', + value: 'detail' + }, + { label: 'Date and time column', value: 'timestamps' }, + { label: 'Incident configuration', value: 'incident_configuration' } + ]; + + case CheckTypes.MONITORING: + case CheckTypes.PARTITIONED: + return [ + { label: 'Daily table status', value: 'table-quality-status-daily' }, + { label: 'Daily checks', value: 'daily' } + ]; + + default: + return []; + } + }, [checkTypes]); + + return ( + +