support for dk and 2.1 and detecting siard-dk version

.github/workflows/CI.yml

@@ -16,10 +16,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up JDK 8
+      - name: Set up JDK 21
         uses: actions/setup-java@v3
         with:
-          java-version: 8
+          java-version: 21
           distribution: 'adopt'
       - name: Cache
         uses: actions/cache@v3

pom.xml

@@ -26,7 +26,7 @@
 
         <version.gwt>2.11.0</version.gwt>
         <scope.gwt-dev>provided</scope.gwt-dev>
-        <version.dbptk>3.0.0</version.dbptk>
+        <version.dbptk>3.1.0-SNAPSHOT</version.dbptk>
         <version.roda>5.4.0</version.roda>
         <version.jersey>3.1.6</version.jersey>
         <version.swagger>2.2.20</version.swagger>
@@ -494,6 +494,11 @@
             <artifactId>dbptk-bindings-siard2-1</artifactId>
             <version>2.0.0</version>
         </dependency>
+        <dependency>
+            <groupId>com.databasepreservation</groupId>
+            <artifactId>dbptk-bindings-siarddk-2020</artifactId>
+            <version>2.0.0</version>
+        </dependency>
         <dependency>
             <groupId>com.databasepreservation</groupId>
             <artifactId>dbptk-model</artifactId>

src/main/java/com/databasepreservation/DBVTK.java

@@ -132,7 +132,7 @@ public FilterRegistrationBean<OnOffFilter> casSingleSignOutFilter() {
     FilterRegistrationBean<OnOffFilter> registrationBean = new FilterRegistrationBean<>();
     registrationBean.setFilter(new OnOffFilter());
     registrationBean.setName("CasSingleSignOutFilter");
-    registrationBean.addInitParameter("inner-filter-class", "org.jasig.cas.client.session.SingleSignOutFilter");
+    registrationBean.addInitParameter("inner-filter-class", "org.apereo.cas.client.session.SingleSignOutFilter");
     registrationBean.addInitParameter("config-prefix", "ui.filter.cas");
     registrationBean.addInitParameter("casServerUrlPrefix", "http://localhost:8888/cas");
     registrationBean.addUrlPatterns("/*");
@@ -151,7 +151,7 @@ public FilterRegistrationBean<OnOffFilter> casValidationFilter() {
     registrationBean.setFilter(new OnOffFilter());
     registrationBean.setName("CasValidationFilter");
     registrationBean.addInitParameter("inner-filter-class",
-      "org.jasig.cas.client.validation.Cas30ProxyReceivingTicketValidationFilter");
+      "org.apereo.cas.client.validation.Cas30ProxyReceivingTicketValidationFilter");
     registrationBean.addInitParameter("config-prefix", "ui.filter.cas");
     registrationBean.addInitParameter("casServerUrlPrefix", "https://localhost:8443/cas");
     registrationBean.addInitParameter("serverName", "https://localhost:8888");
@@ -170,7 +170,7 @@ public FilterRegistrationBean<OnOffFilter> casAuthenticationFilter() {
     FilterRegistrationBean<OnOffFilter> registrationBean = new FilterRegistrationBean<>();
     registrationBean.setFilter(new OnOffFilter());
     registrationBean.setName("CasAuthenticationFilter");
-    registrationBean.addInitParameter("inner-filter-class", "org.jasig.cas.client.authentication.AuthenticationFilter");
+    registrationBean.addInitParameter("inner-filter-class", "org.apereo.cas.client.authentication.AuthenticationFilter");
     registrationBean.addInitParameter("config-prefix", "ui.filter.cas");
     registrationBean.addInitParameter("casServerLoginUrl", "https://localhost:8443/cas/login");
     registrationBean.addUrlPatterns("/login");
@@ -184,7 +184,7 @@ public FilterRegistrationBean<OnOffFilter> casRequestWrapperFilter() {
     registrationBean.setFilter(new OnOffFilter());
     registrationBean.setName("CasRequestWrapperFilter");
     registrationBean.addInitParameter("inner-filter-class",
-      "org.jasig.cas.client.util.HttpServletRequestWrapperFilter");
+      "org.apereo.cas.client.util.HttpServletRequestWrapperFilter");
     registrationBean.addInitParameter("config-prefix", "ui.filter.cas");
     registrationBean.addUrlPatterns("/*");
 

src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java

@@ -21,9 +21,12 @@
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
+import com.databasepreservation.common.client.models.structure.ViewerCell;
+import com.databasepreservation.model.data.Cell;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang.StringUtils;
 import org.roda.core.data.exceptions.GenericException;
@@ -172,7 +175,7 @@ public ResponseEntity<Resource> getReport(@PathVariable(name = "databaseUUID") S
   }
 
   @Override
-  public StringResponse createCollection(String databaseUUID) {
+  public StringResponse createCollection(String databaseUUID, String siardVersion) {
     ControllerAssistant controllerAssistant = new ControllerAssistant() {};
 
     LogEntryState state = LogEntryState.SUCCESS;
@@ -195,9 +198,7 @@ public StringResponse createCollection(String databaseUUID) {
 
     try {
       final ViewerDatabase database = ViewerFactory.getSolrManager().retrieve(ViewerDatabase.class, databaseUUID);
-      StringResponse collection = new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID));
-
-      return collection;
+      return new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID, siardVersion));
     } catch (GenericException | NotFoundException e) {
       state = LogEntryState.FAILURE;
       throw new RESTException(e);
@@ -516,7 +517,8 @@ public ResponseEntity<StreamingResponseBody> exportLOB(
           .equals(row.getCells().get(configTable.getColumnByIndex(columnIndex).getId()).getStoreType())) {
           return handleExternalLobDownload(configTable, row, columnIndex);
         } else {
-          return handleInternalLobDownload(database.getPath(), configTable, row, columnIndex);
+          String version = ViewerFactory.getSolrManager().retrieve(ViewerDatabase.class, databaseUUID).getVersion();
+          return handleInternalLobDownload(database.getPath(), configTable, row, columnIndex, version);
         }
       }
     } catch (NotFoundException | GenericException | IOException e) {
@@ -585,37 +587,44 @@ private ResponseEntity<StreamingResponseBody> handleExternalLobDownload(TableSta
   }
 
   private ResponseEntity<StreamingResponseBody> handleInternalLobDownload(String databasePath,
-    TableStatus tableConfiguration, ViewerRow row, int columnIndex) throws IOException, GenericException {
+    TableStatus tableConfiguration, ViewerRow row, int columnIndex, String version)
+    throws IOException, GenericException {
     String handlebarsFilename = HandlebarsUtils.applyExportTemplate(row, tableConfiguration, columnIndex);
 
     if (ViewerStringUtils.isBlank(handlebarsFilename)) {
       handlebarsFilename = ViewerConstants.SIARD_RECORD_PREFIX + row.getUuid()
         + ViewerConstants.SIARD_LOB_FILE_EXTENSION;
     }
-    String handlebarsMimeType = HandlebarsUtils.applyMimeTypeTemplate(row, tableConfiguration, columnIndex);
 
+    String handlebarsMimeType = HandlebarsUtils.applyMimeTypeTemplate(row, tableConfiguration, columnIndex);
     if (ViewerStringUtils.isBlank(handlebarsMimeType)) {
       handlebarsMimeType = tableConfiguration.getColumnByIndex(columnIndex).getApplicationType();
     }
 
-    if (LobManagerUtils.isLobEmbedded(tableConfiguration, row, columnIndex)) {
-      // handle lob as embedded
-      String lobCellValue = LobManagerUtils.getLobCellValue(tableConfiguration, row, columnIndex);
-      lobCellValue = lobCellValue.replace(ViewerConstants.SIARD_EMBEDDED_LOB_PREFIX, "");
-      String decodedString = new String(Base64.decodeBase64(lobCellValue.getBytes()));
-
+    if (version.equals(ViewerConstants.SIARD_DK_2010) || version.equals(ViewerConstants.SIARD_DK_2020)) {
+      String filePath = row.getCells().get(row.getCells().keySet().toArray()[row.getCells().size() - 1]).getValue();
       return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
-        DownloadUtils.stream(new BufferedInputStream(new ByteArrayInputStream(decodedString.getBytes())))));
+        DownloadUtils.stream(new BufferedInputStream(new FileInputStream(filePath)))));
     } else {
-      // handle lob as internal on separated folder
-      ZipFile zipFile = new ZipFile(databasePath);
-      final ZipEntry entry = zipFile.getEntry(LobManagerUtils.getZipFilePath(tableConfiguration, columnIndex, row));
-      if (entry == null) {
-        throw new GenericException("Zip archive entry is missing");
-      }
+      if (LobManagerUtils.isLobEmbedded(tableConfiguration, row, columnIndex)) {
+        // handle lob as embedded
+        String lobCellValue = LobManagerUtils.getLobCellValue(tableConfiguration, row, columnIndex);
+        lobCellValue = lobCellValue.replace(ViewerConstants.SIARD_EMBEDDED_LOB_PREFIX, "");
+        String decodedString = new String(Base64.decodeBase64(lobCellValue.getBytes()));
+
+        return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
+          DownloadUtils.stream(new BufferedInputStream(new ByteArrayInputStream(decodedString.getBytes())))));
+      } else {
+        // handle lob as internal on separated folder
+        ZipFile zipFile = new ZipFile(databasePath);
+        final ZipEntry entry = zipFile.getEntry(LobManagerUtils.getZipFilePath(tableConfiguration, columnIndex, row));
+        if (entry == null) {
+          throw new GenericException("Zip archive entry is missing");
+        }
 
-      return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
-        DownloadUtils.stream(new BufferedInputStream(zipFile.getInputStream(entry)))));
+        return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
+          DownloadUtils.stream(new BufferedInputStream(zipFile.getInputStream(entry)))));
+      }
     }
   }
 

src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java

@@ -91,42 +91,14 @@ public IndexResult<ViewerDatabase> find(FindRequest findRequest, String localeSt
   }
 
   @Override
-  public IndexResult<ViewerDatabase> findAll(FindRequest findRequest, String localeString) {
-    ControllerAssistant controllerAssistant = new ControllerAssistant() {};
-
-    LogEntryState state = LogEntryState.SUCCESS;
-    User user = controllerAssistant.checkRoles(request);
-
-    if (ViewerConfiguration.getInstance().getApplicationEnvironment().equals(ViewerConstants.APPLICATION_ENV_SERVER)) {
-      if (user.isAdmin() || user.isWhiteList()) {
-
-        return getCrossViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state);
-      } else {
-        List<String> fieldsToReturn = new ArrayList<>();
-        fieldsToReturn.add(ViewerConstants.INDEX_ID);
-        fieldsToReturn.add(ViewerConstants.SOLR_DATABASES_STATUS);
-        fieldsToReturn.add(ViewerConstants.SOLR_DATABASES_METADATA);
-        fieldsToReturn.add(ViewerConstants.SOLR_DATABASES_PERMISSIONS);
-
-        FindRequest userFindRequest = new FindRequest(findRequest.classToReturn, getDatabaseFilterForUser(user),
-          findRequest.sorter, findRequest.sublist, findRequest.facets, findRequest.exportFacets, fieldsToReturn);
-        return getCrossViewerDatabaseIndexResult(userFindRequest, controllerAssistant, user, state,
-          getDatabaseFilterForUser(user));
-      }
-    } else {
-      return getCrossViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state);
-    }
-  }
-
-  @Override
-  public StringResponse create(String path) {
+  public StringResponse create(String path, String siardVersion) {
     final ControllerAssistant controllerAssistant = new ControllerAssistant() {};
 
     LogEntryState state = LogEntryState.SUCCESS;
     User user = controllerAssistant.checkRoles(request);
 
     try {
-      return new StringResponse(SIARDController.loadMetadataFromLocal(path));
+      return new StringResponse(SIARDController.loadMetadataFromLocal(path, siardVersion));
     } catch (GenericException e) {
       state = LogEntryState.FAILURE;
       throw new RESTException(e);

src/main/java/com/databasepreservation/common/api/v1/FileResource.java

@@ -10,11 +10,14 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 import org.roda.core.data.exceptions.AlreadyExistsException;
 import org.roda.core.data.exceptions.GenericException;
@@ -87,8 +90,12 @@ public ResponseEntity<Resource> getSIARDFile(String filename) {
       java.nio.file.Path basePath = Paths.get(ViewerConfiguration.getInstance().getViewerConfigurationAsString("/",
         ViewerConfiguration.PROPERTY_BASE_UPLOAD_PATH));
       java.nio.file.Path siardPath = siardFilesPath.resolve(filename);
-      if (java.nio.file.Files.exists(siardPath) && !java.nio.file.Files.isDirectory(siardPath)
-        && (ViewerConfiguration.checkPathIsWithin(siardPath, siardFilesPath)
+
+      if (java.nio.file.Files.isDirectory(siardPath)) {
+        siardPath = zipDirectory(siardPath);
+      }
+
+      if (java.nio.file.Files.exists(siardPath) && (ViewerConfiguration.checkPathIsWithin(siardPath, siardFilesPath)
           || ViewerConfiguration.checkPathIsWithin(siardPath, basePath))) {
 
         InputStreamResource resource = new InputStreamResource(new FileInputStream(siardPath.toFile()));
@@ -101,12 +108,32 @@ public ResponseEntity<Resource> getSIARDFile(String filename) {
     } catch (NotFoundException | FileNotFoundException e) {
       state = LogEntryState.FAILURE;
       throw new RESTException(e);
+    } catch (IOException e) {
+      throw new RESTException(e);
     } finally {
       // register action
       controllerAssistant.registerAction(user, state, ViewerConstants.CONTROLLER_FILENAME_PARAM, filename);
     }
   }
 
+  private java.nio.file.Path zipDirectory(java.nio.file.Path dirPath) throws IOException {
+    java.nio.file.Path zipFilePath = dirPath.resolveSibling(dirPath.getFileName().toString() + ".zip");
+    try (FileOutputStream fos = new FileOutputStream(zipFilePath.toFile());
+      ZipOutputStream zos = new ZipOutputStream(fos)) {
+      java.nio.file.Files.walk(dirPath).filter(path -> !java.nio.file.Files.isDirectory(path)).forEach(path -> {
+        ZipEntry zipEntry = new ZipEntry(dirPath.relativize(path).toString());
+        try {
+          zos.putNextEntry(zipEntry);
+          java.nio.file.Files.copy(path, zos);
+          zos.closeEntry();
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      });
+    }
+    return zipFilePath;
+  }
+
   @Override
   public void deleteSiardFile(String filename) {
     final ControllerAssistant controllerAssistant = new ControllerAssistant() {};

src/main/java/com/databasepreservation/common/client/ViewerConstants.java

@@ -91,7 +91,7 @@ public class ViewerConstants {
   public static final String DENORMALIZATION_STATUS_PREFIX = "denormalization-";
 
   public static final String INTERNAL_ZIP_LOB_FOLDER = "lobs/";
-
+  public static final String SIARDDK_RESEARCH_INDEX_PATH = "/Schemas/standard/researchIndex.xsd";
   /*
    * SOLR CONFIGSETS
    */
@@ -440,7 +440,9 @@ public class ViewerConstants {
   public static final String SIARD_V10 = "1.0";
   public static final String SIARD_V20 = "2.0";
   public static final String SIARD_V21 = "2.1";
-
+  public static final String SIARD_DK = "dk";
+  public static final String SIARD_DK_2010 = "dk-2010";
+  public static final String SIARD_DK_2020 = "dk-2020";
   /*
    * DBPTK Metadata
    */
@@ -543,6 +545,7 @@ public class ViewerConstants {
   public static final String SIARD_LOB_INSIDE_PREFIX = "internal:";
   public static final String SIARD_EMBEDDED_LOB_PREFIX = "base64:";
   public static final String SIARD_LOB_FILE_EXTENSION = ".bin";
+  public static final String BINARY_LARGE_OBJECT = "BINARY LARGE OBJECT";
 
   public static final String DEFAULT_USERNAME = "admin";
   public static final String DEFAULT_FULL_NAME = "admin";

src/main/java/com/databasepreservation/common/client/common/helpers/HelperUploadSIARDFile.java

@@ -108,7 +108,7 @@ private void uploadMetadataSIARD(String path, FlowPanel panel) {
       Dialogs.showErrors(messages.errorMessagesOpenFile(path), errorMessage, messages.basicActionClose());
       // Toast.showError(messages.errorMessagesOpenFile(), errorMessage);
       panel.remove(loading);
-    }).create(path);
+    }).create(path, ViewerConstants.SIARD_V21);
   }
 
   private void successHandler(Boolean confirm, FlowPanel panel, String databaseUUID, String path) {

src/main/java/com/databasepreservation/common/client/common/lists/DatabaseList.java

@@ -86,7 +86,7 @@ public SafeHtml getValue(ViewerDatabase database) {
     Column<ViewerDatabase, SafeHtml> description = new TooltipColumn<ViewerDatabase>() {
       @Override
       public SafeHtml getValue(ViewerDatabase database) {
-        return database != null && database.getMetadata() != null
+        return database != null && database.getMetadata() != null && database.getMetadata().getDescription() != null
           ? SafeHtmlUtils.fromString(database.getMetadata().getDescription())
           : SafeHtmlUtils.fromString("unknown");
       }
@@ -104,7 +104,7 @@ public SafeHtml getValue(ViewerDatabase database) {
     Column<ViewerDatabase, SafeHtml> dataOwnerColumn = new TooltipColumn<ViewerDatabase>() {
       @Override
       public SafeHtml getValue(ViewerDatabase database) {
-        return database != null && database.getMetadata() != null
+        return database != null && database.getMetadata() != null && database.getMetadata().getDataOwner() != null
           ? SafeHtmlUtils.fromString(database.getMetadata().getDataOwner())
           : SafeHtmlUtils.fromString("unknown");
       }

src/main/java/com/databasepreservation/common/client/common/visualization/browse/RowPanel.java

@@ -412,11 +412,29 @@ private void getNestedHTML(ColumnStatus columnStatus) {
             null, false, new ArrayList<>());
           CollectionService.Util.call((IndexResult<ViewerRow> result) -> {
             if (result.getTotalCount() >= 1) {
+              RowField rowField;
               String json = JSOUtils.cellsToJson(result.getResults().get(0).getCells(), nestedTable);
               String s = JavascriptUtils.compileTemplate(template, json);
-              RowField rowField = RowField.createInstance(columnStatus.getCustomName(), new Label(s));
-              rowField.addColumnDescription(columnStatus.getCustomDescription());
+              if (columnStatus.getTypeName().contains(ViewerConstants.BINARY_LARGE_OBJECT)) {
+                if ((database.getPath() == null || database.getPath().isEmpty())
+                  && !status.getConsolidateProperty().equals(LargeObjectConsolidateProperty.CONSOLIDATED)) {
+                  rowField = RowField.createInstance(new Label(s).getText(),
+                    new HTML(messages.rowPanelTextForLobUnavailable()));
+                } else {
+                  SafeHtml safeHtml = SafeHtmlUtils.EMPTY_SAFE_HTML;
+                  json = JSOUtils.cellsToJson(ViewerConstants.TEMPLATE_LOB_DOWNLOAD_LABEL, messages.row_downloadLOB(),
+                    ViewerConstants.TEMPLATE_LOB_DOWNLOAD_LINK, RestUtils.createExportLobUri(database.getUuid(),
+                      nestedTable.getSchemaName(), nestedTable.getName(), "3309", columnStatus.getColumnIndex()));
+                  safeHtml = SafeHtmlUtils.fromSafeConstant(
+                    JavascriptUtils.compileTemplate(ViewerConstants.DEFAULT_DOWNLOAD_LABEL_TEMPLATE, json));
+
+                  rowField = RowField.createInstance(columnStatus.getCustomName(), new HTML(safeHtml));
+                }
+              } else {
+                rowField = RowField.createInstance(columnStatus.getCustomName(), new Label(s));
+              }
 
+              rowField.addColumnDescription(columnStatus.getCustomDescription());
               panel.add(rowField);
             }
           }).findRows(database.getUuid(), database.getUuid(), nestedTable.getSchemaName(), nestedTable.getName(),

src/main/java/com/databasepreservation/common/client/common/visualization/browse/information/DatabaseInformationPanel.java

@@ -154,7 +154,7 @@ private void loadOnAccess() {
         instances.remove(database.getUuid());
         HistoryManager.gotoSIARDInfo(database.getUuid());
         Dialogs.showErrors(messages.SIARDHomePageDialogTitleForBrowsing(), errorMessage, messages.basicActionClose());
-      }).createCollection(database.getUuid());
+      }).createCollection(database.getUuid(), database.getVersion());
     }
   }