wip - search on all db

keeps · Sep 20, 2024 · 9f8e876 · 9f8e876
1 parent c074614
commit 9f8e876
Show file tree

Hide file tree

Showing 13 changed files with 266 additions and 12 deletions.
diff --git a/deploys/docker-compose-dev.yml b/deploys/docker-compose-dev.yml
@@ -18,6 +18,7 @@ services:
     environment:
       ZK_HOST: zoo:2181
       SOLR_HOST: localhost
+      SOLR_HEAP: 4G
       SOLR_OPTS: "-Dsolr.environment=dev,label=DEV+ENV"
     depends_on:
       - zoo

diff --git a/src/main/java/com/databasepreservation/DBVTK.java b/src/main/java/com/databasepreservation/DBVTK.java
@@ -8,6 +8,7 @@
 package com.databasepreservation;
 
 import java.io.IOException;
+import java.net.http.HttpClient;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -261,6 +262,8 @@ public void addResourceHandlers(ResourceHandlerRegistry registry) {
     }
   }
 
+
+
   // @Bean
   // MultipartConfigElement multipartConfigElement() {
   // MultipartConfigFactory factory = new MultipartConfigFactory();

diff --git a/src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java b/src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java
@@ -195,7 +195,9 @@ public StringResponse createCollection(String databaseUUID) {
 
     try {
       final ViewerDatabase database = ViewerFactory.getSolrManager().retrieve(ViewerDatabase.class, databaseUUID);
-      return new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID));
+      StringResponse collection = new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID));
+
+      return collection;
     } catch (GenericException | NotFoundException e) {
       state = LogEntryState.FAILURE;
       throw new RESTException(e);

diff --git a/src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java b/src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java
@@ -8,7 +8,9 @@
 package com.databasepreservation.common.api.v1;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.roda.core.data.exceptions.GenericException;
@@ -24,11 +26,17 @@
 import com.databasepreservation.common.client.exceptions.RESTException;
 import com.databasepreservation.common.client.index.FindRequest;
 import com.databasepreservation.common.client.index.IndexResult;
+import com.databasepreservation.common.client.index.facets.FacetFieldResult;
+import com.databasepreservation.common.client.index.facets.FacetParameter;
+import com.databasepreservation.common.client.index.facets.FacetValue;
+import com.databasepreservation.common.client.index.facets.Facets;
+import com.databasepreservation.common.client.index.facets.SimpleFacetParameter;
 import com.databasepreservation.common.client.index.filter.AndFiltersParameters;
 import com.databasepreservation.common.client.index.filter.Filter;
 import com.databasepreservation.common.client.index.filter.FilterParameter;
 import com.databasepreservation.common.client.index.filter.OrFiltersParameters;
 import com.databasepreservation.common.client.index.filter.SimpleFilterParameter;
+import com.databasepreservation.common.client.index.sort.Sorter;
 import com.databasepreservation.common.client.models.activity.logs.LogEntryState;
 import com.databasepreservation.common.client.models.status.database.DatabaseStatus;
 import com.databasepreservation.common.client.models.structure.ViewerDatabase;
@@ -39,6 +47,7 @@
 import com.databasepreservation.common.server.ViewerConfiguration;
 import com.databasepreservation.common.server.ViewerFactory;
 import com.databasepreservation.common.server.controller.SIARDController;
+import com.databasepreservation.common.server.index.utils.IterableDatabaseResult;
 import com.databasepreservation.common.utils.ControllerAssistant;
 import com.databasepreservation.common.utils.UserUtility;
 
@@ -62,7 +71,10 @@ public IndexResult<ViewerDatabase> find(FindRequest findRequest, String localeSt
 
     if (ViewerConfiguration.getInstance().getApplicationEnvironment().equals(ViewerConstants.APPLICATION_ENV_SERVER)) {
       if (user.isAdmin() || user.isWhiteList()) {
-        return getViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state);
+
+        return getAllViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state);
+        // return getViewerDatabaseIndexResult(findRequest, controllerAssistant, user,
+        // state);
       } else {
         List<String> fieldsToReturn = new ArrayList<>();
         fieldsToReturn.add(ViewerConstants.INDEX_ID);
@@ -115,6 +127,69 @@ private IndexResult<ViewerDatabase> getViewerDatabaseIndexResult(FindRequest fin
     }
   }
 
+  private IndexResult<ViewerDatabase> getAllViewerDatabaseIndexResult(FindRequest findRequest,
+    ControllerAssistant controllerAssistant, User user, LogEntryState state) {
+    long count = 0;
+    try {
+
+      // List databases
+      Filter availableFilter = new Filter(
+        new SimpleFilterParameter(ViewerConstants.SOLR_DATABASES_STATUS, ViewerDatabaseStatus.AVAILABLE.name()));
+
+      IterableDatabaseResult<ViewerDatabase> databases = ViewerFactory.getSolrManager().findAll(ViewerDatabase.class,
+        availableFilter, Sorter.NONE, findRequest.fieldsToReturn);
+
+      // Search on all collections
+      List<String> collections = new ArrayList<>();
+      Map<String, ViewerDatabase> databaseMap = new HashMap<>();
+      for (ViewerDatabase database : databases) {
+        String collectionName = ViewerConstants.SOLR_INDEX_ROW_COLLECTION_NAME_PREFIX + database.getUuid();
+        collections.add(collectionName);
+        databaseMap.put(database.getUuid(), database);
+      }
+
+      SimpleFacetParameter simpleFacetParameter = new SimpleFacetParameter(ViewerConstants.SOLR_ROWS_DATABASE_UUID,
+        FacetParameter.SORT.COUNT);
+      simpleFacetParameter.setMinCount(1);
+      simpleFacetParameter.setLimit(15);
+      // set offset 0
+      Facets facets = new Facets(simpleFacetParameter);
+
+      final IndexResult<ViewerDatabase> result = ViewerFactory.getSolrManager().find(ViewerDatabase.class, collections,
+        findRequest.filter, findRequest.sorter, findRequest.sublist, facets, findRequest.fieldsToReturn);
+      count = result.getTotalCount();
+
+      IndexResult<ViewerDatabase> result2 = new IndexResult<>();
+      result2.setTotalCount(-1);
+      result2.setLimit(100);
+      result2.setOffset(findRequest.sublist.getFirstElementIndex());
+      List<ViewerDatabase> resultsFromFacet = new ArrayList<>();
+      result2.setResults(resultsFromFacet);
+      // Retrieve the databases HITs
+
+      for (FacetFieldResult facetResult : result.getFacetResults()) {
+        for (FacetValue value : facetResult.getValues()) {
+          String databaseUUID = value.getValue();
+          long searchHits = value.getCount();
+
+          ViewerDatabase vd = databaseMap.get(databaseUUID);
+          vd.setSearchHits(searchHits);
+          resultsFromFacet.add(vd);
+        }
+      }
+      return result2;
+
+    } catch (GenericException | RequestNotValidException e) {
+      state = LogEntryState.FAILURE;
+      throw new RESTException(e);
+    } finally {
+      // register action
+      controllerAssistant.registerAction(user, state, ViewerConstants.CONTROLLER_FILTER_PARAM,
+        JsonUtils.getJsonFromObject(findRequest.filter), ViewerConstants.CONTROLLER_SUBLIST_PARAM,
+        JsonUtils.getJsonFromObject(findRequest.sublist), ViewerConstants.CONTROLLER_RETRIEVE_COUNT, count);
+    }
+  }
+
   private IndexResult<ViewerDatabase> getViewerDatabaseIndexResult(FindRequest findRequest, List<String> fieldsToReturn,
     ControllerAssistant controllerAssistant, User user, LogEntryState state) {
     long count = 0;

diff --git a/src/main/java/com/databasepreservation/common/client/ViewerConstants.java b/src/main/java/com/databasepreservation/common/client/ViewerConstants.java
@@ -210,6 +210,7 @@ public class ViewerConstants {
   /*
    * ROW FIELDS
    */
+  public static final String SOLR_ROWS_DATABASE_UUID = "databaseUUID";
   public static final String SOLR_ROWS_TABLE_ID = "tableId";
   public static final String SOLR_ROWS_TABLE_UUID = "tableUUID";
   public static final String SOLR_ROWS_NESTED_UUID = "nestedUUID";

diff --git a/src/main/java/com/databasepreservation/common/client/models/structure/ViewerDatabase.java b/src/main/java/com/databasepreservation/common/client/models/structure/ViewerDatabase.java
@@ -39,6 +39,8 @@ public class ViewerDatabase extends IsIndexed {
   private String validationSkipped;
   private Set<String> permissions;
 
+  private long searchHits = -1;
+
   public ViewerDatabase() {
   }
 
@@ -216,4 +218,13 @@ public Set<String> getPermissions() {
   public void setPermissions(Set<String> permissions) {
     this.permissions = permissions;
   }
+
+
+  public long getSearchHits() {
+    return searchHits;
+  }
+
+  public void setSearchHits(long searchHits) {
+    this.searchHits = searchHits;
+  }
 }
diff --git a/src/main/java/com/databasepreservation/common/client/models/structure/ViewerRow.java b/src/main/java/com/databasepreservation/common/client/models/structure/ViewerRow.java
@@ -24,6 +24,7 @@ public class ViewerRow extends IsIndexed {
   @Serial
   private static final long serialVersionUID = 5337660979887460340L;
   private String UUID;
+  private String databaseUUID;
   private String tableUUID;
   private String tableId;
   private Map<String, ViewerCell> cells;
@@ -51,6 +52,14 @@ public void setUuid(String UUID) {
     this.UUID = UUID;
   }
 
+  public String getDatabaseUUID() {
+    return databaseUUID;
+  }
+
+  public void setDatabaseUUID(String databaseUUID) {
+    this.databaseUUID = databaseUUID;
+  }
+
   /**
    * @return Map of solrColumnName to cell value as string
    */

diff --git a/src/main/java/com/databasepreservation/common/server/index/DatabaseRowsSolrManager.java b/src/main/java/com/databasepreservation/common/server/index/DatabaseRowsSolrManager.java
@@ -17,14 +17,12 @@
 import java.util.Map;
 import java.util.Set;
 
-import com.databasepreservation.common.server.index.utils.IterableDatabaseResult;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.SolrInputField;
@@ -61,6 +59,7 @@
 import com.databasepreservation.common.server.index.schema.SolrDefaultCollectionRegistry;
 import com.databasepreservation.common.server.index.schema.SolrRowsCollectionRegistry;
 import com.databasepreservation.common.server.index.schema.collections.RowsCollection;
+import com.databasepreservation.common.server.index.utils.IterableDatabaseResult;
 import com.databasepreservation.common.server.index.utils.IterableIndexResult;
 import com.databasepreservation.common.server.index.utils.IterableNestedIndexResult;
 import com.databasepreservation.common.server.index.utils.JsonTransformer;
@@ -182,6 +181,13 @@ public <T extends IsIndexed> IndexResult<T> find(Class<T> classToReturn, Filter
       fieldsToReturn, new HashMap<>());
   }
 
+  public <T extends IsIndexed> IndexResult<T> find(Class<T> classToReturn, List<String> collections, Filter filter,
+    Sorter sorter, Sublist sublist, Facets facets, List<String> fieldsToReturn)
+    throws GenericException, RequestNotValidException {
+    return SolrUtils.find(client, SolrDefaultCollectionRegistry.get(classToReturn), collections, filter, sorter,
+      sublist, facets, fieldsToReturn, new HashMap<>());
+  }
+
   public <T extends IsIndexed> IndexResult<T> find(Class<T> classToReturn, Filter filter, Sorter sorter,
     Sublist sublist, Facets facets) throws GenericException, RequestNotValidException {
     return find(classToReturn, filter, sorter, sublist, facets, new ArrayList<>());

diff --git a/.../java/com/databasepreservation/common/server/index/schema/collections/RowsCollection.java b/.../java/com/databasepreservation/common/server/index/schema/collections/RowsCollection.java
@@ -8,6 +8,7 @@
 package com.databasepreservation.common.server.index.schema.collections;
 
 import static com.databasepreservation.common.client.ViewerConstants.SOLR_INDEX_ROW_COLLECTION_NAME_PREFIX;
+import static com.databasepreservation.common.client.ViewerConstants.SOLR_ROWS_DATABASE_UUID;
 import static com.databasepreservation.common.client.ViewerConstants.SOLR_ROWS_NESTED_TABLE_ID;
 import static com.databasepreservation.common.client.ViewerConstants.SOLR_ROWS_TABLE_ID;
 import static com.databasepreservation.common.client.ViewerConstants.SOLR_ROWS_TABLE_UUID;
@@ -72,6 +73,7 @@ public List<CopyField> getCopyFields() {
   public List<Field> getFields() {
     List<Field> fields = new ArrayList<>(super.getFields());
 
+    fields.add(new Field(SOLR_ROWS_DATABASE_UUID, Field.TYPE_STRING).setIndexed(true).setStored(true));
     fields.add(new Field(SOLR_ROWS_TABLE_ID, Field.TYPE_STRING).setIndexed(true).setStored(true));
     fields.add(new Field(SOLR_ROWS_TABLE_UUID, Field.TYPE_STRING).setIndexed(true).setStored(true));
 
@@ -88,6 +90,7 @@ public SolrInputDocument toSolrDocument(ViewerRow row) throws ViewerException, R
 
     SolrInputDocument doc = super.toSolrDocument(row);
 
+    doc.setField(ViewerConstants.SOLR_ROWS_DATABASE_UUID, row.getDatabaseUUID());
     doc.setField(ViewerConstants.SOLR_ROWS_TABLE_ID, row.getTableId());
     doc.setField(SOLR_ROWS_TABLE_UUID, row.getTableUUID());
     for (Map.Entry<String, ViewerCell> cellEntry : row.getCells().entrySet()) {
@@ -120,6 +123,7 @@ public SolrInputDocument toSolrDocument(ViewerRow row) throws ViewerException, R
   public ViewerRow fromSolrDocument(SolrDocument doc) throws ViewerException {
     ViewerRow viewerRow = super.fromSolrDocument(doc);
 
+    viewerRow.setDatabaseUUID(SolrUtils.objectToString(doc.get(ViewerConstants.SOLR_ROWS_DATABASE_UUID), null));
     viewerRow.setTableId(SolrUtils.objectToString(doc.get(ViewerConstants.SOLR_ROWS_TABLE_ID), null));
     viewerRow.setTableUUID(SolrUtils.objectToString(doc.get(ViewerConstants.SOLR_ROWS_TABLE_UUID), null));
     viewerRow.setNestedUUID(SolrUtils.objectToString(doc.get(ViewerConstants.SOLR_ROWS_NESTED_UUID), null));
@@ -159,6 +163,7 @@ public ViewerRow fromSolrDocument(SolrDocument doc) throws ViewerException {
   private ViewerRow populateNestedRow(SolrDocument doc) throws ViewerException {
     ViewerRow nestedRow = super.fromSolrDocument(doc);
 
+    nestedRow.setDatabaseUUID(SolrUtils.objectToString(doc.get(SOLR_ROWS_DATABASE_UUID), null));
     nestedRow.setTableId(SolrUtils.objectToString(doc.get(SOLR_ROWS_TABLE_ID), null));
     nestedRow.setTableUUID(SolrUtils.objectToString(doc.get(SOLR_ROWS_TABLE_UUID), null));
     nestedRow.setTableId(SolrUtils.objectToString(doc.get(SOLR_ROWS_NESTED_TABLE_ID), null));

diff --git a/src/main/java/com/databasepreservation/common/server/index/utils/SolrAlias.java b/src/main/java/com/databasepreservation/common/server/index/utils/SolrAlias.java
@@ -0,0 +1,60 @@
+package com.databasepreservation.common.server.index.utils;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.io.Serial;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * @author Gabriel Barros <gbarros@keep.pt>
+ */
+public class SolrAlias implements Serializable {
+  @Serial
+  private static final long serialVersionUID = -5247919236401053565L;
+
+  @JsonProperty("name")
+  private String name;
+  @JsonProperty("collections")
+  private List<String> collections;
+
+  public SolrAlias() {
+  }
+
+  public SolrAlias(String aliasName, List<String> collections) {
+    this.name = aliasName;
+    this.collections = collections;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public List<String> getCollections() {
+    return collections;
+  }
+
+  public void setCollections(List<String> collections) {
+    this.collections = collections;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o)
+      return true;
+    if (o == null || getClass() != o.getClass())
+      return false;
+    SolrAlias solrAlias = (SolrAlias) o;
+    return Objects.equals(name, solrAlias.name) && Objects.equals(collections, solrAlias.collections);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(name, collections);
+  }
+}