-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
4 changed files
with
279 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
154 changes: 154 additions & 0 deletions
154
src/main/java/com/databasepreservation/common/server/index/utils/DatabaseResultIterator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| package com.databasepreservation.common.server.index.utils; | ||
|
|
||
| import com.databasepreservation.common.client.index.IndexResult; | ||
| import com.databasepreservation.common.client.index.IsIndexed; | ||
| import com.databasepreservation.common.client.index.filter.Filter; | ||
| import com.databasepreservation.common.client.index.sort.Sorter; | ||
| import org.apache.solr.client.solrj.SolrClient; | ||
| import org.apache.solr.common.params.CursorMarkParams; | ||
| import org.roda.core.data.exceptions.GenericException; | ||
| import org.roda.core.data.exceptions.RequestNotValidException; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.util.HashMap; | ||
| import java.util.Iterator; | ||
| import java.util.List; | ||
| import java.util.NoSuchElementException; | ||
|
|
||
| /** | ||
| * @author Miguel Guimarães <mguimaraes@keep.pt> | ||
| */ | ||
| public class DatabaseResultIterator<T extends IsIndexed> implements Iterator<T> { | ||
|
|
||
| public static final int DEFAULT_PAGE_SIZE = 1000; | ||
| public static final int DEFAULT_RETRIES = 100; | ||
| public static final int DEFAULT_SLEEP_BETWEEN_RETRIES = 10000; | ||
| private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseResultIterator.class); | ||
| private final SolrClient index; | ||
| private final Class<T> classToRetrieve; | ||
| private final Filter filter; | ||
| private final Sorter sorter; | ||
| private final List<String> fieldsToReturn; | ||
| private int pageSize = DEFAULT_PAGE_SIZE; | ||
| private int retries = DEFAULT_RETRIES; | ||
| private int sleepBetweenRetries = DEFAULT_SLEEP_BETWEEN_RETRIES; | ||
| private IndexResult<T> result = null; | ||
| private int indexInResult = 0; | ||
| private String cursorMark = CursorMarkParams.CURSOR_MARK_START; | ||
| private String nextCursorMark = CursorMarkParams.CURSOR_MARK_START; | ||
| private T next = null; | ||
|
|
||
| public DatabaseResultIterator(SolrClient index, Class<T> classToRetrieve, Filter filter, Sorter sorter, | ||
| List<String> fieldsToReturn) { | ||
| this.index = index; | ||
| this.filter = filter; | ||
| this.classToRetrieve = classToRetrieve; | ||
| this.sorter = sorter; | ||
| this.fieldsToReturn = fieldsToReturn; | ||
|
|
||
| getCurrentAndPrepareNext(); | ||
| } | ||
|
|
||
| private T getCurrentAndPrepareNext() { | ||
| T current = next; | ||
|
|
||
| // ensure index result is renewed | ||
| if (result == null || result.getResults().size() == indexInResult) { | ||
| indexInResult = 0; | ||
|
|
||
| cursorMark = nextCursorMark; | ||
| result = null; | ||
| nextCursorMark = null; | ||
| int availableRetries = retries; | ||
|
|
||
| do { | ||
| try { | ||
| Pair<IndexResult<T>, String> page = SolrUtils.find(index, classToRetrieve, filter, sorter, pageSize, | ||
| cursorMark, fieldsToReturn, new HashMap<>()); | ||
| result = page.getFirst(); | ||
| nextCursorMark = page.getSecond(); | ||
|
|
||
| } catch (GenericException | RequestNotValidException e) { | ||
| if (availableRetries > 0) { | ||
| availableRetries--; | ||
| LOGGER.warn("Error getting next page from Solr, retrying in {}ms...", sleepBetweenRetries); | ||
| try { | ||
| Thread.sleep(sleepBetweenRetries); | ||
| } catch (InterruptedException e1) { | ||
| // do nothing | ||
| } | ||
| } else { | ||
| LOGGER.error("Error getting next page from Solr, no more retries.", e); | ||
| throw new NoSuchElementException("Error getting next item in list: " + e.getMessage()); | ||
| } | ||
| } | ||
| } while (result == null); | ||
| } | ||
|
|
||
| if (indexInResult < result.getResults().size()) { | ||
| this.next = result.getResults().get(indexInResult++); | ||
| } else { | ||
| this.next = null; | ||
| } | ||
|
|
||
| return current; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean hasNext() { | ||
| return next != null; | ||
| } | ||
|
|
||
| @Override | ||
| public T next() { | ||
| return getCurrentAndPrepareNext(); | ||
| } | ||
|
|
||
| /** | ||
| * @param pageSize | ||
| * the pageSize to set | ||
| */ | ||
| public void setPageSize(int pageSize) { | ||
| this.pageSize = pageSize; | ||
| } | ||
|
|
||
| /** | ||
| * @return the retries | ||
| */ | ||
| public int getRetries() { | ||
| return retries; | ||
| } | ||
|
|
||
| /** | ||
| * @param retries | ||
| * the retries to set | ||
| */ | ||
| public void setRetries(int retries) { | ||
| this.retries = retries; | ||
| } | ||
|
|
||
| /** | ||
| * @return the sleepBetweenRetries | ||
| */ | ||
| public int getSleepBetweenRetries() { | ||
| return sleepBetweenRetries; | ||
| } | ||
|
|
||
| /** | ||
| * @param sleepBetweenRetries | ||
| * the sleepBetweenRetries to set | ||
| */ | ||
| public void setSleepBetweenRetries(int sleepBetweenRetries) { | ||
| this.sleepBetweenRetries = sleepBetweenRetries; | ||
| } | ||
|
|
||
| /** | ||
| * Gets the total count of objects as reported by underlying Solr requests. | ||
| * | ||
| * @return | ||
| */ | ||
| public long getTotalCount() { | ||
| return result != null ? result.getTotalCount() : -1; | ||
| } | ||
| } |
70 changes: 70 additions & 0 deletions
70
src/main/java/com/databasepreservation/common/server/index/utils/IterableDatabaseResult.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| package com.databasepreservation.common.server.index.utils; | ||
|
|
||
| import com.databasepreservation.common.client.index.IsIndexed; | ||
| import com.databasepreservation.common.client.index.filter.Filter; | ||
| import com.databasepreservation.common.client.index.sort.Sorter; | ||
| import com.databasepreservation.common.utils.CloseableIterable; | ||
| import org.apache.solr.client.solrj.SolrClient; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Iterator; | ||
| import java.util.List; | ||
|
|
||
| /** | ||
| * @author Miguel Guimarães <mguimaraes@keep.pt> | ||
| */ | ||
| public class IterableDatabaseResult<T extends IsIndexed> implements CloseableIterable<T> { | ||
|
|
||
| private static int PAGE_SIZE = -1; | ||
| private static int RETRIES = -1; | ||
| private static int SLEEP_BETWEEN_RETRIES = -1; | ||
|
|
||
| private final DatabaseResultIterator<T> iterator; | ||
|
|
||
| public IterableDatabaseResult(final SolrClient solrClient, final Class<T> returnClass, final Filter filter, | ||
| final Sorter sorter, final List<String> fieldsToReturn) { | ||
| iterator = new DatabaseResultIterator<>(solrClient, returnClass, filter, sorter, fieldsToReturn); | ||
|
|
||
| if (PAGE_SIZE > 0) { | ||
| iterator.setPageSize(PAGE_SIZE); | ||
| } | ||
|
|
||
| if (RETRIES > 0) { | ||
| iterator.setRetries(RETRIES); | ||
| } | ||
|
|
||
| if (SLEEP_BETWEEN_RETRIES > 0) { | ||
| iterator.setSleepBetweenRetries(SLEEP_BETWEEN_RETRIES); | ||
| } | ||
| } | ||
|
|
||
| public static void injectSearchPageSize(int pageSize) { | ||
| PAGE_SIZE = pageSize; | ||
| } | ||
|
|
||
| public static void injectNumberOfRetries(int retries) { | ||
| RETRIES = retries; | ||
| } | ||
|
|
||
| public static void injectSleepBetweenRetries(int sleepTime) { | ||
| SLEEP_BETWEEN_RETRIES = sleepTime; | ||
| } | ||
|
|
||
| @Override | ||
| public Iterator<T> iterator() { | ||
| return iterator; | ||
| } | ||
|
|
||
| @Override | ||
| public void close() throws IOException { | ||
| // do nothing | ||
| } | ||
|
|
||
| /** | ||
| * @see IndexResultIterator#getTotalCount() | ||
| */ | ||
| public long getTotalCount() { | ||
| return iterator.getTotalCount(); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters