diff --git a/oc_search/settings-sample.py b/oc_search/settings-sample.py index 8108360..826bec9 100644 --- a/oc_search/settings-sample.py +++ b/oc_search/settings-sample.py @@ -44,25 +44,23 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'qurl_templatetag', - 'search' + 'search', ] ## Optional applications # 'ramp', # 'smuggler', - MIDDLEWARE = [ "corsheaders.middleware.CorsMiddleware", 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', - "corsheaders.middleware.CorsPostCsrfMiddleware", + "django.middleware.common.CommonMiddleware", 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.middleware.cache.UpdateCacheMiddleware', - 'django.middleware.common.CommonMiddleware', 'django.middleware.cache.FetchFromCacheMiddleware', 'oc_search.middleware.CanadaBilingualMiddleware' ] @@ -116,22 +114,19 @@ DATABASES = { 'default': { - 'ENGINE': '', - }, - 'search': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), } } +JSON_DOWNLOADS_ALLOWED = False + # Smuggler settings SMUGGLER_FIXTURE_DIR = os.path.join(BASE_DIR, 'smuggler') SMUGGLER_EXCLUDE_LIST = ['admin.logentry', 'auth.permission', 'auth.group', 'auth.user', 'contenttypes.contenttype', 'django_celery_results.chordcounter', 'django_celery_results.groupresult', 'django_celery_beat.taskresult'] -DATABASE_ROUTERS = ['search.db_router.SearchRouter'] - # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -236,10 +231,14 @@ 'local': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 'LOCATION': 'oc_search', - } + }, + 'redis': { + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://:password@localhost:6379', + }, } # Object in the local cache expire after this many seconds. Not recommended to be less than 60 seconds. -CACHE_LOCAL_TIMEOUT = 60 * 5 +CACHE_LOCAL_TIMEOUT = 30 SESSION_ENGINE="django.contrib.sessions.backends.file" SESSION_FILE_PATH = os.path.join(BASE_DIR, 'session') @@ -287,13 +286,19 @@ SEARCH_LOGGING_ARCHIVE_FILE = os.path.join(BASE_DIR, 'data', 'search_logs.log') SEARCH_LOGGING_ARCHIVE_AFTER_X_DAYS = 7 -# Used by the Suggested Datasets Search +# Used by the Suggested Datasets Search if enabled SD_COMMENTS_BASE_EN = "http://127.0.0.1:8000/static/sd/" SD_COMMENTS_BASE_FR = "http://127.0.0.1:8000/static/sd/" +SD_SUGGEST_A_DATASET_EN = "https://o127.0.0.1:8000/en/forms/suggest-dataset" +SD_SUGGEST_A_DATASET_FR = "https://127.0.0.1:8000/fr/formulaire/proposez-un-formulaire-densemble-de-donnees" SD_VOTES_BASE_EN = "http://127.0.0.1:8000/static/sd/" SD_VOTES_BASE_FR = "http://127.0.0.1:8000/static/sd/" +SD_RECORD_URL_EN = 'http://127.0.0.1:8000/en/sd/id/' +SD_RECORD_URL_FR = 'http://127.0.0.1:8000/fr/sd/id/' +SD_ALERT_EMAIL_FROM = ['My Name', 'my.email', 'my.domain.org'] + # Used by the import_data_csv console command IMPORT_DATA_CSV_DEFAULT_DEBUG = False diff --git a/oc_search/urls.py b/oc_search/urls.py index 0c3d1a9..7d8f488 100644 --- a/oc_search/urls.py +++ b/oc_search/urls.py @@ -17,6 +17,7 @@ from django.contrib import admin from django.urls import path from django.conf.urls import include +from django.views.decorators.cache import never_cache from search.views import SearchView, RecordView, ExportView, MoreLikeThisView, HomeView, DefaultView, ExportStatusView, DownloadSearchResultsView from ramp.views import RampView @@ -42,8 +43,8 @@ path('search///similaire/', MoreLikeThisView.as_view(), name='MLTForm'), path('rechercher///similaire/', MoreLikeThisView.as_view(), name='MLTForm'), - path('search/search-results///', ExportStatusView.as_view(), name='SearchResultsForm'), - path('rechercher/rapport-de-recherche///', ExportStatusView.as_view(), name='SearchResultsForm'), + path('search/search-results///', never_cache(ExportStatusView.as_view()), name='SearchResultsForm'), + path('rechercher/rapport-de-recherche///', never_cache(ExportStatusView.as_view()), name='SearchResultsForm'), ] if 'ramp' in settings.INSTALLED_APPS: urlpatterns += [ @@ -67,9 +68,9 @@ name='MLTForm'), path(settings.SEARCH_HOST_PATH + '/similaire/', MoreLikeThisView.as_view(), name='MLTForm'), - path(settings.SEARCH_HOST_PATH + 'search-results///', ExportStatusView.as_view(), + path(settings.SEARCH_HOST_PATH + 'search-results///', never_cache(ExportStatusView.as_view()), name='SearchResultsForm'), - path(settings.SEARCH_HOST_PATH + 'rapport-de-recherche///', ExportStatusView.as_view(), + path(settings.SEARCH_HOST_PATH + 'rapport-de-recherche///', never_cache(ExportStatusView.as_view()), name='SearchResultsForm'), ] if 'ramp' in settings.INSTALLED_APPS: diff --git a/requirements.txt b/requirements.txt index ee26336..35b0f36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,27 @@ -Babel==2.12.1 -bleach==6.0.0 -celery==5.2.7 +Babel==2.14.0 +bleach==6.1.0 +celery==5.3.6 ckanapi==4.7 -django==3.2.21 +django==4.2.9 django-celery-beat==2.5.0 django-celery-results==2.5.1 -django-cors-headers==3.14.0 +django-cors-headers==4.3.1 django-jazzmin==2.6.0 django-qurl-templatetag==0.0.14 -django-redis-cache==3.0.1 django-redis-sessions==0.6.2 django-smuggler==1.0.4 -docutils==0.20.1 -email-validator==2.0.0.post2 +email-validator==2.1.0.post1 inflection==0.5.1 -markdown2==2.4.10 +markdown2==2.4.12 nltk==3.8.1 -pandas==1.4.4 +pandas==2.2.0 psycopg2==2.9.9 python-dateutil==2.8.2 PyYAML==6.0.1 -scikit-learn==1.3.2 -Unidecode==1.3.7 -uWSGI==2.0.19.1; sys_platform == 'linux' +redis==5.0.1 +scikit-learn==1.4.0 +setuptools>=69.0.3 +Unidecode==1.3.8 +uWSGI==2.0.23; sys_platform == 'linux' #git+https://github.com/thriuin/SolrClient.git@master -redis==3.5.3 # not directly required, but cannot use the latest as suggested by Snyk due to incompatability with django-redis-cache -setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/search/management/commands/create_solr_core.py b/search/management/commands/create_solr_core.py index bffa6b6..876eaad 100644 --- a/search/management/commands/create_solr_core.py +++ b/search/management/commands/create_solr_core.py @@ -15,8 +15,6 @@ "filters": [ {"class": "solr.WordDelimiterGraphFilterFactory", "preserveOriginal": "1", "splitOnCaseChange": "0", "catenateAll": "1"}, {"class": "solr.FlattenGraphFilterFactory"}, - {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true", "synonyms": "lang/synonyms_en.txt"}, - {"class": "solr.FlattenGraphFilterFactory"}, {"class": "solr.StopFilterFactory", "words": "lang/stopwords_en.txt", "ignoreCase": "true"}, {"class": "solr.LowerCaseFilterFactory"}, {"class": "solr.EnglishPossessiveFilterFactory"}, @@ -29,8 +27,10 @@ "tokenizer": {"class": "solr.WhitespaceTokenizerFactory"}, "filters": [ {"class": "solr.WordDelimiterGraphFilterFactory", "preserveOriginal": "1", "splitOnCaseChange": "0", "catenateAll": "1"}, + {"class": "solr.FlattenGraphFilterFactory"}, {"class": "solr.StopFilterFactory", "words": "lang/stopwords_en.txt", "ignoreCase": "true"}, {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true", "synonyms": "lang/synonyms_en.txt"}, + {"class": "solr.FlattenGraphFilterFactory"}, {"class": "solr.LowerCaseFilterFactory"}, {"class": "solr.EnglishPossessiveFilterFactory"}, {"class": "solr.KeywordMarkerFilterFactory", "protected": "protwords.txt"}, @@ -44,19 +44,30 @@ "name": "search_text_fr", "class": "solr.TextField", "positionIncrementGap": "100", - "analyzer": { + "indexAnalyzer": { "charFilters": [{"class": "solr.HTMLStripCharFilterFactory"}], "tokenizer": {"class": "solr.StandardTokenizerFactory"}, "filters": [ - {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true", "synonyms": "lang/synonyms_fr.txt"}, - {"class": "solr.FlattenGraphFilterFactory"}, {"class": "solr.ElisionFilterFactory", "articles": "lang/contractions_fr.txt", "ignoreCase": "true"}, {"class": "solr.LowerCaseFilterFactory"}, {"class": "solr.StopFilterFactory", "format": "snowball", "words": "lang/stopwords_fr.txt", "ignoreCase": "true"}, - {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true", "synonyms": "lang/synonyms_fr.txt"}, {"class": "solr.FrenchLightStemFilterFactory"} ] - } + }, + "queryAnalyzer": { + "charFilters": [{"class": "solr.HTMLStripCharFilterFactory"}], + "tokenizer": {"class": "solr.StandardTokenizerFactory"}, + "filters": [ + {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true", + "synonyms": "lang/synonyms_fr.txt"}, + {"class": "solr.FlattenGraphFilterFactory"}, + {"class": "solr.ElisionFilterFactory", "articles": "lang/contractions_fr.txt", "ignoreCase": "true"}, + {"class": "solr.LowerCaseFilterFactory"}, + {"class": "solr.StopFilterFactory", "format": "snowball", "words": "lang/stopwords_fr.txt", + "ignoreCase": "true"}, + {"class": "solr.FrenchLightStemFilterFactory"} + ] + }, } diff --git a/search/views.py b/search/views.py index 0b06343..c954ac7 100644 --- a/search/views.py +++ b/search/views.py @@ -1,7 +1,6 @@ import collections import csv from datetime import datetime -from django.views.decorators.cache import never_cache from django.conf import settings from django.core.cache import caches from django.core.exceptions import ObjectDoesNotExist @@ -784,7 +783,7 @@ class ExportStatusView(View): def __init__(self): super().__init__() - @never_cache + def get(self, request: HttpRequest, lang='en', search_type='', task_id=''): translation.activate(lang) response_dict = {"file_url": ""}