OPEN-3021: Upgrading to Django 4.2.9

oc_search/settings-sample.py

@@ -44,25 +44,23 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'qurl_templatetag',
-    'search'
+    'search',
 ]
 
 ## Optional applications
 # 'ramp',
 # 'smuggler',
-
 MIDDLEWARE = [
     "corsheaders.middleware.CorsMiddleware",
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
-    "corsheaders.middleware.CorsPostCsrfMiddleware",
+    "django.middleware.common.CommonMiddleware",
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django.middleware.locale.LocaleMiddleware',
     'django.middleware.cache.UpdateCacheMiddleware',
-    'django.middleware.common.CommonMiddleware',
     'django.middleware.cache.FetchFromCacheMiddleware',
     'oc_search.middleware.CanadaBilingualMiddleware'
 ]
@@ -116,22 +114,19 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': '',
-    },
-    'search': {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
     }
 }
 
+JSON_DOWNLOADS_ALLOWED = False
+
 # Smuggler settings
 SMUGGLER_FIXTURE_DIR = os.path.join(BASE_DIR, 'smuggler')
 SMUGGLER_EXCLUDE_LIST = ['admin.logentry', 'auth.permission', 'auth.group', 'auth.user',
                          'contenttypes.contenttype',
                          'django_celery_results.chordcounter', 'django_celery_results.groupresult', 'django_celery_beat.taskresult']
 
-DATABASE_ROUTERS = ['search.db_router.SearchRouter']
-
 # Password validation
 # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
 
@@ -236,10 +231,14 @@
     'local': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
         'LOCATION': 'oc_search',
-    }
+    },
+    'redis': {
+        'BACKEND': 'django.core.cache.backends.redis.RedisCache',
+        'LOCATION': 'redis://:password@localhost:6379',
+    },
 }
 # Object in the local cache expire after this many seconds. Not recommended to be less than 60 seconds.
-CACHE_LOCAL_TIMEOUT = 60 * 5
+CACHE_LOCAL_TIMEOUT = 30
 
 SESSION_ENGINE="django.contrib.sessions.backends.file"
 SESSION_FILE_PATH = os.path.join(BASE_DIR, 'session')
@@ -287,13 +286,19 @@
 SEARCH_LOGGING_ARCHIVE_FILE = os.path.join(BASE_DIR, 'data', 'search_logs.log')
 SEARCH_LOGGING_ARCHIVE_AFTER_X_DAYS = 7
 
-# Used by the Suggested Datasets Search
+# Used by the Suggested Datasets Search if enabled
 
 SD_COMMENTS_BASE_EN = "http://127.0.0.1:8000/static/sd/"
 SD_COMMENTS_BASE_FR = "http://127.0.0.1:8000/static/sd/"
+SD_SUGGEST_A_DATASET_EN = "https://o127.0.0.1:8000/en/forms/suggest-dataset"
+SD_SUGGEST_A_DATASET_FR = "https://127.0.0.1:8000/fr/formulaire/proposez-un-formulaire-densemble-de-donnees"
 SD_VOTES_BASE_EN = "http://127.0.0.1:8000/static/sd/"
 SD_VOTES_BASE_FR = "http://127.0.0.1:8000/static/sd/"
 
+SD_RECORD_URL_EN = 'http://127.0.0.1:8000/en/sd/id/'
+SD_RECORD_URL_FR = 'http://127.0.0.1:8000/fr/sd/id/'
+SD_ALERT_EMAIL_FROM = ['My Name', 'my.email', 'my.domain.org']
+
 # Used by the import_data_csv console command
 
 IMPORT_DATA_CSV_DEFAULT_DEBUG = False

oc_search/urls.py

@@ -17,6 +17,7 @@
 from django.contrib import admin
 from django.urls import path
 from django.conf.urls import include
+from django.views.decorators.cache import never_cache
 from search.views import SearchView, RecordView, ExportView, MoreLikeThisView, HomeView, DefaultView, ExportStatusView, DownloadSearchResultsView
 from ramp.views import RampView
 
@@ -42,8 +43,8 @@
         path('search/<str:lang>/<str:search_type>/similaire/<str:record_id>', MoreLikeThisView.as_view(),
              name='MLTForm'),
         path('rechercher/<str:lang>/<str:search_type>/similaire/<str:record_id>', MoreLikeThisView.as_view(), name='MLTForm'),
-        path('search/search-results/<str:lang>/<str:search_type>/<str:task_id>', ExportStatusView.as_view(), name='SearchResultsForm'),
-        path('rechercher/rapport-de-recherche/<str:lang>/<str:search_type>/<str:task_id>', ExportStatusView.as_view(), name='SearchResultsForm'),
+        path('search/search-results/<str:lang>/<str:search_type>/<str:task_id>', never_cache(ExportStatusView.as_view()), name='SearchResultsForm'),
+        path('rechercher/rapport-de-recherche/<str:lang>/<str:search_type>/<str:task_id>', never_cache(ExportStatusView.as_view()), name='SearchResultsForm'),
     ]
     if 'ramp' in settings.INSTALLED_APPS:
         urlpatterns += [
@@ -67,9 +68,9 @@
              name='MLTForm'),
         path(settings.SEARCH_HOST_PATH + '<str:search_type>/similaire/<str:record_id>', MoreLikeThisView.as_view(),
              name='MLTForm'),
-        path(settings.SEARCH_HOST_PATH + 'search-results/<str:lang>/<str:search_type>/<str:task_id>', ExportStatusView.as_view(),
+        path(settings.SEARCH_HOST_PATH + 'search-results/<str:lang>/<str:search_type>/<str:task_id>', never_cache(ExportStatusView.as_view()),
              name='SearchResultsForm'),
-        path(settings.SEARCH_HOST_PATH + 'rapport-de-recherche/<str:lang>/<str:search_type>/<str:task_id>', ExportStatusView.as_view(),
+        path(settings.SEARCH_HOST_PATH + 'rapport-de-recherche/<str:lang>/<str:search_type>/<str:task_id>', never_cache(ExportStatusView.as_view()),
              name='SearchResultsForm'),
     ]
     if 'ramp' in settings.INSTALLED_APPS:

requirements.txt

@@ -1,29 +1,27 @@
-Babel==2.12.1
-bleach==6.0.0
-celery==5.2.7
+Babel==2.14.0
+bleach==6.1.0
+celery==5.3.6
 ckanapi==4.7
-django==3.2.21
+django==4.2.9
 django-celery-beat==2.5.0
 django-celery-results==2.5.1
-django-cors-headers==3.14.0
+django-cors-headers==4.3.1
 django-jazzmin==2.6.0
 django-qurl-templatetag==0.0.14
-django-redis-cache==3.0.1
 django-redis-sessions==0.6.2
 django-smuggler==1.0.4
-docutils==0.20.1
-email-validator==2.0.0.post2
+email-validator==2.1.0.post1
 inflection==0.5.1
-markdown2==2.4.10
+markdown2==2.4.12
 nltk==3.8.1
-pandas==1.4.4
+pandas==2.2.0
 psycopg2==2.9.9
 python-dateutil==2.8.2
 PyYAML==6.0.1
-scikit-learn==1.3.2
-Unidecode==1.3.7
-uWSGI==2.0.19.1; sys_platform == 'linux'
+redis==5.0.1
+scikit-learn==1.4.0
+setuptools>=69.0.3
+Unidecode==1.3.8
+uWSGI==2.0.23; sys_platform == 'linux'
 #git+https://github.com/thriuin/SolrClient.git@master
 
-redis==3.5.3 # not directly required, but cannot use the latest as suggested by Snyk due to incompatability with django-redis-cache
-setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability

search/management/commands/create_solr_core.py

@@ -15,8 +15,6 @@
         "filters": [
             {"class": "solr.WordDelimiterGraphFilterFactory", "preserveOriginal": "1", "splitOnCaseChange": "0", "catenateAll": "1"},
             {"class": "solr.FlattenGraphFilterFactory"},
-            {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true", "synonyms": "lang/synonyms_en.txt"},
-            {"class": "solr.FlattenGraphFilterFactory"},
             {"class": "solr.StopFilterFactory", "words": "lang/stopwords_en.txt", "ignoreCase": "true"},
             {"class": "solr.LowerCaseFilterFactory"},
             {"class": "solr.EnglishPossessiveFilterFactory"},
@@ -29,8 +27,10 @@
         "tokenizer": {"class": "solr.WhitespaceTokenizerFactory"},
         "filters": [
             {"class": "solr.WordDelimiterGraphFilterFactory", "preserveOriginal": "1", "splitOnCaseChange": "0", "catenateAll": "1"},
+            {"class": "solr.FlattenGraphFilterFactory"},
             {"class": "solr.StopFilterFactory", "words": "lang/stopwords_en.txt", "ignoreCase": "true"},
             {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true", "synonyms": "lang/synonyms_en.txt"},
+            {"class": "solr.FlattenGraphFilterFactory"},
             {"class": "solr.LowerCaseFilterFactory"},
             {"class": "solr.EnglishPossessiveFilterFactory"},
             {"class": "solr.KeywordMarkerFilterFactory", "protected": "protwords.txt"},
@@ -44,19 +44,30 @@
     "name": "search_text_fr",
     "class": "solr.TextField",
     "positionIncrementGap": "100",
-    "analyzer": {
+    "indexAnalyzer": {
         "charFilters": [{"class": "solr.HTMLStripCharFilterFactory"}],
         "tokenizer": {"class": "solr.StandardTokenizerFactory"},
         "filters": [
-            {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true", "synonyms": "lang/synonyms_fr.txt"},
-            {"class": "solr.FlattenGraphFilterFactory"},
             {"class": "solr.ElisionFilterFactory", "articles": "lang/contractions_fr.txt", "ignoreCase": "true"},
             {"class": "solr.LowerCaseFilterFactory"},
             {"class": "solr.StopFilterFactory", "format": "snowball", "words": "lang/stopwords_fr.txt", "ignoreCase": "true"},
-            {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true", "synonyms": "lang/synonyms_fr.txt"},
             {"class": "solr.FrenchLightStemFilterFactory"}
         ]
-    }
+    },
+    "queryAnalyzer": {
+        "charFilters": [{"class": "solr.HTMLStripCharFilterFactory"}],
+        "tokenizer": {"class": "solr.StandardTokenizerFactory"},
+        "filters": [
+            {"class": "solr.SynonymGraphFilterFactory", "expand": "true", "ignoreCase": "true",
+             "synonyms": "lang/synonyms_fr.txt"},
+            {"class": "solr.FlattenGraphFilterFactory"},
+            {"class": "solr.ElisionFilterFactory", "articles": "lang/contractions_fr.txt", "ignoreCase": "true"},
+            {"class": "solr.LowerCaseFilterFactory"},
+            {"class": "solr.StopFilterFactory", "format": "snowball", "words": "lang/stopwords_fr.txt",
+             "ignoreCase": "true"},
+            {"class": "solr.FrenchLightStemFilterFactory"}
+        ]
+    },
 }
 
 

search/views.py

@@ -1,7 +1,6 @@
 import collections
 import csv
 from datetime import datetime
-from django.views.decorators.cache import never_cache
 from django.conf import settings
 from django.core.cache import caches
 from django.core.exceptions import ObjectDoesNotExist
@@ -784,7 +783,7 @@ class ExportStatusView(View):
     def __init__(self):
         super().__init__()
 
-    @never_cache
+
     def get(self, request: HttpRequest, lang='en', search_type='', task_id=''):
         translation.activate(lang)
         response_dict = {"file_url": ""}