Merge pull request #73 from rok4/develop

Release 2.0.1

.github/dependabot.yml

@@ -2,6 +2,7 @@ version: 2
 updates:
   - package-ecosystem: pip
     directory: "/"
+    target-branch: develop
     schedule:
       interval: monthly
       time: "17:00"
@@ -11,6 +12,7 @@ updates:
 
   - package-ecosystem: "github-actions"
     directory: "/"
+    target-branch: develop
     schedule:
       interval: monthly
       time: "22:00"

.github/workflows/build-and-release.yaml

@@ -16,7 +16,7 @@ jobs:
       job_status: ${{ job.status }}
     steps:
       - name: Checkout project
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Create Release
         id: create_release
@@ -49,7 +49,7 @@ jobs:
 
     steps:
       - name: Checkout project
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup python
         uses: actions/setup-python@v4
@@ -164,7 +164,7 @@ jobs:
     steps:
 
       - name: Checkout project on gh-pages
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: 'gh-pages'
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -213,7 +213,7 @@ jobs:
 
     steps:
       - name: Remove release and tag
-        uses: dev-drprasad/delete-tag-and-release@v0.2.0
+        uses: dev-drprasad/delete-tag-and-release@v1.0.1
         with:
           tag_name: ${{ github.ref_name }}
           delete_release: true

.github/workflows/build-docs.yaml

@@ -13,7 +13,7 @@ jobs:
 
     steps:
       - name: Checkout project
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: 'gh-pages'
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -64,4 +64,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v2

.pre-commit-config.yaml

@@ -22,13 +22,13 @@ repos:
         args: [--markdown-linebreak-ext=md]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.0.281"
+    rev: "v0.0.291"
     hooks:
       - id: ruff
         args: ["--fix-only", "--target-version=py38"]
 
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.9.1
     hooks:
       - id: black
         args: ["--target-version=py38"]
@@ -40,7 +40,7 @@ repos:
         args: ["--profile", "black", "--filter-files"]
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+    rev: v3.13.0
     hooks:
       - id: pyupgrade
         args:

CHANGELOG.md

@@ -1,3 +1,13 @@
+## 2.0.1
+
+### [Added]
+
+* `storage` : le cache de lecture est configurable en taille (avec ROK4_READING_LRU_CACHE_SIZE) et en temps de rétention (avec ROK4_READING_LRU_CACHE_TTL)
+
+### [Security]
+
+* Montée de version de pillow (faille de sécurité liée à libwebp)
+
 ## 2.0.0
 
 ### [Fixed]

src/rok4/storage.py

@@ -9,6 +9,12 @@
 
 According to functions, all storage types are not necessarily available.
 
+Readings uses a LRU cache system with a TTL. It's possible to configure it with environment variables :
+- ROK4_READING_LRU_CACHE_SIZE : Number of cached element. Default 64. Set 0 or a negative integer to configure a cache without bound. A power of two make cache more efficient.
+- ROK4_READING_LRU_CACHE_TTL : Validity duration of cached element, in seconds. Default 300. 0 or negative integer to disable time validity.
+
+To disable cache, set ROK4_READING_LRU_CACHE_SIZE to 1 and ROK4_READING_LRU_CACHE_TTL to 0.
+
 Using CEPH storage requires environment variables :
 - ROK4_CEPH_CONFFILE
 - ROK4_CEPH_USERNAME
@@ -69,10 +75,34 @@
 __S3_CLIENTS = {}
 __S3_DEFAULT_CLIENT = None
 
+__LRU_SIZE = 64
+__LRU_TTL = 300
+
+try:
+    __LRU_SIZE = int(os.environ["ROK4_READING_LRU_CACHE_SIZE"])
+    if __LRU_SIZE < 1:
+        __LRU_SIZE = None
+except ValueError:
+    pass
+except KeyError:
+    pass
+
+try:
+    __LRU_TTL = int(os.environ["ROK4_READING_LRU_CACHE_TTL"])
+    if __LRU_TTL < 0:
+        __LRU_TTL = 0
+except ValueError:
+    pass
+except KeyError:
+    pass
+
 
 def __get_ttl_hash():
-    """Return the same value withing 5 minutes time period"""
-    return round(time.time() / 300)
+    """Return the time string rounded according to time-to-live value"""
+    if __LRU_TTL == 0:
+        return time.time()
+    else:
+        return round(time.time() / __LRU_TTL)
 
 
 def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", str]], str, str]:
@@ -294,7 +324,7 @@ def get_data_str(path: str) -> str:
     return get_data_binary(path).decode("utf-8")
 
 
-@lru_cache(maxsize=50)
+@lru_cache(maxsize=__LRU_SIZE)
 def __get_cached_data_binary(path: str, ttl_hash: int, range: Tuple[int, int] = None) -> str:
     """Load data into a binary string, using a LRU cache