diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9327b14..867bf21 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,6 +2,7 @@ version: 2 updates: - package-ecosystem: pip directory: "/" + target-branch: develop schedule: interval: monthly time: "17:00" @@ -11,6 +12,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" + target-branch: develop schedule: interval: monthly time: "22:00" diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index eb7796c..b464b69 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -16,7 +16,7 @@ jobs: job_status: ${{ job.status }} steps: - name: Checkout project - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create Release id: create_release @@ -49,7 +49,7 @@ jobs: steps: - name: Checkout project - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup python uses: actions/setup-python@v4 @@ -164,7 +164,7 @@ jobs: steps: - name: Checkout project on gh-pages - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: 'gh-pages' token: ${{ secrets.GITHUB_TOKEN }} @@ -213,7 +213,7 @@ jobs: steps: - name: Remove release and tag - uses: dev-drprasad/delete-tag-and-release@v0.2.0 + uses: dev-drprasad/delete-tag-and-release@v1.0.1 with: tag_name: ${{ github.ref_name }} delete_release: true diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml index 459496b..53fd700 100644 --- a/.github/workflows/build-docs.yaml +++ b/.github/workflows/build-docs.yaml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout project - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: 'gh-pages' token: ${{ secrets.GITHUB_TOKEN }} @@ -64,4 +64,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v1 + uses: actions/deploy-pages@v2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5bc54ba..f32290e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,13 +22,13 @@ repos: args: [--markdown-linebreak-ext=md] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.0.281" + rev: "v0.0.291" hooks: - id: ruff args: ["--fix-only", "--target-version=py38"] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.9.1 hooks: - id: black args: ["--target-version=py38"] @@ -40,7 +40,7 @@ repos: args: ["--profile", "black", "--filter-files"] - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 + rev: v3.13.0 hooks: - id: pyupgrade args: diff --git a/CHANGELOG.md b/CHANGELOG.md index 59c51ed..31e31a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## 2.0.1 + +### [Added] + +* `storage` : le cache de lecture est configurable en taille (avec ROK4_READING_LRU_CACHE_SIZE) et en temps de rétention (avec ROK4_READING_LRU_CACHE_TTL) + +### [Security] + +* Montée de version de pillow (faille de sécurité liée à libwebp) + ## 2.0.0 ### [Fixed] diff --git a/src/rok4/storage.py b/src/rok4/storage.py index fc7a775..a3c6975 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -9,6 +9,12 @@ According to functions, all storage types are not necessarily available. +Readings uses a LRU cache system with a TTL. It's possible to configure it with environment variables : +- ROK4_READING_LRU_CACHE_SIZE : Number of cached element. Default 64. Set 0 or a negative integer to configure a cache without bound. A power of two make cache more efficient. +- ROK4_READING_LRU_CACHE_TTL : Validity duration of cached element, in seconds. Default 300. 0 or negative integer to disable time validity. + +To disable cache, set ROK4_READING_LRU_CACHE_SIZE to 1 and ROK4_READING_LRU_CACHE_TTL to 0. + Using CEPH storage requires environment variables : - ROK4_CEPH_CONFFILE - ROK4_CEPH_USERNAME @@ -69,10 +75,34 @@ __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None +__LRU_SIZE = 64 +__LRU_TTL = 300 + +try: + __LRU_SIZE = int(os.environ["ROK4_READING_LRU_CACHE_SIZE"]) + if __LRU_SIZE < 1: + __LRU_SIZE = None +except ValueError: + pass +except KeyError: + pass + +try: + __LRU_TTL = int(os.environ["ROK4_READING_LRU_CACHE_TTL"]) + if __LRU_TTL < 0: + __LRU_TTL = 0 +except ValueError: + pass +except KeyError: + pass + def __get_ttl_hash(): - """Return the same value withing 5 minutes time period""" - return round(time.time() / 300) + """Return the time string rounded according to time-to-live value""" + if __LRU_TTL == 0: + return time.time() + else: + return round(time.time() / __LRU_TTL) def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", str]], str, str]: @@ -294,7 +324,7 @@ def get_data_str(path: str) -> str: return get_data_binary(path).decode("utf-8") -@lru_cache(maxsize=50) +@lru_cache(maxsize=__LRU_SIZE) def __get_cached_data_binary(path: str, ttl_hash: int, range: Tuple[int, int] = None) -> str: """Load data into a binary string, using a LRU cache