✨ added CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX env to facilitate testing (

⚠️ devops) (ITISFoundation#5251)
mguidon · Jan 18, 2024 · c0e329d · c0e329d
1 parent b4088ad
commit c0e329d
Show file tree

Hide file tree

Showing 8 changed files with 16 additions and 7 deletions.
diff --git a/.env-devel b/.env-devel
@@ -38,6 +38,7 @@ CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=5
 CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null
 CLUSTERS_KEEPER_TASK_INTERVAL=30
 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null
+CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX=""
 
 
 DASK_SCHEDULER_HOST=dask-scheduler

diff --git a/services/clusters-keeper/.env-devel b/services/clusters-keeper/.env-devel
@@ -8,6 +8,7 @@ PRIMARY_EC2_INSTANCES_KEY_NAME=XXXXXXXXXX
 PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS=XXXXXXXXXX
 PRIMARY_EC2_INSTANCES_SUBNET_ID=XXXXXXXXXX
 EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY=XXXXXXXXXX
+CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX="testing"
 LOG_FORMAT_LOCAL_DEV_ENABLED=True
 RABBIT_HOST=rabbit
 RABBIT_PASSWORD=test

diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
@@ -219,6 +219,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
         auto_default_from_env=True
     )
 
+    CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX: str = Field(
+        ...,
+        description="set a prefix to all machines created (useful for testing)",
+    )
+
     CLUSTERS_KEEPER_RABBITMQ: RabbitSettings | None = Field(auto_default_from_env=True)
 
     CLUSTERS_KEEPER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True

diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py
@@ -25,13 +25,13 @@ def get_cluster_name(
     wallet_id: WalletID | None,
     is_manager: bool,
 ) -> str:
-    return f"{CLUSTER_NAME_PREFIX}{'manager' if is_manager else 'worker'}-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}"
+    return f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}{CLUSTER_NAME_PREFIX}{'manager' if is_manager else 'worker'}-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}"
 
 
 def _minimal_identification_tag(app_settings: ApplicationSettings) -> EC2Tags:
     return {
-        AWSTagKey(".".join([_APPLICATION_TAG_KEY, "deploy"])): AWSTagValue(
-            app_settings.SWARM_STACK_NAME
+        AWSTagKey(".".join([_APPLICATION_TAG_KEY, "deploy",])): AWSTagValue(
+            f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}{app_settings.SWARM_STACK_NAME}"
         )
     }
 

diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py
@@ -100,6 +100,7 @@ def app_environment(
             "CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID": faker.pystr(),
             "CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY": faker.pystr(),
             "CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES": "{}",
+            "CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX": faker.pystr(),
             "PRIMARY_EC2_INSTANCES_KEY_NAME": faker.pystr(),
             "PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps(
                 faker.pylist(allowed_types=(str,))

diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters.py b/services/clusters-keeper/tests/unit/test_modules_clusters.py
@@ -76,9 +76,9 @@ async def _assert_cluster_instance_created(
     assert all("Value" in x for x in instance_ec2_tags)
 
     _EXPECTED_TAGS: dict[str, str] = {
-        f"{_APPLICATION_TAG_KEY}.deploy": app_settings.SWARM_STACK_NAME,
+        f"{_APPLICATION_TAG_KEY}.deploy": f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}{app_settings.SWARM_STACK_NAME}",
         f"{_APPLICATION_TAG_KEY}.version": f"{APP_VERSION}",
-        "Name": f"{CLUSTER_NAME_PREFIX}manager-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}",
+        "Name": f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}{CLUSTER_NAME_PREFIX}manager-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}",
         "user_id": f"{user_id}",
         "wallet_id": f"{wallet_id}",
         "osparc-tag": "the pytest tag is here",

diff --git a/services/clusters-keeper/tests/unit/test_utils_ec2.py b/services/clusters-keeper/tests/unit/test_utils_ec2.py
@@ -41,14 +41,14 @@ def test_get_cluster_name(
         get_cluster_name(
             app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=True
         )
-        == f"osparc-computational-cluster-manager-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}"
+        == f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}osparc-computational-cluster-manager-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}"
     )
     # worker
     assert (
         get_cluster_name(
             app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=False
         )
-        == f"osparc-computational-cluster-worker-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}"
+        == f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}osparc-computational-cluster-worker-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}"
     )
 
 

diff --git a/services/docker-compose.yml b/services/docker-compose.yml
@@ -151,6 +151,7 @@ services:
       - CLUSTERS_KEEPER_EC2_ENDPOINT=${CLUSTERS_KEEPER_EC2_ENDPOINT}
       - CLUSTERS_KEEPER_EC2_REGION_NAME=${CLUSTERS_KEEPER_EC2_REGION_NAME}
       - CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY=${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY}
+      - CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX=${CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}
       - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED}
       - CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=${CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES}
       - PRIMARY_EC2_INSTANCES_ALLOWED_TYPES=${PRIMARY_EC2_INSTANCES_ALLOWED_TYPES}