fix: move restore to main in restore roles and refactor check availab…

…le storage

roles/available_storage/tasks/main.yml

@@ -1,11 +1,11 @@
 - name: Get disk information
-  shell: df -B 1 "{{ AVAILABLE_STORAGE_DISK_TO_CHECK }}" | awk 'NR==2 {print $2","$3}'
+  shell: df -BM "{{ AVAILABLE_STORAGE_DISK_TO_CHECK }}" | awk 'NR==2 {print $2","$3}'
   register: disk_info
 
 - name: Extract values
   set_fact:
-    disk_size: disk_info.stdout.split(',')[0] | int
-    disk_used: disk_info.stdout.split(',')[1] | int
+    disk_size: disk_info.stdout.split(',')[0].replace('M', '') | int
+    disk_used: disk_info.stdout.split(',')[1].replace('M', '') | int
 
 - name: Calculate free storage
   set_fact:

roles/mongo_backup/defaults/main.yml

@@ -1,10 +1,8 @@
 ---
-
 MONGO_BACKUP_USER: "admin"
 MONGO_BACKUP_PASSWORD: "mongo_password"
 MONGO_BACKUP_ALL_DATABASES: true
-MONGO_BACKUP_DATABASE:
-- edxapp
+MONGO_BACKUP_DATABASE: "edxapp"
 MONGO_BACKUP_ROOT: "/var/edunext_tmp/mongo"
 MONGO_BACKUP_DATE: "{{ ansible_date_time.date }}"
 MONGO_BACKUP_LOCATION: "{{ MONGO_BACKUP_ROOT }}/{{ MONGO_BACKUP_DATE }}.d"

roles/mongo_backup/tasks/backup.yml

@@ -6,11 +6,6 @@
     path: "{{ MONGO_BACKUP_LOCATION }}"
     state: directory
 
-- name: Convert Mongo backups databases to list to provide backwards compatibility
-  set_fact:
-    MYSQL_BACKUP_DATABASES: "{{ MONGO_BACKUP_DATABASE | split }}"
-  when: MONGO_BACKUP_DATABASE is string
-
 # To clone the databases, we use the commands recommended by MongoDB:
 # https://www.mongodb.com/docs/database-tools/mongodump/#copy-and-clone-databases
 - name: Create mongo database backup
@@ -21,7 +16,7 @@
     --gzip
     --archive={{ MONGO_BACKUP_LOCATION }}/{{ MONGO_BACKUP_DATE }}_mongo.gz
     {% if not MONGO_BACKUP_ALL_DATABASES %}
-    --db={{ MONGO_BACKUP_DATABASES | join(" ") }}
+    --db={{ MONGO_BACKUP_DATABASE }}
     {% endif %}
 
 - name: Give the server time to recover

roles/mongo_clone/defaults/main.yml

@@ -1,5 +1,6 @@
 MONGO_CLONE_USER: admin
 MONGO_CLONE_PASSWORD: admin_pass
+MONGO_CLONE_TARGET_PATH: /edx/var/mongo/mongodb
 MONGO_CLONE_ROOT: /var/edunext_tmp/mongo
 MONGO_CLONE_DB_LIST:
 - edxapp

roles/mongo_clone/tasks/clone.yml

@@ -7,8 +7,7 @@
     MONGO_BACKUP_PRE_CLEAN_ROOT: true
     MONGO_BACKUP_USER: "{{ MONGO_CLONE_USER }}"
     MONGO_BACKUP_PASSWORD: "{{ MONGO_CLONE_PASSWORD }}"
-    MONGO_BACKUP_DATABASE:
-    - "{{ item }}"
+    MONGO_BACKUP_DATABASE: "{{ item }}"
     MONGO_BACKUP_ROOT: "{{ MONGO_CLONE_ROOT }}"
     MONGO_BACKUP_STORAGE_OPTIONS:
       EXTERNAL_STORAGE_TYPE: ""

roles/mongo_clone/tasks/main.yml

@@ -1,3 +1,9 @@
+- name: Launch Mongo check available storage
+  include_role:
+    name: available_storage
+  vars:
+    AVAILABLE_STORAGE_DISK_TO_CHECK: "{{ MONGO_CLONE_TARGET_PATH }}"
+
 - name: Backup and restore every single database in the clone list
   include_tasks: clone.yml
   loop: "{{ MONGO_CLONE_DB_LIST }}"

roles/mongo_restore/tasks/main.yml

@@ -1,13 +1,17 @@
-- name: Launch Mongo check available storage
-  include_role:
-    name: available_storage
-  vars:
-    AVAILABLE_STORAGE_DISK_TO_CHECK: "{{ MONGO_VOLUMES[0].device }}"
-
-- name: Launch restore MongoDB Backup
-  include_tasks: restore.yml
-  vars:
-    MONGO_BACKUP_FILE: "{{ MONGO_RESTORE_BACKUP_LOCATION }}/{{ MONGO_RESTORE_BACKUP_DATE }}_mongo.gz"
+# To clone the databases, we use the commands recommended by MongoDB:
+# https://www.mongodb.com/docs/database-tools/mongodump/#copy-and-clone-databases
+- name: Restore database
+  shell: >
+    mongorestore
+    --authenticationDatabase admin
+    -u {{ MONGO_RESTORE_USER  }} -p '{{ MONGO_RESTORE_PASSWORD }}'
+    --gzip
+    --archive={{ MONGO_RESTORE_BACKUP_LOCATION }}/{{ MONGO_RESTORE_BACKUP_DATE }}_mongo.gz
+    --numParallelCollections={{ MONGO_RESTORE_NUM_PARALLEL_COLLECTIONS }}
+    {% if MONGO_RESTORE_DATABASE %}
+    --nsFrom={{ MONGO_RESTORE_DATABASE }}.*
+    --nsTo={{ MONGO_RESTORE_CLONE_DATABASE }}.*
+    {% endif %}
 
 - name: Clean artifact path
   file:

roles/mysql_clone/defaults/main.yml

@@ -1,5 +1,6 @@
 MYSQL_CLONE_USER: admin
 MYSQL_CLONE_PASSWORD: admin_pass
+MYSQL_CLONE_TARGET_PATH: /var/lib/mysql
 MYSQL_CLONE_ROOT: /var/edunext_tmp/mysql
 MYSQL_CLONE_DB_LIST:
 - edxapp

roles/mysql_clone/tasks/main.yml

@@ -1,3 +1,9 @@
+- name: Launch MySQL check available storage
+  include_role:
+    name: available_storage
+  vars:
+    AVAILABLE_STORAGE_DISK_TO_CHECK: "{{ MYSQL_CLONE_TARGET_PATH }}"
+
 - name: Backup and restore every single database in the clone list
   include_tasks: clone.yml
   loop: "{{ MYSQL_CLONE_DB_LIST }}"

roles/mysql_restore/tasks/main.yml

@@ -1,14 +1,13 @@
-- name: Upload the backup to a remote storage
-  include_role:
-    name: available_storage
-  vars:
-    AVAILABLE_STORAGE_DISK_TO_CHECK: "{{ MYSQL_VOLUMES[0].device }}"
+# TODO: Verify that the file is indeed a .gz file
+- name: Unzip backup file
+  shell: "gzip -d {{ MYSQL_RESTORE_BACKUP_LOCATION }}/{{ MYSQL_RESTORE_BACKUP_DATE }}_mysql.sql.gz"
+  args:
+    chdir: "{{ MYSQL_RESTORE_BACKUP_LOCATION }}"
 
-- name: Launch restore MySQL Backup
-  include_tasks: restore.yml
-  vars:
-    MYSQL_BACKUP_FILE: "{{ MYSQL_RESTORE_BACKUP_LOCATION }}/{{ MYSQL_RESTORE_BACKUP_DATE }}_mysql.sql"
-    MYSQL_BACKUP_FILE_LOCATION: "{{ MYSQL_RESTORE_BACKUP_LOCATION }}"
+- name: Restore database
+  shell: >
+    mysql -u {{ MYSQL_RESTORE_USER }} -p{{ MYSQL_RESTORE_PASSWORD }}
+    {% if MYSQL_RESTORE_DATABASE %}{{ MYSQL_RESTORE_DATABASE }}{% endif %} < {{ MYSQL_BACKUP_FILE }}
 
 - name: Clean artifact path
   file: