Merge pull request #31 from SrZorro/master

Fixes #27 and corrects comments from pull #27

Dockerfile

@@ -46,11 +46,11 @@ RUN apk upgrade --update && \
     touch /etc/volumerize/remove-all-inc-of-but-n-full /etc/volumerize/remove-all-but-n-full /etc/volumerize/startContainers /etc/volumerize/stopContainers \
       /etc/volumerize/backup /etc/volumerize/backupIncremental /etc/volumerize/backupFull /etc/volumerize/restore \
       /etc/volumerize/periodicBackup /etc/volumerize/verify /etc/volumerize/cleanup /etc/volumerize/remove-older-than /etc/volumerize/cleanCacheLocks \
-      /etc/volumerize/postexecute /etc/volumerize/prexecute && \
+      /etc/volumerize/prepoststrategy && \
     chmod +x /etc/volumerize/remove-all-inc-of-but-n-full /etc/volumerize/remove-all-but-n-full /etc/volumerize/startContainers /etc/volumerize/stopContainers \
       /etc/volumerize/backup /etc/volumerize/backupIncremental /etc/volumerize/backupFull /etc/volumerize/restore \
       /etc/volumerize/periodicBackup /etc/volumerize/verify /etc/volumerize/cleanup /etc/volumerize/remove-older-than /etc/volumerize/cleanCacheLocks \
-      /etc/volumerize/postexecute /etc/volumerize/prexecute && \
+      /etc/volumerize/prepoststrategy && \
     # Install Jobber
     export JOBBER_HOME=/tmp/jobber && \
     export JOBBER_LIB=$JOBBER_HOME/lib && \

README.md

@@ -407,13 +407,20 @@ $ docker run -d \
 
 # Post scripts and pre scripts (prepost strategies)
 
-If you mount a folder in `/prexecute` containing `.sh` files, they will be executed in alphabetical order before the backup and after the docker stop process.
 
-If you mount a folder in `/postexecute` containing `.sh` files, they will be executed in alphabetical order after the backup and before the docker start process.
+Pre-scripts must be located at `/preexecute/$duplicity_action/$your_scripts_here`.
 
-In this scripts the global variable `BACKUP_TYPE` contains the type of backup or restore, so it will contain one of this options `backup, backupIncremental, backupFull or restore` so you can customize your scripts to do one thing or another depending of the type of backup or if its a restore.
+Post-scripts must be located at `/postexecute/$duplicity_action/$your_scripts_here`.
 
-For an example and implementation for mysqldump and restore look at /prepost_strategies/mysql/README.md
+`$duplicity_action` folder must be named `backup`, `restore` or `verify`.
+
+> Note: `backup` action is the same for the scripts `backup`, `backupFull`, `backupIncremental` and `periodicBackup`.
+
+All `.sh` files located in the `$duplicity_action` folder will be executed in alphabetical order.
+
+When using prepost strategies, this will be the execution flow: `pre-scripts -> stop containers -> duplicity action -> start containers -> post-scripts`.
+
+Some premade strategies are available at [prepost strategies](prepost_strategies).
 
 # Container Scripts
 
@@ -431,8 +438,11 @@ This image creates at container startup some convenience scripts.
 | stopContainers | Stops the specified Docker containers |
 | remove-older-than | Delete older backups |
 | cleanCacheLocks | Cleanup of old Cache locks. |
-| prexecute | Execute all .sh files in /prexecute folder. |
-| postexecute | Execute all .sh files in /postexecute folder. |
+| prepoststrategy `$execution_phase` `$duplicity_action` | Execute all `.sh` files for the specified exeuction phase and duplicity action in alphabetical order. |
+
+`$execution_phase` must be `preAction` or `postAction`.
+
+`$duplicity_action` must be `backup`, `verify` or `restpore`.
 
 Example triggering script inside running container:
 

imagescripts/create_jobber.sh

@@ -13,9 +13,11 @@ cat > ${JOBBER_SCRIPT_DIR}/periodicBackup <<_EOF_
 
 set -o errexit
 
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup
 source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers
 ${DUPLICITY_COMMAND} ${JOBBER_PARAMETER_PROXY} ${DUPLICITY_MODE} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET}
 source ${VOLUMERIZE_SCRIPT_DIR}/startContainers
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup
 _EOF_
 
 JOBBER_CRON_SCHEDULE='0 0 4 * * *'

imagescripts/create_scripts.sh

@@ -8,29 +8,28 @@ source $CUR_DIR/base.sh
 
 readonly PARAMETER_PROXY='$@'
 
-cat > ${VOLUMERIZE_SCRIPT_DIR}/prexecute << '_EOF_'
+cat > ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy << '_EOF_'
 #!/bin/bash
 
 set -o errexit
 
-if [ -d "/prexecute" ]; then
-    for f in /prexecute/*; do
-        case "$f" in
-            *.sh) echo "running $f"; . "$f" ;;
-            *)    echo "ignoring $f" ;;
-        esac
-        echo
-    done
-fi
-_EOF_
+strategy_path=""
 
-cat > ${VOLUMERIZE_SCRIPT_DIR}/postexecute << '_EOF_'
-#!/bin/bash
+case $1 in
+    preAction ) strategy_path=/preexecute ;;
+    postAction ) strategy_path=/postexecute ;;
+    *) echo "Error: prepoststrategy first parameter 'execution phase' must be preAction or postAction"; exit 1 ;;
+esac
 
-set -o errexit
+case $2 in
+    backup | verify | restore ) ;;
+    *) echo "Error: porepoststrategy second parameter 'duplicity action' must be backup, verify or restore"; exit 1 ;;
+esac
+
+strategy_path=$strategy_path/$2
 
-if [ -d "/postexecute" ]; then
-    for f in /postexecute/*; do
+if [ -d "$strategy_path" ]; then
+    for f in $strategy_path/*; do
         case "$f" in
             *.sh) echo "running $f"; . "$f" ;;
             *)    echo "ignoring $f" ;;
@@ -40,65 +39,63 @@ if [ -d "/postexecute" ]; then
 fi
 _EOF_
 
-
 cat > ${VOLUMERIZE_SCRIPT_DIR}/backup <<_EOF_
 #!/bin/bash
 
 set -o errexit
 
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup
 source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers
-export BACKUP_TYPE=\$(basename -- "\$0")
-source ${VOLUMERIZE_SCRIPT_DIR}/prexecute
 ${DUPLICITY_COMMAND} ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET}
-source ${VOLUMERIZE_SCRIPT_DIR}/postexecute
 source ${VOLUMERIZE_SCRIPT_DIR}/startContainers
+PREPOSTSTRATEGY=/postexecute/backup
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup
 _EOF_
 
 cat > ${VOLUMERIZE_SCRIPT_DIR}/backupIncremental <<_EOF_
 #!/bin/bash
 
 set -o errexit
 
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup
 source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers
-export BACKUP_TYPE=\$(basename -- "\$0")
-source ${VOLUMERIZE_SCRIPT_DIR}/prexecute
 ${DUPLICITY_COMMAND} incremental ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET}
-source ${VOLUMERIZE_SCRIPT_DIR}/postexecute
 source ${VOLUMERIZE_SCRIPT_DIR}/startContainers
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup
 _EOF_
 
 cat > ${VOLUMERIZE_SCRIPT_DIR}/backupFull <<_EOF_
 #!/bin/bash
 
 set -o errexit
 
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup
 source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers
-export BACKUP_TYPE=\$(basename -- "\$0")
-source ${VOLUMERIZE_SCRIPT_DIR}/prexecute
 ${DUPLICITY_COMMAND} full ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET}
-source ${VOLUMERIZE_SCRIPT_DIR}/postexecute
 source ${VOLUMERIZE_SCRIPT_DIR}/startContainers
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup
 _EOF_
 
 cat > ${VOLUMERIZE_SCRIPT_DIR}/restore <<_EOF_
 #!/bin/bash
 
 set -o errexit
 
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction restore
 source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers
-export BACKUP_TYPE=\$(basename -- "\$0")
-source ${VOLUMERIZE_SCRIPT_DIR}/prexecute
 ${DUPLICITY_COMMAND} restore --force ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE}
-source ${VOLUMERIZE_SCRIPT_DIR}/postexecute
 source ${VOLUMERIZE_SCRIPT_DIR}/startContainers
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction restore
 _EOF_
 
 cat > ${VOLUMERIZE_SCRIPT_DIR}/verify <<_EOF_
 #!/bin/bash
 
 set -o errexit
 
-exec ${DUPLICITY_COMMAND} verify --compare-data ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE}
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction verify
+${DUPLICITY_COMMAND} verify --compare-data ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE}
+${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction verify
 _EOF_
 
 cat > ${VOLUMERIZE_SCRIPT_DIR}/cleanup <<_EOF_

prepost_strategies/mongodb/Dockerfile

@@ -0,0 +1,7 @@
+FROM blacklabelops/volumerize
+
+RUN apk add --no-cache \
+    mongodb-tools
+
+COPY postexecute /postexecute
+COPY preexecute /preexecute

prepost_strategies/mongodb/README.md

@@ -0,0 +1,55 @@
+# Using a prepost strategy to create MongoDB backups
+
+Volumerize can execute scripts before and after the backup process.
+
+With this prepost strategy you can create dump of your MongoDB containers and save it with Volumerize.
+
+## Environment Variables
+
+Aside of the required environment variables by Volumerize, this prepost strategy will require a couple of extra variables.
+MONGO_USERNAME MONGO_PASSWORD MONGO_HOST MONGO_PORT
+| Name           | Description                                                |
+| -------------- | ---------------------------------------------------------- |
+| MONGO_USERNAME | Username of the user who will perform the restore or dump. |
+| MONGO_PASSWORD | Password of the user who will perform the restore or dump. |
+| MONGO_HOST     | MongoDB IP or domain.                                      |
+| MONGO_PORT     | MongoDB port.                                              |
+
+## Example with Docker Compose
+
+```YAML
+version: "3"
+
+services:
+  mongodb:
+    image: mongo
+    ports:
+      - 27017:27017
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=root
+      - MONGO_INITDB_ROOT_PASSWORD=1234
+    volumes:
+      - mongodb:/data/db
+
+  volumerize:
+    build: ./prepost_strategies/mongodb/
+    environment:
+      - VOLUMERIZE_SOURCE=/source
+      - VOLUMERIZE_TARGET=file:///backup
+      - MONGO_USERNAME=root
+      - MONGO_PASSWORD=1234
+      - MONGO_PORT=27017
+      - MONGO_HOST=mongodb
+    volumes:
+      - volumerize-cache:/volumerize-cache
+      - backup:/backup
+    depends_on:
+      - mongodb
+
+volumes:
+  volumerize-cache:
+  mongodb:
+  backup:
+```
+
+Then execute `docker-compose exec volumerize backup` to create a backup of your database and `docker-compose exec volumerize restore` to restore it from your backup.

prepost_strategies/mongodb/postexecute/restore/mongorestore.sh

@@ -0,0 +1,7 @@
+source /preexecute/utils/check-env.sh
+
+check_env "mongorestore" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT"
+
+echo "mongorestore starts"
+mongorestore --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" ${VOLUMERIZE_SOURCE}
+echo "Import done"

prepost_strategies/mongodb/preexecute/backup/mongodump.sh

@@ -0,0 +1,9 @@
+source /preexecute/utils/check-env.sh
+
+check_env "mongodump" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT"
+
+echo "Creating $VOLUMERIZE_SOURCE folder if not exists"
+mkdir -p $VOLUMERIZE_SOURCE
+
+echo "mongodump starts"
+mongodump --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" --out ${VOLUMERIZE_SOURCE}

prepost_strategies/mysql/prexecute/utils/check-env.sh → prepost_strategies/mongodb/preexecute/utils/check-env.sh

@@ -6,8 +6,8 @@
  # @param1 - Name of the failed environment variable
  ##
 function _check_env_failed {
-    echo "[!] ENV VAR $1 IS NOT SET"
-    printf "\n====ENVIRONMENT VARIABLES FAILED====\n\n"
+    echo "Environment variable $1 is not set."
+    echo "Environment variables failed, exit 1"
     exit 1
 }
 
@@ -17,7 +17,7 @@ function _check_env_failed {
  # @param1 - Name of the environment variable
  ##
 function _check_env_ok {
-    echo "Env var $1 OK"
+    echo "Env var $1 ok."
 }
 
 ## 
@@ -26,7 +26,7 @@ function _check_env_ok {
  # @param2 to ∞ - Environment variables to check
  ##
 function check_env {
-    printf "\n====CHECKING ENVIRONMENT VARIABLES FOR $1====\n\n"
+    echo "Checking environment variables for $1."
 
     for e_var in "$@"; do
         if [ $e_var = $1 ]; then continue; fi # Jump first arg
@@ -39,5 +39,5 @@ function check_env {
         fi
 
     done
-    printf "\n====ENVIRONMENT VARIABLES OK====\n\n"
+    echo "Environment variables ok."
 }

prepost_strategies/mysql/README.md

@@ -42,7 +42,7 @@ services:
       - MYSQL_DATABASE=somedatabase
     volumes:
       - volumerize-cache:/volumerize-cache
-      - .backup:/backup
+      - backup:/backup
     depends_on:
       - mariadb
 

prepost_strategies/mysql/postexecute/restore/mysqlimport.sh

@@ -0,0 +1,7 @@
+source /preexecute/utils/check-env.sh
+
+check_env "mysqlimport" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_DATABASE"
+
+echo "mysql import starts"
+pv ${VOLUMERIZE_SOURCE}/dump-${MYSQL_DATABASE}.sql | mysql -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} $MYSQL_DATABASE
+echo "Import done"

prepost_strategies/mysql/preexecute/backup/mysqldump.sh

@@ -0,0 +1,12 @@
+source /preexecute/utils/check-env.sh
+
+check_env "Mysqldump" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_DATABASE"
+
+echo "Creating $VOLUMERIZE_SOURCE folder if not exists"
+mkdir -p $VOLUMERIZE_SOURCE
+
+# Based on this answer https://stackoverflow.com/a/32361604
+SIZE_BYTES=$(mysql --skip-column-names -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} $MYSQL_DATABASE -e "SELECT ROUND(SUM(data_length * 0.8), 0) FROM information_schema.TABLES WHERE table_schema='${MYSQL_DATABASE}';")
+
+echo "mysqldump starts (Progress is aproximated)"
+mysqldump --databases "${MYSQL_DATABASE}" --single-transaction --add-drop-database --user="${MYSQL_USERNAME}" --password="${MYSQL_PASSWORD}" --host="${MYSQL_HOST}" | pv --progress --size "$SIZE_BYTES" > ${VOLUMERIZE_SOURCE}/dump-${MYSQL_DATABASE}.sql

prepost_strategies/mysql/preexecute/utils/check-env.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+## 
+ # @private
+ # Executed if a variable is not set or is empty string
+ # @param1 - Name of the failed environment variable
+ ##
+function _check_env_failed {
+    echo "Environment variable $1 is not set."
+    echo "Environment variables failed, exit 1"
+    exit 1
+}
+
+## 
+ # @private
+ # Executed if a variable is setted
+ # @param1 - Name of the environment variable
+ ##
+function _check_env_ok {
+    echo "Env var $1 ok."
+}
+
+## 
+ # Use it to check if environment variables are set
+ # @param1      - Name of the context
+ # @param2 to ∞ - Environment variables to check
+ ##
+function check_env {
+    echo "Checking environment variables for $1."
+
+    for e_var in "$@"; do
+        if [ $e_var = $1 ]; then continue; fi # Jump first arg
+
+        # Check if env var is setted, if not raise error
+        if [ "${!e_var}" = "" ]; then 
+            _check_env_failed $e_var; 
+        else 
+            _check_env_ok $e_var; 
+        fi
+
+    done
+    echo "Environment variables ok."
+}