From 39eea48f637cbd95bbdd6ea42d7e9e4b9b018dc1 Mon Sep 17 00:00:00 2001 From: Guslington Date: Tue, 15 Jan 2019 11:01:27 +1100 Subject: [PATCH 1/2] fixes for sqs offloading for sharing and copying regions --- shelvery/engine.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/shelvery/engine.py b/shelvery/engine.py index a29681e..c45a2a4 100644 --- a/shelvery/engine.py +++ b/shelvery/engine.py @@ -452,15 +452,16 @@ def do_copy_backup(self, map_args={}, **kwargs): """ kwargs.update(map_args) - + backup_id = kwargs['BackupId'] + origin_region = kwargs['OriginRegion'] # if backup is not available, exit and rely on recursive lambda call copy backup # in non lambda mode this should never happen if RuntimeConfig.is_offload_queueing(self): - if not self.is_backup_available(backup_region, backup_id): - self.store_backup_data(self.get_backup_resource(backup_region, backup_id)) + if not self.is_backup_available(origin_region,backup_id): + self.copy_backup(self.get_backup_resource(origin_region, backup_id)) else: - if not self.wait_backup_available(backup_region=kwargs['OriginRegion'], - backup_id=kwargs['BackupId'], + if not self.wait_backup_available(backup_region=origin_region, + backup_id=backup_id, lambda_method='do_copy_backup', lambda_args=kwargs): return @@ -544,21 +545,21 @@ def do_copy_backup(self, map_args={}, **kwargs): def do_share_backup(self, map_args={}, **kwargs): """Share backup with other AWS account, actual implementation""" kwargs.update(map_args) - + backup_id = kwargs['BackupId'] + backup_region = kwargs['Region'] + backup_resource = self.get_backup_resource(backup_region, backup_id) # if backup is not available, exit and rely on recursive lambda call do share backup # in non lambda mode this should never happen if RuntimeConfig.is_offload_queueing(self): if not self.is_backup_available(backup_region, backup_id): - self.store_backup_data(self.get_backup_resource(backup_region, backup_id)) + self.share_backup(self.get_backup_resource(backup_region, backup_id)) else: - if not self.wait_backup_available(backup_region=kwargs['Region'], - backup_id=kwargs['BackupId'], + if not self.wait_backup_available(backup_region=backup_region, + backup_id=backup_id, lambda_method='do_share_backup', lambda_args=kwargs): return - backup_region = kwargs['Region'] - backup_id = kwargs['BackupId'] destination_account_id = kwargs['AwsAccountId'] self.logger.info(f"Do share backup {backup_id} ({backup_region}) with {destination_account_id}") try: @@ -615,12 +616,12 @@ def do_store_backup_data(self, map_args={}, **kwargs): kwargs.update(map_args) backup_id = kwargs['BackupId'] backup_region = kwargs['BackupRegion'] - + backup_resource = self.get_backup_resource(backup_region, backup_id) # if backup is not available, exit and rely on recursive lambda call write metadata # in non lambda mode this should never happen if RuntimeConfig.is_offload_queueing(self): if not self.is_backup_available(backup_region, backup_id): - self.store_backup_data(self.get_backup_resource(backup_region, backup_id)) + self.store_backup_data(backup_resource) else: if not self.wait_backup_available(backup_region=backup_region, backup_id=backup_id, @@ -628,7 +629,6 @@ def do_store_backup_data(self, map_args={}, **kwargs): lambda_args=kwargs): return - backup_resource = self.get_backup_resource(backup_region, backup_id) if backup_resource.account_id is None: backup_resource.account_id = self.account_id bucket = self._get_data_bucket(backup_resource.region) From 646df6873aa293fce7ddf117fe3afa3cbfa028a2 Mon Sep 17 00:00:00 2001 From: Guslington Date: Tue, 15 Jan 2019 13:14:28 +1100 Subject: [PATCH 2/2] version bump to 0.8.1 --- README.md | 6 +++--- deploy-sam-template.sh | 2 +- setup.py | 2 +- shelvery/__init__.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 931e563..4fe5aef 100644 --- a/README.md +++ b/README.md @@ -391,10 +391,10 @@ $ shelvery ebs pull_shared_backups ## Waiting on backups to complete -By default shelvery will wait by sleeping and then querying the aws api for a complete status. +By default shelvery will wait by sleeping and then querying the aws api for a complete status. If this is not your preferred method you can offload the sleep to SQS to save costs on lambda compute. -You can set the sqs url and the wait period (seconds) before lambda is invoked to check on the status of the backup. +You can set the sqs url and the wait period (seconds) before lambda is invoked to check on the status of the backup. If the backup is not complete it will be passed back to sqs to wait for the same period. ```text @@ -414,4 +414,4 @@ shelvery_sqs_queue_wait_period=300 2. Run the `deploy-sam-template.sh` script with the options to deploy the template in the target account. - `-b` [required] source bucket to deploy the sam package to - - `-v` [optional] shelvery version to deploy, defaults to `0.8.0` + - `-v` [optional] shelvery version to deploy, defaults to `0.8.1` diff --git a/deploy-sam-template.sh b/deploy-sam-template.sh index e47aa5a..6f9d9ab 100755 --- a/deploy-sam-template.sh +++ b/deploy-sam-template.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -SHELVERY_VERSION=0.8.0 +SHELVERY_VERSION=0.8.1 while getopts ":b:v:a:" opt; do case $opt in diff --git a/setup.py b/setup.py index 2743edd..98dcc1d 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup -setup(name='shelvery', version='0.8.0', author='Base2Services R&D', +setup(name='shelvery', version='0.8.1', author='Base2Services R&D', author_email='itsupport@base2services.com', url='http://github.com/base2Services/shelvery-aws-backups', classifiers=[ diff --git a/shelvery/__init__.py b/shelvery/__init__.py index 9865b94..6bb042d 100644 --- a/shelvery/__init__.py +++ b/shelvery/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.8.0' +__version__ = '0.8.1' LAMBDA_WAIT_ITERATION = 'lambda_wait_iteration' S3_DATA_PREFIX = 'backups' SHELVERY_DO_BACKUP_TAGS = ['True', 'true', '1', 'TRUE']