From 2cce341ab78e4e35c9299141caac69ae639cd488 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 23 Dec 2016 10:37:58 -0500 Subject: [PATCH 1/2] use new startTime and endTime metadata if present --- scripts/logfetch/logfetch_base.py | 4 ++-- scripts/logfetch/s3_logs.py | 15 +++++++++++++-- scripts/setup.py | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/logfetch/logfetch_base.py b/scripts/logfetch/logfetch_base.py index b49def02aa..2fd61915ff 100644 --- a/scripts/logfetch/logfetch_base.py +++ b/scripts/logfetch/logfetch_base.py @@ -50,7 +50,7 @@ def all_tasks_for_request(args, request): elif len(active_tasks) == 0: return historical_tasks else: - return active_tasks + [h for h in historical_tasks if is_task_in_date_range(args, int(str(h['updatedAt'])[0:-3]), int(str(h['taskId']['startedAt'])[0:-3]))] + return active_tasks + [h for h in historical_tasks if date_range_overlaps(args, int(str(h['updatedAt'])[0:-3]), int(str(h['taskId']['startedAt'])[0:-3]))] else: return active_tasks @@ -72,7 +72,7 @@ def is_in_date_range(args, timestamp): else: return False if timstamp_datetime < args.start else True -def is_task_in_date_range(args, start, end): +def date_range_overlaps(args, start, end): start_datetime = datetime.utcfromtimestamp(start) end_datetime = datetime.utcfromtimestamp(end) if args.end: diff --git a/scripts/logfetch/s3_logs.py b/scripts/logfetch/s3_logs.py index fcd88c8a24..37cdf94074 100644 --- a/scripts/logfetch/s3_logs.py +++ b/scripts/logfetch/s3_logs.py @@ -23,8 +23,8 @@ def download_s3_logs(args): async_requests = [] all_logs = [] for log_file in logs: - filename = log_file['key'].rsplit("/", 1)[1] - if logfetch_base.is_in_date_range(args, int(str(log_file['lastModified'])[0:-3])): + if log_file_in_date_range(args, log_file): + filename = log_file['key'].rsplit("/", 1)[1] if not args.logtype or log_matches(args, filename): logfetch_base.log(colored('Including log {0}'.format(filename), 'blue') + '\n', args, True) if not already_downloaded(args.dest, filename): @@ -48,6 +48,17 @@ def download_s3_logs(args): all_logs = modify_download_list(all_logs) return all_logs +def log_file_in_date_range(args, log_file): + if 'startTime' in log_file: + if 'endTime' in log_file: + return logfetch_base.date_range_overlaps(args, int(str(log_file['startTime'])[0:-3]), int(str(log_file['endTime'])[0:-3])) + else: + return logfetch_base.date_range_overlaps(args, int(str(log_file['startTime'])[0:-3]), int(str(log_file['lastModified'])[0:-3])) + elif 'endTime' in log_file: + return logfetch_base.is_in_date_range(args, int(str(log_file['endTime'])[0:-3])) + else: + return logfetch_base.is_in_date_range(args, int(str(log_file['lastModified'])[0:-3])) + def modify_download_list(all_logs): for index, log in enumerate(all_logs): if log.endswith('.gz') and not os.path.isfile(log) and os.path.isfile(log[:-3]): diff --git a/scripts/setup.py b/scripts/setup.py index ec0f76d96a..542d9d284e 100644 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -11,7 +11,7 @@ setup( name='singularity-logfetch', - version='0.28.0', + version='0.29.0', description='Singularity log fetching and searching', author="HubSpot", author_email='singularity-users@googlegroups.com', From 2940b6c83b4f6ecb8d10c2841768f71bbf262cd8 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 23 Dec 2016 10:40:05 -0500 Subject: [PATCH 2/2] don't update progress bar in verbose mode --- scripts/logfetch/live_logs.py | 2 +- scripts/logfetch/s3_logs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/logfetch/live_logs.py b/scripts/logfetch/live_logs.py index 362e8f0fd2..014a8690ee 100644 --- a/scripts/logfetch/live_logs.py +++ b/scripts/logfetch/live_logs.py @@ -50,7 +50,7 @@ def download_live_logs(args): elif args.logtype: logfetch_base.log(colored('Excluding log {0}, doesn\'t match {1}'.format(log_file, args.logtype), 'magenta') + '\n', args, True) tasks_check_progress += 1 - logfetch_base.update_progress_bar(tasks_check_progress, tasks_check_goal, 'Log Finder', args.silent) + logfetch_base.update_progress_bar(tasks_check_progress, tasks_check_goal, 'Log Finder', args.silent or args.verbose) if async_requests: logfetch_base.log(colored('\nStarting {0} live logs downloads\n'.format(len(async_requests)), 'cyan'), args, False) diff --git a/scripts/logfetch/s3_logs.py b/scripts/logfetch/s3_logs.py index 37cdf94074..c2bfef0e49 100644 --- a/scripts/logfetch/s3_logs.py +++ b/scripts/logfetch/s3_logs.py @@ -82,7 +82,7 @@ def logs_for_all_requests(args): s3_logs = logfetch_base.get_json_response(s3_task_logs_uri(args, task), args, s3_params) logs = logs + s3_logs if s3_logs else logs tasks_progress += 1 - logfetch_base.update_progress_bar(tasks_progress, tasks_goal, 'S3 Log Finder', args.silent) + logfetch_base.update_progress_bar(tasks_progress, tasks_goal, 'S3 Log Finder', args.silent or args.verbose) logfetch_base.log(colored('\nAlso searching s3 history...\n', 'cyan'), args, False) for request in logfetch_base.all_requests(args): s3_logs = logfetch_base.get_json_response(s3_request_logs_uri(args, request), args, s3_params)