From 1337564747779c937505592297bdc131ca50b6a4 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 30 Aug 2022 11:02:43 +0000 Subject: [PATCH 01/28] fix: docker/nginx/Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-ALPINE315-CURL-2938004 - https://snyk.io/vuln/SNYK-ALPINE315-CURL-2938004 - https://snyk.io/vuln/SNYK-ALPINE315-PCRE2-2869383 - https://snyk.io/vuln/SNYK-ALPINE315-PCRE2-2869384 - https://snyk.io/vuln/SNYK-ALPINE315-ZLIB-2976173 --- docker/nginx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index 5acbebcdc..b9c1c7abb 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.21.6-alpine +FROM nginx:1.22-alpine COPY docker/nginx/nginx.conf /etc/nginx/nginx.conf From c5d0c35a7c3208c83d96a300ae66005aed276f04 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 30 Aug 2022 11:03:37 +0000 Subject: [PATCH 02/28] fix: docker/rekono/Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946427 - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946428 - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946723 - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946727 - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946729 --- docker/rekono/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/rekono/Dockerfile b/docker/rekono/Dockerfile index c33628171..7c96a58ec 100644 --- a/docker/rekono/Dockerfile +++ b/docker/rekono/Dockerfile @@ -25,7 +25,7 @@ WORKDIR /code ENV REKONO_HOME /rekono -FROM node:17.6.0-alpine as frontend +FROM node:17.7.2-alpine as frontend RUN mkdir /frontend COPY --from=backend /code/frontend /frontend From b5fe299b7b722615651fb642e2e56b0389da2535 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 30 Aug 2022 11:04:15 +0000 Subject: [PATCH 03/28] fix: docker/rekono/Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946427 - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946428 - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946723 - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946727 - https://snyk.io/vuln/SNYK-UPSTREAM-NODE-2946729 --- docker/rekono/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/rekono/Dockerfile b/docker/rekono/Dockerfile index 7c96a58ec..dd734d6ab 100644 --- a/docker/rekono/Dockerfile +++ b/docker/rekono/Dockerfile @@ -25,7 +25,7 @@ WORKDIR /code ENV REKONO_HOME /rekono -FROM node:17.7.2-alpine as frontend +FROM node:18.6.0-alpine as frontend RUN mkdir /frontend COPY --from=backend /code/frontend /frontend From dac4f5f04b02bb24df31c33d20d02c3a3d2a1483 Mon Sep 17 00:00:00 2001 From: cbk914 Date: Tue, 30 Aug 2022 13:40:46 +0200 Subject: [PATCH 04/28] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0b0f4a60b..62dc4dfb8 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ docker-compose up -d If you need more than one tool running at the same time, you can set the number of executions-worker instances: ``` -docker-compose up -d --scale executions-worker=5 +docker-compose up -d --scale execution-worker=5 ``` Go to https://127.0.0.1/ From e4e4eb1eb3423d6741a6e4e0184fb23bc0c42418 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Wed, 31 Aug 2022 14:59:54 +0200 Subject: [PATCH 05/28] Update CONTRIBUTING documentation --- CONTRIBUTING.md | 10 +++++++--- README.md | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bca5da53d..c8b508974 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,7 @@ Please, don't report security vulnerabilities in GitHub Issues. See our [Securit ## Contributing to Rekono -You can create Pull Requests to the `develop` branch of this project. All the Pull Requests should be reviewed and approved before been merged. After that, your code will be included on the next Rekono release. +**You can create Pull Requests to the `develop` branch of this project**. All the Pull Requests should be reviewed and approved before been merged. After that, your code will be included on the next Rekono release. In this section you can see how to achieve that and the things that you should to take into account. @@ -30,7 +30,7 @@ coverage run manage.py test ### Add support for a new hacking tool -The support of external hacking tools in Rekono is based on two steps: +The support of external hacking tools in Rekono is based on the following steps: 1. Define the hacking tools in the [tools/fixture](https://github.com/pablosnt/rekono/tree/main/rekono/tools/fixtures) files. There are one file for each required entity: @@ -54,6 +54,10 @@ The support of external hacking tools in Rekono is based on two steps: - Override the method `parse_output_file` or `parse_plain_output` depending on the tool output type. +3. Implement unit tests to check the parser correct working. You can use some [tool reports](https://github.com/pablosnt/rekono/tree/main/rekono/testing/data/reports) as example for that. + +4. Add the tool reference in the [README.md](https://github.com/pablosnt/rekono#supported-tools). + ### CI/CD This project has the following checks in _Continuous Integration_: @@ -79,5 +83,5 @@ pre-commit install There are some guidelines to keep the code clean and ensure the correct working of the application: - Comment your code, specially to document the classes and methods. -- Make unit tests of all your code to ensure its correct working. It's important to keep the testing coverage over a 95% coverage. +- Make unit tests for all your code to ensure its correct working. It's important to keep the testing coverage over a 95% coverage. - Don't include code vulnerabilities or vulnerable libraries. diff --git a/README.md b/README.md index 62dc4dfb8..0b0f4a60b 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ docker-compose up -d If you need more than one tool running at the same time, you can set the number of executions-worker instances: ``` -docker-compose up -d --scale execution-worker=5 +docker-compose up -d --scale executions-worker=5 ``` Go to https://127.0.0.1/ From b890c6f42793fc5c6fc6cc8d15b31861dee5fde0 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Wed, 31 Aug 2022 16:01:20 +0200 Subject: [PATCH 06/28] Fix unique exploit storage in database --- rekono/findings/models.py | 9 +++++---- rekono/tools/tools/metasploit.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/rekono/findings/models.py b/rekono/findings/models.py index fb74e0026..278401182 100644 --- a/rekono/findings/models.py +++ b/rekono/findings/models.py @@ -3,9 +3,6 @@ from defectdojo.constants import DD_DATE_FORMAT from django.db import models from executions.models import Execution -from findings.enums import (DataType, OSType, PathType, PortStatus, Protocol, - Severity) -from findings.utils import get_unique_filter from input_types.base import BaseInput from input_types.enums import InputKeyword from input_types.utils import get_url @@ -14,6 +11,10 @@ from targets.utils import get_target_type from tools.models import Input, Tool +from findings.enums import (DataType, OSType, PathType, PortStatus, Protocol, + Severity) +from findings.utils import get_unique_filter + # Create your models here. @@ -600,7 +601,7 @@ class Exploit(Finding): key_fields: List[Dict[str, Any]] = [ # Unique field list {'name': 'vulnerability_id', 'is_base': True}, {'name': 'technology_id', 'is_base': True}, - {'name': 'edb_id', 'is_base': False} + {'name': 'reference', 'is_base': False} ] def parse(self, accumulated: Dict[str, Any] = {}) -> Dict[str, Any]: diff --git a/rekono/tools/tools/metasploit.py b/rekono/tools/tools/metasploit.py index d66400437..56efe7d09 100644 --- a/rekono/tools/tools/metasploit.py +++ b/rekono/tools/tools/metasploit.py @@ -1,4 +1,5 @@ from findings.models import Exploit + from tools.tools.base_tool import BaseTool From cb9d25c3ef8b0f9e6c4d472f25afdf62692f458c Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Wed, 31 Aug 2022 17:34:24 +0200 Subject: [PATCH 07/28] Include parentheses as valid character in input validation --- rekono/frontend/src/backend/RekonoApi.vue | 4 ++-- rekono/security/input_validation.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rekono/frontend/src/backend/RekonoApi.vue b/rekono/frontend/src/backend/RekonoApi.vue index b3bd4ed42..52af617dc 100644 --- a/rekono/frontend/src/backend/RekonoApi.vue +++ b/rekono/frontend/src/backend/RekonoApi.vue @@ -76,8 +76,8 @@ export default { ], cancellableStatuses: ['Requested', 'Running'], timeUnits: ['Weeks', 'Days', 'Hours', 'Minutes'], - nameRegex: /^[\wÀ-ÿ\s.\-[\]]*$/, - textRegex: /^[\wÀ-ÿ\s.:,+\-'"?¿¡!#%$€[\]]*$/, + nameRegex: /^[\wÀ-ÿ\s.\-[\]()]*$/, + textRegex: /^[\wÀ-ÿ\s.:,+\-'"?¿¡!#%$€[\]()]*$/, pathRegex: /^[\w./#?&%]*$/, cveRegex: /^CVE-[0-9]{4}-[0-9]{1,7}$/, defectDojoEnabled: process.env.VUE_APP_DEFECTDOJO.toLowerCase() === 'true', diff --git a/rekono/security/input_validation.py b/rekono/security/input_validation.py index 2d5d065d1..3ed889950 100644 --- a/rekono/security/input_validation.py +++ b/rekono/security/input_validation.py @@ -5,8 +5,8 @@ logger = logging.getLogger() # Rekono logger -NAME_REGEX = r'[\wÀ-ÿ\s\.\-\[\]]*' # Regex for names validation -TEXT_REGEX = r'[\wÀ-ÿ\s\.:,+\-\'"?¿¡!#%$€\[\]]*' # Regex for text validation +NAME_REGEX = r'[\wÀ-ÿ\s\.\-\[\]()]*' # Regex for names validation +TEXT_REGEX = r'[\wÀ-ÿ\s\.:,+\-\'"?¿¡!#%$€\[\]()]*' # Regex for text validation PATH_REGEX = r'[\w\./#?&%$\\]*' # Regex for path validation CVE_REGEX = 'CVE-[0-9]{4}-[0-9]{1,7}' # Regex for CVE validation From 0d3bbe22ee657e5863ffdd628fef72ce2d6da143 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Sat, 3 Sep 2022 16:31:11 +0200 Subject: [PATCH 08/28] Optimize generation of multiple executions from previous findings --- .github/workflows/unit-testing.yml | 2 + rekono/executions/utils.py | 193 +++++++++++------------------ 2 files changed, 71 insertions(+), 124 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 6ead0fe5c..2573cb2dd 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -2,6 +2,8 @@ name: Unit testing on: workflow_dispatch: pull_request: + paths: + - 'rekono/**' env: REQUIRED_COVERAGE: 95 diff --git a/rekono/executions/utils.py b/rekono/executions/utils.py index bd16df91f..b2f06b5d0 100644 --- a/rekono/executions/utils.py +++ b/rekono/executions/utils.py @@ -4,103 +4,58 @@ from input_types.base import BaseInput from input_types.models import InputType from stringcase import snakecase -from tools.models import Argument, Tool +from tools.models import Argument, Input, Tool -def get_related_executions( - related_input_types: List[InputType], - executions: List[List[BaseInput]] -) -> Dict[int, Dict[str, Any]]: - '''Get executions with related input types. - - Args: - related_input_types (List[InputType]): Related input types to search in the execution list - executions (List[List[BaseInput]]): List with the inputs already associated to each execution - - Returns: - Dict[int, Dict[str, Any]]: Related executions data, including index, related field name and related inputs - ''' - # Will save the executions (indexes) where the inputs can be assigned based on the related input types - relations: Dict[int, Dict[str, Any]] = {} - for relation in related_input_types: # For each related input type - for index, exec_inputs in enumerate(executions): # For each execution - for i in exec_inputs: # For each input assigned to execution - related_model = relation.get_related_model_class() - callback_target = relation.get_callback_target_class() - target_field_name = snakecase(cast(Any, callback_target).__name__) if callback_target else None - # Input related to the input type model - is_model = isinstance(i, cast(Any, related_model)) if related_model else False - # Input related to the input type target - is_target = isinstance(i, cast(Any, callback_target)) if callback_target else False - if is_model or is_target: - if index in relations: - relations[index]['inputs'].append(i) # Add input to the relations - else: - relations[index] = { # Create a new relation based on index - # Input field to access the related input - 'field': relation.name.lower() if is_model else target_field_name, - 'inputs': [i] # Related input list - } - if relations: - break # Relations found - return relations - - -def is_execution_in_list(to_check: List[BaseInput], executions: List[List[BaseInput]]) -> bool: - '''Check if execution is already in the execution list. - - Args: - to_check (List[BaseInput]): Execution to check - executions (List[List[BaseInput]]): Execution list - - Returns: - bool: Indicate if execution already exists or not - ''' - existing = False - for execution in executions: # For each execution - for index, item in enumerate(execution): # For each base input in execution - if item != to_check[index]: # Execution doesn't match - existing = False - break - else: # Execution match - existing = True - if existing: # Execution found - break - return existing - - -def add_input( - argument: Argument, - base_input: BaseInput, - executions: List[List[BaseInput]], - indexes: List[int] -) -> List[List[BaseInput]]: - '''Assign base input to the properly executions based on argument 'multiple' field. - - Args: - argument (Argument): Argument related to the input type - base_input (BaseInput): BaseInput. It can be a Finding, a Resource or a Target - executions (List[List[BaseInput]]): List with the inputs already associated to each execution - indexes (List[int]): Indexes of the executions list where the base input can be assigned - - Returns: - List[List[BaseInput]]: Executions list after assign the base input - ''' - for index in indexes: # For each selected index - if argument.multiple: - # Argument multiple is True, so input should be assigned in all selected executions (indexes) - executions[index].append(base_input) +def get_executions_with_relations(base_inputs: Dict[InputType, List[BaseInput]], tool: Tool) -> List[List[BaseInput]]: + executions: List[List[BaseInput]] = [[]] # BaseInput list for each execution + # It's required because base inputs will be assigned to executions based on relationships between them + input_relations = utils.get_relations_between_input_types() # Get relations between input types + # For each input type, and his related input types + for input_type, related_input_types in list(reversed(input_relations.items())): + if input_type not in base_inputs: + continue + argument = Argument.objects.filter(tool=tool, inputs__type=input_type).order_by('inputs__order').first() + if related_input_types: + for base_input in base_inputs[input_type]: + for index, execution_list in enumerate(executions.copy()): + assigned = False + for related_input_type in related_input_types: + base_inputs_by_class = [bi for bi in execution_list if bi.__class__ == base_input.__class__] + related_target = related_input_type.get_callback_target_class() + related_target_field = snakecase(cast(Any, related_target).__name__) if related_target else None + if ( + ( + hasattr(base_input, related_input_type.name.lower()) and + getattr(base_input, related_input_type.name.lower()) in execution_list + ) or + ( + hasattr(base_input, related_target_field) and + getattr(base_input, related_target_field) in execution_list + ) + ): + if argument.multiple or len(base_inputs_by_class) == 0: + executions[index].append(base_input) + assigned = True + break + elif not argument.multiple and len(base_inputs_by_class) > 0: + new_execution = execution_list.copy() + new_execution.remove(base_inputs_by_class[0]) + new_execution.append(base_input) + executions.append(new_execution) + assigned = True + break + if assigned: + break + elif argument.multiple: + for item in range(len(executions)): + executions[item].extend(base_inputs[input_type]) else: - # Argument multiple is False, so input should be assigned to executions without inputs of same type - # Filter inputs in the current index, removing inputs of the same types - execution_copy = [f for f in executions[index] if type(f) != type(base_input)] - if len(execution_copy) == len(executions[index]): # No inputs of same type in this index - executions[index].append(base_input) # Assign base input to the current index - else: # Inputs with same type in this index - # New execution is created from the filtered index and the input - execution_copy.append(base_input) - if not is_execution_in_list(execution_copy, executions): # Check if execution exists - executions.append(execution_copy) + new_executions: List[List[BaseInput]] = [] + for base_input in base_inputs[input_type]: + for execution_list in executions: + new_executions.append(list(execution_list + [base_input])) + executions = new_executions return executions @@ -108,38 +63,28 @@ def get_executions_from_findings(base_inputs: List[BaseInput], tool: Tool) -> Li '''Get needed executions for a tool based on a given a input (Finding, Resource or Target) list. Args: - inputs (List[BaseInput]): BaseInput list + base_inputs (List[BaseInput]): BaseInput list tool (Tool): Tool that will be executed Returns: List[List[BaseInput]]: List of inputs to be passed for each tool execution ''' - executions: List[List[BaseInput]] = [[]] # BaseInput list for each execution - # It's required because base inputs will be assigned to executions based on relationships between them - input_relations = utils.get_relations_between_input_types() # Get relations between input types - # For each input type, and his related input types - for input_type, related_input_types in list(reversed(input_relations.items())): - # Get properly argument for input type - argument = Argument.objects.filter(tool=tool, inputs__type=input_type).order_by('inputs__order').first() - if not argument: - continue # No argument found - related_model = input_type.get_related_model_class() - callback_target = input_type.get_callback_target_class() - # Filter base inputs based on the input type model or target - filtered = [f for f in base_inputs if ( - (related_model and isinstance(f, cast(Any, related_model))) or - (callback_target and isinstance(f, cast(Any, callback_target))) - )] - if not filtered: - continue # No base inputs found - relations = get_related_executions(related_input_types, executions) # Get executions with related inputs - for base_input in filtered: # For each base input - # By default, can be assigned to all executions - indexes = list(range(len(executions))) - if relations: # If relations found - # Filter relations to get indexes with related inputs to the current base input - related = [i for i, v in relations.items() if getattr(base_input, v['field']) in v['inputs']] - if len(related) > 0: # Related inputs found - indexes = related - executions = add_input(argument, base_input, executions, indexes) # Assign base input to executions - return executions + tool_inputs: List[Input] = Input.objects.filter(argument__tool=tool).all() + filtered_base_inputs: Dict[InputType, List[BaseInput]] = {} + for tool_input in tool_inputs: + base_input_list = [ + bi for bi in base_inputs if bi.__class__ in [ + c for c in [tool_input.type.get_related_model_class(), tool_input.type.get_callback_target_class()] if c + ] + ] + if base_input_list: + filtered_base_inputs[tool_input.type] = base_input_list + if len(filtered_base_inputs.keys()) > 1: + return get_executions_with_relations(filtered_base_inputs, tool) + elif len(filtered_base_inputs.keys()) == 1: + argument = Argument.objects.filter(tool=tool, inputs__type=list(filtered_base_inputs.keys())[0]).first() + if argument.multiple: + return list(filtered_base_inputs.values()) + else: + return cast(List[List[BaseInput]], [[bi] for bi in list(filtered_base_inputs.values())]) + return [base_inputs] From 0a090a920a203eb920f32f28e4a5f6b4527de55b Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Sun, 4 Sep 2022 22:35:05 +0200 Subject: [PATCH 09/28] Fix executions generation --- rekono/executions/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rekono/executions/utils.py b/rekono/executions/utils.py index b2f06b5d0..f8cd8088a 100644 --- a/rekono/executions/utils.py +++ b/rekono/executions/utils.py @@ -74,7 +74,7 @@ def get_executions_from_findings(base_inputs: List[BaseInput], tool: Tool) -> Li for tool_input in tool_inputs: base_input_list = [ bi for bi in base_inputs if bi.__class__ in [ - c for c in [tool_input.type.get_related_model_class(), tool_input.type.get_callback_target_class()] if c + tool_input.type.get_related_model_class(), tool_input.type.get_callback_target_class() ] ] if base_input_list: @@ -86,5 +86,5 @@ def get_executions_from_findings(base_inputs: List[BaseInput], tool: Tool) -> Li if argument.multiple: return list(filtered_base_inputs.values()) else: - return cast(List[List[BaseInput]], [[bi] for bi in list(filtered_base_inputs.values())]) + return [[bi] for bi in list(filtered_base_inputs.values())[0]] return [base_inputs] From 7f8792ea35c520d0b074cc9cc75f7880c43b6dee Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Sun, 4 Sep 2022 22:48:54 +0200 Subject: [PATCH 10/28] Bump python-libnmap to version 0.7.3 --- .snyk | 12 ------------ requirements.txt | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.snyk b/.snyk index bfb4a6910..2b62034f9 100644 --- a/.snyk +++ b/.snyk @@ -2,16 +2,4 @@ version: v1.25.0 # ignores vulnerabilities until expiry date; change duration by modifying expiry date ignore: - SNYK-PYTHON-PYTHONLIBNMAP-2808864: - - '*': - reason: >- - python-libnmap is only used to parse nmap output. It's not used to - execute nmap. No fix available at the moment - expires: 2022-09-04T17:10:10.695Z - created: 2022-06-04T17:10:10.697Z - SNYK-JS-MOMENT-2944238: - - '*': - reason: No fix available yet - expires: 2022-09-07T21:30:04.511Z - created: 2022-07-08T21:30:04.520Z patch: {} diff --git a/requirements.txt b/requirements.txt index ec0111116..dae7da35b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pycryptodome==3.11.0 psycopg2==2.9.1 pyjwt==2.4.0 python-magic==0.4.24 -python-libnmap==0.7.2 +python-libnmap==0.7.3 python-telegram-bot==13.7 pyyaml==6.0.0 requests==2.26.0 From 18bb20391a83ad0f9a83bd64174abecb16126dd9 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 6 Sep 2022 22:20:42 +0200 Subject: [PATCH 11/28] Ensure dependencies between docker containers --- docker-compose.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a8f686636..4a54d4dbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,7 +78,7 @@ services: networks: - internal depends_on: - - postgres + - initialize - redis environment: - RKN_DB_HOST=postgres @@ -105,7 +105,7 @@ services: cap_add: - NET_ADMIN depends_on: - - postgres + - initialize - redis environment: - RKN_DB_HOST=postgres @@ -127,7 +127,7 @@ services: - internal - external depends_on: - - postgres + - initialize - redis environment: - RKN_DB_HOST=postgres @@ -152,6 +152,7 @@ services: - internal - external depends_on: + - initialize - redis - postfix environment: @@ -180,7 +181,7 @@ services: - internal - external depends_on: - - postgres + - initialize - redis environment: - RKN_DB_HOST=postgres @@ -214,7 +215,7 @@ services: - internal - external depends_on: - - postgres + - initialize - redis environment: - RKN_DB_HOST=postgres From db69290b35423bca3b7add0970da79682e3446bd Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Wed, 7 Sep 2022 00:06:28 +0200 Subject: [PATCH 12/28] Complete CONTRIBUTING.md --- CONTRIBUTING.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c8b508974..7cfd0d3e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,10 +54,14 @@ The support of external hacking tools in Rekono is based on the following steps: - Override the method `parse_output_file` or `parse_plain_output` depending on the tool output type. -3. Implement unit tests to check the parser correct working. You can use some [tool reports](https://github.com/pablosnt/rekono/tree/main/rekono/testing/data/reports) as example for that. +3. Implement [unit tests](https://github.com/pablosnt/rekono/tree/main/rekono/testing/tools) to check the parser correct working. You can add your [tool reports](https://github.com/pablosnt/rekono/tree/main/rekono/testing/data/reports) as example for that. 4. Add the tool reference in the [README.md](https://github.com/pablosnt/rekono#supported-tools). +5. Add tool installation to the [Kali Linux Dockerfile](https://github.com/pablosnt/rekono/blob/main/docker/kali/Dockerfile). + +6. Add tool installation to the [Rekono CLI](https://github.com/pablosnt/rekono-cli/blob/main/rekono/installation/tools.py). + ### CI/CD This project has the following checks in _Continuous Integration_: From eceb5da28c0f7a5903ad74f29676d08e28d7582f Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Wed, 7 Sep 2022 00:06:50 +0200 Subject: [PATCH 13/28] Fix Sslscan parser to prevent errors after scan HTTP sites --- rekono/tools/tools/sslscan.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rekono/tools/tools/sslscan.py b/rekono/tools/tools/sslscan.py index c823f7e86..debcc9875 100644 --- a/rekono/tools/tools/sslscan.py +++ b/rekono/tools/tools/sslscan.py @@ -3,6 +3,7 @@ from findings.enums import Severity from findings.models import Technology, Vulnerability + from tools.tools.base_tool import BaseTool @@ -25,7 +26,10 @@ def get_technology(self, technologies: List[Technology], sslversion: str) -> Uni def parse_output_file(self) -> None: '''Parse tool output file to create finding entities.''' technologies: List[Technology] = [] - root = parser.parse(self.path_output).getroot() # Report root + try: + root = parser.parse(self.path_output).getroot() # Report root + except parser.ParseError: + return tests = root.findall('ssltest') # Get test for test in tests: # For each test for item in test: # For each item From 5ff51371f32251cdbda36de6bfdcf34364aa7b23 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 13 Sep 2022 22:30:30 +0200 Subject: [PATCH 14/28] Clean code and improve unit tests for calculation of executions --- rekono/executions/utils.py | 72 +++++++++++++------ .../test_executions_from_findings.py | 16 ++++- 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/rekono/executions/utils.py b/rekono/executions/utils.py index f8cd8088a..76699ffbb 100644 --- a/rekono/executions/utils.py +++ b/rekono/executions/utils.py @@ -7,7 +7,19 @@ from tools.models import Argument, Input, Tool -def get_executions_with_relations(base_inputs: Dict[InputType, List[BaseInput]], tool: Tool) -> List[List[BaseInput]]: +def get_executions_from_findings_with_relationships( + base_inputs: Dict[InputType, List[BaseInput]], + tool: Tool +) -> List[List[BaseInput]]: + '''Get needed executions for a tool based on a given inputs with relationships between them. + + Args: + base_inputs (Dict[InputType, List[BaseInput]]): InputTypes for this tool and related input list + tool (Tool): Tool that will be executed + + Returns: + List[List[BaseInput]]: List of inputs to be passed for each tool execution + ''' executions: List[List[BaseInput]] = [[]] # BaseInput list for each execution # It's required because base inputs will be assigned to executions based on relationships between them input_relations = utils.get_relations_between_input_types() # Get relations between input types @@ -15,52 +27,61 @@ def get_executions_with_relations(base_inputs: Dict[InputType, List[BaseInput]], for input_type, related_input_types in list(reversed(input_relations.items())): if input_type not in base_inputs: continue + # Get argument by tool and input type argument = Argument.objects.filter(tool=tool, inputs__type=input_type).order_by('inputs__order').first() - if related_input_types: - for base_input in base_inputs[input_type]: - for index, execution_list in enumerate(executions.copy()): + if related_input_types: # Input with related input types + for base_input in base_inputs[input_type]: # For each input + for index, execution_list in enumerate(executions.copy()): # For each execution list assigned = False - for related_input_type in related_input_types: + for related_input_type in related_input_types: # For each related input type + # Check number of inputs of the same type in this execution base_inputs_by_class = [bi for bi in execution_list if bi.__class__ == base_input.__class__] + # Get target class from related input type related_target = related_input_type.get_callback_target_class() + # Get field name to the related target related_target_field = snakecase(cast(Any, related_target).__name__) if related_target else None if ( ( + # Check if input has a relationship hasattr(base_input, related_input_type.name.lower()) and getattr(base_input, related_input_type.name.lower()) in execution_list ) or ( + # Check if input has a relationship with a target hasattr(base_input, related_target_field) and getattr(base_input, related_target_field) in execution_list ) ): if argument.multiple or len(base_inputs_by_class) == 0: + # Add input in current execution executions[index].append(base_input) assigned = True break elif not argument.multiple and len(base_inputs_by_class) > 0: - new_execution = execution_list.copy() - new_execution.remove(base_inputs_by_class[0]) - new_execution.append(base_input) + # Duplicate current execution + new_execution = execution_list.copy() # Copy input list + new_execution.remove(base_inputs_by_class[0]) # Remove input with same type + new_execution.append(base_input) # Add input executions.append(new_execution) assigned = True break if assigned: break elif argument.multiple: + # Input type without relationships and argument that allows multiple inputs for item in range(len(executions)): - executions[item].extend(base_inputs[input_type]) - else: + executions[item].extend(base_inputs[input_type]) # Add inputs in all executions + else: # Input type without relationships new_executions: List[List[BaseInput]] = [] - for base_input in base_inputs[input_type]: - for execution_list in executions: - new_executions.append(list(execution_list + [base_input])) + for base_input in base_inputs[input_type]: # For each input + for execution_list in executions: # For each execution + new_executions.append(list(execution_list + [base_input])) # Add input to the execution executions = new_executions return executions def get_executions_from_findings(base_inputs: List[BaseInput], tool: Tool) -> List[List[BaseInput]]: - '''Get needed executions for a tool based on a given a input (Finding, Resource or Target) list. + '''Get needed executions for a tool based on a given input (Finding, Resource or Target) list. Args: base_inputs (List[BaseInput]): BaseInput list @@ -69,7 +90,7 @@ def get_executions_from_findings(base_inputs: List[BaseInput], tool: Tool) -> Li Returns: List[List[BaseInput]]: List of inputs to be passed for each tool execution ''' - tool_inputs: List[Input] = Input.objects.filter(argument__tool=tool).all() + tool_inputs: List[Input] = Input.objects.filter(argument__tool=tool).all() # Get inputs by tool filtered_base_inputs: Dict[InputType, List[BaseInput]] = {} for tool_input in tool_inputs: base_input_list = [ @@ -78,13 +99,18 @@ def get_executions_from_findings(base_inputs: List[BaseInput], tool: Tool) -> Li ] ] if base_input_list: - filtered_base_inputs[tool_input.type] = base_input_list - if len(filtered_base_inputs.keys()) > 1: - return get_executions_with_relations(filtered_base_inputs, tool) - elif len(filtered_base_inputs.keys()) == 1: - argument = Argument.objects.filter(tool=tool, inputs__type=list(filtered_base_inputs.keys())[0]).first() - if argument.multiple: - return list(filtered_base_inputs.values()) + filtered_base_inputs[tool_input.type] = base_input_list # Relation between inputs and classes + if len(filtered_base_inputs.keys()) > 1: # Multiple input types + # Get executions from inputs with maybe relationships + return get_executions_from_findings_with_relationships(filtered_base_inputs, tool) + elif len(filtered_base_inputs.keys()) == 1: # Only one input type + # Get argument by tool and input type + argument = Argument.objects.filter( + tool=tool, inputs__type=list(filtered_base_inputs.keys())[0] + ).order_by('inputs__order').first() + if argument.multiple: # Argument with multiple inputs + return list(filtered_base_inputs.values()) # One execution with all inputs else: - return [[bi] for bi in list(filtered_base_inputs.values())[0]] + return [[bi] for bi in list(filtered_base_inputs.values())[0]] # One execution for each input + # By default, one execution with all inputs return [base_inputs] diff --git a/rekono/testing/executions/test_executions_from_findings.py b/rekono/testing/executions/test_executions_from_findings.py index 3f10a10fe..83c33e776 100644 --- a/rekono/testing/executions/test_executions_from_findings.py +++ b/rekono/testing/executions/test_executions_from_findings.py @@ -3,8 +3,6 @@ from django.db.models import Model from django.test import TestCase from django.utils import timezone -from executions.models import Execution -from executions.utils import get_executions_from_findings from findings.models import Host, Path, Port from input_types.base import BaseInput from input_types.models import InputType @@ -18,6 +16,9 @@ from tools.enums import IntensityRank, Stage from tools.models import Argument, Configuration, Input, Tool +from executions.models import Execution +from executions.utils import get_executions_from_findings + class ExecutionsFromFindingsTest(TestCase): '''Test cases for get_executions_from_findings CRITICAL feature.''' @@ -157,6 +158,17 @@ def test_with_findings(self) -> None: executions = get_executions_from_findings(findings, self.tool) self.assertEqual(expected, executions) + def test_with_only_one_finding_type(self) -> None: + '''Test get_executions_from_findings feature with findings. Simulates new executions from previous findings.''' + host_1 = self.create_finding(Host, address='10.10.10.1') + host_2 = self.create_finding(Host, address='10.10.10.2') + host_3 = self.create_finding(Host, address='10.10.10.3') + host_4 = self.create_finding(Host, address='10.10.10.4') + findings = [host_1, host_2, host_3, host_4] # Finding list to pass as argument + expected = [[host_1], [host_2], [host_3], [host_4]] # Expected executions + executions = get_executions_from_findings(findings, self.tool) + self.assertEqual(expected, executions) + def test_with_targets(self) -> None: '''Test get_executions_from_findings feature with targets. Simulates initial new executions.''' # Target ports with some target endpoints From 05c0d40ac43e74c0c756e85d9ba2dbda13d8ba2c Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 13 Sep 2022 22:50:11 +0200 Subject: [PATCH 15/28] Include retry mechanism to prevent unexpected errors during external API requests --- rekono/defectdojo/api.py | 40 +++++++++++++++++++++++++++---------- rekono/findings/nvd_nist.py | 18 ++++++++++++++--- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/rekono/defectdojo/api.py b/rekono/defectdojo/api.py index 4f94f94bb..d993768c6 100644 --- a/rekono/defectdojo/api.py +++ b/rekono/defectdojo/api.py @@ -1,13 +1,15 @@ import logging from datetime import datetime, timedelta from typing import Any, Tuple +from urllib.parse import urlparse import requests -from defectdojo.constants import DD_DATE_FORMAT, DD_DATETIME_FORMAT from findings.enums import Severity from projects.models import Project - from rekono.settings import DEFECT_DOJO as config +from requests.adapters import HTTPAdapter, Retry + +from defectdojo.constants import DD_DATE_FORMAT, DD_DATETIME_FORMAT # Mapping between Rekono and Defect-Dojo severities SEVERITY_MAPPING = { @@ -33,6 +35,11 @@ def __init__(self): self.product_type = config.get('PRODUCT_TYPE') # Product type name for Rekono self.test_type = config.get('TEST_TYPE') # Test type name for Rekono self.test = config.get('TEST') # Test name for Rekono + schema = urlparse(self.api_url_pattern).scheme # Get API schema + self.http_session = requests.Session() # Create HTTP session + # Configure retry protocol to prevent unexpected errors + retries = Retry(total=10, backoff_factor=1, status_forcelist=[500, 502, 503, 504]) + self.http_session.mount(f'{schema}://', HTTPAdapter(max_retries=retries)) def request( self, @@ -60,15 +67,26 @@ def request( 'User-Agent': 'Rekono', # Rekono User-Agent 'Authorization': f'Token {self.api_key}' # Authentication via API key } - response = requests.request( # Defect-Dojo API request - method=method, - url=f'{self.url}{endpoint}', - headers=headers, - params=params, - data=data, - files=files, - verify=self.verify_tls - ) + try: + response = self.http_session.request( # Defect-Dojo API request + method=method, + url=f'{self.url}{endpoint}', + headers=headers, + params=params, + data=data, + files=files, + verify=self.verify_tls + ) + except requests.exceptions.ConnectionError: + response = self.http_session.request( # Defect-Dojo API request + method=method, + url=f'{self.url}{endpoint}', + headers=headers, + params=params, + data=data, + files=files, + verify=self.verify_tls + ) logger.info(f'[Defect-Dojo] {method.upper()} /api/v2{endpoint} > HTTP {response.status_code}') if response.status_code == expected_status: return True, response.json() # Successful request diff --git a/rekono/findings/nvd_nist.py b/rekono/findings/nvd_nist.py index a8b85560c..200f11f8f 100644 --- a/rekono/findings/nvd_nist.py +++ b/rekono/findings/nvd_nist.py @@ -1,6 +1,9 @@ import logging +from urllib.parse import urlparse import requests +from requests.adapters import HTTPAdapter, Retry + from findings.enums import Severity # Mapping between severity values and CVSS values @@ -34,6 +37,12 @@ def __init__(self, cve: str) -> None: self.cwe = self.parse_cwe() if self.raw_cve_info else None # CVE weakness as CWE code # CVE severity based on CVSS score self.severity = self.parse_severity() if self.raw_cve_info else Severity.MEDIUM + schema = urlparse(self.api_url_pattern).scheme # Get API schema + self.http_session = requests.Session() # Create HTTP session + # Configure retry protocol to prevent unexpected errors + # Free NVD NIST API has a rate limit of 10 requests by second + retries = Retry(total=10, backoff_factor=3, status_forcelist=[403, 500, 502, 503, 504]) + self.http_session.mount(f'{schema}://', HTTPAdapter(max_retries=retries)) def request(self) -> dict: '''Get information from a CVE using the NVD NIST API Rest. @@ -41,9 +50,12 @@ def request(self) -> dict: Returns: dict: Raw NVD NIST CVE information ''' - res = requests.get(self.api_url_pattern.format(cve=self.cve)) - logger.info(f'[NVD NIST] GET {self.cve} > HTTP {res.status_code}') - return res.json()['result']['CVE_Items'][0] if res.status_code == 200 else {} + try: + response = self.http_session.get(self.api_url_pattern.format(cve=self.cve)) + except requests.exceptions.ConnectionError: + response = self.http_session.get(self.api_url_pattern.format(cve=self.cve)) + logger.info(f'[NVD NIST] GET {self.cve} > HTTP {response.status_code}') + return response.json()['result']['CVE_Items'][0] if response.status_code == 200 else {} def parse_description(self) -> str: '''Get description from raw CVE information. From bcd729559efd898862fc6c869f374009169fcdb3 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 13 Sep 2022 22:52:55 +0200 Subject: [PATCH 16/28] Adapt findings queue timeout --- rekono/rekono/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rekono/rekono/settings.py b/rekono/rekono/settings.py index 1cd9750a8..7f7b8b51a 100644 --- a/rekono/rekono/settings.py +++ b/rekono/rekono/settings.py @@ -357,7 +357,7 @@ 'HOST': os.getenv(RKN_RQ_HOST, CONFIG.RQ_HOST), 'PORT': os.getenv(RKN_RQ_PORT, CONFIG.RQ_PORT), 'DB': 0, - 'DEFAULT_TIMEOUT': 300 # 5 minutes + 'DEFAULT_TIMEOUT': 1200 # 20 minutes }, 'emails-queue': { 'HOST': os.getenv(RKN_RQ_HOST, CONFIG.RQ_HOST), From 5af5358bcb6a988ae215fdd1cd2616b1bba4ff0f Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 13 Sep 2022 23:09:21 +0200 Subject: [PATCH 17/28] Fix Snyk file --- rekono/frontend/.snyk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rekono/frontend/.snyk b/rekono/frontend/.snyk index 8a95290f8..d100aa76a 100644 --- a/rekono/frontend/.snyk +++ b/rekono/frontend/.snyk @@ -5,6 +5,6 @@ ignore: SNYK-JS-MOMENT-2944238: - '*': reason: No fix available yet - expires: 2022-09-07T21:30:04.511Z - created: 2022-07-08T21:30:04.520Z + expires: 2022-12-13T21:07:02.600Z + created: 2022-09-13T21:07:02.607Z patch: {} From 0e77df24340d0e11d36a02be705640fc6b19a30e Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 13 Sep 2022 23:14:30 +0200 Subject: [PATCH 18/28] Fix in Defect-Dojo API calls --- rekono/defectdojo/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rekono/defectdojo/api.py b/rekono/defectdojo/api.py index d993768c6..3f89e0805 100644 --- a/rekono/defectdojo/api.py +++ b/rekono/defectdojo/api.py @@ -35,7 +35,7 @@ def __init__(self): self.product_type = config.get('PRODUCT_TYPE') # Product type name for Rekono self.test_type = config.get('TEST_TYPE') # Test type name for Rekono self.test = config.get('TEST') # Test name for Rekono - schema = urlparse(self.api_url_pattern).scheme # Get API schema + schema = urlparse(self.url).scheme # Get API schema self.http_session = requests.Session() # Create HTTP session # Configure retry protocol to prevent unexpected errors retries = Retry(total=10, backoff_factor=1, status_forcelist=[500, 502, 503, 504]) From 00c814b7d23d104016643bb318ef63f9e0294a3d Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Wed, 14 Sep 2022 17:24:20 +0200 Subject: [PATCH 19/28] Fix error in API requests to NVD NIST --- rekono/defectdojo/api.py | 2 +- rekono/findings/nvd_nist.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/rekono/defectdojo/api.py b/rekono/defectdojo/api.py index 3f89e0805..5bffb4831 100644 --- a/rekono/defectdojo/api.py +++ b/rekono/defectdojo/api.py @@ -38,7 +38,7 @@ def __init__(self): schema = urlparse(self.url).scheme # Get API schema self.http_session = requests.Session() # Create HTTP session # Configure retry protocol to prevent unexpected errors - retries = Retry(total=10, backoff_factor=1, status_forcelist=[500, 502, 503, 504]) + retries = Retry(total=10, backoff_factor=1, status_forcelist=[500, 502, 503, 504, 599]) self.http_session.mount(f'{schema}://', HTTPAdapter(max_retries=retries)) def request( diff --git a/rekono/findings/nvd_nist.py b/rekono/findings/nvd_nist.py index 200f11f8f..6dca2d6fa 100644 --- a/rekono/findings/nvd_nist.py +++ b/rekono/findings/nvd_nist.py @@ -37,12 +37,6 @@ def __init__(self, cve: str) -> None: self.cwe = self.parse_cwe() if self.raw_cve_info else None # CVE weakness as CWE code # CVE severity based on CVSS score self.severity = self.parse_severity() if self.raw_cve_info else Severity.MEDIUM - schema = urlparse(self.api_url_pattern).scheme # Get API schema - self.http_session = requests.Session() # Create HTTP session - # Configure retry protocol to prevent unexpected errors - # Free NVD NIST API has a rate limit of 10 requests by second - retries = Retry(total=10, backoff_factor=3, status_forcelist=[403, 500, 502, 503, 504]) - self.http_session.mount(f'{schema}://', HTTPAdapter(max_retries=retries)) def request(self) -> dict: '''Get information from a CVE using the NVD NIST API Rest. @@ -50,10 +44,16 @@ def request(self) -> dict: Returns: dict: Raw NVD NIST CVE information ''' + schema = urlparse(self.api_url_pattern).scheme # Get API schema + session = requests.Session() # Create HTTP session + # Configure retry protocol to prevent unexpected errors + # Free NVD NIST API has a rate limit of 10 requests by second + retries = Retry(total=10, backoff_factor=3, status_forcelist=[403, 500, 502, 503, 504, 599]) + session.mount(f'{schema}://', HTTPAdapter(max_retries=retries)) try: - response = self.http_session.get(self.api_url_pattern.format(cve=self.cve)) + response = session.get(self.api_url_pattern.format(cve=self.cve)) except requests.exceptions.ConnectionError: - response = self.http_session.get(self.api_url_pattern.format(cve=self.cve)) + response = session.get(self.api_url_pattern.format(cve=self.cve)) logger.info(f'[NVD NIST] GET {self.cve} > HTTP {response.status_code}') return response.json()['result']['CVE_Items'][0] if response.status_code == 200 else {} From 4a53f6c62be2275958ef6e466b7ea43eccc04fd7 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Sat, 17 Sep 2022 18:41:57 +0200 Subject: [PATCH 20/28] Reduce sleep time between API retries --- rekono/defectdojo/api.py | 2 +- rekono/findings/nvd_nist.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rekono/defectdojo/api.py b/rekono/defectdojo/api.py index 5bffb4831..dacb00149 100644 --- a/rekono/defectdojo/api.py +++ b/rekono/defectdojo/api.py @@ -38,7 +38,7 @@ def __init__(self): schema = urlparse(self.url).scheme # Get API schema self.http_session = requests.Session() # Create HTTP session # Configure retry protocol to prevent unexpected errors - retries = Retry(total=10, backoff_factor=1, status_forcelist=[500, 502, 503, 504, 599]) + retries = Retry(total=10, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504, 599]) self.http_session.mount(f'{schema}://', HTTPAdapter(max_retries=retries)) def request( diff --git a/rekono/findings/nvd_nist.py b/rekono/findings/nvd_nist.py index 6dca2d6fa..68b4f072c 100644 --- a/rekono/findings/nvd_nist.py +++ b/rekono/findings/nvd_nist.py @@ -48,7 +48,7 @@ def request(self) -> dict: session = requests.Session() # Create HTTP session # Configure retry protocol to prevent unexpected errors # Free NVD NIST API has a rate limit of 10 requests by second - retries = Retry(total=10, backoff_factor=3, status_forcelist=[403, 500, 502, 503, 504, 599]) + retries = Retry(total=10, backoff_factor=1, status_forcelist=[403, 500, 502, 503, 504, 599]) session.mount(f'{schema}://', HTTPAdapter(max_retries=retries)) try: response = session.get(self.api_url_pattern.format(cve=self.cve)) From 1d527846c9c24637f0d5ced6d185650485282d11 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Sun, 18 Sep 2022 12:51:01 +0200 Subject: [PATCH 21/28] Reduce sleep time between API retries --- rekono/defectdojo/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rekono/defectdojo/api.py b/rekono/defectdojo/api.py index dacb00149..a2ac70b90 100644 --- a/rekono/defectdojo/api.py +++ b/rekono/defectdojo/api.py @@ -38,7 +38,7 @@ def __init__(self): schema = urlparse(self.url).scheme # Get API schema self.http_session = requests.Session() # Create HTTP session # Configure retry protocol to prevent unexpected errors - retries = Retry(total=10, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504, 599]) + retries = Retry(total=10, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504, 599]) self.http_session.mount(f'{schema}://', HTTPAdapter(max_retries=retries)) def request( From 4cb10e521e6bbe05065c5d4f2c6d90ae01a1714c Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Mon, 19 Sep 2022 19:52:59 +0200 Subject: [PATCH 22/28] Mock NVD NIST API for unit testing --- .secrets.baseline | 4 +- rekono/testing/api/external/test_nvd_nist.py | 8 +++ rekono/testing/executions/test_base_tool.py | 11 ++- rekono/testing/mocks/nvd_nist.py | 72 ++++++++++++++++++++ 4 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 rekono/testing/mocks/nvd_nist.py diff --git a/.secrets.baseline b/.secrets.baseline index 343a524ea..15d607bb6 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -748,7 +748,7 @@ "filename": "rekono/testing/executions/test_base_tool.py", "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 295, + "line_number": 297, "is_secret": false } ], @@ -803,5 +803,5 @@ } ] }, - "generated_at": "2022-04-24T18:37:45Z" + "generated_at": "2022-09-19T16:59:42Z" } diff --git a/rekono/testing/api/external/test_nvd_nist.py b/rekono/testing/api/external/test_nvd_nist.py index f58438006..26c845f5c 100644 --- a/rekono/testing/api/external/test_nvd_nist.py +++ b/rekono/testing/api/external/test_nvd_nist.py @@ -1,6 +1,11 @@ +from unittest import mock + from django.test import TestCase from findings.enums import Severity from findings.nvd_nist import NvdNist +from testing.mocks.nvd_nist import (nvd_nist_not_found, + nvd_nist_success_cvss_2, + nvd_nist_success_cvss_3) class NvdNistTest(TestCase): @@ -25,14 +30,17 @@ def get_cve_data(self, cve: str, severity: str) -> None: self.assertEqual(nvd_nist.cve_reference_pattern.format(cve=cve), nvd_nist.reference) self.assertEqual(severity, nvd_nist.severity) + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_success_cvss_3) # Mocks NVD NIST response def test_get_cve_data(self) -> None: '''Test get CVE data from NVD NIST feature.''' self.get_cve_data(self.cve, Severity.CRITICAL) + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_not_found) # Mocks NVD NIST response def test_cve_data_not_found(self) -> None: '''Test get not found CVE data from NVD NIST feature.''' self.get_cve_data(self.not_found_cve, Severity.MEDIUM) + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_success_cvss_2) # Mocks NVD NIST response def test_get_old_cve_data(self) -> None: '''Test get old CVE data from NVD NIST feature.''' self.get_cve_data(self.old_cve, Severity.HIGH) diff --git a/rekono/testing/executions/test_base_tool.py b/rekono/testing/executions/test_base_tool.py index 6af9b4e34..1bd5519a0 100644 --- a/rekono/testing/executions/test_base_tool.py +++ b/rekono/testing/executions/test_base_tool.py @@ -5,7 +5,6 @@ import django_rq from django.test import TestCase from django.utils import timezone -from executions.models import Execution from findings.enums import DataType, Protocol, Severity from findings.models import (OSINT, Credential, Exploit, Finding, Host, Path, Port, Technology, Vulnerability) @@ -22,6 +21,7 @@ from tasks.models import Task from testing.mocks.defectdojo import (defect_dojo_error, defect_dojo_success, defect_dojo_success_multiple) +from testing.mocks.nvd_nist import nvd_nist_success_cvss_3 from tools.enums import IntensityRank, Stage from tools.exceptions import ToolExecutionException from tools.models import Argument, Configuration, Input, Intensity, Tool @@ -29,6 +29,8 @@ from tools.utils import get_tool_class_by_name from users.models import User +from executions.models import Execution + class BaseToolTest(TestCase): '''Test cases for Base Tool operations.''' @@ -531,11 +533,13 @@ def process_findings(self, imported_in_defectdojo: bool) -> None: self.assertEqual(imported_in_defectdojo, execution.imported_in_defectdojo) @mock.patch('defectdojo.api.DefectDojo.request', defect_dojo_success) # Mocks Defect-Dojo response + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_success_cvss_3) # Mocks NVD NIST response def test_process_findings_with_defectdojo_target_engagement(self) -> None: '''Test process_findings feature with import in Defect-Dojo using target engagement.''' self.process_findings(True) @mock.patch('defectdojo.api.DefectDojo.request', defect_dojo_success) # Mocks Defect-Dojo response + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_success_cvss_3) # Mocks NVD NIST response def test_process_findings_with_defectdojo_product_engagement(self) -> None: '''Test process_findings feature with import in Defect-Dojo using product engagement.''' self.project.defectdojo_engagement_id = 1 # Product engagement Id @@ -545,6 +549,7 @@ def test_process_findings_with_defectdojo_product_engagement(self) -> None: @mock.patch('defectdojo.api.DefectDojo.request', defect_dojo_success) # Mocks Defect-Dojo response @mock.patch('defectdojo.api.DefectDojo.get_product', defect_dojo_error) + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_success_cvss_3) # Mocks NVD NIST response def test_process_findings_with_defectdojo_engagement_not_found(self) -> None: '''Test process_findings feature with import in Defect-Dojo using not found engagement.''' self.project.defectdojo_engagement_id = 1 # Product engagement Id @@ -553,6 +558,7 @@ def test_process_findings_with_defectdojo_engagement_not_found(self) -> None: self.process_findings(False) @mock.patch('defectdojo.api.DefectDojo.request', defect_dojo_success) # Mocks Defect-Dojo response + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_success_cvss_3) # Mocks NVD NIST response def test_process_findings_with_defectdojo_findings_import(self) -> None: '''Test process_findings feature with import in Defect-Dojo using the Rekono findings.''' self.nmap.defectdojo_scan_type = None # Import findings instead executions @@ -561,6 +567,7 @@ def test_process_findings_with_defectdojo_findings_import(self) -> None: @mock.patch('defectdojo.api.DefectDojo.request', defect_dojo_success) # Mocks Defect-Dojo response @mock.patch('defectdojo.api.DefectDojo.get_rekono_test_type', defect_dojo_success_multiple) + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_success_cvss_3) # Mocks NVD NIST response def test_process_findings_with_existing_defectdojo_test_type(self) -> None: '''Test process_findings feature with import in Defect-Dojo using existing test type.''' self.nmap.defectdojo_scan_type = None # Import findings instead executions @@ -569,6 +576,7 @@ def test_process_findings_with_existing_defectdojo_test_type(self) -> None: @mock.patch('defectdojo.api.DefectDojo.request', defect_dojo_success) # Mocks Defect-Dojo response @mock.patch('defectdojo.api.DefectDojo.create_rekono_test_type', defect_dojo_error) + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_success_cvss_3) # Mocks NVD NIST response def test_process_findings_with_errors_in_defectdojo_test_type_creation(self) -> None: '''Test process_findings feature with unexpected error during Defect-Dojo test type creation.''' self.nmap.defectdojo_scan_type = None # Import findings instead executions @@ -576,6 +584,7 @@ def test_process_findings_with_errors_in_defectdojo_test_type_creation(self) -> self.process_findings(False) @mock.patch('defectdojo.api.DefectDojo.request', defect_dojo_error) # Mocks Defect-Dojo response + @mock.patch('findings.nvd_nist.NvdNist.request', nvd_nist_success_cvss_3) # Mocks NVD NIST response def test_process_findings_with_unvailable_defectdojo(self) -> None: '''Test process_findings feature with unavailable Defect-Dojo instance.''' self.process_findings(False) diff --git a/rekono/testing/mocks/nvd_nist.py b/rekono/testing/mocks/nvd_nist.py new file mode 100644 index 000000000..132bcca15 --- /dev/null +++ b/rekono/testing/mocks/nvd_nist.py @@ -0,0 +1,72 @@ +from typing import Any + +'''Mock for NVD NIST API integration implemented on findings.nvd_nist package.''' + + +nvd_nist_base_success = { # NVD NIST base Response + 'cve': { + 'description': { + 'description_data': [ + { + 'lang': 'en', + 'value': 'description' + } + ] + }, + 'problemtype': { + 'problemtype_data': [ + { + 'description': [ + { + 'value': 'CWE-200' + } + ] + } + ] + } + }, + 'impact': {} +} + + +def nvd_nist_success_cvss_3(*args: Any, **kwargs: Any) -> dict: + '''Get mocked response from CVE with CVSS 3. + + Returns: + dict: NVD NIST response + ''' + response = nvd_nist_base_success.copy() + response['impact'] = { + 'baseMetricV3': { + 'cvssV3': { + 'baseScore': 9 + } + } + } + return response + + +def nvd_nist_success_cvss_2(*args: Any, **kwargs: Any) -> dict: + '''Get mocked response from CVE with CVSS 2. + + Returns: + dict: NVD NIST response + ''' + response = nvd_nist_base_success.copy() + response['impact'] = { + 'baseMetricV2': { + 'cvssV2': { + 'baseScore': 8 + } + } + } + return response + + +def nvd_nist_not_found(*args: Any, **kwargs: Any) -> dict: + '''Get mocked response from not found CVE + + Returns: + dict: Empty response + ''' + return {} From 4e3947fead85dbfd663714a73e595c5bd2f4cf58 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Mon, 19 Sep 2022 22:38:41 +0200 Subject: [PATCH 23/28] Debug workflow error --- .github/workflows/check-installation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-installation.yml b/.github/workflows/check-installation.yml index 481433beb..9c2e78d2c 100644 --- a/.github/workflows/check-installation.yml +++ b/.github/workflows/check-installation.yml @@ -21,8 +21,8 @@ jobs: - name: Test default credentials run: | - sleep 2m - curl -X POST --insecure -H 'Content-Type: application/json' https://127.0.0.1/api/token/ --data '{"username": "'"${RKN_USERNAME}"'", "password": "'"${RKN_PASSWORD}"'"}' + sleep 2ms RESPONSE=$(curl -X POST --insecure -H 'Content-Type: application/json' https://127.0.0.1/api/token/ --data '{"username": "'"${RKN_USERNAME}"'", "password": "'"${RKN_PASSWORD}"'"}' -o /dev/null -w '%{http_code}\n' -s) + docker logs echo $RESPONSE if [ $RESPONSE != 200 ]; then exit 10; fi \ No newline at end of file From 76c4a9e1c276eb6e848c919ae17a39a1f7eff730 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Mon, 19 Sep 2022 23:35:18 +0200 Subject: [PATCH 24/28] Fix bash syntax --- .github/workflows/check-installation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-installation.yml b/.github/workflows/check-installation.yml index 9c2e78d2c..da0526121 100644 --- a/.github/workflows/check-installation.yml +++ b/.github/workflows/check-installation.yml @@ -21,7 +21,7 @@ jobs: - name: Test default credentials run: | - sleep 2ms + sleep 2m RESPONSE=$(curl -X POST --insecure -H 'Content-Type: application/json' https://127.0.0.1/api/token/ --data '{"username": "'"${RKN_USERNAME}"'", "password": "'"${RKN_PASSWORD}"'"}' -o /dev/null -w '%{http_code}\n' -s) docker logs echo $RESPONSE From 9f20847f09fbc2051f4a91f006d8e582624bd479 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Mon, 19 Sep 2022 23:47:54 +0200 Subject: [PATCH 25/28] Bump Rekono version to 1.0.1 --- rekono/frontend/package-lock.json | 4 ++-- rekono/frontend/package.json | 2 +- rekono/rekono/__init__.py | 2 -- rekono/rekono/settings.py | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/rekono/frontend/package-lock.json b/rekono/frontend/package-lock.json index 3f4922736..78d7a791b 100644 --- a/rekono/frontend/package-lock.json +++ b/rekono/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "rekono", - "version": "1.0.0", + "version": "1.0.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "rekono", - "version": "1.0.0", + "version": "1.0.1", "dependencies": { "axios": "^0.26.0", "bootstrap": "^4.5.3", diff --git a/rekono/frontend/package.json b/rekono/frontend/package.json index 25b30aa56..ada5144ba 100644 --- a/rekono/frontend/package.json +++ b/rekono/frontend/package.json @@ -1,6 +1,6 @@ { "name": "rekono", - "version": "1.0.0", + "version": "1.0.1", "private": true, "scripts": { "serve": "vue-cli-service serve", diff --git a/rekono/rekono/__init__.py b/rekono/rekono/__init__.py index 91f20cd1b..547c1646f 100644 --- a/rekono/rekono/__init__.py +++ b/rekono/rekono/__init__.py @@ -1,3 +1 @@ '''Rekono main module.''' - -VERSION = '1.0.0' diff --git a/rekono/rekono/settings.py b/rekono/rekono/settings.py index 7f7b8b51a..7dc4a7c98 100644 --- a/rekono/rekono/settings.py +++ b/rekono/rekono/settings.py @@ -39,7 +39,7 @@ ################################################################################ DESCRIPTION = 'Execute full pentesting processes combining multiple hacking tools automatically' # Rekono description -VERSION = '1.0.0' # Rekono version +VERSION = '1.0.1' # Rekono version ################################################################################ From 4a81f95124c4adf07bf09f06359270efb4b2d22a Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 20 Sep 2022 00:07:47 +0200 Subject: [PATCH 26/28] Debug workflow --- .github/workflows/check-installation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-installation.yml b/.github/workflows/check-installation.yml index da0526121..60669d9ac 100644 --- a/.github/workflows/check-installation.yml +++ b/.github/workflows/check-installation.yml @@ -21,8 +21,8 @@ jobs: - name: Test default credentials run: | - sleep 2m + sleep 5m RESPONSE=$(curl -X POST --insecure -H 'Content-Type: application/json' https://127.0.0.1/api/token/ --data '{"username": "'"${RKN_USERNAME}"'", "password": "'"${RKN_PASSWORD}"'"}' -o /dev/null -w '%{http_code}\n' -s) - docker logs + docker-compose logs echo $RESPONSE if [ $RESPONSE != 200 ]; then exit 10; fi \ No newline at end of file From caa904cf73f8474bbebe633a0c72349b96ed7a19 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 20 Sep 2022 15:15:30 +0200 Subject: [PATCH 27/28] Remove check installation workflow because it doesn't work with the current docker-compose version in GitHub runners --- .github/workflows/check-installation.yml | 28 ------------------------ 1 file changed, 28 deletions(-) delete mode 100644 .github/workflows/check-installation.yml diff --git a/.github/workflows/check-installation.yml b/.github/workflows/check-installation.yml deleted file mode 100644 index 481433beb..000000000 --- a/.github/workflows/check-installation.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Check installation -on: - workflow_dispatch: - -env: - RKN_USERNAME: rekono - RKN_PASSWORD: rekono - -jobs: - docker: - name: Test Docker environment - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Launch docker environment - run: | - docker-compose build - docker-compose up -d - - - name: Test default credentials - run: | - sleep 2m - curl -X POST --insecure -H 'Content-Type: application/json' https://127.0.0.1/api/token/ --data '{"username": "'"${RKN_USERNAME}"'", "password": "'"${RKN_PASSWORD}"'"}' - RESPONSE=$(curl -X POST --insecure -H 'Content-Type: application/json' https://127.0.0.1/api/token/ --data '{"username": "'"${RKN_USERNAME}"'", "password": "'"${RKN_PASSWORD}"'"}' -o /dev/null -w '%{http_code}\n' -s) - echo $RESPONSE - if [ $RESPONSE != 200 ]; then exit 10; fi \ No newline at end of file From 09ad32f66b93ce07d00049fbd87e785ccfaa9d52 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Date: Tue, 20 Sep 2022 15:38:41 +0200 Subject: [PATCH 28/28] Update changelog file --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dec66a71a..584c34ae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.1] - 2022-09-20 + +### Fixed + +- Retry requests to Defect-Dojo API after unexpected errors (https://github.com/pablosnt/rekono/pull/39) +- Retry requests to NVD NIST API to avoid blocks by the API rate limit and after unexpected errors (https://github.com/pablosnt/rekono/pull/39) +- Save unique exploits based on its `reference` instead of `edb_id` (https://github.com/pablosnt/rekono/pull/30) +- Prevent unexpected errors parsing malformed Sslscan reports (https://github.com/pablosnt/rekono/pull/27) + +### Changed + +- Optimize calculation of executions from previous findings to make process executions faster (https://github.com/pablosnt/rekono/pull/27) +- Allow parentheses in text values like names and descriptions (https://github.com/pablosnt/rekono/pull/29) + +### Security + +- Bump `nginx` Docker image version to `1.22-alpine` (https://github.com/pablosnt/rekono/pull/25/files) +- Bump `node` Docker image version to `18.6.0-alpine` (https://github.com/pablosnt/rekono/pull/25/files) +- Bump `python-libnmap` version to `0.7.3` (https://github.com/pablosnt/rekono/pull/31) + + ## [1.0.0] - 2022-08-19 ### Added