Merge pull request #725 from Spredzy/sync20210611

Beta1: Resync

Reviewed-by: https://github.com/apps/ansible-zuul

ansible_runner/config/_base.py

@@ -58,7 +58,7 @@ def __init__(self,
                  project_dir=None, artifact_dir=None, fact_cache_type='jsonfile', fact_cache=None,
                  process_isolation=False, process_isolation_executable=None,
                  container_image=None, container_volume_mounts=None, container_options=None, container_workdir=None,
-                 ident=None, rotate_artifacts=0, timeout=None, ssh_key=None, quiet=False, json_mode=False):
+                 ident=None, rotate_artifacts=0, timeout=None, ssh_key=None, quiet=False, json_mode=False, check_job_event_data=False):
         # common params
         self.host_cwd = host_cwd
         self.envvars = envvars
@@ -82,6 +82,7 @@ def __init__(self,
         self.passwords = passwords
         self.settings = settings
         self.timeout = timeout
+        self.check_job_event_data = check_job_event_data
 
         # setup initial environment
         if private_data_dir:
@@ -118,10 +119,8 @@ def __init__(self,
         else:
             self.cwd = os.getcwd()
 
-        if not os.path.exists(self.private_data_dir):
-            os.makedirs(self.private_data_dir, mode=0o700)
-        if not os.path.exists(self.artifact_dir):
-            os.makedirs(self.artifact_dir, mode=0o700)
+        os.makedirs(self.private_data_dir, exist_ok=True, mode=0o700)
+        os.makedirs(self.artifact_dir, exist_ok=True, mode=0o700)
 
     _CONTAINER_ENGINES = ('docker', 'podman')
 

ansible_runner/interface.py

@@ -197,6 +197,9 @@ def run(**kwargs):
     :param fact_cache_type: A string of the type of fact cache to use.  Defaults to 'jsonfile'.
     :param omit_event_data: Omits extra ansible event data from event payload (stdout and event still included)
     :param only_failed_event_data: Omits extra ansible event data unless it's a failed event (stdout and event still included)
+    :param check_job_event_data: Check if job events data is completely generated. If event data is not completely generated and if
+                                 value is set to 'True' it will raise 'AnsibleRunnerException' exception,
+                                 if set to 'False' it log a debug message and continue execution. Default value is 'False'
     :type private_data_dir: str
     :type ident: str
     :type json_mode: bool
@@ -244,6 +247,7 @@ def run(**kwargs):
     :type fact_cache_type: str
     :type omit_event_data: bool
     :type only_failed_event_data: bool
+    :type check_job_event_data: bool
 
     :returns: A :py:class:`ansible_runner.runner.Runner` object, or a simple object containing `rc` if run remotely
     '''
@@ -340,6 +344,10 @@ def run_command(executable_cmd, cmdline_args=None, **kwargs):
     :param finished_callback: An optional callback that will be invoked at shutdown after process cleanup.
     :param status_handler: An optional callback that will be invoked any time the status changes (e.g...started, running, failed, successful, timeout)
     :param artifacts_handler: An optional callback that will be invoked at the end of the run to deal with the artifacts from the run.
+    :param check_job_event_data: Check if job events data is completely generated. If event data is not completely generated and if
+                                 value is set to 'True' it will raise 'AnsibleRunnerException' exception,
+                                 if set to 'False' it log a debug message and continue execution. Default value is 'False'
+
     :type executable_cmd: str
     :type cmdline_args: list
     :type input_fd: file descriptor
@@ -372,15 +380,17 @@ def run_command(executable_cmd, cmdline_args=None, **kwargs):
     :type finished_callback: function
     :type status_handler: function
     :type artifacts_handler: function
+    :type check_job_event_data: bool
 
-    :returns: Retunes a tuple of response and error string. In case if ``runner_mode`` is set to ``pexpect`` the error value is empty as
+    :returns: Returns a tuple of response, error string and return code.
+              In case if ``runner_mode`` is set to ``pexpect`` the error value is empty as
               ``pexpect`` uses same output descriptor for stdout and stderr.
     '''
     r = init_command_config(executable_cmd, cmdline_args=cmdline_args, **kwargs)
     r.run()
     response = r.stdout.read()
     error = r.stderr.read()
-    return response, error
+    return response, error, r.rc
 
 
 def run_command_async(executable_cmd, cmdline_args=None, **kwargs):
@@ -471,6 +481,10 @@ def get_plugin_docs(plugin_names, plugin_type=None, response_format=None, snippe
     :param finished_callback: An optional callback that will be invoked at shutdown after process cleanup.
     :param status_handler: An optional callback that will be invoked any time the status changes (e.g...started, running, failed, successful, timeout)
     :param artifacts_handler: An optional callback that will be invoked at the end of the run to deal with the artifacts from the run.
+    :param check_job_event_data: Check if job events data is completely generated. If event data is not completely generated and if
+                                 value is set to 'True' it will raise 'AnsibleRunnerException' exception,
+                                 if set to 'False' it log a debug message and continue execution. Default value is 'False'
+
     :type plugin_names: list
     :type plugin_type: str
     :type response_format: str
@@ -504,6 +518,7 @@ def get_plugin_docs(plugin_names, plugin_type=None, response_format=None, snippe
     :type finished_callback: function
     :type status_handler: function
     :type artifacts_handler: function
+    :type check_job_event_data: bool
 
     :returns: Returns a tuple of response and error string. In case if ``runner_mode`` is set to ``pexpect`` the error value is empty as
               ``pexpect`` uses same output descriptor for stdout and stderr. If the value of ``response_format`` is ``json``
@@ -583,6 +598,10 @@ def get_plugin_list(list_files=None, response_format=None, plugin_type=None, pla
     :param finished_callback: An optional callback that will be invoked at shutdown after process cleanup.
     :param status_handler: An optional callback that will be invoked any time the status changes (e.g...started, running, failed, successful, timeout)
     :param artifacts_handler: An optional callback that will be invoked at the end of the run to deal with the artifacts from the run.
+    :param check_job_event_data: Check if job events data is completely generated. If event data is not completely generated and if
+                                 value is set to 'True' it will raise 'AnsibleRunnerException' exception,
+                                 if set to 'False' it log a debug message and continue execution. Default value is 'False'
+
     :type list_files: bool
     :type plugin_type: str
     :type response_format: str
@@ -615,6 +634,7 @@ def get_plugin_list(list_files=None, response_format=None, plugin_type=None, pla
     :type finished_callback: function
     :type status_handler: function
     :type artifacts_handler: function
+    :type check_job_event_data: bool
 
     :returns: Returns a tuple of response and error string. In case if ``runner_mode`` is set to ``pexpect`` the error value is empty as
               ``pexpect`` uses same output descriptor for stdout and stderr. If the vaue of ``response_format`` is ``json``
@@ -697,6 +717,9 @@ def get_inventory(action, inventories, response_format=None, host=None, playbook
     :param finished_callback: An optional callback that will be invoked at shutdown after process cleanup.
     :param status_handler: An optional callback that will be invoked any time the status changes (e.g...started, running, failed, successful, timeout)
     :param artifacts_handler: An optional callback that will be invoked at the end of the run to deal with the artifacts from the run.
+    :param check_job_event_data: Check if job events data is completely generated. If event data is not completely generated and if
+                                 value is set to 'True' it will raise 'AnsibleRunnerException' exception,
+                                 if set to 'False' it log a debug message and continue execution. Default value is 'False'
     :type action: str
     :type inventories: list
     :type response_format: str
@@ -733,6 +756,7 @@ def get_inventory(action, inventories, response_format=None, host=None, playbook
     :type finished_callback: function
     :type status_handler: function
     :type artifacts_handler: function
+    :type check_job_event_data: bool
 
     :returns: Returns a tuple of response and error string. In case if ``runner_mode`` is set to ``pexpect`` the error value is
               empty as ``pexpect`` uses same output descriptor for stdout and stderr. If the vaue of ``response_format`` is ``json``
@@ -807,6 +831,9 @@ def get_ansible_config(action, config_file=None, only_changed=None, **kwargs):
     :param finished_callback: An optional callback that will be invoked at shutdown after process cleanup.
     :param status_handler: An optional callback that will be invoked any time the status changes (e.g...started, running, failed, successful, timeout)
     :param artifacts_handler: An optional callback that will be invoked at the end of the run to deal with the artifacts from the run.
+    :param check_job_event_data: Check if job events data is completely generated. If event data is not completely generated and if
+                                 value is set to 'True' it will raise 'AnsibleRunnerException' exception,
+                                 if set to 'False' it log a debug message and continue execution. Default value is 'False'
     :type action: str
     :type config_file: str
     :type only_changed: bool
@@ -837,6 +864,7 @@ def get_ansible_config(action, config_file=None, only_changed=None, **kwargs):
     :type finished_callback: function
     :type status_handler: function
     :type artifacts_handler: function
+    :type check_job_event_data: bool
 
     :returns: Returns a tuple of response and error string. In case if ``runner_mode`` is set to ``pexpect`` the error value is
               empty as ``pexpect`` uses same output descriptor for stdout and stderr.

ansible_runner/runner.py

@@ -72,8 +72,12 @@ def event_callback(self, event_data):
                     event_data.update(partial_event_data)
                     if self.remove_partials:
                         os.remove(partial_filename)
-                except IOError:
-                    debug("Failed to open ansible stdout callback plugin partial data file {}".format(partial_filename))
+                except IOError as e:
+                    msg = "Failed to open ansible stdout callback plugin partial data" \
+                          " file {} with error {}".format(partial_filename, str(e))
+                    debug(msg)
+                    if self.config.check_job_event_data:
+                        raise AnsibleRunnerException(msg)
 
                 # prefer 'created' from partial data, but verbose events set time here
                 if 'created' not in event_data:

setup.cfg

@@ -14,6 +14,8 @@ console_scripts =
 [files]
 packages =
     ansible_runner
+data-files =
+    share/ansible-runner/utils = utils/*
 
 [pep8]
 # E201 - Whitespace after '('

test/integration/test_interface.py

@@ -142,12 +142,13 @@ def test_run_command(test_data_dir):
     private_data_dir = os.path.join(test_data_dir, 'debug')
     inventory = os.path.join(private_data_dir, 'inventory', 'inv_1')
     playbook = os.path.join(private_data_dir, 'project', 'debug.yml')
-    out, err = run_command(
+    out, err, rc = run_command(
         private_data_dir=private_data_dir,
         executable_cmd='ansible-playbook',
         cmdline_args=[playbook, '-i', inventory]
     )
     assert "Hello world!" in out
+    assert rc == 0
     assert err == ''
 
 
@@ -160,13 +161,14 @@ def test_run_ansible_command_within_container(test_data_dir, container_runtime_i
         'process_isolation': True,
         'container_image': defaults.default_container_image
     }
-    out, err = run_command(
+    rc, out, err = run_command(
         private_data_dir=private_data_dir,
         executable_cmd='ansible-playbook',
         cmdline_args=[playbook, '-i', inventory],
         **container_kwargs
     )
     assert "Hello world!" in out
+    assert rc == 0
     assert err == ''
 
 
@@ -180,14 +182,15 @@ def test_run_script_within_container(test_data_dir, container_runtime_installed)
         'container_image': defaults.default_container_image,
         'container_volume_mounts': container_volume_mounts
     }
-    out, _ = run_command(
+    out, _, rc = run_command(
         private_data_dir=private_data_dir,
         executable_cmd='python3',
         cmdline_args=[os.path.join(script_path, 'test_ee.py')],
         **container_kwargs
     )
 
     assert "os-release" in out
+    assert rc == 0
 
 
 def test_run_command_async(test_data_dir):

test/unit/config/test__base.py

@@ -242,7 +242,8 @@ def test_wrap_args_with_ssh_agent_silent():
 
 @patch('os.path.isdir', return_value=False)
 @patch('os.path.exists', return_value=True)
-def test_container_volume_mounting_with_Z(mock_isdir, mock_exists, tmpdir):
+@patch('os.makedirs', return_value=True)
+def test_container_volume_mounting_with_Z(mock_isdir, mock_exists, mock_makedirs, tmpdir):
     rc = BaseConfig(private_data_dir=str(tmpdir))
     os.path.isdir = Mock()
     rc.container_volume_mounts = ['project_path:project_path:Z']

test/unit/config/test_runner.py

@@ -538,7 +538,8 @@ def new_exists(path):
 
 @patch('os.path.isdir', return_value=False)
 @patch('os.path.exists', return_value=True)
-def test_process_isolation_settings(mock_isdir, mock_exists):
+@patch('os.makedirs', return_value=True)
+def test_process_isolation_settings(mock_isdir, mock_exists, mock_makedirs):
     rc = RunnerConfig('/')
     rc.artifact_dir = '/tmp/artifacts'
     rc.playbook = 'main.yaml'

test/unit/test_runner.py

@@ -11,7 +11,7 @@
 import sys
 
 from ansible_runner import Runner
-from ansible_runner.exceptions import CallbackError
+from ansible_runner.exceptions import CallbackError, AnsibleRunnerException
 from ansible_runner.config.runner import RunnerConfig
 
 HERE, FILENAME = os.path.split(__file__)
@@ -103,6 +103,18 @@ def test_env_vars(rc, value):
         assert value in f.read()
 
 
+def test_event_callback_data_check(rc):
+    rc.ident = "testident"
+    rc.check_job_event_data = True
+    runner = Runner(config=rc, remove_partials=False)
+    runner.event_handler = mock.Mock()
+
+    with pytest.raises(AnsibleRunnerException) as exc:
+        runner.event_callback(dict(uuid="testuuid", counter=0))
+
+    assert "Failed to open ansible stdout callback plugin partial data" in str(exc)
+
+
 def test_event_callback_interface_has_ident(rc):
     rc.ident = "testident"
     runner = Runner(config=rc, remove_partials=False)