Add inspection effectiveness

.gitignore

@@ -130,3 +130,5 @@ dmypy.json
 
 # LibreOffice locks
 .~lock.*#
+
+.idea/

config.yml

@@ -96,3 +96,4 @@ inspection:
   cluster:
     cluster_selection: cluster
     interval: 5
+  effectiveness: 0.65

popsborder/inputs.py

@@ -643,3 +643,19 @@ def load_skip_lot_consignment_records(filename, tracked_properties):
             level = row["compliance_level"]
             records[tuple(combo)] = text_to_value(level)
     return records
+
+
+def get_validated_effectiveness(config, verbose=False):
+    """Set the effectiveness of the inspector.
+
+    :param config: Configuration file
+    :param verbose: Print the message if True
+    """
+    if isinstance(config, dict):
+        if "effectiveness" in config["inspection"]:
+            if 0 <= config["inspection"]["effectiveness"] <= 1:
+                return config["inspection"]["effectiveness"]
+            else:
+                if verbose:
+                    print("Effectiveness out of range: it should be between 0 and 1.")
+    return 1

popsborder/inspections.py

@@ -27,6 +27,8 @@
 
 import numpy as np
 
+from .inputs import get_validated_effectiveness
+
 
 def inspect_first(consignment):
     """Inspect only the first box in the consignment"""
@@ -410,6 +412,8 @@ def inspect(config, consignment, n_units_to_inspect, detailed):
         config, consignment, n_units_to_inspect
     )
 
+    effectiveness = get_validated_effectiveness(config)
+
     # Inspect selected boxes, count opened boxes, inspected items, and contaminated
     # items to detection and completion
     ret = types.SimpleNamespace(
@@ -454,7 +458,7 @@ def inspect(config, consignment, n_units_to_inspect, detailed):
                     ret.items_inspected_completion += 1
                     if not detected:
                         ret.items_inspected_detection += 1
-                    if item:
+                    if item and random.random() < effectiveness:
                         # Count all contaminated items in sample, regardless of
                         # detected variable
                         ret.contaminated_items_completion += 1
@@ -488,7 +492,7 @@ def inspect(config, consignment, n_units_to_inspect, detailed):
                     boxes_opened_detection.append(
                         math.floor(item_index / items_per_box)
                     )
-                if consignment.items[item_index]:
+                if consignment.items[item_index] and random.random() < effectiveness:
                     # Count every contaminated item in sample
                     ret.contaminated_items_completion += 1
                     if not detected:
@@ -522,7 +526,7 @@ def inspect(config, consignment, n_units_to_inspect, detailed):
                     ret.inspected_item_indexes.append(item_index)
                 if not detected:
                     ret.items_inspected_detection += 1
-                if item:
+                if item and random.random() < effectiveness:
                     # Count every contaminated item in sample
                     ret.contaminated_items_completion += 1
                     # If first contaminated box inspected,

tests/test_effectiveness.py

@@ -0,0 +1,159 @@
+"""Test effectiveness"""
+
+import types
+
+import pytest
+
+from popsborder.inputs import get_validated_effectiveness
+from popsborder.inputs import load_configuration_yaml_from_text
+from popsborder.simulation import run_simulation
+
+CONFIG = """\
+consignment:
+  generation_method: parameter_based
+  parameter_based:
+    origins:
+    - Netherlands
+    - Mexico
+    flowers:
+    - Hyacinthus
+    - Rosa
+    - Gerbera
+    ports:
+    - NY JFK CBP
+    - FL Miami Air CBP
+    boxes:
+      min: 1
+      max: 100
+  items_per_box:
+    default: 100
+contamination:
+  contamination_unit: items
+  contamination_rate:
+    distribution: beta
+    parameters:
+    - 4
+    - 60
+  arrangement: random_box
+  random_box:
+    probability: 0.2
+    ratio: 0.5
+inspection:
+  unit: boxes
+  within_box_proportion: 1
+  sample_strategy: proportion
+  tolerance_level: 0
+  min_boxes: 0
+  proportion:
+    value: 0.02
+  hypergeometric:
+    detection_level: 0.05
+    confidence_level: 0.95
+  fixed_n: 10
+  selection_strategy: random
+  cluster:
+    cluster_selection: random
+    interval: 3
+"""
+
+ret = types.SimpleNamespace(
+    inspected_item_indexes=[],
+    boxes_opened_completion=0,
+    boxes_opened_detection=0,
+    items_inspected_completion=0,
+    items_inspected_detection=0,
+    contaminated_items_completion=0,
+    contaminated_items_detection=0,
+    contaminated_items_missed=0,
+)
+
+config = load_configuration_yaml_from_text(CONFIG)
+num_consignments = 100
+detailed = False
+
+
+def test_set_effectiveness_no_key():
+    """Test config has no effectiveness key"""
+    effectiveness = get_validated_effectiveness(config)
+    assert effectiveness == 1
+
+
+def test_set_effectiveness_out_of_range():
+    """Test effectiveness out of range"""
+    for val in [-1, 1.1, 2.5]:
+        config["inspection"]["effectiveness"] = val
+        effectiveness = get_validated_effectiveness(config)
+        assert effectiveness == 1
+
+
+def test_set_effectiveness_in_range():
+    """Test effectiveness in range"""
+    for val in [0, 0.5, 1]:
+        config["inspection"]["effectiveness"] = val
+        effectiveness = get_validated_effectiveness(config)
+        assert effectiveness == val
+
+
+class TestEffectiveness:
+    """There are two types of inspection methodologies:
+    1) counting contaminated items in the first contaminated box
+        * the unit is a boxes or items with a "cluster" selection strategy
+        * inspection_effectiveness is calculated. It should be close enough to
+        effectiveness in the configuration file.
+    2) counting the first contaminated item.
+        * the unit is item with other than "cluster" selection strategy
+        * inspection_effectiveness is 0 since only count first contaminated item. For
+        this, the number of items missed before detection is calculated. Simulation
+        result is average of how many missed contaminated items before first
+        contaminated item is detected.
+    """
+
+    @pytest.fixture()
+    def setup(self):
+        min_boxes = 30
+        max_boxes = 150
+        # config = load_configuration_yaml_from_text(CONFIG)
+        config["consignment"]["parameter_based"]["boxes"]["min"] = min_boxes
+        config["consignment"]["parameter_based"]["boxes"]["max"] = max_boxes
+        config["inspection"]["effectiveness"] = 0.9
+        yield config
+
+    def test_effectiveness_unit_box(self, setup):
+        """Test effectiveness with inspection method boxes."""
+        for seed in range(10):
+            result = run_simulation(
+                config=config, num_simulations=3, num_consignments=100, seed=seed
+            )
+        assert result.pct_contaminant_unreported_if_detection > 0
+
+    def test_effectiveness_unit_items_random(self, setup):
+        """Test effectiveness with inspection method items with random selection
+        strategy.
+        """
+        config["inspection"]["unit"] = "items"
+        for seed in range(10):
+            result = run_simulation(
+                config=config, num_simulations=3, num_consignments=100, seed=seed
+            )
+        assert result.pct_contaminant_unreported_if_detection > 0
+
+    def test_effectiveness_unit_items_cluster(self, setup):
+        """Test effectiveness with inspection method items with cluster selection
+        strategy.
+        """
+        config["inspection"]["unit"] = "items"
+        config["inspection"]["selection_strategy"] = "cluster"
+        for seed in range(10):
+            result = run_simulation(
+                config=config, num_simulations=3, num_consignments=100, seed=seed
+            )
+        assert result.pct_contaminant_unreported_if_detection > 0
+
+    def test_effectiveness_none(self, setup):
+        """Test effectiveness not set in the configuration file."""
+        del config["inspection"]["effectiveness"]
+        for seed in range(10):
+            result = run_simulation(
+                config=config, num_simulations=3, num_consignments=100, seed=seed
+            )
+        assert result.pct_contaminant_unreported_if_detection > 0