add test for signed partial job results #786

openeogeotrellis/load_stac.py

@@ -142,6 +142,7 @@ def operator_value(criterion: Dict[str, object]) -> (str, object):
                            else None)
 
     if dependency_job_info:
+        # TODO: handle polling own job results as well (~ GpsBatchJobs.poll_job_dependencies)
         logger.info(f"load_stac of results of own job {dependency_job_info.id}")
         intersecting_items = []
 

tests/data/stac/issue786_partial_job_results/collection.json

@@ -0,0 +1,47 @@
+{
+  "id": "collection",
+  "type": "Collection",
+  "description": "description",
+  "license": "proprietary",
+  "stac_version": "1.0.0",
+  "extent": {
+    "spatial": {
+      "bbox": [
+        [
+          -180,
+          -90,
+          180,
+          90
+        ]
+      ]
+    },
+    "temporal": {
+      "interval": [
+        [
+          "1982-08-22T00:00:00.000Z",
+          null
+        ]
+      ]
+    }
+  },
+  "links": [
+    {
+      "rel": "item",
+      "href": "results/items/item01.json"
+    }
+  ],
+  "summaries": {
+    "eo:bands": [
+        {
+          "name": "band1"
+        },
+        {
+          "name": "band2"
+        },
+        {
+          "name": "band3"
+        }
+      ]
+  },
+  "openeo:status": "$openeo_status"
+}

tests/data/stac/issue786_partial_job_results/item01.json

@@ -0,0 +1,77 @@
+{
+  "stac_version": "1.0.0",
+  "stac_extensions": [],
+  "type": "Feature",
+  "id": "item",
+  "bbox": [
+    5.0,
+    50.0,
+    6.0,
+    51.0
+  ],
+  "geometry": {
+    "type": "Polygon",
+    "coordinates": [
+      [
+        [
+          5.0,
+          50.0
+        ],
+        [
+          5.0,
+          51.0
+        ],
+        [
+          6.0,
+          51.0
+        ],
+        [
+          6.0,
+          50.0
+        ],
+        [
+          5.0,
+          50.0
+        ]
+      ]
+    ]
+  },
+  "properties": {
+    "start_datetime": "2022-03-04T00:00:00Z",
+    "end_datetime": "2022-03-04T00:00:00Z",
+    "proj:epsg": 4326,
+    "proj:bbox": [
+      5.0,
+      50.0,
+      6.0,
+      51.0
+    ],
+    "proj:shape": [
+      10,
+      10
+    ]
+  },
+  "links": [
+    {
+      "rel": "collection",
+      "href": "collection.json"
+    }
+  ],
+  "assets": {
+    "asset1": {
+      "href": "asset01.tiff",
+      "type": "image/tiff; application=geotiff",
+      "eo:bands": [
+        {
+          "name": "band1"
+        },
+        {
+          "name": "band2"
+        },
+        {
+          "name": "band3"
+        }
+      ]
+    }
+  }
+}

tests/test_api_result.py

@@ -4063,6 +4063,68 @@ def test_load_stac_from_unsigned_job_results_respects_proj_metadata(self, api110
             assert ds.shape == tuple(expected_shape)
             assert tuple(ds.bounds) == tuple(map(pytest.approx, expected_bbox))
 
+    @gps_config_overrides(job_dependencies_poll_interval_seconds=0)
+    def test_load_stac_from_partial_job_results_basic(self, api110, urllib_mock, tmp_path, caplog):
+        """load_stac from partial job results Collection (signed case)"""
+
+        caplog.set_level("DEBUG")
+
+        def collection_json(path):
+            # stateful response callback to allow different responses for the same URL: first 'running', then 'finished'
+            dependency_finished = False
+
+            def collection_json_with_openeo_status(_: urllib.request.Request):
+                nonlocal dependency_finished
+
+                openeo_status = 'finished' if dependency_finished else 'running'
+                dependency_finished = True
+                return UrllibMocker.Response(
+                    data=get_test_data_file(path).read_text().replace("$openeo_status", openeo_status))
+
+            return collection_json_with_openeo_status
+
+        def item_json(path):
+            return (
+                get_test_data_file(path).read_text()
+                .replace(
+                    "asset01.tiff",
+                    f"file://{get_test_data_file('binary/load_stac/collection01/asset01.tif').absolute()}"
+                )
+            )
+
+        urllib_mock.register("GET",
+                             "https://openeo.test/openeo/jobs/j-2402094545c945c09e1307503aa58a3a/results?partial=true",
+                             response=collection_json("stac/issue786_partial_job_results/collection.json"))
+        urllib_mock.get("https://openeo.test/openeo/jobs/j-2402094545c945c09e1307503aa58a3a/results/items/item01.json",
+                        data=item_json("stac/issue786_partial_job_results/item01.json"))
+
+        process_graph = {
+            "loadstac1": {
+                "process_id": "load_stac",
+                "arguments": {
+                    "url": "https://openeo.test/openeo/jobs/j-2402094545c945c09e1307503aa58a3a/results?partial=true"
+                }
+            },
+            "saveresult1": {
+                "process_id": "save_result",
+                "arguments": {"data": {"from_node": "loadstac1"}, "format": "GTiff"},
+                "result": True
+            },
+        }
+
+        api110.result(process_graph).assert_status_code(200)
+
+        assert ("OpenEO batch job results status for"
+                " https://openeo.test/openeo/jobs/j-2402094545c945c09e1307503aa58a3a/results?partial=true: running"
+                in caplog.messages)
+        assert ("OpenEO batch job results status for"
+                " https://openeo.test/openeo/jobs/j-2402094545c945c09e1307503aa58a3a/results?partial=true: finished"
+                in caplog.messages)
+
+    def test_load_stac_from_unsigned_partial_job_results(self):
+        """load_stac from partial job results Collection (unsigned case)"""
+        raise NotImplementedError("TODO")
+
 
 class TestEtlApiReporting:
     @pytest.fixture(autouse=True)