Merge pull request #65 from NREL/bnb/dev

path updates for kestrel runs
NREL · Jan 5, 2024 · 5a5676c · 5a5676c
2 parents 9df56d5 + b3d684a
commit 5a5676c
Show file tree

Hide file tree

Showing 9 changed files with 158 additions and 140 deletions.
diff --git a/nsrdb/config/pipeline_legacy/config_east_nsrdb.json b/nsrdb/config/pipeline_legacy/config_east_nsrdb.json
@@ -44,12 +44,12 @@
     "name": "nsrdb_east",
     "nsrdb_freq": "10min",
     "nsrdb_grid": "/projects/pxs/reference_grids/nsrdb_meta_2km_east_-105.csv",
-    "out_dir": "/lustre/eaglefs/projects/pxs/processing/2019/east/",
+    "out_dir": "/projects/pxs/processing/2019/east/",
     "year": 2019
   },
   "eagle": {
     "alloc": "pxs",
-    "feature": "--qos=high",
+    "feature": "--qos=normal",
     "memory": 178,
     "walltime": 48
   }

diff --git a/nsrdb/config/pipeline_legacy/config_east_pipeline.json b/nsrdb/config/pipeline_legacy/config_east_pipeline.json
@@ -6,19 +6,19 @@
   "name": "east",
   "pipeline": [
     {
-      "data-model": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "data-model": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     },
     {
-      "collect-data-model": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "collect-data-model": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     },
     {
-      "cloud-fill": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "cloud-fill": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     },
     {
-      "all-sky": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "all-sky": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     },
     {
-      "collect-final": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "collect-final": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     }
   ]
 }
diff --git a/nsrdb/config/pipeline_legacy/config_west_conus_nsrdb.json b/nsrdb/config/pipeline_legacy/config_west_conus_nsrdb.json
@@ -41,13 +41,13 @@
     "log_level": "DEBUG",
     "name": "nsrdb_west_conus",
     "nsrdb_freq": "5min",
-    "nsrdb_grid": "/lustre/eaglefs/projects/pxs/reference_grids/nsrdb_meta_2km_conus_west.csv",
-    "out_dir": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/",
+    "nsrdb_grid": "/projects/pxs/reference_grids/nsrdb_meta_2km_conus_west.csv",
+    "out_dir": "/projects/pxs/processing/2019/west_conus/",
     "year": 2019
   },
   "eagle": {
     "alloc": "pxs",
-    "feature": "--qos=high",
+    "feature": "--qos=normal",
     "memory": 83,
     "walltime": 4
   }

diff --git a/nsrdb/config/pipeline_legacy/config_west_conus_pipeline.json b/nsrdb/config/pipeline_legacy/config_west_conus_pipeline.json
@@ -6,19 +6,19 @@
   "name": "west_conus",
   "pipeline": [
     {
-      "data-model": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "data-model": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     },
     {
-      "collect-data-model": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "collect-data-model": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     },
     {
-      "cloud-fill": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "cloud-fill": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     },
     {
-      "all-sky": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "all-sky": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     },
     {
-      "collect-final": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "collect-final": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     }
   ]
 }
diff --git a/nsrdb/config/templates/config_nsrdb_pre2018.json b/nsrdb/config/templates/config_nsrdb_pre2018.json
@@ -13,43 +13,43 @@
     ],
     "factory_kwargs": {
       "cld_opd_dcomp": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cld_press_acha": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cld_reff_dcomp": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cloud_fraction": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cloud_probability": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cloud_type": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "refl_0_65um_nom": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "refl_0_65um_nom_stddev_3x3": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "refl_3_75um_nom": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "surface_albedo": {
         "source_dir": "/projects/pxs/ancillary/albedo/nsrdb_%year%/"
       },
       "temp_11_0um_nom": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "temp_11_0um_nom_stddev_3x3": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "temp_3_75um_nom": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       }
     },
     "max_workers": null,