path updates for kestrel runs

nsrdb/config/pipeline_legacy/config_east_nsrdb.json

@@ -44,12 +44,12 @@
     "name": "nsrdb_east",
     "nsrdb_freq": "10min",
     "nsrdb_grid": "/projects/pxs/reference_grids/nsrdb_meta_2km_east_-105.csv",
-    "out_dir": "/lustre/eaglefs/projects/pxs/processing/2019/east/",
+    "out_dir": "/projects/pxs/processing/2019/east/",
     "year": 2019
   },
   "eagle": {
     "alloc": "pxs",
-    "feature": "--qos=high",
+    "feature": "--qos=normal",
     "memory": 178,
     "walltime": 48
   }

nsrdb/config/pipeline_legacy/config_east_pipeline.json

@@ -6,19 +6,19 @@
   "name": "east",
   "pipeline": [
     {
-      "data-model": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "data-model": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     },
     {
-      "collect-data-model": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "collect-data-model": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     },
     {
-      "cloud-fill": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "cloud-fill": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     },
     {
-      "all-sky": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "all-sky": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     },
     {
-      "collect-final": "/lustre/eaglefs/projects/pxs/processing/2019/east/config_nsrdb_east.json"
+      "collect-final": "/projects/pxs/processing/2019/east/config_nsrdb_east.json"
     }
   ]
 }

nsrdb/config/pipeline_legacy/config_west_conus_nsrdb.json

@@ -41,13 +41,13 @@
     "log_level": "DEBUG",
     "name": "nsrdb_west_conus",
     "nsrdb_freq": "5min",
-    "nsrdb_grid": "/lustre/eaglefs/projects/pxs/reference_grids/nsrdb_meta_2km_conus_west.csv",
-    "out_dir": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/",
+    "nsrdb_grid": "/projects/pxs/reference_grids/nsrdb_meta_2km_conus_west.csv",
+    "out_dir": "/projects/pxs/processing/2019/west_conus/",
     "year": 2019
   },
   "eagle": {
     "alloc": "pxs",
-    "feature": "--qos=high",
+    "feature": "--qos=normal",
     "memory": 83,
     "walltime": 4
   }

nsrdb/config/pipeline_legacy/config_west_conus_pipeline.json

@@ -6,19 +6,19 @@
   "name": "west_conus",
   "pipeline": [
     {
-      "data-model": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "data-model": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     },
     {
-      "collect-data-model": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "collect-data-model": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     },
     {
-      "cloud-fill": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "cloud-fill": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     },
     {
-      "all-sky": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "all-sky": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     },
     {
-      "collect-final": "/lustre/eaglefs/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
+      "collect-final": "/projects/pxs/processing/2019/west_conus/config_nsrdb_west_conus.json"
     }
   ]
 }

nsrdb/config/templates/config_nsrdb_pre2018.json

@@ -13,43 +13,43 @@
     ],
     "factory_kwargs": {
       "cld_opd_dcomp": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cld_press_acha": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cld_reff_dcomp": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cloud_fraction": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cloud_probability": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "cloud_type": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "refl_0_65um_nom": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "refl_0_65um_nom_stddev_3x3": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "refl_3_75um_nom": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "surface_albedo": {
         "source_dir": "/projects/pxs/ancillary/albedo/nsrdb_%year%/"
       },
       "temp_11_0um_nom": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "temp_11_0um_nom_stddev_3x3": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       },
       "temp_3_75um_nom": {
-        "pattern": "/lustre/eaglefs/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
+        "pattern": "/projects/pxs/HDF/%satellite%/%year%/{doy}/level2/*_%year%_*.h5"
       }
     },
     "max_workers": null,

nsrdb/utilities/extract_surfrad.py

@@ -4,14 +4,13 @@
 
 @author: gbuster
 """
+import os
+
 import h5py
 import numpy as np
-import os
 import pandas as pd
-
 from farms import SZA_LIM
 
-
 DAT_COLS = ('year',
             'jday',
             'month',
@@ -104,7 +103,7 @@ def get_dat_table(d, flist):
         table = []
 
         # get readlines iterator
-        with open(os.path.join(d, fname), 'r') as f:
+        with open(os.path.join(d, fname)) as f:
             lines = f.readlines()
 
         # iterate through lines
@@ -161,7 +160,7 @@ def get_lw1_table(d, flist):
     for i, fname in enumerate(flist):
 
         # get readlines iterator
-        with open(os.path.join(d, fname), 'r') as f:
+        with open(os.path.join(d, fname)) as f:
             lines = f.readlines()
 
         # iterate through lines
@@ -302,7 +301,7 @@ def extract_all(root_dir, dir_out, years=range(1998, 2018), file_flag='.dat',
 
 
 if __name__ == '__main__':
-    root_dir = '/lustre/eaglefs/projects/pxs/surfrad/raw'
-    dir_out = '/lustre/eaglefs/projects/pxs/surfrad/h5'
+    root_dir = '/projects/pxs/surfrad/raw'
+    dir_out = '/projects/pxs/surfrad/h5'
     site_codes = ('bon', 'dra', 'fpk', 'gwn', 'psu', 'sxf', 'tbl')
     extract_all(root_dir, dir_out, site_codes=site_codes)

nsrdb/utilities/file_utils.py

@@ -3,26 +3,24 @@
 
 @author: gbuster
 """
-from concurrent.futures import as_completed
 import gzip
 import logging
 import os
+import re
 import shlex
 import shutil
-import re
-from subprocess import Popen, PIPE, run
 import time
-from urllib.request import urlopen
+from concurrent.futures import as_completed
+from subprocess import PIPE, Popen, run
 from urllib.error import URLError
+from urllib.request import urlopen
+
+import numpy as np
 import pandas as pd
 from packaging import version
-
-
 from rex.utilities.execution import SpawnProcessPool
 from rex.utilities.loggers import init_logger
 
-import numpy as np
-
 logger = logging.getLogger(__name__)
 
 DIR = os.path.dirname(os.path.realpath(__file__))
@@ -210,7 +208,7 @@ def convert_h4(path4, f_h4, path5, f_h5):
     h5 = os.path.join(path5, f_h5)
 
     if not os.path.exists(h4):
-        raise IOError('Could not locate file for conversion to h5: {}'
+        raise OSError('Could not locate file for conversion to h5: {}'
                       .format(h4))
     if os.path.exists(h5):
         logger.info('Target h5 file already exists, may have already been '
@@ -281,7 +279,7 @@ def convert_list_serial(conversion_list):
     """Convert h4 to h5 files in serial based on the conversion list.
 
     Parameters
-    -------
+    ----------
     conversion_list : list
         List of paths and files to convert for input to convert4to5.
         Format is: conversion_list = [[path4, f_h4, path5, f_h5], ...]
@@ -297,7 +295,7 @@ def convert_list_parallel(conversion_list, n_workers=2):
     """Convert h4 to h5 files in parallel based on the conversion list.
 
     Parameters
-    -------
+    ----------
     conversion_list : list
         List of paths and files to convert for input to convert4to5.
         Format is: conversion_list = [[path4, f_h4, path5, f_h5], ...]
@@ -364,8 +362,8 @@ def convert_directory(path4, path5, n_workers=1):
 
 
 if __name__ == '__main__':
-    path4 = '/lustre/eaglefs/projects/pxs/uwisc/2018_west/'
-    path5 = '/lustre/eaglefs/projects/pxs/uwisc/2018_west_h5/'
+    path4 = '/projects/pxs/uwisc/2018_west/'
+    path5 = '/projects/pxs/uwisc/2018_west_h5/'
     init_logger(__name__, log_level='INFO',
                 log_file=os.path.join(path5, 'convert.log'))
     convert_directory(path4, path5, n_workers=36)

nsrdb/utilities/movers.py

@@ -3,15 +3,15 @@
 
 @author: gbuster
 """
-import h5py
 import logging
-import numpy as np
 import os
-import pandas as pd
 import time
 from warnings import warn
 
-from rex.utilities.hpc import SLURM, PBS
+import h5py
+import numpy as np
+import pandas as pd
+from rex.utilities.hpc import PBS, SLURM
 from rex.utilities.loggers import init_logger
 
 from nsrdb.utilities.file_utils import repack_h5
@@ -421,7 +421,7 @@ def peregrine(fun_str, arg_str, alloc='pxs', queue='batch-h',
 
 def eagle(fun_str, arg_str, alloc='pxs', memory=96,
           walltime=10, node_name='mover',
-          stdout_path='/lustre/eaglefs/scratch/gbuster/data_movers/'):
+          stdout_path='//scratch/gbuster/data_movers/'):
     """Kick off an eagle job to execute a mover function.
 
     Parameters

nsrdb/utilities/update_nsrdb_versions.py

@@ -3,10 +3,11 @@
 
 @author: gbuster
 """
-import h5py
 import os
 
-root_dir = '/lustre/eaglefs/projects/pxs/nsrdb/v3.0.1/'
+import h5py
+
+root_dir = '/projects/pxs/nsrdb/v3.0.1/'
 
 versions = {"nsrdb_1998.h5": "3.0.6",
             "nsrdb_1999.h5": "3.0.6",