Merge pull request #1610 from gustavo-marques/dev-ncar-main-candidate…

…-2023-09-13

NCAR to main (2023-09-14)

.github/workflows/coupled-api.yml

@@ -28,7 +28,3 @@ jobs:
     - name: Compile MOM6 for the NUOPC driver
       shell: bash
       run: make check_mom6_api_nuopc -j
-
-    - name: Compile MOM6 for the MCT driver
-      shell: bash
-      run: make check_mom6_api_mct -j

.testing/Makefile

@@ -255,7 +255,6 @@ build/opt/Makefile: MOM_ENV=$(PATH_FMS) $(OPT_FCFLAGS) $(MOM_LDFLAGS)
 build/opt_target/Makefile: MOM_ENV=$(PATH_FMS) $(OPT_FCFLAGS) $(MOM_LDFLAGS)
 build/coupled/Makefile: MOM_ENV=$(PATH_FMS) $(SYMMETRIC_FCFLAGS) $(MOM_LDFLAGS)
 build/nuopc/Makefile: MOM_ENV=$(PATH_FMS) $(SYMMETRIC_FCFLAGS) $(MOM_LDFLAGS)
-build/mct/Makefile: MOM_ENV=$(PATH_FMS) $(SYMMETRIC_FCFLAGS) $(MOM_LDFLAGS)
 build/cov/Makefile: MOM_ENV=$(PATH_FMS) $(COV_FCFLAGS) $(COV_LDFLAGS)
 build/unit/Makefile: MOM_ENV=$(PATH_FMS) $(COV_FCFLAGS) $(COV_LDFLAGS)
 
@@ -269,7 +268,6 @@ build/opt/Makefile: MOM_ACFLAGS=
 build/opt_target/Makefile: MOM_ACFLAGS=
 build/coupled/Makefile: MOM_ACFLAGS=--with-driver=FMS_cap
 build/nuopc/Makefile: MOM_ACFLAGS=--with-driver=nuopc_cap
-build/mct/Makefile: MOM_ACFLAGS=--with-driver=mct_cap
 build/cov/Makefile: MOM_ACFLAGS=
 build/unit/Makefile: MOM_ACFLAGS=--with-driver=unit_tests
 
@@ -370,11 +368,6 @@ build/coupled/ocean_model_MOM.o: build/coupled/Makefile
 	cd $(@D) && make $(@F)
 check_mom6_api_coupled: build/coupled/ocean_model_MOM.o
 
-# MCT driver
-build/mct/mom_ocean_model_mct.o: build/mct/Makefile
-	cd $(@D) && make $(@F)
-check_mom6_api_mct: build/mct/mom_ocean_model_mct.o
-
 
 #---
 # Testing

config_src/drivers/nuopc_cap/mom_cap.F90

@@ -28,9 +28,11 @@ module MOM_cap_mod
 use MOM_cap_methods,          only: mom_import, mom_export, mom_set_geomtype, mod2med_areacor
 use MOM_cap_methods,          only: med2mod_areacor, state_diagnose
 use MOM_cap_methods,          only: ChkErr
+use MOM_ensemble_manager,     only: ensemble_manager_init
 
 #ifdef CESMCOUPLED
 use shr_log_mod,             only: shr_log_setLogUnit
+use nuopc_shr_methods,       only: get_component_instance
 #endif
 use time_utils_mod,           only: esmf2fms_time
 
@@ -147,7 +149,8 @@ module MOM_cap_mod
 logical :: cesm_coupled = .false.
 type(ESMF_GeomType_Flag) :: geomtype
 #endif
-character(len=8) :: restart_mode = 'alarms'
+character(len=8)  :: restart_mode = 'alarms'
+character(len=16) :: inst_suffix = ''
 real(8) :: timere
 
 contains
@@ -434,6 +437,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
                                                                  ! (same as restartfile if single restart file)
   character(len=*), parameter            :: subname='(MOM_cap:InitializeAdvertise)'
   character(len=32)                      :: calendar
+  character(len=:), allocatable          :: rpointer_filename
+  integer                                :: inst_index
   real(8)                                :: MPI_Wtime, timeiads
 !--------------------------------
 
@@ -465,6 +470,13 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
   CALL ESMF_TimeIntervalGet(TINT, S=DT_OCEAN, RC=rc)
   if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
+#ifdef CESMCOUPLED
+  call get_component_instance(gcomp, inst_suffix, inst_index, rc)
+  if (ChkErr(rc,__LINE__,u_FILE_u)) return
+  call ensemble_manager_init(inst_suffix)
+  rpointer_filename = 'rpointer.ocn'//trim(inst_suffix)
+#endif
+
   ! reset shr logging to my log file
   if (localPet==0) then
     call NUOPC_CompAttributeGet(gcomp, name="diro", &
@@ -474,11 +486,20 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
          isPresent=isPresentLogfile, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     if (isPresentDiro .and. isPresentLogfile) then
-         call NUOPC_CompAttributeGet(gcomp, name="diro", value=diro, rc=rc)
-         if (ChkErr(rc,__LINE__,u_FILE_u)) return
-         call NUOPC_CompAttributeGet(gcomp, name="logfile", value=logfile, rc=rc)
-         if (ChkErr(rc,__LINE__,u_FILE_u)) return
-         open(newunit=stdout,file=trim(diro)//"/"//trim(logfile))
+      call NUOPC_CompAttributeGet(gcomp, name="diro", value=diro, rc=rc)
+      if (ChkErr(rc,__LINE__,u_FILE_u)) return
+      call NUOPC_CompAttributeGet(gcomp, name="logfile", value=logfile, rc=rc)
+      if (ChkErr(rc,__LINE__,u_FILE_u)) return
+
+      if (cesm_coupled) then
+        ! Multiinstance logfile name needs a correction
+        if(len_trim(inst_suffix) > 0) then
+          n = index(logfile, '.')
+          logfile = logfile(1:n-1)//trim(inst_suffix)//logfile(n:)
+        endif
+      endif
+
+      open(newunit=stdout,file=trim(diro)//"/"//trim(logfile))
     else
       stdout = output_unit
     endif
@@ -535,12 +556,6 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
 
   time0 = set_date (YEAR,MONTH,DAY,HOUR,MINUTE,SECOND)
 
-
-  ! rsd need to figure out how to get this without share code
-  !call shr_nuopc_get_component_instance(gcomp, inst_suffix, inst_index)
-  !inst_name = "OCN"//trim(inst_suffix)
-
-
   if (is_root_pe()) then
     write(stdout,*) subname//'start time: y,m,d-',year,month,day,'h,m,s=',hour,minute,second
   endif
@@ -595,9 +610,9 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
 
       if (localPet == 0) then
         ! this hard coded for rpointer.ocn right now
-        open(newunit=readunit, file='rpointer.ocn', form='formatted', status='old', iostat=iostat)
+        open(newunit=readunit, file=rpointer_filename, form='formatted', status='old', iostat=iostat)
         if (iostat /= 0) then
-          call ESMF_LogSetError(ESMF_RC_FILE_OPEN, msg=subname//' ERROR opening rpointer.ocn', &
+          call ESMF_LogSetError(ESMF_RC_FILE_OPEN, msg=subname//' ERROR opening '//rpointer_filename, &
                  line=__LINE__, file=u_FILE_u, rcToReturn=rc)
           return
         endif
@@ -607,7 +622,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
             if (len(trim(restartfiles))>1 .and. iostat<0) then
               exit ! done reading restart files list.
             else
-              call ESMF_LogSetError(ESMF_RC_FILE_READ, msg=subname//' ERROR reading rpointer.ocn', &
+              call ESMF_LogSetError(ESMF_RC_FILE_READ, msg=subname//' ERROR reading '//rpointer_filename, &
                  line=__LINE__, file=u_FILE_u, rcToReturn=rc)
               return
             endif
@@ -630,7 +645,12 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
   endif
 
   ocean_public%is_ocean_pe = .true.
-  call ocean_model_init(ocean_public, ocean_state, time0, time_start, input_restart_file=trim(adjustl(restartfiles)))
+  if (cesm_coupled .and. len_trim(inst_suffix)>0) then
+    call ocean_model_init(ocean_public, ocean_state, time0, time_start, &
+      input_restart_file=trim(adjustl(restartfiles)), inst_index=inst_index)
+  else
+    call ocean_model_init(ocean_public, ocean_state, time0, time_start, input_restart_file=trim(adjustl(restartfiles)))
+  endif
 
   ! GMM, this call is not needed in CESM. Check with EMC if it can be deleted.
   call ocean_model_flux_init(ocean_state)
@@ -1514,6 +1534,7 @@ subroutine ModelAdvance(gcomp, rc)
   character(len=128)                     :: fldname
   character(len=*),parameter             :: subname='(MOM_cap:ModelAdvance)'
   character(len=8)                       :: suffix
+  character(len=:), allocatable          :: rpointer_filename
   integer                                :: num_rest_files
   real(8)                                :: MPI_Wtime, timers
 
@@ -1688,20 +1709,26 @@ subroutine ModelAdvance(gcomp, rc)
         call ESMF_VMGet(vm, localPet=localPet, rc=rc)
         if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
+        rpointer_filename = 'rpointer.ocn'//trim(inst_suffix)
+
         write(restartname,'(A,".mom6.r.",I4.4,"-",I2.2,"-",I2.2,"-",I5.5)') &
              trim(casename), year, month, day, seconds
         call ESMF_LogWrite("MOM_cap: Writing restart :  "//trim(restartname), ESMF_LOGMSG_INFO)
         ! write restart file(s)
         call ocean_model_restart(ocean_state, restartname=restartname, num_rest_files=num_rest_files)
         if (localPet == 0) then
            ! Write name of restart file in the rpointer file - this is currently hard-coded for the ocean
-          open(newunit=writeunit, file='rpointer.ocn', form='formatted', status='unknown', iostat=iostat)
+          open(newunit=writeunit, file=rpointer_filename, form='formatted', status='unknown', iostat=iostat)
           if (iostat /= 0) then
             call ESMF_LogSetError(ESMF_RC_FILE_OPEN, &
-                 msg=subname//' ERROR opening rpointer.ocn', line=__LINE__, file=u_FILE_u, rcToReturn=rc)
+                 msg=subname//' ERROR opening '//rpointer_filename, line=__LINE__, file=u_FILE_u, rcToReturn=rc)
             return
           endif
-          write(writeunit,'(a)') trim(restartname)//'.nc'
+          if (len_trim(inst_suffix) == 0) then
+            write(writeunit,'(a)') trim(restartname)//'.nc'
+          else
+            write(writeunit,'(a)') trim(restartname)//'.'//trim(inst_suffix)//'.nc'
+          endif
 
           if (num_rest_files > 1) then
             ! append i.th restart file name to rpointer

config_src/drivers/nuopc_cap/mom_ocean_model_nuopc.F90

@@ -41,7 +41,7 @@ module MOM_ocean_model_nuopc
 use MOM_time_manager,        only : operator(/=), operator(<=), operator(>=)
 use MOM_time_manager,        only : operator(<), real_to_time_type, time_type_to_real
 use MOM_interpolate,         only : time_interp_external_init
-use MOM_tracer_flow_control, only : call_tracer_flux_init
+use MOM_tracer_flow_control, only : tracer_flow_control_CS, call_tracer_flux_init, call_tracer_set_forcing
 use MOM_unit_scaling,        only : unit_scale_type
 use MOM_variables,           only : surface
 use MOM_verticalGrid,        only : verticalGrid_type
@@ -210,6 +210,8 @@ module MOM_ocean_model_nuopc
   type(marine_ice_CS), pointer :: &
     marine_ice_CSp => NULL()  !< A pointer to the control structure for the
                               !! marine ice effects module.
+  type(tracer_flow_control_CS), pointer :: &
+    tracer_flow_CSp => NULL()  !< A pointer to the tracer flow control structure
   type(wave_parameters_CS), pointer, public :: &
     Waves => NULL()           !< A pointer to the surface wave control structure
   type(surface_forcing_CS), pointer :: &
@@ -229,7 +231,7 @@ module MOM_ocean_model_nuopc
 !!   This subroutine initializes both the ocean state and the ocean surface type.
 !! Because of the way that indicies and domains are handled, Ocean_sfc must have
 !! been used in a previous call to initialize_ocean_type.
-subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, input_restart_file)
+subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, input_restart_file, inst_index)
   type(ocean_public_type), target, &
                        intent(inout) :: Ocean_sfc !< A structure containing various publicly
                                 !! visible ocean surface properties after initialization,
@@ -246,6 +248,7 @@ subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, i
                                               !! tracer fluxes, and can be used to spawn related
                                               !! internal variables in the ice model.
   character(len=*), optional, intent(in) :: input_restart_file !< If present, name of restart file to read
+  integer, optional :: inst_index !< Ensemble index provided by the cap (instead of FMS ensemble manager)
 
   ! Local variables
   real :: Rho0        ! The Boussinesq ocean density, in kg m-3.
@@ -255,7 +258,6 @@ subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, i
                       !! min(HFrz, OBLD), where OBLD is the boundary layer depth.
                       !! If HFrz <= 0 (default), melt potential will not be computed.
   logical :: use_melt_pot !< If true, allocate melt_potential array
-  logical :: use_CFC  !< If true, allocated arrays for surface CFCs.
 
 
 ! This include declares and sets the variable "version".
@@ -283,7 +285,8 @@ subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, i
   call initialize_MOM(OS%Time, Time_init, param_file, OS%dirs, OS%MOM_CSp, &
                       OS%restart_CSp, Time_in, offline_tracer_mode=OS%offline_tracer_mode, &
                       input_restart_file=input_restart_file, &
-                      diag_ptr=OS%diag, count_calls=.true., waves_CSp=OS%Waves)
+                      diag_ptr=OS%diag, count_calls=.true., tracer_flow_CSp=OS%tracer_flow_CSp, &
+                      waves_CSp=OS%Waves, ensemble_num=inst_index)
   call get_MOM_state_elements(OS%MOM_CSp, G=OS%grid, GV=OS%GV, US=OS%US, C_p=OS%C_p, &
                               C_p_scaled=OS%fluxes%C_p, use_temp=use_temperature)
 
@@ -376,16 +379,14 @@ subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, i
     use_melt_pot=.false.
   endif
 
-  call get_param(param_file, mdl, "USE_CFC_CAP", use_CFC, &
-                 default=.false., do_not_log=.true.)
   call get_param(param_file, mdl, "USE_WAVES", OS%Use_Waves, &
        "If true, enables surface wave modules.", default=.false.)
 
   !   Consider using a run-time flag to determine whether to do the diagnostic
   ! vertical integrals, since the related 3-d sums are not negligible in cost.
   call allocate_surface_state(OS%sfc_state, OS%grid, use_temperature, &
                               do_integrals=.true., gas_fields_ocn=gas_fields_ocn, &
-                              use_meltpot=use_melt_pot, use_cfcs=use_CFC)
+                              use_meltpot=use_melt_pot)
 
   call surface_forcing_init(Time_in, OS%grid, OS%US, param_file, OS%diag, &
                             OS%forcing_CSp, OS%restore_salinity, OS%restore_temp, OS%use_waves)
@@ -611,6 +612,11 @@ subroutine update_ocean_model(Ice_ocean_boundary, OS, Ocean_sfc, &
     call finish_MOM_initialization(OS%Time, OS%dirs, OS%MOM_CSp, OS%restart_CSp)
   endif
 
+  if (do_thermo) &
+    call call_tracer_set_forcing(OS%sfc_state, OS%fluxes, OS%Time, &
+                                 real_to_time_type(dt_coupling), OS%grid, OS%US, OS%GV%Rho0, &
+                                 OS%tracer_flow_CSp)
+
   call disable_averaging(OS%diag)
   Master_time = OS%Time ; Time1 = OS%Time