Skip to content

Commit

Permalink
Improve ADJ diagnostics (#835)
Browse files Browse the repository at this point in the history
* move call to DIAGSTATS_INI_IO

from diagnostics_init_fixed.F to diagnostics_init_varia.F

* move call to TURNOFF_MODEL_IO (for DIVA)

- move call to TURNOFF_MODEL_IO from ctrl_init_fixed.F (for DIVA run) to the_model_main.F
- minor improvement in the_model_main.F (test for file "costfinal" only once,
  improve indentation, ...)

* switch "diag_pkgStatus" on/off

- switch diag_pkgStatus to ready2fillDiags also in first backward sweep call
- switch it off in last DIAGNOSTICS_WRITE call (as it used to be)
- try to do the same in diagnostics_write_adj.F (for now, commented out)

* update "gdiag" description

* cleaning

- remove 500+ lines of MITgcm general description that don't get updated when
  main version (in model/src) is; add a relevant comment/description ;
- avoid changing back and forth "writeBinaryPrec" by using WRITE_REC_3D_RL ;
- remove Divided-Adjoint pieces of code ; remove some un-used var ;
- import some improvements from model/src version.

* switch arg order (myThid last)

for small S/R (and used only for DIVA) cost_final_[re]store.F

* Adjust report to STDOUT

write few more (useful) info.

* New small S/R for diagnostics with DIVA

Just to know when DIAGNOSTICS_SWITCH_ONOFF is called for the first time

* switch on diagnostics (including some ADJ diags)

* report more output

* Adjust comments/descriptions

* adjust comments

* document fixing ADJ-diags in DIVA runs
  • Loading branch information
jm-c authored Jul 17, 2024
1 parent e37161e commit acacc28
Show file tree
Hide file tree
Showing 23 changed files with 568 additions and 959 deletions.
7 changes: 7 additions & 0 deletions doc/tag-index
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
Notes on tags used in MITgcmUV
==============================

o pkg/diagnostics:
- to allow to use ADJ-diags in DIVA runs, change call sequence regarding
TURNOFF_MODEL_IO and DIAGSTATS_INI_IO and simplify (fewer tests for
"costfinal") the_model_main.F (+ shorten pkg/openad version) ;
- restore the use of "diag_pkgStatus" (switch & check) in AD backward sweep
with new small S/R for DIVA runs ;
- turn on FWD and ADJ-diags in secondary lab_sea AD test "noseaice".
o pkg/bling:
- clean-up "bling_light.F" (fix typo related to PHYTO_SELF_SHADING option,
move k-loop outside) ; add CHL to bling_ad_check_lev{2,3,4}_dir.h and
Expand Down
140 changes: 70 additions & 70 deletions model/src/the_model_main.F
Original file line number Diff line number Diff line change
Expand Up @@ -606,19 +606,19 @@ SUBROUTINE THE_MODEL_MAIN(myThid)
C myIter :: Iteration counter for this thread
INTEGER myIter
_RL myTime
LOGICAL exst
LOGICAL costFinalExist
LOGICAL lastdiva
CEOP

C-- set default:
exst = .TRUE.
costFinalExist = .TRUE.
lastdiva = .TRUE.

#ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_ENTER('THE_MODEL_MAIN',myThid)
#endif

#if defined(USE_PAPI) || defined(USE_PCL_FLOPS_SFP) || defined(USE_PCL_FLOPS) || defined(USE_PCL)
#if ( defined USE_PAPI || defined USE_PCL_FLOPS_SFP || defined USE_PCL_FLOPS || defined USE_PCL )
CALL TIMER_CONTROL('','INIT','THE_MODEL_MAIN',myThid)
#endif
C-- This timer encompasses the whole code
Expand All @@ -635,81 +635,83 @@ SUBROUTINE THE_MODEL_MAIN(myThid)
myTime = startTime
myIter = nIter0

#if ( defined (ALLOW_ADMTLM) )
#ifdef ALLOW_ADMTLM

STOP 'should never get here; ADMTLM_DSVD calls ADMTLM_DRIVER'

#elif ( defined (ALLOW_AUTODIFF))
#elif ( defined ALLOW_AUTODIFF )

# ifdef ALLOW_CTRL
# ifndef EXCLUDE_CTRL_PACK
IF ( useCTRL ) THEN
inquire( file='costfinal', exist=exst )
IF ( .NOT. exst ) THEN
IF ( (optimcycle.NE.0 .OR. .NOT.doinitxx)
& .AND. doMainUnpack ) THEN
CALL TIMER_START('CTRL_UNPACK [THE_MODEL_MAIN]',myThid)
CALL CTRL_UNPACK( .TRUE. , myThid )
CALL TIMER_STOP ('CTRL_UNPACK [THE_MODEL_MAIN]',myThid)
ENDIF
ENDIF
ENDIF
# endif /* EXCLUDE_CTRL_PACK */
# endif
# ifndef EXCLUDE_CTRL_PACK
INQUIRE( FILE='costfinal', EXIST=costFinalExist )
IF ( .NOT.costFinalExist ) THEN
IF ( (optimcycle.NE.0 .OR. .NOT.doinitxx)
& .AND. doMainUnpack ) THEN
CALL TIMER_START('CTRL_UNPACK [THE_MODEL_MAIN]',myThid)
CALL CTRL_UNPACK( .TRUE. , myThid )
CALL TIMER_STOP ('CTRL_UNPACK [THE_MODEL_MAIN]',myThid)
ENDIF
ENDIF
# elif ( defined ALLOW_DIVIDED_ADJOINT )
INQUIRE( FILE='costfinal', EXIST=costFinalExist )
# endif /* EXCLUDE_CTRL_PACK & ALLOW_DIVIDED_ADJOINT */
ENDIF
# endif /* ALLOW_CTRL */

# ifdef ALLOW_COST
CALL COST_DEPENDENT_INIT ( myThid )
# endif

# if defined( ALLOW_TANGENTLINEAR_RUN )

# ifndef ALLOW_TAPENADE
# if ( defined ALLOW_TANGENTLINEAR_RUN )

# ifdef ALLOW_DEBUG
# ifndef ALLOW_TAPENADE
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('G_THE_MAIN_LOOP',myThid)
# endif
# endif
CALL TIMER_START('G_THE_MAIN_LOOP [TANGENT RUN]',myThid)
CALL G_THE_MAIN_LOOP ( myTime, myIter, myThid )
CALL TIMER_STOP ('G_THE_MAIN_LOOP [TANGENT RUN]',myThid)
# endif /* ndef ALLOW_TAPENADE */

# endif /* ndef ALLOW_TAPENADE */

# elif defined( ALLOW_ADJOINT_RUN )
# elif ( defined ALLOW_ADJOINT_RUN )

# ifdef ALLOW_TAPENADE
CALL THE_MAIN_LOOP_B( myTime, myIter, myThid )
# else /* ALLOW_TAPENADE */
# ifdef ALLOW_TAPENADE
CALL THE_MAIN_LOOP_B( myTime, myIter, myThid )
# else /* ALLOW_TAPENADE */

# ifdef ALLOW_DIVIDED_ADJOINT
# ifdef ALLOW_DIVIDED_ADJOINT
C-- The following assumes the TAF option '-pure'
inquire( file='costfinal', exist=exst )
IF ( .NOT. exst) THEN
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('MDTHE_MAIN_LOOP',myThid)
# endif
CALL TIMER_START('MDTHE_MAIN_LOOP [MD RUN]', myThid)
CALL MDTHE_MAIN_LOOP ( myTime, myIter, myThid )
CALL TIMER_STOP ('MDTHE_MAIN_LOOP [MD RUN]', myThid)
CALL COST_FINAL_STORE ( myThid, lastdiva )
IF ( .NOT.costFinalExist ) THEN
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('MDTHE_MAIN_LOOP',myThid)
# endif
CALL TIMER_START('MDTHE_MAIN_LOOP [MD RUN]', myThid)
CALL MDTHE_MAIN_LOOP ( myTime, myIter, myThid )
CALL TIMER_STOP ('MDTHE_MAIN_LOOP [MD RUN]', myThid)
CALL COST_FINAL_STORE ( lastdiva, myThid )
ELSE
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('ADTHE_MAIN_LOOP',myThid)
# endif
CALL TIMER_START('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
CALL ADTHE_MAIN_LOOP ( myThid )
CALL TIMER_STOP ('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
CALL COST_FINAL_RESTORE ( myThid, lastdiva )
C-- for DIVA, avoid forward-related output in adjoint part
CALL TURNOFF_MODEL_IO( 1, myThid )
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('ADTHE_MAIN_LOOP',myThid)
# endif
CALL TIMER_START('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
CALL ADTHE_MAIN_LOOP ( myThid )
CALL TIMER_STOP ('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
CALL COST_FINAL_RESTORE ( lastdiva, myThid )
ENDIF

# else /* ALLOW_DIVIDED_ADJOINT undef */
# ifdef ALLOW_DEBUG
# else /* ALLOW_DIVIDED_ADJOINT undef */
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('ADTHE_MAIN_LOOP',myThid)
# endif
# endif
CALL TIMER_START('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
CALL ADTHE_MAIN_LOOP ( myThid )
CALL TIMER_STOP ('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
# endif /* ALLOW_DIVIDED_ADJOINT */
# endif /* ALLOW_TAPENADE */
# endif /* ALLOW_DIVIDED_ADJOINT */

# endif /* ALLOW_TAPENADE */

# else /* forward run only within AD setting */

Expand All @@ -725,27 +727,25 @@ SUBROUTINE THE_MODEL_MAIN(myThid)

# ifdef ALLOW_CTRL
# ifndef EXCLUDE_CTRL_PACK
IF ( useCTRL ) THEN
IF ( lastdiva .AND. doMainPack ) THEN
CALL TIMER_START('CTRL_PACK [THE_MODEL_MAIN]',myThid)
CALL CTRL_PACK( .FALSE. , myThid )
CALL TIMER_STOP ('CTRL_PACK [THE_MODEL_MAIN]',myThid)
IF ( ( optimcycle.EQ.0 .OR. (.NOT. doMainUnpack) )
& .AND. myIter.EQ.nIter0 ) THEN
CALL TIMER_START('CTRL_PACK [THE_MODEL_MAIN]',myThid)
CALL CTRL_PACK( .TRUE. , myThid )
CALL TIMER_STOP ('CTRL_PACK [THE_MODEL_MAIN]',myThid)
ENDIF
ENDIF
IF ( useCTRL .AND. lastdiva .AND. doMainPack ) THEN
CALL TIMER_START('CTRL_PACK [THE_MODEL_MAIN]',myThid)
CALL CTRL_PACK( .FALSE. , myThid )
CALL TIMER_STOP ('CTRL_PACK [THE_MODEL_MAIN]',myThid)
IF ( ( optimcycle.EQ.0 .OR. (.NOT. doMainUnpack) )
& .AND. myIter.EQ.nIter0 ) THEN
CALL TIMER_START('CTRL_PACK [THE_MODEL_MAIN]',myThid)
CALL CTRL_PACK( .TRUE. , myThid )
CALL TIMER_STOP ('CTRL_PACK [THE_MODEL_MAIN]',myThid)
ENDIF
ENDIF
# endif /* EXCLUDE_CTRL_PACK */
# endif /* ALLOW_CTRL */

# ifdef ALLOW_GRDCHK
IF ( useGrdchk .AND. lastdiva ) THEN
CALL TIMER_START('GRDCHK_MAIN [THE_MODEL_MAIN]',myThid)
CALL GRDCHK_MAIN( myThid )
CALL TIMER_STOP ('GRDCHK_MAIN [THE_MODEL_MAIN]',myThid)
CALL TIMER_START('GRDCHK_MAIN [THE_MODEL_MAIN]',myThid)
CALL GRDCHK_MAIN( myThid )
CALL TIMER_STOP ('GRDCHK_MAIN [THE_MODEL_MAIN]',myThid)
ENDIF
# endif

Expand All @@ -762,13 +762,13 @@ SUBROUTINE THE_MODEL_MAIN(myThid)
#endif /* ALLOW_TANGENTLINEAR_RUN ALLOW_ADJOINT_RUN ALLOW_ADMTLM */

#ifdef ALLOW_STREAMICE
IF (useStreamIce) THEN
IF ( useStreamIce ) THEN
CALL STREAMICE_FINALIZE_PETSC
ENDIF
#endif

#ifdef ALLOW_MNC
IF (useMNC) THEN
IF ( useMNC ) THEN
C Close all open NetCDF files
_BEGIN_MASTER( myThid )
CALL MNC_FILE_CLOSE_ALL( myThid )
Expand All @@ -781,8 +781,8 @@ SUBROUTINE THE_MODEL_MAIN(myThid)

C-- Write timer statistics
IF ( myThid .EQ. 1 ) THEN
CALL TIMER_PRINTALL( myThid )
CALL COMM_STATS
CALL TIMER_PRINTALL( myThid )
CALL COMM_STATS
ENDIF

C-- Check threads synchronization :
Expand Down
102 changes: 47 additions & 55 deletions pkg/cost/cost_final_restore.F
Original file line number Diff line number Diff line change
Expand Up @@ -3,68 +3,60 @@
# include "AUTODIFF_OPTIONS.h"
#endif

subroutine cost_final_restore ( mythid, lastdiva )
CBOP
C !ROUTINE: COST_FINAL_RESTORE

c ==================================================================
c SUBROUTINE cost_final_restore
c ==================================================================
c
c Restore cost function value
c This is required in conjunction with DIVA
c [email protected] 5-Mar-2003
c
c ==================================================================
c SUBROUTINE cost_final_restore
c ==================================================================
C !INTERFACE:
SUBROUTINE COST_FINAL_RESTORE( lastDiva, myThid )

implicit none

c == global variables ==
C !DESCRIPTION:
C Restore cost function value
C This is required in conjunction with DIVA
C [email protected] 5-Mar-2003

C !USES:
IMPLICIT NONE
C == Global variables ===
#include "EEPARAMS.h"
#include "SIZE.h"

#include "cost.h"

c == routine arguments ==

integer mythid
logical lastdiva

# ifdef ALLOW_DIVIDED_ADJOINT

c == local variables ==
logical exst

integer idivbeg, idivend
c == end of interface ==

c-- Each process has calculated the global part for itself.
cph IF (myProcId .eq. 0) THEN
_BEGIN_MASTER( mythid )
c
inquire(file='divided.ctrl',exist=exst)
if (exst) then
open(unit=76,file='divided.ctrl',form='formatted')
read(unit=76,fmt=*) idivbeg,idivend
close(unit=76)
else
idivbeg = -2
endif
c
if ( idivbeg .EQ. 0 ) then
lastdiva = .TRUE.
open(unit=76,file='costfinal')
read(76,*) fc
close(76)
else
lastdiva = .FALSE.
endif
c
_END_MASTER( mythid )
cph ENDIF
C !INPUT/OUTPUT PARAMETERS:
C lastDiva :: True if last AD run from DIVA sequence, False otherwise
C myThid :: my Thread Id number
LOGICAL lastDiva
INTEGER myThid

#ifdef ALLOW_DIVIDED_ADJOINT
C !LOCAL VARIABLES:
LOGICAL exst
INTEGER idivbeg, idivend
CEOP

_BEGIN_MASTER( myThid )

INQUIRE(FILE='divided.ctrl',EXIST=exst)
IF (exst) THEN
OPEN(UNIT=76,FILE='divided.ctrl',FORM='formatted')
READ(UNIT=76,FMT=*) idivbeg,idivend
CLOSE(UNIT=76)
ELSE
idivbeg = -2
ENDIF

IF ( idivbeg .EQ. 0 ) THEN
lastDiva = .TRUE.
OPEN(UNIT=76,FILE='costfinal')
READ(76,*) fc
CLOSE(76)
ELSE
lastDiva = .FALSE.
ENDIF

_END_MASTER( myThid )
_BARRIER

#endif
#endif /* ALLOW_DIVIDED_ADJOINT */

end
RETURN
END
Loading

0 comments on commit acacc28

Please sign in to comment.