1 """!Utilities for ensemble-based data assimilation.
3 This module contains utilities for doing various forms of
4 ensemble-based data assimilation. It manages a two-dimension
5 ensemble-time array of Task objects that know how to run each analysis
6 cycle for each member. This module also provides a way of presenting
7 the previous forecast cycle's ensda objects such that this cycle can
8 use them. This is all built on top of the classes in the
9 hwrf.fcsttask module."""
13 __all__=[
'DAEnsemble',
'FromPriorCycle',
'FromGFSENKF',
14 'write_ensda_flag_file',
'read_ensda_flag_file',
15 'CycleTDRCheck',
'AlwaysRunENSDA',
'enada_pre_object_for']
26 from hwrf.numerics import to_datetime_rel, to_timedelta, TimeArray, to_datetime
34 """!Represents a two-dimensional ensemble-vs-time array of
35 hwrf.hwrftask.HWRFTask objects."""
36 def __init__(self,dstore,conf,section,anlintime=None,
37 taskname=
None,**kwargs):
38 """!Constructor for DAEnsemble
39 @param dstore the produtil.datastore.Datastore database to use
40 @param conf the hwrf.config.HWRFConfig for configuration info
41 @param section the section to use in conf
42 @param anlintime the analysis input time
43 @param taskname the name of this task in the database
44 @param kwargs passed to the superclass constructor"""
45 super(DAEnsemble,self).
__init__(dstore,conf,section,taskname,
50 anlintime=to_datetime_rel(anlintime,conf.cycle)
54 cycling_interval=conf.getfloat(
'config',
'cycling_interval')*3600.0
56 self.
__tstep=to_timedelta(self.
confstr(
'da_cycle',cycling_interval))
57 assert(self.
__tstep>to_timedelta(1800))
58 endtime=to_datetime_rel(cycling_interval,conf.cycle)
66 """!The time at the beginning of the first ensemble step."""
71 """!The time at the end of the last ensemble step."""
76 """!The number of members of the ensemble."""
81 """!The number of ensemble DA time steps."""
86 """!Iterates over all ensemble analysis input times."""
87 for t
in self.__members.times():
92 """!Iterates over all ensemble analysis output times."""
94 for t
in self.__members.times():
102 """!Iterates over all member ids."""
107 """!sets the HWRFTask to use to use for one cycle of one member
109 Tells member enkfmem to use the specified task to produce
110 output whose input analysis time is atime.
111 @param atime the analysis time, a datetime.datetime
112 @param enkfmem the enkf member id
113 @param task the HWRFTask to use"""
114 self.__memberids.add(enkfmem)
118 """!iterate over members for a specified analysis input time
120 Iterates over all members at the specified anlintime,
121 yielding (id,member) tuples.
122 @param atime the analysis time, a datetime.datetime"""
125 yield (enkfmem,memstep)
128 """!iterate over members at the final analysis output time.
130 Iterates over all members at the final analysis output time,
131 yielding (id,member) tuples."""
136 """!iterate over analysis cycles for a specific member
138 Iterates over (time,EnsembleDAMemberStep) pairs for the
140 for (t,a)
in self.__members.iteritems():
144 """!get the analysis cycle for the specified member and time
146 Returns the da cycle for member enkfmem at analysis input
148 @param atime the analysis time, a datetime.datetime
149 @param enkfmem the ensemble id"""
153 """!iterate over all needed input data, for hwrf.input
155 Calls inputiter for all steps of all ENKF members. This is
156 for use by the hwrf.input to figure out what input data is
157 required for the DA ensemble."""
158 for t,e
in self.__members.iteritems():
159 for m
in e.itervalues():
160 for d
in m.inputiter():
164 """!print detailed diagnostics
166 Sends detailed diagnostics about all members to the print()
167 statement. This is intended for debugging only."""
168 for (time,stuff)
in self.__members.iteritems():
169 for (enkfid,memberstep)
in stuff.iteritems():
170 print "self.__members[%s][%s]=%s"%(
171 repr(time),repr(enkfid),repr(memberstep))
172 t=self.__members.lasttime
173 print 'last time t is %s'%(repr(t),)
175 print 'self.__members[t][%s]=%s'%(
176 repr(enkfid),repr(memberstep))
181 """!Represents an ensemble member from the previous forecast cycle.
183 This is used to generate UpstreamFile objects for the previous
184 cycle's ensemble of FromGFSENKF simulations."""
185 def __init__(self,dstore,conf,section,domains,enkfmem,anlouttime,
187 """!FromPriorCycle constructor
189 Makes a new FromPriorCycle object.
190 @param dstore the produtil.datastore.Datastore database object
191 @param conf the hwrf.config.HWRFConfig with configuration info
192 @param section the section to use within conf
193 @param domains the list or tuple of hwrf.wrf.WRFDomain objects
194 to obtain from the prior cycle, in order of
196 @param enkfmem the enkf member ID
197 @param anlouttime the output time at the end of the analysis
198 @param kwargs passed to the superclass constructor"""
199 super(FromPriorCycle,self).
__init__(dstore,conf,section,**kwargs)
205 """!Iterates over all products
207 Iterates over products produced by the prior forecast cycle's
209 @param domains if provided and non-None, only products from these
211 @param kwargs ignored """
213 if domains
is None: domains=self.
__domains
215 for domain
in domains:
223 """!return the product for the specified domain and analysis time
225 Returns the product for the wrfinput_d01 file for the
226 specified domain and time. This is simply a wrapper around
227 get_product(domain,atime)
228 @param domain the domain of interest
229 @param atime the analysis time as a datetime.datetime"""
232 """!returns the wrfanl file's product"""
235 """!Returns a product for the specified domain and time.
237 Creates a new produtil.datastore.UpstreamFile for the wrfinput
238 or wrfanl file for the given domain, at the given analysis
239 time. The analysis time must be the anlouttime.
240 @return a newly created UpstreamFile for the file of interest
241 @param domain the domain of interest
242 @param atime the analysis time"""
246 logger.info(
'Wrong atime: %s vs %s'
250 logger.info(
'Invalid domain: %s not in %s'%(
251 str(domain),
', '.join([str(x)
for x
in self.
__domains])))
254 '{oldcom}/{oldvit[stormnamelc]}{oldvit[stormid3lc]}.{oldvit[YMDH]}.'
255 'ensda_{enkfid:03d}.wrfinput_d{domid:02d}',enkfid=self.
__enkfmem,
256 domid=int(domain.get_grid_id()))
257 logger.info(
'Domain %s atime %s enkfmem %s loc %s'%(
258 str(domain),str(atime),repr(self.
__enkfmem),repr(loc)))
260 prodname=os.path.basename(loc),
265 def get_track(self,atime):
269 logger.info(
'Wrong atime: %s vs %s'
274 '{oldcom}/{oldvit[stormnamelc]}{oldvit[stormid3lc]}.{oldvit[YMDH]}.'
275 'trak.hwrf.atcfunix.mem'+enkfid)
276 logger.info(
'atime %s enkfmem %s loc %s'%(
277 str(atime),repr(self.
__enkfmem),repr(loc)))
279 prodname=os.path.basename(loc),
287 """! Forecast ensemble member based on the GFS ENKF.
289 Runs one member of an ensemble DA forecast ensemble, using a
290 member of the GFS ENKF ensemble as initial and boundary
291 conditions. Some data from the earlier deterministic forecast
292 jobs is reused to simplify the process."""
294 def __init__(self,dstore,conf,section,detinit,enkfmem,sim,
295 taskname=
None,track=
None,relocate=
None,priorcycle=
None,
298 """!Constructor for FromGFSENKF
300 @param dstore the produtil.datastore.Datastore database to use
301 @param conf the hwrf.config.HWRFConfig that provides
303 @param section the section in conf to use
304 @param detinit the deterministic initialization, an
305 hwrf.init.HWRFInit or hwrf.init.InitBeforeGSI object.
306 @param enkfmem the ensemble member id
307 @param sim the hwrf.wrf.WRFSimulation object
308 @param taskname the name of the task within the database
309 @param kwargs passed to the parent class constructor"""
310 """!Constructor for FromGFSENKF
312 @param dstore the produtil.datastore.Datastore database to use
313 @param conf the hwrf.config.HWRFConfig that provides
315 @param section the section in conf to use
316 @param detinit the deterministic initialization, an
317 hwrf.init.HWRFInit or hwrf.init.InitBeforeGSI object.
318 @param enkfmem the ensemble member id
319 @param sim the hwrf.wrf.WRFSimulation object
320 @param taskname the name of the task within the database
321 @param kwargs passed to the parent class constructor"""
322 super(FromGFSENKF,self).
__init__(dstore,conf,section,taskname,
325 if track
is None or relocate
is None:
327 'You must explicitly specify the track and relocate '
328 'arguments to FromGFSENKF.__init__ and they must be boolean '
331 self.
track=bool(track)
367 """!The analysis input time."""
368 return self.__wrf.simstart()
371 """!Create the wrf() and fcst
373 This function, called from the constructor, creates the
374 hwrf.wrf.WRFSimulation and hwrf.fcsttask.AnalysisCycle used
376 @param detinit the deterministic model initialization, an
377 hwrf.init.HWRFInit or hwrf.init.InitBeforeGSI.
378 @param sim the hwrf.wrf.WRFSimulation passed to the constructor."""
382 workdir=self.
workdir+
'/fcst',keeprun=
True)
386 self.fcst.sim.set_active_io_form_to(2)
389 """!Adds input sources to the forecast object.
391 Adds metgrid, geogrid, wrfinput, wrfbdy, wrfanl, and coupler
392 fort.65 input to the fcst member variable.
393 @param delinit the deterministic model initialization, an
394 hwrf.init.HWRFInit or hwrf.init.InitBeforeGSI."""
395 """!Adds input sources to the forecast object.
397 Adds metgrid, geogrid, wrfinput, wrfbdy, wrfanl, and coupler
398 fort.65 input to the fcst member variable.
399 @param delinit the deterministic model initialization, an
400 hwrf.init.HWRFInit or hwrf.init.InitBeforeGSI."""
407 for domain
in self.
sim:
408 if domain.is_moad():
continue
409 self.fcst.add_wrfanl(self.
wrfanl,domain)
410 self.fcst.sim.add_output(
'history',start=0,step=3600*6,end=6*3600)
414 """!The wrf simulation made by make_wrf()"""
418 """!Passes control to the hwrf.prep.PrepHybrid.inputiter().
420 Iterates over the prep member's inputiter, a
421 hwrf.prep.PrepHybrid.inputiter(). Yields all input data
422 information needed by the hwrf.input module to pull input
424 for d
in self.prep.inputiter():
yield d
427 """!Creates initialization tasks.
429 Called from the constructor. Creates the initialization
430 tasks, prep, realinit and wrfanl; and links to the
431 deterministic init geogrid and metgrid tasks. This is called
432 by the constructor, to create the needed inputs to the fcst
434 @param detinit the deterministic model initialization, an
435 hwrf.init.HWRFInit or hwrf.init.InitBeforeGSI."""
442 geodat=self.geogrid.geodat(moad)
444 realsection=self.
confstr(
'realinit')
445 prepsection=self.
confstr(
'prep_hybrid')
454 taskname=self.
taskname+
'.real',keeprun=
False,
458 .add_prep_hybrid(self.
prep)
461 self.realinit.sim.set_active_io_form_to(2)
463 self.prep.tvset(
'enkfmem',self.
enkfmem)
470 workdir=self.
workdir+
'/wrfanl',outdir=self.
outdir+
'/wrfanl')\
477 def _make_track(self):
478 """Makes the gribber and tracker member variables."""
480 ensdadoms=[ d
for d
in self.fcst.sim ]
484 self.
confstr(
'post'),6*3600,needcrtm=
False,
485 streams=[
'history'],taskname=self.
taskname+
'.post',
491 'part',hwrf.tracker.tracker_subset,
None)
493 lon=self.
conf[
'config',
'domlon'])
495 lon=self.storminfo.lon)
496 basin=self.storminfo.pubbasin2
497 if ((basin.upper()==
'AL' or basin.upper()==
'EP') \
498 and domloc.ewcenter<360.0):
499 domloc.ewcenter+=360.0
501 wgrib=alias(exe(self.conf.getexe(
'wgrib'))))
502 r.sync_frequently=self.conf.getbool(
'config',
'sync_frequently',
True)
503 r.add(
'd2',clatlon(grid2,res=[0.045,0.045],size=[30.,30.],
504 scan=136,n=[667,667]))
505 r.add(
'trkgrid',grid2*r.grid(
'd2'))
507 basedir=self.
outdir+
'/regribber'
508 enkf000=
'%03d'%int(self.
enkfmem)
509 r.to_intercom(
'{out_prefix}.hwrftrk.grbf{fahr:02d}.mem'+enkf000,
'hwrftrk')
512 start=self.
anlintime,step=6*3600,end=6*3600,
513 taskname=self.
taskname+
'.regribber',
518 step=6*3600,end=6*3600,outdir=self.
outdir+
'/tracker',
519 workdir=self.
workdir+
'/tracker')
521 self.tracker.send_atcfunix(
522 'track0',
'{com}/{out_prefix}.trak.hwrf.atcfunix.mem'+enkf000)
525 """!Iterates over all forecast products.
527 Passes control to hwrf.fcsttask.AnalysisCycle.products() to
528 iterate over all products that match the specified arguments.
529 @param kwargs passed to hwrf.fcsttask.AnalysisCycle.products()"""
530 for p
in self.fcst.products(**kwargs):
534 """!Runs the initialization and forecast for this ensemble member.
536 Runs the prep, realinit, wrfanl and fcst member tasks, using input
537 from the GFS ENKF, and the deterministic initialization."""
540 logger.info(
'Run ensemble member in %s'%(where,))
548 with self.dstore.transaction()
as t:
553 if self.gribber.is_completed():
556 msg=
'Error regridding inputs to tracker. See earlier log messages for details.'
561 def _make_relocate_kwargs(self,track,modin,dest_dir,priorcycle):
562 """Makes a dict containing the keyword arguments to send in to
563 the constructor for the hwrf.relocate task(s).
564 modin - the modin argument to the constructor
565 dest_dir - the directory in which to run the relocate"""
567 ensdadoms=[ d
for d
in self.fcst.sim ]
569 sim=self.fcst.sim,domains=ensdadoms,
570 modin=modin,dest_dir=dest_dir,
571 workdir=self.
workdir+
'/relocate',
572 outdir=self.
outdir+
'/relocate')
574 if priorcycle
is not None: kwargs.update(ensda=priorcycle)
575 if track: kwargs.update(parentTrack=self.
tracker,trackName=
'track0')
578 def _make_relocate(self,track,modin,priorcycle):
579 """Makes the relocation, rstage1, rstage2 and rstage3 member
581 track - the track argument to the constructor
582 modin - the modin argument to the constructor"""
584 dest_dir=os.path.join(self.
workdir,
'relocate')
589 taskname_pattern=self.
taskname+
'relocate.stage'+
'%d',
592 self.
rstage1=self.relocation.rstage1
593 self.
rstage3=self.relocation.rstage3
596 """Runs the relocate jobs, if present."""
597 if 'rstage1' in self.__dict__:
598 self.rstage1.delete_temp()
600 if 'rstage3' in self.__dict__:
602 if self.rstage1.scrub
and self.rstage3.scrub:
603 self.rstage3.delete_temp()
607 """!Writes the stormX.run_ensda flag file.
609 Writs the storm*.run_ensda flag file for this cycle. The purpose
610 of the file is to tell the workflow and scripting layers whether
611 the ensemble needs to be run. The file will contain a single
612 line: RUN_ENSDA=YES or RUN_ENSDA=NO. Will also log a message to
613 the jlogfile at INFO level telling which was written.
614 @param flag_file the full path to the flag file
615 @param run_ensda True or False: should the ENSDA be run? """
616 if run_ensda
is True:
617 with open(flag_file,
'wt')
as f:
618 f.write(
'RUN_ENSDA=YES\n')
619 produtil.log.jlogger.info(
'Will run HWRF ENSDA for this cycle.')
621 with open(flag_file,
'wt')
as f:
622 f.write(
'RUN_ENSDA=NO\n')
623 produtil.log.jlogger.info(
'Disabled HWRF ENSDA for this cycle.')
626 """!Reads the stormX.run_ensda flag file
628 This function is used by the scripting and workflow layers to
629 determine if the data assimilation ensemble should be run. Reads
630 the storm*.run_ensda flag file line by line, searching for a
631 single line RUN_ENSDA=YES or RUN_ENSDA=NO. Returns True for YES
632 or False for No, based on the last such line seen. Returns None
635 with open(flag_file,
'rt')
as f:
637 if line.find(
'RUN_ENSDA=YES')>=0:
639 elif line.find(
'RUN_ENSDA=NO')>=0:
645 """!Determines if Tail Doppler Radar (TDR) data is available.
647 This class checks to see if a specified cycle has Tail Doppler
648 Radar data available. This is the condition used to decide
649 whether to run the DA ensemble in the 2015 Operational HWRF."""
650 def __init__(self,dstore,conf,section,cycle_rel,**kwargs):
651 """!CycleTDRCheck constructor.
653 Create a new CycleTDRCheck which will look for TDR for the
654 specified cycle. The cycle_rel is anything accepted by
655 to_datetime_rel's second argument.
656 @param dstore the produtil.datastore.Datastore database object
657 @param conf the hwrf.config.HWRFConfig with configuration info
658 @param section the section to use within conf
659 @param cycle_rel specifies the cycle. This must be a number
660 of hours relative to the current cycle (conf.cycle) analysis
661 time. For example, -6*3600 would be the prior cycle and
662 48*3600 would be two days from now.
663 @param kwargs passed to the superclass constructor"""
664 super(CycleTDRCheck,self).
__init__(dstore,conf,section,**kwargs)
665 incat_name=self.
confstr(
'catalog')
668 self.
tgtcycle=to_datetime_rel(cycle_rel,conf.cycle)
671 dataset=self.
confstr(
'dataset',
'tdr')
672 item=self.
confstr(
'item',
'gdas1_bufr')
673 obstype=self.
confstr(
'obstype',
'tldplr')
684 """!Should ENSDA be run?
686 If self.run was called in this process, returns the cached
687 result of that. Otherwise, reads the run_ensda flag file from
688 COM. Uses hwrf.ensda.read_ensda_flag_file()"""
694 """!Iterates over needed files from upstream workflows.
696 This iterator's purpose is to provide "all files external
697 to this workflow" that are needed by the task. In this case,
698 of course, it is only the TDR bufr_d file. Hence, there is a
699 single "yield" statement that requests the TDR."""
703 """!creates the storm*.run_ensda file
705 Creates the storm1.run_ensda flag file with RUN_ENSDA=YES if
706 the TDR data is available, and RUN_ENSDA=NO otherwise."""
717 """!Write the ensda flag file.
719 Calls hwrf.ensda.write_ensda_flag_file to write the flag file.
720 @param run_ensda True means the ensemble should be run, False
721 if it should not be run."""
725 """!Check if TDR data is available for this cycle
727 Checks the on-disk input data to see if the TDR data is
730 atime=to_datetime(self.conf.cycle)
733 item=self.conf.get(
'tdr_new_obstype',
'item')
734 ds=os.path.join(self.
getdir(
'intercom'),
'bufrprep')
735 it=self._in_catalog.parse(item,atime=atime,
736 logger=logger,obstype=
'tldplr')
737 there=os.path.join(ds,it)
738 logger.info(
'TDR bufrprep should be at %s'%there)
739 if isnonempty(there):
744 def _actually_run(self):
745 """!Search on disk for TDR or a trigger file.
747 Do not call this routine directly; it is an internal
748 implementation routine. This routine contains the code that
749 actually determines if TDR is present or not. The "run"
750 routine calls _actually_run in a try-finally block to ensure
751 the run_ensda flag file is created no matter what."""
754 if self.
realtime and os.path.isdir(
'/dcom/us007003'):
756 logger.info(
'TDR data is available for current cycle %s!'
757 'Enabling ENSDA.'%self.conf.cycle.strftime(
'%Y%m%d%H'))
760 logger.info(
'There will be TDR data for cycle %s!'
761 'Enabling ENSDA.'%self.tgtcycle.strftime(
'%Y%m%d%H'))
764 logger.warning(
'ensda trigger file is not found. '
765 'TDR data is not available for current cycle'
766 'Will continue without ENSDA.')
771 if there_it_is
is None:
773 'Configuration error: DataCatalog %s does not know how '
774 'to find TDR data. Will continue without ENSDA.'
777 elif not isnonempty(there_it_is):
779 '%s: %s Tail Doppler Radar bufr_d file is empty or '
780 'non-existant. Will continue without ENSDA.'
781 %(there_it_is,self.tgtcycle.strftime(
'%Y%m%d%H')))
784 logger.info(
'%s: TDR data found for cycle %s! Enabling ENSDA.'
785 %(there_it_is,self.tgtcycle.strftime(
'%Y%m%d%H')))
789 """!Read TDR trigger file for operational run
791 Reads the TDR trigger file sent by the Aircraft Operations
792 Center (AOC) before a NOAA P3 Orion flight. This is used in
793 NCEP Operations when running the operational HWRF, to
794 determine if TDR is going to be available soon."""
797 ymdh=atime.strftime(
'%Y%m%d%H')
798 basin=self.storminfo.pubbasin2
799 if int(ymdh[0:4]) <= 2013:
801 elif basin.upper()==
'AL':
802 self.
_stormid=
'%s%02d' % (
'1',self.storminfo.stnum)
803 elif basin.upper()==
'EP':
804 self.
_stormid=
'%s%02d' % (
'2',self.storminfo.stnum)
805 elif basin.upper()==
'CP':
806 self.
_stormid=
'%s%02d' % (
'3',self.storminfo.stnum)
810 input_catalog=self.conf.get(
'config',
'fcst_catalog')
811 dcom=self.conf.get(input_catalog,
'dcom',
'/dcom/us007003')
812 if os.path.isdir(dcom):
813 btime=to_datetime_rel(-24*3600,atime)
814 tank1=os.path.join(dcom,atime.strftime(
"%Y%m%d"),
'b006/xx070')
815 tank2=os.path.join(dcom,btime.strftime(
"%Y%m%d"),
'b006/xx070')
816 logger.info(
'Locations: tank1 at %s tank2 at %s'%(tank1,tank2))
818 numtry=self.
confint(
'numofcheck',1)
819 timeinv=self.
confint(
'checksecinv',300)
823 if isnonempty(tank1):
824 logger.info(
'tank1 exist')
825 make_symlink(tank1,
'tldplrbufr',force=
True,logger=logger)
827 if not isnonempty(
'runensda')
and ymdh[8:10]==
'00' \
828 and isnonempty(tank2):
829 make_symlink(tank2,
'tldplrbufr',force=
True,logger=logger)
834 maxwait=timeinv,sleeptime=stime,min_size=1,
835 min_mtime_age=5,min_atime_age=
None,
836 min_ctime_age=
None,min_fraction=1.0):
837 logger.info(
'found trigger file')
840 if isnonempty(
'runensda'):
843 totalwait=timeinv*(numtry-1)/60.0
844 logger.info(
'waited for %s minutes, ensda trigger'
845 ' is not found'%str(totalwait))
848 logger.warning(
'%s does not exist. This is not wcoss.'
849 'real-time ensda trigger can only be run on wcoss'%dcom)
853 """!Runs the hwrf_readtdrtrigger program.
855 Runs the hwrf_readtdrtrigger program to decide if the TDR
856 trigger file is for this storm or not.
857 @param stmidin the storm
858 @param tgtcycle the cycle of interest"""
859 self.
log().info(
'readensdatrigger')
861 fprog =
'hwrf_readtdrtrigger'
868 """!Used in place of CycleTDRCheck to force ENSDA to always run.
870 This subclass of CycleTDRCheck instructs ENSDA to run whether
871 TDR is available or not."""
875 Always returns True, indicating that ENSDA should always be
876 run even if the world is imploding and pigs are raining from
877 clouds of cotton candy.
880 def _actually_run(self):
881 """!Runs the TDR check and ignores its results.
883 Calls the superclass _actually_run, so that the TDR check is
884 done, and then returns True regardless of the TDR
885 availability. Logs a warning about this to the
886 produtil.log.jlogger.
888 if not super(AlwaysRunENSDA,self)._actually_run():
889 msg=
"OVERRIDE: Will run ENSDA anyway due to "\
890 "configuration settings."
891 self.
log().warning(msg)
897 """!Generates a CycleTDRCheck or AlwaysRunENSDA based on
898 configuration settings.
900 Reads the [config] section ensda_when option to decide what TDR
901 check class should be used.
902 * ensda_when="tdr_next_cycle" - create a CycleTDRCheck
903 * ensda_when="always" - create an AlwaysRunENSDA
904 * ensda_when=anything else - raise an exception
905 @param ds,conf,section,next_cycle - passed to the constructor"""
906 ensda_when=conf.getstr(
'config',
'ensda_when',
'tdr_next_cycle').lower()
907 if ensda_when==
'tdr_next_cycle':
909 ds,conf,
'tdrcheck',next_cycle)
910 elif ensda_when==
'always':
912 ds,conf,
'tdrcheck',next_cycle)
914 raise ValueError(
'The ensda_when option must be set to tdr_next_cycle or always (case-insensitive) not %s.'%(repr(ensda_when),))
Change directory, handle temporary directories.
This module provides a set of utility functions to do filesystem operations.
Create namelists, monitor wrf simulations, generate filenames.
def get_wrfinput(self, domain, atime)
return the product for the specified domain and analysis time
def should_run_ensda(self)
Should ENSDA be run?
def confstrinterp(self, string, section=None, kwargs)
Alias for self.icstr for backward compatibility.
def tdr_this_cycle(self)
Check if TDR data is available for this cycle.
Runs regrib operations on many input times, sending output to an hwrf.gribtask.GRIBTask.
def dump(self)
print detailed diagnostics
runs wrf for an analysis cycle
A wrapper around PostOneWRF that posts many WRF output times.
def getexe
Alias for hwrf.config.HWRFConfig.get() for the "exe" section.
Represents a two-dimensional ensemble-vs-time array of hwrf.hwrftask.HWRFTask objects.
taskname
Read-only property: the name of this task.
def inputiter(self)
Passes control to the hwrf.prep.PrepHybrid.inputiter().
The base class of tasks run by the HWRF system.
This task runs the GFDL Vortex Tracker on HWRF output.
conf
This HWRFTask's hwrf.config.HWRFConfig object.
dstore
Read-only property, an alias for getdatastore(), the Datastore in which this Datum resides...
def nmembers(self)
The number of members of the ensemble.
a HWRFTask subclass that runs real_nmm
storminfo
The hwrf.storminfo.StormInfo describing the vitals information for the storm processed by this HWRFTa...
prep
the hwrf.prep.PrepHybrid that processes the GFS ENKF forecast and analysis spectral data ...
def run(self)
creates the storm*.run_ensda file
metgrid
the metgrid task from the deterministic initialization
def __init__(self, dstore, conf, section, domains, enkfmem, anlouttime, kwargs)
FromPriorCycle constructor.
def inputiter(self)
iterate over all needed input data, for hwrf.input
def make_fcst(self, detinit)
Adds input sources to the forecast object.
def make_init(self, detinit)
Creates initialization tasks.
def set_member(self, atime, enkfmem, task)
sets the HWRFTask to use to use for one cycle of one member
def checkrun(arg, logger=None, kwargs)
This is a simple wrapper round run that raises ExitStatusException if the program exit status is non-...
def should_run_ensda(self)
Returns True.
tgtcycle
the cycle for whom TDR data is checked
def write_flag_file(self, run_ensda)
Write the ensda flag file.
def __init__(self, dstore, conf, section, detinit, enkfmem, sim, taskname=None, track=None, relocate=None, priorcycle=None, kwargs)
Constructor for FromGFSENKF.
geogrid
the geogrid task from the deterministic initialization
A GRIBBase that subsets GRIB files, keeping only certain parameters.
enkfmem
the enkf member id
A shell-like syntax for running serial, MPI and OpenMP programs.
Base class of tasks run by HWRF.
def getdir
Alias for hwrf.config.HWRFConfig.get() for the "dir" section.
def run(self)
Runs the initialization and forecast for this ensemble member.
outdir
The directory in which this task should deliver its final output.
PrepHybrid runs the prep_hybrid program to transform GFS spectral files to the HWRF grid...
def anlintime(self)
The analysis input time.
Runs the real_nmm or wrf executables.
def __init__(self, dstore, conf, section, cycle_rel, kwargs)
CycleTDRCheck constructor.
Stores products and tasks in an sqlite3 database file.
def to_datetime(d)
Converts the argument to a datetime.
def write_ensda_flag_file(flag_file, run_ensda)
Writes the stormX.run_ensda flag file.
def sim(self)
The wrf simulation made by make_wrf()
This subclass of TempDir takes a directory name, instead of generating one automatically.
Time manipulation and other numerical routines.
realinit
the hwrf.fcsttask.RealNMM that generates wrfinput_d01 and wrfbdy_d01 files for input to the fcst and ...
Forecast ensemble member based on the GFS ENKF.
An hwrf.hwrftask.HWRFTask that performs regribbing operations.
def _make_relocate(self, track, modin, priorcycle)
workdir
The directory in which this task should be run.
def confint
Alias for self.conf.getint for section self.section.
def steps_for_member(self, enkfmem)
iterate over analysis cycles for a specific member
def products(self, kwargs)
Iterates over all forecast products.
Represents a specific location on the earth as a latitude, longitude pair.
def scrub(self)
Should temporary files be deleted as soon as they are not needed?
def log
Obtain a logging domain.
def member_ids(self)
Iterates over all member ids.
Represents an ensemble member from the previous forecast cycle.
def anlouttime(self)
The time at the end of the last ensemble step.
def anlintime(self)
The time at the beginning of the first ensemble step.
def anlouttimes(self)
Iterates over all ensemble analysis output times.
def inputiter(self)
Iterates over needed files from upstream workflows.
def members_at_time(self, atime)
iterate over members for a specified analysis input time
A time-indexed array that can only handle equally spaced times.
def enada_pre_object_for(ds, conf, section, next_cycle)
Generates a CycleTDRCheck or AlwaysRunENSDA based on configuration settings.
def members_at_anlouttime(self)
iterate over members at the final analysis output time.
def nsteps(self)
The number of ensemble DA time steps.
def member(self, atime, enkfmem)
get the analysis cycle for the specified member and time
def _actually_run(self)
Search on disk for TDR or a trigger file.
def read_trigger_file(self)
Read TDR trigger file for operational run.
def get_product(self, domain, atime)
Returns a product for the specified domain and time.
wrfanl
the hwrf.fcsttask.WRFAnl that generates input wrfanl_d* files for input to the fcst ...
vinttave
An alias for GRIB1VintTave.
Runs the Unified Post Processor on outputs from the WRF-NMM, producing E grid GRIB files as EGRIB1Pro...
This module contains tasks to prepare input for the GFDL Vortex Tracker, run the tracker and deliver ...
def _make_relocate_kwargs(self, track, modin, dest_dir, priorcycle)
Exceptions raised by the hwrf package.
def exe(name, kwargs)
Returns a prog.ImmutableRunner object that represents a large serial program that must be run on a co...
generate and manipulate wrf namelists, predict output filenames
def products(self, domains=None, kwargs)
Iterates over all products.
def iteritems(self)
Iterates over all metadata (key,value) pairs for this Datum, including "available" and "location"...
def confstr
Alias for self.conf.getstr for section self.section.
Describes regribbing operations using an algebraic structure.
def make_wrf(self, detinit, sim)
Create the wrf() and fcst.
def igrb1(task, kwargs)
This is a convenient alias for the GRIB1Selector constructor.
runs a short WRF simulation to generate wrfanl files named "ghost"
Exceptions for hwrf.regrib.GRIBTask for certain internal errors.
def read_ensda_flag_file(flag_file)
Reads the stormX.run_ensda flag file.
def anlintimes(self)
Iterates over all ensemble analysis input times.
Runs the prep_hybrid program on GFS spectral files.
def taskvars(self)
The dict of object-local values used for string substitution.
def realtime(self)
Is this job a real-time forecast job?
def get_wrfanl(self, domain, atime)
returns the wrfanl file's product
Determines if Tail Doppler Radar (TDR) data is available.
def __init__(self, dstore, conf, section, anlintime=None, taskname=None, kwargs)
Constructor for DAEnsemble.
fcst
the forecast task, an hwrf.fcsttask.AnalysisCycle
def readensdatrigger(self, stmidin, tgtcycle)
Runs the hwrf_readtdrtrigger program.
Represents a Product created by an external workflow.
Used in place of CycleTDRCheck to force ENSDA to always run.
This represents all three stages of the relocate.