forked from SBFRF/cmtb
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathRunWorkFlow_CSHORE.py
218 lines (178 loc) · 9.46 KB
/
RunWorkFlow_CSHORE.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
# -*- coding: utf-8 -*-
#!/home/number/anaconda2/bin/python
import matplotlib
matplotlib.use('Agg')
import os, getopt, sys, shutil, logging
import numpy as np
from subprocess import check_output
import datetime as DT
from frontback.frontBackCSHORE import CSHORE_analysis, CSHOREsimSetup
from prepdata import inputOutput
from getdatatestbed.getDataFRF import getDataTestBed
import yaml
import platform
def master_CSHORE_run(inputDict):
"""will run CSHORE with any version prefix given start, end, and a simulation duration
Args:
inputDict: keys are:
:key pFlag - plots or not (boolean)
:key analyzeFlag - analyze results or not (boolean)
:key generateFlag - generate input files or not (boolean)
:key runFlag - run the simulation or not (boolean)
:key start_date - date I am starting the simulation (format '2018-01-15T00:00:00Z')
:key end_date - date I am ending the simulation (format '2018-01-15T00:00:00Z')
:key WD - path to the working directory the user wants
:key netCDFdir - path to the netCDF save location specified by the user
:key THREDDS - which THREDDS server are we using, 'FRF' or 'CHL'
:key version_prefix - right now we have 'FIXED', 'MOBILE', or 'MOBILE_RESET'
:key duration - how long you want the simulations to run in hours (24 by default)
Returns:
None
"""
version_prefix = inputDict['version_prefix']
endTime = inputDict['endTime']
startTime = inputDict['startTime']
simulationDuration = inputDict['simulationDuration']
THREDDS = inputDict['THREDDS']
workingDir = inputDict['workingDirectory']
generateFlag = inputDict['generateFlag']
runFlag = inputDict['runFlag']
analyzeFlag = inputDict['analyzeFlag']
sorceCodePATH = inputDict['modelExecutable']
if startTime == '$': # this signifies daily or "live" run
endTime = DT.datetime.now().strftime('%Y-%m-%dT00:00:00Z')
startTime = (DT.datetime.strptime(endTime, '%Y-%m-%dT00:00:00Z') - DT.timedelta(seconds=simulationDuration*60)).strftime('%Y-%m-%dT00:00:00Z')
# version check
prefixList = np.array(['FIXED', 'MOBILE', 'MOBILE_RESET'])
assert (version_prefix == prefixList).any(), "Please enter a valid version prefix\n Prefix assigned = %s must be in List %s" % (version_prefix, prefixList)
# __________________input vars________________________________
codeDir = os.getcwd()
if workingDir[-1] == '/':
outDataBase =os.path.join(workingDir, 'CSHORE', version_prefix)
else:
outDataBase = os.path.join(workingDir, 'CSHORE', version_prefix)
TOD = 0 # 0=start simulations at 0000
LOG_FILENAME = os.path.join(inputDict['logfileLoc'], 'CSHORE/%s/logs/CMTB_BatchRun_Log_%s_%s_%s.log' %(version_prefix, version_prefix, startTime.replace(':',''), endTime.replace(':','')))
#
# try:
# logging.basicConfig(filename=LOG_FILENAME, level=logging.DEBUG)
# except IOError:
# try:
# os.makedirs(os.path.join(outDataBase,'logs')) # in python 3, exist_ok=True)
# except OSError: #folder exists
# pass
# LOG_FILENAME = os.path.join(outDataBase, 'logs/CMTB_BatchRun_Log_%s_%s_%s.log' %(version_prefix, startTime.replace(':',''), endTime.replace(':','')))
# logging.basicConfig(filename=LOG_FILENAME, level=logging.DEBUG)
# logging.debug('\n-------------------\nTraceback Error Log for:\n\nSimulation Started: %s\n-------------------\n' % (DT.datetime.now()))
# ____________________________________________________________
# establishing the resolution of the input datetime
d2 = DT.datetime.strptime(endTime, '%Y-%m-%dT%H:%M:%SZ') + DT.timedelta(TOD / 24., 0, 0)
d1 = DT.datetime.strptime(startTime, '%Y-%m-%dT%H:%M:%SZ') + DT.timedelta(TOD / 24., 0, 0)
# if the version is MOBILE then I do NOT want to check this, because MOBILE continuously
# evolves and NEVER resets the bathymetry
if version_prefix != 'MOBILE':
# pull the latest bathymetry
cmtb_data = getDataTestBed(d1, d1+DT.timedelta(minutes=1), THREDDS)
bathy_data = cmtb_data.getBathyIntegratedTransect()
b_time = bathy_data['time']
try: # try to pull the .nc file of the previous run.
Time_O = (DT.datetime.strptime(startTime, '%Y-%m-%dT%H:%M:%SZ') - DT.timedelta(days=1)).strftime('%Y-%m-%dT%H%M%SZ')
# initialize the class
cshore_io_O = inputOutput.cshoreIO()
# get into the directory I need
params0, bc0, veg0, hydro0, sed0, morpho0, meta0 = cshore_io_O.load_CSHORE_results(outDataBase + Time_O)
# what SURVEY TIME did this thing use??
prev_mod_stime = meta0['bathy_surv_stime']
# ok, this says that if:
# 1 - the previous model used a survey older than the latest survey
# 2 - the previous model started AFTER the latest survey (i.e., it should have used the latest survey)
if (b_time > prev_mod_stime) and (Time_O > b_time):
d1_N = b_time.replace(microsecond=0, second=0, minute=0, hour=0)
if d1_N != b_time:
d1_N = d1_N + DT.timedelta(days=1)
# this means we rounded it down and have to add back a day to start on the 00:00:00 after the survey
# reset the first day of the simulations to be the day after or of the latest survey
# (depending on if the survey time is 00:00:00 or 12:00:00)
del d1
d1 = d1_N
del d1_N
except IOError:
# this means that this is the first time this has been run, so you don't have to worry about it.
pass
# This is the portion that creates a list of simulation end times (start times?)
dt_DT = DT.timedelta(0, simulationDuration * 60 * 60) # timestep in datetime
# make List of Datestring items, for simulations
a = [d1]
dateStringList = [a[0].strftime("%Y-%m-%dT%H:%M:%SZ")]
for i in range(int(np.ceil((d2-d1).total_seconds()/dt_DT.total_seconds()))-1):
a.append(a[-1] + dt_DT)
dateStringList.append(a[-1].strftime("%Y-%m-%dT%H:%M:%SZ"))
errors, errorDates = [],[]
# change this to be the same as the data folder?
# os.chdir(workingDir) # is this right? NO it's not
curdir = os.getcwd()
for time in dateStringList:
try:
print '----------------------Begin {} ---------------------------'.format(time)
if generateFlag == True:
CSHOREsimSetup(startTime=time, inputDict=inputDict)
datadir = os.path.join(outDataBase, ''.join(time.split(':'))) # moving to the new simulation's folder
if runFlag == True:
os.chdir(datadir)# changing locations to where data should be downloaded to
shutil.copy2(sorceCodePATH, datadir)
print 'Bathy Interpolation done\n Beginning Simulation'
check_output(os.path.join('./', sorceCodePATH.split('/')[-1]), shell=True)
# as this is written the script has to be in the working directory, not in a sub-folder!
# run analyze and archive script
os.chdir(curdir)
if analyzeFlag == True:
print '**\nBegin Analyze Script'
CSHORE_analysis(startTime=time, inputDict=inputDict)
# not sure i want this so i commented it out for now
"""
if pFlag == True and DT.date.today() == d2:
# move files
moveFnames = glob.glob(curdir + 'CMTB*.png')
moveFnames.extend(glob.glob(curdir + 'CMTB*.gif'))
for file in moveFnames:
shutil.move(file, '/mnt/gaia/CMTB')
print 'moved %s ' % file
"""
print('----------------------SUCCESS--------------------')
except Exception, e:
os.chdir(curdir)
print ' << ERROR >> HAPPENED IN THIS TIME STEP '
print e
logging.exception('\nERROR FOUND @ %s\n' %time, exc_info=True)
if __name__ == "__main__":
opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
print '___________________\n________________\n___________________\n________________\n___________________\n________________\n'
print 'USACE FRF Coastal Model Test Bed : CSHORE'
# It will throw and error and tell the user where to go look for the example yaml
try:
# assume the user gave the path
yamlLoc = args[0]
with open(os.path.join(yamlLoc), 'r') as f:
inputDict = yaml.load(f)
except:
raise IOError('Input YAML file required. See yaml_files/TestBedExampleInputs/CSHORE_Input_example for example yaml file.')
# add in defaults for inputDict
if 'THREDDS' not in inputDict.keys():
inputDict['THREDDS'] = 'FRF'
if 'bathyLoc' not in inputDict.keys():
inputDict['bathyLoc'] = 'integrated_bathy'
if 'profileNumber' not in inputDict.keys():
inputDict['profileNumber'] = 960
if 'duration' not in inputDict.keys():
inputDict['duration'] = 24
if 'generateFlag' not in inputDict.keys():
inputDict['generateFlag'] = True
if 'pFlag' not in inputDict.keys():
inputDict['pFlag'] = False
if 'runFlag' not in inputDict.keys():
inputDict['runFlag'] = True
if 'analyzeFlag' not in inputDict.keys():
inputDict['analyzeFlag'] = True
if 'logfileLoc' not in inputDict.keys():
inputDict['logfileLoc'] = inputDict['workingDirectory']
master_CSHORE_run(inputDict=inputDict)