From 679c4ccdbff00dd5fc57144983d4d19117b4bba2 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 26 Feb 2015 16:45:42 -0800 Subject: [PATCH 01/47] Support for python 3. Added pycompat.py to allow for seamless use of python builtins and a few modules that have had name changes since python 2. --- rvic/clib/rvic_convolution.c | 32 ++--- rvic/convert.py | 15 +- rvic/convolution.py | 56 ++++---- rvic/core/aggregate.py | 46 +++--- rvic/core/config.py | 9 +- rvic/core/convert.py | 85 ++++------- rvic/core/convolution_wrapper.py | 6 +- rvic/core/history.py | 210 ++++++++++------------------ rvic/core/log.py | 4 +- rvic/core/make_uh.py | 74 +++++----- rvic/core/{mpi.py => multi_proc.py} | 4 +- rvic/core/param_file.py | 57 ++++---- rvic/core/plots.py | 14 +- rvic/core/pycompat.py | 38 +++++ rvic/core/read_forcing.py | 88 ++++++++---- rvic/core/remap.py | 2 +- rvic/core/share.py | 13 +- rvic/core/time_utility.py | 15 +- rvic/core/utilities.py | 42 +++--- rvic/core/variables.py | 101 +++++++------ rvic/core/write.py | 78 +++++------ rvic/parameters.py | 90 ++++++------ setup.py | 1 - tests/run_tests.py | 17 ++- tools/find_pour_points.py | 43 +++--- 25 files changed, 584 insertions(+), 556 deletions(-) rename rvic/core/{mpi.py => multi_proc.py} (96%) create mode 100644 rvic/core/pycompat.py diff --git a/rvic/clib/rvic_convolution.c b/rvic/clib/rvic_convolution.c index 7be64b5..bc57087 100644 --- a/rvic/clib/rvic_convolution.c +++ b/rvic/clib/rvic_convolution.c @@ -1,16 +1,17 @@ #include -void convolve(const int nsources, /*scalar - number of sources*/ - const int noutlets, /*scalar - length of subset*/ - const int subset_length, /*scalar - length of subset*/ - const int x_size, - const int* source2outlet_ind, /*1d array - source to outlet mapping*/ - const int* source_y_ind, /*1d array - source y location*/ - const int* source_x_ind, /*1d array - source x location*/ - const int* source_time_offset, /*1d array - source time offset*/ - const double* unit_hydrograph, /*2d array[times][sources] - unit hydrographs*/ - const double* aggrunin, /*2d array[ysize][xsize] - vic runoff flux*/ - double* ring) /*2d array[times][outlets] - convolution ring*/ +void +convolve(const int nsources, /*scalar - number of sources*/ + const int noutlets, /*scalar - length of subset*/ + const int subset_length, /*scalar - length of subset*/ + const int x_size, + const int *source2outlet_ind, /*1d array - source to outlet mapping*/ + const int *source_y_ind, /*1d array - source y location*/ + const int *source_x_ind, /*1d array - source x location*/ + const int *source_time_offset, /*1d array - source time offset*/ + const double *unit_hydrograph, /*2d array[times][sources] - unit hydrographs*/ + const double *aggrunin, /*2d array[ysize][xsize] - vic runoff flux*/ + double *ring) /*2d array[times][outlets] - convolution ring*/ { int s, i, j; /*counters*/ int y, x, offset, outlet; /*2d indicies*/ @@ -18,15 +19,14 @@ void convolve(const int nsources, /*scalar - number of sources*/ /*Loop through all sources*/ for (s = 0; s < nsources; s++) { - outlet = source2outlet_ind[s]; y = source_y_ind[s]; x = source_x_ind[s]; offset = source_time_offset[s]; - //1d index location - //2d-->1d indexing goes like this: ind = y*x_size + x - xyind = y*x_size + x; + // 1d index location + // 2d-->1d indexing goes like this: ind = y*x_size + x + xyind = y * x_size + x; /* Do the convolution */ // i is the position in the unit hydrograph @@ -34,7 +34,7 @@ void convolve(const int nsources, /*scalar - number of sources*/ for (i = 0; i < subset_length; i++) { j = i + offset; - //1d index locations + // 1d index locations rind = j * noutlets + outlet; uhind = i * nsources + s; diff --git a/rvic/convert.py b/rvic/convert.py index 0007619..b31ad70 100644 --- a/rvic/convert.py +++ b/rvic/convert.py @@ -3,12 +3,12 @@ """ from logging import getLogger -from core.log import init_logger, close_logger, LOG_NAME -from core.utilities import make_directories, copy_inputs, read_domain -from core.utilities import tar_inputs -from core.convert import read_station_file, read_uhs_files, move_domain -from core.param_file import finish_params -from core.config import read_config +from .core.log import init_logger, close_logger, LOG_NAME +from .core.utilities import make_directories, copy_inputs, read_domain +from .core.utilities import tar_inputs +from .core.convert import read_station_file, read_uhs_files, move_domain +from .core.param_file import finish_params +from .core.config import read_config # -------------------------------------------------------------------- # @@ -59,7 +59,8 @@ def uhs2param_init(config_file): # copy inputs to $case_dir/inputs and update configuration config_dict = copy_inputs(config_file, directories['inputs']) options = config_dict['OPTIONS'] - config_dict['POUR_POINTS'] = {'FILE_NAME': config_dict['UHS_FILES']['STATION_FILE']} + config_dict['POUR_POINTS'] = { + 'FILE_NAME': config_dict['UHS_FILES']['STATION_FILE']} config_dict['ROUTING']['FILE_NAME'] = 'unknown' config_dict['UH_BOX'] = {'FILE_NAME': 'unknown'} # ---------------------------------------------------------------- # diff --git a/rvic/convolution.py b/rvic/convolution.py index bcad366..854b1bd 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -16,15 +16,16 @@ import os from collections import OrderedDict from logging import getLogger -from core.log import init_logger, close_logger, LOG_NAME -from core.utilities import make_directories, read_domain -from core.utilities import write_rpointer, tar_inputs -from core.variables import Rvar -from core.time_utility import Dtime -from core.read_forcing import DataModel -from core.history import Tape -from core.share import NcGlobals, RVIC_TRACERS -from core.config import read_config +from .core.log import init_logger, close_logger, LOG_NAME +from .core.utilities import make_directories, read_domain +from .core.utilities import write_rpointer, tar_inputs +from .core.variables import Rvar +from .core.time_utility import Dtime +from .core.read_forcing import DataModel +from .core.history import Tape +from .core.share import NcGlobals, RVIC_TRACERS +from .core.config import read_config +from .core.pycompat import iteritems # -------------------------------------------------------------------- # @@ -83,7 +84,7 @@ def convolution_init(config_file): # ---------------------------------------------------------------- # # Copy Inputs to $case_dir/inputs and update configuration - #config_dict = copy_inputs(config_file, directories['inputs']) + # config_dict = copy_inputs(config_file, directories['inputs']) options = config_dict['OPTIONS'] # ---------------------------------------------------------------- # @@ -108,8 +109,8 @@ def convolution_init(config_file): # ---------------------------------------------------------------- # # Read Domain File domain = config_dict['DOMAIN'] - dom_data, dom_vatts, dom_gatts = read_domain(domain, - lat0_is_min=data_model.lat0_is_min) + dom_data, dom_vatts, dom_gatts = read_domain( + domain, lat0_is_min=data_model.lat0_is_min) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -173,19 +174,20 @@ def convolution_init(config_file): # make sure history file fields are all in list form if numtapes == 1: - for var, value in history.iteritems(): + for var, value in iteritems(history): if not isinstance(value, list): history[var] = list([value]) - global_atts = NcGlobals(title='RVIC history file', - casename=options['CASEID'], - casestr=options['CASESTR'], - RvicPourPointsFile=os.path.split(rout_var.RvicPourPointsFile)[1], - RvicUHFile=os.path.split(rout_var.RvicUHFile)[1], - RvicFdrFile=os.path.split(rout_var.RvicFdrFile)[1], - RvicDomainFile=os.path.split(domain['FILE_NAME'])[1]) + global_atts = NcGlobals( + title='RVIC history file', + casename=options['CASEID'], + casestr=options['CASESTR'], + RvicPourPointsFile=os.path.split(rout_var.RvicPourPointsFile)[1], + RvicUHFile=os.path.split(rout_var.RvicUHFile)[1], + RvicFdrFile=os.path.split(rout_var.RvicFdrFile)[1], + RvicDomainFile=os.path.split(domain['FILE_NAME'])[1]) - for j in xrange(numtapes): + for j in range(numtapes): tapename = 'Tape.{0}'.format(j) log.info('setting up History %s', tapename) hist_tapes[tapename] = Tape(time_handle.time_ord, @@ -208,7 +210,7 @@ def convolution_init(config_file): glob_ats=global_atts) # loop over again and print summary - for tapename, tape in hist_tapes.iteritems(): + for tapename, tape in iteritems(hist_tapes): log.info('==========%s==========', tapename) log.info(tape) tape.write_initial() @@ -275,7 +277,7 @@ def convolution_run(hist_tapes, data_model, rout_var, dom_data, time_handle, data2tape['storage'] = rout_var.get_storage() # Update the history Tape(s) - for tapename, tape in hist_tapes.iteritems(): + for tapename, tape in iteritems(hist_tapes): log.debug('Updating Tape: %s', tapename) tape.update(data2tape, time_ord) # ------------------------------------------------------------ # @@ -286,7 +288,7 @@ def convolution_run(hist_tapes, data_model, rout_var, dom_data, time_handle, # History files history_files = [] history_restart_files = [] - for tapename, tape in hist_tapes.iteritems(): + for tapename, tape in iteritems(hist_tapes): log.debug('Writing Restart File for Tape: %s', tapename) # hist_fname, rest_fname = tape.write_restart() history_files.append(tape.filename) @@ -304,7 +306,7 @@ def convolution_run(hist_tapes, data_model, rout_var, dom_data, time_handle, if not stop_flag: timestamp, time_ord, stop_flag, \ rest_flag = time_handle.advance_timestep() - #check that we're still inline with convolution + # check that we're still inline with convolution if end_timestamp != timestamp: raise ValueError('timestamps do not match after convolution') else: @@ -313,7 +315,7 @@ def convolution_run(hist_tapes, data_model, rout_var, dom_data, time_handle, # ---------------------------------------------------------------- # # Make sure we write out the last history file - for tapename, tape in hist_tapes.iteritems(): + for tapename, tape in iteritems(hist_tapes): log.debug('Closing Tape: %s', tapename) tape.finish() # ---------------------------------------------------------------- # @@ -336,7 +338,7 @@ def convolution_final(time_handle, hist_tapes): log.info("-----------------------------------------------------------") log.info('Done with streamflow convolution') log.info('Processed %i timesteps' % time_handle.timesteps) - for name, tape in hist_tapes.iteritems(): + for name, tape in iteritems(hist_tapes): log.info('Wrote %i history files from %s' % (tape.files_count, name)) # log.info('Routed to %i points' % time_handle.points) log.info("-----------------------------------------------------------") diff --git a/rvic/core/aggregate.py b/rvic/core/aggregate.py index 424553b..4537a60 100644 --- a/rvic/core/aggregate.py +++ b/rvic/core/aggregate.py @@ -4,12 +4,12 @@ """ import numpy as np -from collections import OrderedDict -from share import FILLVALUE_F -from utilities import find_nearest, latlon2yx -from variables import Point +from .share import FILLVALUE_F +from .utilities import find_nearest, latlon2yx +from .variables import Point from logging import getLogger -from log import LOG_NAME +from .log import LOG_NAME +from .pycompat import OrderedDict, iteritems, zip # -------------------------------------------------------------------- # # create logger @@ -33,7 +33,7 @@ def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): fdr_srcarea = fdr_data[config_dict['ROUTING']['SOURCE_AREA_VAR']] # ---------------------------------------------------------------- # - #Find Destination grid cells + # Find Destination grid cells log.info('Finding addresses now...') routys, routxs = latlon2yx(plats=lats, plons=lons, @@ -81,7 +81,7 @@ def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): # ---------------------------------------------------------------- # # Sort based on outlet total source area pour_point.source_area - outlets = OrderedDict(sorted(outlets.items(), + outlets = OrderedDict(sorted(list(iteritems(outlets)), key=lambda t: t[1].upstream_area, reverse=True)) # ---------------------------------------------------------------- # @@ -102,8 +102,8 @@ def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): log.info('NUMBER OF POUR POINTS IN INPUT LIST: %i' % num) log.info('NUMBER OF POINTS TO AGGREGATE TO: %i' % key_count) log.info('NUMBER OF POUR POINTS AGGREGATED: %i' % pp_count) - log.info('EFFECIENCY OF: %.2f %%' % (100.*pp_count / num)) - log.info('UNASSIGNED POUR POINTS: %i \n' % (num-pp_count)) + log.info('EFFECIENCY OF: %.2f %%' % (100. * pp_count / num)) + log.info('UNASSIGNED POUR POINTS: %i \n' % (num - pp_count)) log.info('-------------------------------------------------------------\n') # ---------------------------------------------------------------- # @@ -139,8 +139,10 @@ def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False): # ---------------------------------------------------------------- # # make output arrays for lons/lats and initialize fraction/unit_hydrograph # pad output arrays so there is a space =pad around inputs - lats = np.arange((lat_min-res*pad), (lat_max+res*(pad+1)), res)[::-1] - lons = np.arange((lon_min-res*pad), (lon_max+res*(pad+1)), res) + lats = np.arange((lat_min - res * pad), + (lat_max + res * (pad + 1)), res)[::-1] + lons = np.arange((lon_min - res * pad), + (lon_max + res * (pad + 1)), res) fraction = np.zeros((len(lats), len(lons)), dtype=np.float64) unit_hydrograph = np.zeros((tshape, len(lats), len(lons)), @@ -153,18 +155,18 @@ def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False): # find target index locations of all corners for both datasets # Not that the lat inds are inverted ilat_min_ind = find_nearest(lats, np.max(in_data['lat'])) - ilat_max_ind = find_nearest(lats, np.min(in_data['lat']))+1 + ilat_max_ind = find_nearest(lats, np.min(in_data['lat'])) + 1 ilon_min_ind = find_nearest(lons, np.min(in_data['lon'])) - ilon_max_ind = find_nearest(lons, np.max(in_data['lon']))+1 + ilon_max_ind = find_nearest(lons, np.max(in_data['lon'])) + 1 log.debug('in_data fraction shape: %s', str(in_data['fraction'].shape)) if agg_data: alat_min_ind = find_nearest(lats, np.max(agg_data['lat'])) - alat_max_ind = find_nearest(lats, np.min(agg_data['lat']))+1 + alat_max_ind = find_nearest(lats, np.min(agg_data['lat'])) + 1 alon_min_ind = find_nearest(lons, np.min(agg_data['lon'])) - alon_max_ind = find_nearest(lons, np.max(agg_data['lon']))+1 + alon_max_ind = find_nearest(lons, np.max(agg_data['lon'])) + 1 log.debug('agg_data fraction shape: %s', str(agg_data['fraction'].shape)) @@ -173,12 +175,18 @@ def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False): # ---------------------------------------------------------------- # # Place data - fraction[ilat_min_ind:ilat_max_ind, ilon_min_ind:ilon_max_ind] += in_data['fraction'] - unit_hydrograph[:, ilat_min_ind:ilat_max_ind, ilon_min_ind:ilon_max_ind] += in_data['unit_hydrograph'] + fraction[ilat_min_ind:ilat_max_ind, + ilon_min_ind:ilon_max_ind] += in_data['fraction'] + unit_hydrograph[:, + ilat_min_ind:ilat_max_ind, + ilon_min_ind:ilon_max_ind] += in_data['unit_hydrograph'] if agg_data: - fraction[alat_min_ind:alat_max_ind, alon_min_ind:alon_max_ind] += agg_data['fraction'] - unit_hydrograph[:, alat_min_ind:alat_max_ind, alon_min_ind:alon_max_ind] += agg_data['unit_hydrograph'] + fraction[alat_min_ind:alat_max_ind, + alon_min_ind:alon_max_ind] += agg_data['fraction'] + unit_hydrograph[:, alat_min_ind:alat_max_ind, + alon_min_ind:alon_max_ind] += \ + agg_data['unit_hydrograph'] # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # diff --git a/rvic/core/config.py b/rvic/core/config.py index 31cbf99..b263071 100644 --- a/rvic/core/config.py +++ b/rvic/core/config.py @@ -4,8 +4,7 @@ """ import os -from collections import OrderedDict -from ConfigParser import SafeConfigParser +from .pycompat import OrderedDict, SafeConfigParser class Config(object): @@ -70,11 +69,11 @@ def config_type(value): return os.path.expandvars(value) else: try: - return map(float, val_list) + return list(map(float, val_list)) except: pass try: - return map(int, val_list) + return list(map(int, val_list)) except: return val_list # -------------------------------------------------------------------- # @@ -84,7 +83,7 @@ def config_type(value): def isfloat(x): """Test of value is a float""" try: - a = float(x) + float(x) except ValueError: return False else: diff --git a/rvic/core/convert.py b/rvic/core/convert.py index 04bd4d9..dc0496d 100644 --- a/rvic/core/convert.py +++ b/rvic/core/convert.py @@ -5,9 +5,10 @@ import re import numpy as np import logging -from log import LOG_NAME -from variables import Point -from share import FILLVALUE_I +from .log import LOG_NAME +from .variables import Point +from .share import FILLVALUE_I +from .pycompat import iteritems # -------------------------------------------------------------------- # # create logger @@ -35,13 +36,13 @@ def read_station_file(file_name, dom_data, config_dict): uhs_file = f.readline().strip() # move to zero based index - y = int(y)-1 - x = int(x)-1 + y = int(y) - 1 + x = int(x) - 1 # make sure files exist log.info('On station: %s, active: %s', name, active) uhs2_file = os.path.join(config_dict['UHS_FILES']['ROUT_DIR'], - name+'.uh_s2') + name + '.uh_s2') if active == '1': if os.path.isfile(uhs_file): active = True @@ -60,8 +61,10 @@ def read_station_file(file_name, dom_data, config_dict): outlets[i].uhs_file = uhs_file outlets[i].cell_id = i outlets[i].outlet_decomp_ind = dom_data['cell_ids'][y, x] - outlets[i].lon = dom_data[config_dict['DOMAIN']['LONGITUDE_VAR']][y, x] - outlets[i].lat = dom_data[config_dict['DOMAIN']['LATITUDE_VAR']][y, x] + outlets[i].lon = \ + dom_data[config_dict['DOMAIN']['LONGITUDE_VAR']][y, x] + outlets[i].lat = \ + dom_data[config_dict['DOMAIN']['LATITUDE_VAR']][y, x] else: log.info('%s not active... skipping', name) f.close() @@ -82,7 +85,7 @@ def read_uhs_files(outlets, dom_data, config_dict): line2: ... """ if config_dict['UHS_FILES']['ROUT_PROGRAM'] == 'C': - for cell_id, outlet in outlets.iteritems(): + for cell_id, outlet in iteritems(outlets): log.info('Reading outlet %i: %s', cell_id, outlet.name) log.debug(outlet.uhs_file) f = open(outlet.uhs_file, 'r') @@ -98,63 +101,33 @@ def read_uhs_files(outlets, dom_data, config_dict): uh = [] # loop over the source points - for j in xrange(num_sources): + for j in range(num_sources): line = re.sub(' +', ' ', f.readline()) lon, lat, fracs, x, y = line.split() # move to zero based index - y = int(y)-1 - x = int(x)-1 + y = int(y) - 1 + x = int(x) - 1 outlets[cell_id].lon_source[j] = float(lon) outlets[cell_id].lat_source[j] = float(lat) outlets[cell_id].fractions[j] = float(fracs) outlets[cell_id].x_source[j] = int(x) outlets[cell_id].y_source[j] = int(y) line = re.sub(' +', ' ', f.readline()) - uh.append(map(float, line.split())) + uh.append(list(map(float, line.split()))) - outlets[cell_id].unit_hydrograph = np.rot90(np.array(uh, - dtype=np.float64), - k=-1) - outlets[cell_id].source_decomp_ind = dom_data['cell_ids'][outlets[cell_id].y_source, outlets[cell_id].x_source] + outlets[cell_id].unit_hydrograph = np.rot90( + np.array(uh, dtype=np.float64), k=-1) + outlets[cell_id].source_decomp_ind = \ + dom_data['cell_ids'][outlets[cell_id].y_source, + outlets[cell_id].x_source] f.close() - outlets[cell_id].cell_id_source = dom_data['cell_ids'][outlets[cell_id].y_source, outlets[cell_id].x_source] + outlets[cell_id].cell_id_source = \ + dom_data['cell_ids'][outlets[cell_id].y_source, + outlets[cell_id].x_source] elif config_dict['UHS_FILES']['ROUT_PROGRAM'] == 'Fortran': raise ValueError('Fortran conversion not working...') - # log.info('parsing fortran uhs files') - # # setup for finding x, y inds - # dom_lon = dom_data[config_dict['DOMAIN']['LONGITUDE_VAR']] - # dom_lat = dom_data[config_dict['DOMAIN']['LATITUDE_VAR']] - # combined = np.dstack(([dom_lat.ravel(), dom_lon.ravel()]))[0] - # mytree = cKDTree(combined) - - # for cell_id, outlet in outlets.iteritems(): - # # read lons, lats, and unit hydrographs - # log.debug('reading %s' %outlet.ll_file) - # lons, lats = np.genfromtxt(outlet.ll_file, dtype="f8", unpack=True) - - # log.debug('reading %s' %outlet.uhs_file) - # uh = np.genfromtxt(outlet.uhs_file, dtype="f8") - # outlets[cell_id].unit_hydrograph = np.rot90(uh, k=-1) - # if len(lons) != uh.shape[0]: - # raise ValueError('length mismatch in ll file and uhs file %s %s' %(lons.shape, uh.shape)) - - # # now find the y_source and x_sources - # points = list(np.vstack((np.array(lats), np.array(lons))).transpose()) - # dist, indexes = mytree.query(points, k=1) - # yinds, xinds = np.unravel_index(np.array(indexes), dom_lat.shape) - - # # finally, get the fractions and the source_decomp_ind from the domain data - # outlets[cell_id].cell_id_source = dom_data['cell_ids'][yinds, xinds] - # outlets[cell_id].fractions = dom_data[config_dict['DOMAIN']['FRACTION_VAR']][yinds, xinds] - # print 'fractions', outlets[cell_id].fractions - - # # now store all the data - # outlets[cell_id].lat_source = lats - # outlets[cell_id].lon_source = lons - # outlets[cell_id].y_source = yinds - # outlets[cell_id].x_source = xinds else: raise ValueError('UHS_FILES[ROUT_PROGRAM] must be either C or Fortran') @@ -192,7 +165,7 @@ def move_domain(dom_data, new_dom_data, outlets): # ---------------------------------------------------------------- # # Adjust locations - for cell_id, outlet in outlets.iteritems(): + for cell_id, outlet in iteritems(outlets): outlets[cell_id].domy = new_y[outlet.domy] outlets[cell_id].domx = new_x[outlet.domx] @@ -200,8 +173,12 @@ def move_domain(dom_data, new_dom_data, outlets): outlets[cell_id].y_source = new_y[outlet.y_source] outlets[cell_id].x_source = new_x[outlet.x_source] - outlets[cell_id].outlet_decomp_ind = new_dom_data['cell_ids'][outlets[cell_id].domy, outlets[cell_id].domx] - outlets[cell_id].source_decomp_ind = new_dom_data['cell_ids'][outlets[cell_id].y_source, outlets[cell_id].x_source] + outlets[cell_id].outlet_decomp_ind = \ + new_dom_data['cell_ids'][outlets[cell_id].domy, + outlets[cell_id].domx] + outlets[cell_id].source_decomp_ind = \ + new_dom_data['cell_ids'][outlets[cell_id].y_source, + outlets[cell_id].x_source] # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # diff --git a/rvic/core/convolution_wrapper.py b/rvic/core/convolution_wrapper.py index 2f55225..6711d5c 100644 --- a/rvic/core/convolution_wrapper.py +++ b/rvic/core/convolution_wrapper.py @@ -16,10 +16,12 @@ try: _convolution = np.ctypeslib.load_library(SHAREDOBJECT, LIBPATH) except ImportError as ie: - print('looking for shared object {0} in {1}'.format(SHAREDOBJECT, LIBPATH)) + print(('looking for shared object {0} in {1}'.format(SHAREDOBJECT, + LIBPATH))) raise ImportError(ie) except OSError as oe: - print('looking for shared object {0} in {1}'.format(SHAREDOBJECT, LIBPATH)) + print(('looking for shared object {0} in {1}'.format(SHAREDOBJECT, + LIBPATH))) raise ImportError(oe) _args = [ctypes.c_int, diff --git a/rvic/core/history.py b/rvic/core/history.py index 75fde4f..22537e5 100644 --- a/rvic/core/history.py +++ b/rvic/core/history.py @@ -18,12 +18,14 @@ import numpy as np from netCDF4 import Dataset, date2num, num2date, stringtochar from datetime import datetime -from time_utility import ord_to_datetime +from .time_utility import ord_to_datetime from logging import getLogger -from log import LOG_NAME -from share import SECSPERDAY, HOURSPERDAY, TIMEUNITS, NC_INT, NC_FLOAT, NC_CHAR -from share import NC_DOUBLE, WATERDENSITY, MONTHSPERYEAR -import share +from .log import LOG_NAME +from .share import SECSPERDAY, HOURSPERDAY, TIMEUNITS +from .share import NC_INT, NC_FLOAT, NC_CHAR +from .share import NC_DOUBLE, WATERDENSITY, MONTHSPERYEAR +from .pycompat import iteritems +from . import share # -------------------------------------------------------------------- # @@ -39,8 +41,8 @@ class Tape(object): # ---------------------------------------------------------------- # # Init - def __init__(self, time_ord, caseid, Rvar, tape_num=0, - fincl=['streamflow'], mfilt=1, ndens=2, nhtfrq=0, + def __init__(self, time_ord, caseid, rvar, tape_num=0, + fincl=['streamflow'], mfilt=1, ndens=2, nhtfrq=0, avgflag='A', units='kg m-2 s-1', file_format='NETCDF4_CLASSIC', outtype='grid', grid_lons=False, grid_lats=False, grid_area=None, out_dir='.', @@ -66,7 +68,7 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0, self._out_dir = out_dir self._glob_ats = glob_ats - self.__get_rvar(Rvar) # Get the initial Rvar fields + self.__get_rvar(rvar) # Get the initial rvar fields self._grid_shape = grid_area.shape self._out_data = {} @@ -160,20 +162,29 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0, # ------------------------------------------------------------ # # Determine the format of the output filename if self._avgflag == 'I': - self._fname_format = os.path.join(out_dir, - "%s.rvic.h%s%s.%%Y-%%m-%%d-%%H-%%M-%%S.nc" % (self._caseid, self._tape_num, self._avgflag.lower())) + self._fname_format = os.path.join( + out_dir, "%s.rvic.h%s%s.%%Y-%%m-%%d-%%H-%%M-%%S.nc" % + (self._caseid, self._tape_num, self._avgflag.lower())) else: if self._nhtfrq == 0: - self._fname_format = os.path.join(out_dir, - "%s.rvic.h%s%s.%%Y-%%m.nc" % (self._caseid, self._tape_num, self._avgflag.lower())) - elif (self._nhtfrq == -24) or (nhtfrq*self._dt == SECSPERDAY): - self._fname_format = os.path.join(out_dir, - "%s.rvic.h%s%s.%%Y-%%m-%%d.nc" % (self._caseid, self._tape_num, self._avgflag.lower())) + self._fname_format = os.path.join( + out_dir, + "%s.rvic.h%s%s.%%Y-%%m.nc" % + (self._caseid, self._tape_num, self._avgflag.lower())) + elif (self._nhtfrq == -24) or (nhtfrq * self._dt == SECSPERDAY): + self._fname_format = os.path.join( + out_dir, + "%s.rvic.h%s%s.%%Y-%%m-%%d.nc" % + (self._caseid, self._tape_num, self._avgflag.lower())) else: - self._fname_format = os.path.join(out_dir, - "%s.rvic.h%s%s.%%Y-%%m-%%d-%%H.nc" % (self._caseid, self._tape_num, self._avgflag.lower())) - self._rest_fname_format = os.path.join(out_dir, - "%s.rvic.rh%s.%%Y-%%m-%%d-%%H-%%M-%%S.nc" % (self._caseid, self._tape_num)) + self._fname_format = os.path.join( + out_dir, + "%s.rvic.h%s%s.%%Y-%%m-%%d-%%H.nc" % + (self._caseid, self._tape_num, self._avgflag.lower())) + self._rest_fname_format = os.path.join( + out_dir, + "%s.rvic.rh%s.%%Y-%%m-%%d-%%H-%%M-%%S.nc" % + (self._caseid, self._tape_num)) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -335,7 +346,7 @@ def __next_write_out_data(self): if mfilt < 1: mfilt = 1 - self._out_data_write = mfilt-1 + self._out_data_write = mfilt - 1 self._out_times = np.empty(mfilt, dtype=np.float64) if self._avgflag != 'I': self._out_time_bnds = np.empty((mfilt, 2), dtype=np.float64) @@ -365,10 +376,12 @@ def __update_out_data(self): # Grid the fields self._out_data[field][self._out_data_i, self._outlet_y_ind, - self._outlet_x_ind] = self._temp_data[field][:] * self._units_mult + self._outlet_x_ind] = \ + self._temp_data[field][:] * self._units_mult # ---------------------------------------------------- # else: - self._out_data[field][self._out_data_i, :] = self._temp_data[field] * self._units_mult + self._out_data[field][self._out_data_i, :] = \ + self._temp_data[field] * self._units_mult # ------------------------------------------------------------ # self._out_times[self._out_data_i] = self._write_ord @@ -453,15 +466,18 @@ def __next_update_out_data(self): # Get next file names and timeord if self._avgflag == 'I': self._write_ord = b1 - self.filename = num2date(b1, TIMEUNITS, - calendar=self._calendar).strftime(self._fname_format) + self.filename = num2date( + b1, TIMEUNITS, + calendar=self._calendar).strftime(self._fname_format) else: self._time_bnds = np.array([[b0, b1]]) self._write_ord = np.average(self._time_bnds) - self.filename = num2date(b0, TIMEUNITS, - calendar=self._calendar).strftime(self._fname_format) - self.rest_filename = num2date(b1, TIMEUNITS, - calendar=self._calendar).strftime(self._fname_format) + self.filename = num2date( + b0, TIMEUNITS, + calendar=self._calendar).strftime(self._fname_format) + self.rest_filename = num2date( + b1, TIMEUNITS, + calendar=self._calendar).strftime(self._fname_format) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -490,23 +506,23 @@ def __write_grid(self): # ------------------------------------------------------------ # # Time Variable - time = f.createDimension('time', None) + f.createDimension('time', None) time = f.createVariable('time', self._ncprec, ('time',)) - time[:] = self._out_times[:self._out_data_i+1] - for key, val in share.time.__dict__.iteritems(): + time[:] = self._out_times[:self._out_data_i + 1] + for key, val in iteritems(share.time.__dict__): if val: setattr(time, key, val) time.calendar = self._calendar if self._avgflag != 'I': - nv = f.createDimension('nv', 2) + f.createDimension('nv', 2) time.bounds = 'time_bnds' time_bnds = f.createVariable('time_bnds', self._ncprec, ('time', 'nv',), **self.ncvaropts) - time_bnds[:, :] = self._out_time_bnds[:self._out_data_i+1] + time_bnds[:, :] = self._out_time_bnds[:self._out_data_i + 1] # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -523,11 +539,11 @@ def __write_grid(self): xc[:, :] = self._grid_lons yc[:, :] = self._grid_lats - for key, val in share.xc.__dict__.iteritems(): + for key, val in iteritems(share.xc.__dict__): if val: setattr(xc, key, val) - for key, val in share.yc.__dict__.iteritems(): + for key, val in iteritems(share.yc.__dict__): if val: setattr(yc, key, val) @@ -544,11 +560,11 @@ def __write_grid(self): lon[:] = self._grid_lons lat[:] = self._grid_lats - for key, val in share.lon.__dict__.iteritems(): + for key, val in iteritems(share.lon.__dict__): if val: setattr(lon, key, val) - for key, val in share.lat.__dict__.iteritems(): + for key, val in iteritems(share.lat.__dict__): if val: setattr(lat, key, val) # ------------------------------------------------------------ # @@ -560,9 +576,9 @@ def __write_grid(self): for field in self._fincl: var = f.createVariable(field, self._ncprec, tcoords, **self.ncvaropts) - var[:, :] = self._out_data[field][:self._out_data_i+1] + var[:, :] = self._out_data[field][:self._out_data_i + 1] - for key, val in getattr(share, field).__dict__.iteritems(): + for key, val in iteritems(getattr(share, field).__dict__): if val: setattr(var, key, val) var.units = self._units @@ -573,7 +589,7 @@ def __write_grid(self): # ------------------------------------------------------------ # # write global attributes self._glob_ats.update() - for key, val in self._glob_ats.atts.iteritems(): + for key, val in iteritems(self._glob_ats.atts): if val: setattr(f, key, val) # ------------------------------------------------------------ # @@ -593,35 +609,35 @@ def __write_array(self): # ------------------------------------------------------------ # # Time Variable - time = f.createDimension('time', None) + f.createDimension('time', None) time = f.createVariable('time', self._ncprec, ('time',), **self.ncvaropts) - time[:] = self._out_times[:self._out_data_i+1] - for key, val in share.time.__dict__.iteritems(): + time[:] = self._out_times[:self._out_data_i + 1] + for key, val in iteritems(share.time.__dict__): if val: setattr(time, key, val) time.calendar = self._calendar if self._avgflag != 'I': - nv = f.createDimension('nv', 2) + f.createDimension('nv', 2) time.bounds = 'time_bnds' time_bnds = f.createVariable('time_bnds', self._ncprec, ('time', 'nv',), **self.ncvaropts) - time_bnds[:, :] = self._out_time_bnds[:self._out_data_i+1] + time_bnds[:, :] = self._out_time_bnds[:self._out_data_i + 1] # ------------------------------------------------------------ # # ------------------------------------------------------------ # # Setup Coordinate Variables coords = ('outlets',) - outlets = f.createDimension('outlets', self._num_outlets) + f.createDimension('outlets', self._num_outlets) nocoords = coords + ('nc_chars',) char_names = stringtochar(self._outlet_name) - chars = f.createDimension(nocoords[1], char_names.shape[1]) + f.createDimension(nocoords[1], char_names.shape[1]) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -646,27 +662,27 @@ def __write_array(self): outlet_decomp_ind[:] = self._outlet_decomp_ind onm[:, :] = char_names - for key, val in share.outlet_lon.__dict__.iteritems(): + for key, val in iteritems(share.outlet_lon.__dict__): if val: setattr(outlet_lon, key, val) - for key, val in share.outlet_lat.__dict__.iteritems(): + for key, val in iteritems(share.outlet_lat.__dict__): if val: setattr(outlet_lat, key, val) - for key, val in share.outlet_y_ind.__dict__.iteritems(): + for key, val in iteritems(share.outlet_y_ind.__dict__): if val: setattr(outlet_y_ind, key, val) - for key, val in share.outlet_x_ind.__dict__.iteritems(): + for key, val in iteritems(share.outlet_x_ind.__dict__): if val: setattr(outlet_x_ind, key, val) - for key, val in share.outlet_decomp_ind.__dict__.iteritems(): + for key, val in iteritems(share.outlet_decomp_ind.__dict__): if val: setattr(outlet_decomp_ind, key, val) - for key, val in share.outlet_name.__dict__.iteritems(): + for key, val in iteritems(share.outlet_name.__dict__): if val: setattr(onm, key, val) # ------------------------------------------------------------ # @@ -678,9 +694,9 @@ def __write_array(self): for field in self._fincl: var = f.createVariable(field, self._ncprec, tcoords, **self.ncvaropts) - var[:, :] = self._out_data[field][:self._out_data_i+1] + var[:, :] = self._out_data[field][:self._out_data_i + 1] - for key, val in getattr(share, field).__dict__.iteritems(): + for key, val in iteritems(getattr(share, field).__dict__): if val: setattr(var, key, val) var.units = self._units @@ -689,7 +705,7 @@ def __write_array(self): # ------------------------------------------------------------ # # write global attributes self._glob_ats.update() - for key, val in self._glob_ats.atts.iteritems(): + for key, val in iteritems(self._glob_ats.atts): if val: setattr(f, key, val) f.featureType = "timeSeries" @@ -697,86 +713,4 @@ def __write_array(self): f.close() log.info('Finished writing %s', self.filename) # ---------------------------------------------------------------- # - - # ---------------------------------------------------------------- # - # write initial flux - # def write_restart(self): - # """Write history tape state, matches CESM-RVIC format""" - # """dims nx, ny, allrof, string_length, fname_lenp2, fname_len, - # len1, scalar, max_chars, max_nflds, max_flds""" - - # # ------------------------------------------------------------ # - # # Open file - # f = Dataset(self.rest_filename, 'w', self._file_format) - # # ------------------------------------------------------------ # - - # # ------------------------------------------------------------ # - # # Dimensions - # nx = f.createDimension('nx', self._grid_shape[1]) - # ny = f.createDimension('ny', self._grid_shape[0]) - # allrof = f.createDimension('allrof', ) - # string_length = f.createDimension('string_length', 8) - # fname_lenp2 = f.createDimension('fname_lenp2', 34) - # fname_len = f.createDimension('fname_len', 32) - # len1 = f.createDimension('len1', 1) - # scalar = f.createDimension('scalar', 1) - # max_chars = f.createDimension('max_chars', 128) - # max_nflds = f.createDimension('max_nflds', 2) - # max_flds = f.createDimension('max_flds', 15) - # # ------------------------------------------------------------ # - - # # ------------------------------------------------------------ # - # # Write Fields - # restvars = OrderedDict() - # restvars['nhtfrq'] = f.createVariable('nhtfrq', NC_INT, ('scalar', )) - # restvars['mfilt'] = f.createVariable('mfilt', NC_INT, ('scalar', )) - # restvars['ncprec'] = f.createVariable('ncprec', NC_INT, ('scalar', )) - # restvars['fincl'] = f.createVariable('fincl', NC_CHAR, ('max_flds', 'fname_lenp2',)) - # restvars['fexcl'] = f.createVariable('fexcl', NC_CHAR, ('max_flds', 'fname_lenp2',)) - # restvars['nflds'] = f.createVariable('nflds', NC_INT, ('scalar', )) - # restvars['ntimes'] = f.createVariable('ntimes', NC_INT, ('scalar', )) - # restvars['is_endhist'] = f.createVariable('is_endhist', NC_INT, ('scalar', )) - # restvars['begtime'] = f.createVariable('begtime', NC_DOUBLE, ('scalar', )) - # restvars['hpindex'] = f.createVariable('hpindex', NC_INT, ('max_nflds', )) - # restvars['avgflag'] = f.createVariable('avgflag', NC_CHAR, ('max_nflds', 'len1',)) - # restvars['name'] = f.createVariable('name', NC_CHAR, ('max_nflds', 'fname_len', )) - # restvars['long_name'] = f.createVariable('long_name', NC_CHAR, ('max_nflds', 'max_chars', )) - # restvars['units'] = f.createVariable('units', NC_CHAR, ('max_nflds', 'max_chars', )) - - # restvars['nhtfrq'][:] = self._nhtfrq - # restvars['mfilt'][:] = self._mfilt - # restvars['ncprec'][:] = self._ndens - # # restvars['fincl'][:, :] = self._fincl - # restvars['fexcl'][:, :] = self._fexcl - # restvars['nflds'][:] = len(self._fincl) - # restvars['ntimes'][:] = 1 - # restvars['is_endhist'][:] = 0 - # restvars['begtime'][:] = self._begtime - # restvars['hpindex'][:] = 'help' - # restvars['avgflag'][:, :] = self._avgflag - # restvars['name'][:, :] = self._fincl - # restvars['long_name'][:, :] = 'help' - # restvars['units'][:, :] = 'help' - - # for name, var in restvas.iteritems(): - # ncvar = getattr(share, name) - # for key, val in ncvar.__dict__.iteritems(): - # if val: - # setattr(var, key, val) - # # ------------------------------------------------------------ # - # # ------------------------------------------------------------ # - - # # ------------------------------------------------------------ # - # # write global attributes - # self._glob_ats.update() - # for key, val in self._glob_ats.atts.iteritems(): - # if val: - # setattr(f, key, val) - # f.title = "RVIC Restart History information, required to continue a simulation" - # f.comment = "This entire file NOT needed for drystart, startup, or branch simulations" - # f.featureType = "timeSeries" - # # ------------------------------------------------------------ # - - # return self.filename, self.rest_filename - # # ---------------------------------------------------------------- # # -------------------------------------------------------------------- # diff --git a/rvic/core/log.py b/rvic/core/log.py index e612f42..d7a79b9 100644 --- a/rvic/core/log.py +++ b/rvic/core/log.py @@ -43,8 +43,8 @@ def init_logger(log_dir='./', log_level='DEBUG', verbose=False): # ---------------------------------------------------------------- # # create log file handler if log_dir: - log_file = os.path.join(log_dir, 'RVIC-'+strftime("%Y%m%d-%H%M%S", - gmtime())+'.log') + log_file = os.path.join(log_dir, 'RVIC-' + strftime("%Y%m%d-%H%M%S", + gmtime()) + '.log') fh = logging.FileHandler(log_file) fh.setLevel(log_level) fh.setFormatter(FORMATTER) diff --git a/rvic/core/make_uh.py b/rvic/core/make_uh.py index e894fc6..8771cc6 100644 --- a/rvic/core/make_uh.py +++ b/rvic/core/make_uh.py @@ -18,9 +18,10 @@ import numpy as np import logging from scipy.interpolate import interp1d -from utilities import latlon2yx -from share import SECSPERDAY -from log import LOG_NAME +from .utilities import latlon2yx +from .share import SECSPERDAY +from .log import LOG_NAME +from .pycompat import zip, range # -------------------------------------------------------------------- # # create logger @@ -42,20 +43,22 @@ def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): # ---------------------------------------------------------------- # # Find Basin Dims and ID - basin_id = fdr_data[rout_dict['BASIN_ID_VAR']][pour_point.routy, pour_point.routx] + basin_id = fdr_data[rout_dict['BASIN_ID_VAR']][pour_point.routy, + pour_point.routx] log.info('Input Latitude: %f', pour_point.lat) log.info('Input Longitude: %f', pour_point.lon) log.info('Global Basid ID: %i', basin_id) - y_inds, x_inds = np.nonzero(fdr_data[rout_dict['BASIN_ID_VAR']] == basin_id) + y_inds, x_inds = \ + np.nonzero(fdr_data[rout_dict['BASIN_ID_VAR']] == basin_id) y = np.arange(len(fdr_data[rout_dict['LATITUDE_VAR']])) x = np.arange(len(fdr_data[rout_dict['LONGITUDE_VAR']])) x_min = min(x[x_inds]) - x_max = max(x[x_inds])+1 + x_max = max(x[x_inds]) + 1 y_min = min(y[y_inds]) - y_max = max(y[y_inds])+1 + y_max = max(y[y_inds]) + 1 # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -63,9 +66,12 @@ def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): basin = {} basin['lat'] = fdr_data[rout_dict['LATITUDE_VAR']][y_min:y_max] basin['lon'] = fdr_data[rout_dict['LONGITUDE_VAR']][x_min:x_max] - basin['basin_id'] = fdr_data[rout_dict['BASIN_ID_VAR']][y_min:y_max, x_min:x_max] - basin['flow_direction'] = fdr_data[rout_dict['FLOW_DIRECTION_VAR']][y_min:y_max, x_min:x_max] - basin['flow_distance'] = fdr_data[rout_dict['FLOW_DISTANCE_VAR']][y_min:y_max, x_min:x_max] + basin['basin_id'] = fdr_data[rout_dict['BASIN_ID_VAR']][y_min:y_max, + x_min:x_max] + basin['flow_direction'] = \ + fdr_data[rout_dict['FLOW_DIRECTION_VAR']][y_min:y_max, x_min:x_max] + basin['flow_distance'] = \ + fdr_data[rout_dict['FLOW_DISTANCE_VAR']][y_min:y_max, x_min:x_max] basin['velocity'] = fdr_data['velocity'][y_min:y_max, x_min:x_max] basin['diffusion'] = fdr_data['diffusion'][y_min:y_max, x_min:x_max] @@ -114,8 +120,8 @@ def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): # Find timestep (timestep is determined from uh_BOX input file) input_interval = find_ts(uh_t) rout_data['unit_hydrograph_dt'] = input_interval - t_cell = int(rout_dict['CELL_FLOWDAYS']*SECSPERDAY/input_interval) - t_uh = int(rout_dict['BASIN_FLOWDAYS']*SECSPERDAY/input_interval) + t_cell = int(rout_dict['CELL_FLOWDAYS'] * SECSPERDAY / input_interval) + t_uh = int(rout_dict['BASIN_FLOWDAYS'] * SECSPERDAY / input_interval) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -152,11 +158,9 @@ def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): # ---------------------------------------------------------------- # # Agregate to output timestep rout_data['unit_hydrograph'], \ - rout_data['timesteps'] = adjust_uh_timestep(uh_s, t_uh, - input_interval, - rout_dict['OUTPUT_INTERVAL'], - catchment['x_inds'], - catchment['y_inds']) + rout_data['timesteps'] = adjust_uh_timestep( + uh_s, t_uh, input_interval, rout_dict['OUTPUT_INTERVAL'], + catchment['x_inds'], catchment['y_inds']) # ---------------------------------------------------------------- # return rout_data # -------------------------------------------------------------------- # @@ -168,7 +172,7 @@ def find_ts(uh_t): """ Determines the (input_interval) based on the timestep given in uhfile """ - input_interval = uh_t[1]-uh_t[0] + input_interval = uh_t[1] - uh_t[0] log.debug('Input Timestep = %i seconds', input_interval) return input_interval # -------------------------------------------------------------------- # @@ -187,7 +191,7 @@ def read_direction(fdr, dy, dx): to_y = np.zeros_like(fdr, dtype=np.int16) to_x = np.zeros_like(fdr, dtype=np.int16) - valid_dirs = dy.keys() + valid_dirs = list(dy.keys()) for (y, x), d in np.ndenumerate(fdr): if d in valid_dirs: @@ -243,8 +247,8 @@ def search_catchment(to_y, to_x, pour_point, basin_ids, basin_id): for yy, xx in zip(byinds, bxinds): if in_catch[yy, xx] >= 0: # set the old path to zero - pathy[:cells+1] = 0 - pathx[:cells+1] = 0 + pathy[:cells + 1] = 0 + pathx[:cells + 1] = 0 # reset the cells counter cells = 0 @@ -260,14 +264,14 @@ def search_catchment(to_y, to_x, pour_point, basin_ids, basin_id): or in_catch[yy, xx] == 1: # get the path inds - py = pathy[:cells+1] - px = pathx[:cells+1] + py = pathy[:cells + 1] + px = pathx[:cells + 1] # Add path to in_catch and count_ds in_catch[py, px] = 1 # reverse the path count - temp_path = path_count[:cells+1][::-1] + temp_path = path_count[:cells + 1][::-1] count_ds[py, px] = (count_ds[yy, xx] + temp_path) break @@ -320,18 +324,18 @@ def make_uh(dt, t_cell, y_inds, x_inds, velocity, diffusion, xmask): log.debug('Making uh for each cell') uh = np.zeros((t_cell, xmask.shape[0], xmask.shape[1]), dtype=np.float64) - time = np.arange(dt, t_cell*dt+dt, dt, dtype=np.float64) + time = np.arange(dt, t_cell * dt + dt, dt, dtype=np.float64) for y, x in zip(y_inds, x_inds): xm = xmask[y, x] v = velocity[y, x] d = diffusion[y, x] - exponent = -1*np.power(v*time-xm, 2)/(4*d*time) - green = xm/(2*time*np.sqrt(np.pi*time*d))*np.exp(exponent) + exponent = -1 * np.power(v * time - xm, 2) / (4 * d * time) + green = xm / (2 * time * np.sqrt(np.pi * time * d)) * np.exp(exponent) # Normalize - uh[:, y, x] = green/green.sum() + uh[:, y, x] = green / green.sum() return uh # -------------------------------------------------------------------- # @@ -380,7 +384,7 @@ def make_grid_uh(t_uh, t_cell, uh_river, uh_box, to_y, to_x, y_inds, x_inds, log.debug("Making unit_hydrograph grid") unit_hydrograph = np.zeros((t_uh, uh_river.shape[1], uh_river.shape[2])) - irf_temp = np.zeros(t_uh+t_cell, dtype=np.float64) + irf_temp = np.zeros(t_uh + t_cell, dtype=np.float64) for (y, x, d) in zip(y_inds, x_inds, count_ds): irf_temp[:] = 0.0 @@ -388,7 +392,8 @@ def make_grid_uh(t_uh, t_cell, uh_river, uh_box, to_y, to_x, y_inds, x_inds, yy = to_y[y, x] xx = to_x[y, x] irf_temp = np.convolve(uh_box, uh_river[:, yy, xx]) - unit_hydrograph[:, y, x] = irf_temp[:t_uh]/np.sum(irf_temp[:t_uh]) + unit_hydrograph[:, y, x] = \ + irf_temp[:t_uh] / np.sum(irf_temp[:t_uh]) else: irf_temp[:len(uh_box)] = uh_box[:] unit_hydrograph[:, y, x] = irf_temp[:t_uh] @@ -414,14 +419,15 @@ def adjust_uh_timestep(unit_hydrograph, t_uh, input_interval, output_interval, elif np.remainder(output_interval, input_interval) == 0: log.debug('Aggregating to %i from %i seconds', output_interval, input_interval) - fac = int(output_interval/input_interval) - t_uh_out = int(t_uh/fac) + fac = int(output_interval / input_interval) + t_uh_out = int(t_uh / fac) ts_new = np.arange(t_uh_out) uh_out = np.zeros((t_uh_out, unit_hydrograph.shape[1], unit_hydrograph.shape[2]), dtype=np.float64) for (y, x) in zip(y_inds, x_inds): - for t in xrange(t_uh_out): - uh_out[t, y, x] = unit_hydrograph[t*fac:t*fac+fac, y, x].sum() + for t in range(t_uh_out): + uh_out[t, y, x] = unit_hydrograph[t * fac:t * fac + fac, + y, x].sum() elif np.remainder(input_interval, output_interval): log.debug('Interpolating unit hydrograph from input_interval: %i to ' 'output_interval: %i', input_interval, output_interval) diff --git a/rvic/core/mpi.py b/rvic/core/multi_proc.py similarity index 96% rename from rvic/core/mpi.py rename to rvic/core/multi_proc.py index a244ead..cb6e134 100644 --- a/rvic/core/mpi.py +++ b/rvic/core/multi_proc.py @@ -1,8 +1,8 @@ """ -mpi.py +multi_proc.py """ -from log import LOG_NAME +from .log import LOG_NAME import multiprocessing from multiprocessing.pool import Pool import traceback diff --git a/rvic/core/param_file.py b/rvic/core/param_file.py index 45260a2..d9a4299 100644 --- a/rvic/core/param_file.py +++ b/rvic/core/param_file.py @@ -3,12 +3,13 @@ """ import numpy as np import logging -from log import LOG_NAME -from write import write_param_file -from share import NcGlobals, SECSPERDAY, MAX_NC_CHARS +from .log import LOG_NAME +from .write import write_param_file +from .share import NcGlobals, SECSPERDAY, MAX_NC_CHARS +from .pycompat import iteritems, range, zip +from . import plots import os from datetime import date -import plots # -------------------------------------------------------------------- # @@ -44,7 +45,8 @@ def finish_params(outlets, dom_data, config_dict, directories): # subset (shorten time base) if options['SUBSET_DAYS'] and \ options['SUBSET_DAYS'] < routing['BASIN_FLOWDAYS']: - subset_length = options['SUBSET_DAYS']*SECSPERDAY/routing['OUTPUT_INTERVAL'] + subset_length = (options['SUBSET_DAYS'] * + SECSPERDAY / routing['OUTPUT_INTERVAL']) outlets, full_time_length, \ before, after = subset(outlets, subset_length=subset_length) @@ -65,7 +67,7 @@ def finish_params(outlets, dom_data, config_dict, directories): else: log.info('Not subsetting because either SUBSET_DAYS is null or ' 'SUBSET_DAYS dom_fractions) log.info('Adjust fractions for %s grid cells', len(yi)) - ratio_fraction[yi, xi] = dom_fractions[yi, xi]/fractions[yi, xi] + ratio_fraction[yi, xi] = dom_fractions[yi, xi] / fractions[yi, xi] # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # # Adjust fracs based on ratio_fraction - for cell_id, outlet in outlets.iteritems(): + for cell_id, outlet in iteritems(outlets): y = outlet.y_source x = outlet.x_source if adjust: @@ -294,7 +301,7 @@ def subset(outlets, subset_length=None): log.info('subsetting unit-hydrographs now...') log.debug('Subset Length: %s', subset_length) - for i, (cell_id, outlet) in enumerate(outlets.iteritems()): + for i, (cell_id, outlet) in enumerate(iteritems(outlets)): if i == 0: full_time_length = outlet.unit_hydrograph.shape[0] log.debug('Subset Length: %s', subset_length) @@ -311,10 +318,10 @@ def subset(outlets, subset_length=None): out_uh = np.zeros((subset_length, outlet.unit_hydrograph.shape[1]), dtype=np.float64) - d_left = -1*subset_length/2 - d_right = subset_length/2 + d_left = -1 * subset_length / 2 + d_right = subset_length / 2 - for j in xrange(outlet.unit_hydrograph.shape[1]): + for j in range(outlet.unit_hydrograph.shape[1]): # find index position of maximum maxind = np.argmax(outlet.unit_hydrograph[:, j]) @@ -371,7 +378,7 @@ def group(outlets, subset_length): n_outlets = len(outlets) n_sources = 0 - for outlet in outlets.itervalues(): + for outlet in outlets.values(): n_sources += len(outlet.y_source) gd = {} @@ -410,7 +417,7 @@ def group(outlets, subset_length): # ---------------------------------------------------------------- # # place outlet and source vars into gd dictionary a = 0 - for i, (cell_id, outlet) in enumerate(outlets.iteritems()): + for i, (cell_id, outlet) in enumerate(iteritems(outlets)): b = a + len(outlet.y_source) log.debug('unit_hydrograph.shape %s', outlet.unit_hydrograph.shape) # -------------------------------------------------------- # diff --git a/rvic/core/plots.py b/rvic/core/plots.py index 43fb59a..cb925c4 100644 --- a/rvic/core/plots.py +++ b/rvic/core/plots.py @@ -3,7 +3,7 @@ """ import os import logging -from log import LOG_NAME +from .log import LOG_NAME import numpy as np from datetime import date try: @@ -65,7 +65,7 @@ def _fractions_grid(data, dom_x, dom_y, title, case_id, plot_dir): data = np.ma.array(data, mask=mask) cmap = matplotlib.cm.cool - cmap.set_bad(color=u'w') + cmap.set_bad(color='w') fig = plt.figure() plt.pcolormesh(data, cmap=cmap) @@ -96,7 +96,7 @@ def _fractions_map(data, dom_x, dom_y, title, case_id, plot_dir): pfname = os.path.join(plot_dir, file_name) fig = plt.figure(figsize=(8, 8)) - ax = fig.add_axes([0.1, 0.1, 0.8, 0.8]) + fig.add_axes([0.1, 0.1, 0.8, 0.8]) dom_x[dom_x < 0] += 360.0 @@ -104,12 +104,11 @@ def _fractions_map(data, dom_x, dom_y, title, case_id, plot_dir): data = np.ma.array(data, mask=mask) cmap = matplotlib.cm.cool - cmap.set_bad(color=u'w') + cmap.set_bad(color='w') # define projection - midx = int(dom_x.shape[1]/2) - midy = int(dom_x.shape[0]/2) - print midx, midy + midx = int(dom_x.shape[1] / 2) + midy = int(dom_x.shape[0] / 2) projection = {'projection': 'stere', 'lon_0': dom_x[-1, midx], @@ -136,7 +135,6 @@ def _fractions_map(data, dom_x, dom_y, title, case_id, plot_dir): m.drawmeridians(meridians, labels=[0, 0, 0, 1]) x, y = m(dom_x, dom_y) # compute map proj coordinates. - print 'passed 1' cs = m.pcolormesh(x, y, data, cmap=cmap) m.colorbar(cs, location='right', pad="5%") plt.title(title) diff --git a/rvic/core/pycompat.py b/rvic/core/pycompat.py new file mode 100644 index 0000000..a23dadb --- /dev/null +++ b/rvic/core/pycompat.py @@ -0,0 +1,38 @@ +import sys + +PY3 = sys.version_info[0] >= 3 + +if PY3: + basestring = str + unicode_type = str + bytes_type = bytes + + def iteritems(d): + return iter(d.items()) + + def itervalues(d): + return iter(d.values()) + range = range + zip = zip + from configparser import SafeConfigParser +else: + # Python 2 + basestring = basestring + unicode_type = unicode + bytes_type = str + + def iteritems(d): + return d.iteritems() + + def itervalues(d): + return d.itervalues() + range = xrange + from itertools import izip as zip + from ConfigParser import SafeConfigParser +try: + from cyordereddict import OrderedDict +except ImportError: + try: + from collections import OrderedDict + except ImportError: + from ordereddict import OrderedDict diff --git a/rvic/core/read_forcing.py b/rvic/core/read_forcing.py index 518c794..70bbb9f 100644 --- a/rvic/core/read_forcing.py +++ b/rvic/core/read_forcing.py @@ -8,9 +8,10 @@ from bisect import bisect_right from netCDF4 import Dataset, date2num, date2index, num2date from logging import getLogger -from log import LOG_NAME -from time_utility import ord_to_datetime -from share import MMPERMETER, CMPERMETER, WATERDENSITY, TIMEUNITS, SECSPERDAY +from .log import LOG_NAME +from .time_utility import ord_to_datetime +from .share import MMPERMETER, CMPERMETER, WATERDENSITY, TIMEUNITS, SECSPERDAY +from .pycompat import range # -------------------------------------------------------------------- # # create logger @@ -48,8 +49,8 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, start = [int(start)] end = [int(end)] else: - start = map(int, start.split('-')) - end = map(int, end.split('-')) + start = list(map(int, start.split('-'))) + end = list(map(int, end.split('-'))) # single files without year in them if not start: @@ -57,7 +58,7 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, # yearly files elif len(start) == 1: - for year in xrange(start[0], end[0]+1): + for year in range(start[0], end[0] + 1): self.files.append(os.path.join(self.path, file_str.replace('$YYYY', "{0:04d}".format(year)))) @@ -67,8 +68,13 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, year = start[0] month = start[1] while True: - self.files.append(os.path.join(self.path, - file_str.replace('$YYYY', "{0:04d}".format(year)).replace('$MM', "{0:02d}".format(month)))) + self.files.append( + os.path.join( + self.path, + file_str.replace( + '$YYYY', + "{0:04d}".format(year)).replace( + '$MM', "{0:02d}".format(month)))) if year == end[0] and month == end[1]: break else: @@ -84,8 +90,16 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, month = start[1] day = start[2] while True: - self.files.append(os.path.join(self.path, - file_str.replace('$YYYY', "{0:04d}".format(year)).replace('$MM', "{0:02d}".format(month)).replace('$DD', "{0:02d}".format(day)))) + self.files.append( + os.path.join( + self.path, + file_str.replace( + '$YYYY', + "{0:04d}".format(year)).replace( + '$MM', + "{0:02d}".format(month)).replace( + '$DD', + "{0:02d}".format(day)))) if year == end[0] and month == end[1] and day == end[2]: break else: @@ -150,7 +164,7 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, t1 = date2num(num2date(time_series[1], self.time_units, calendar=self.calendar), TIMEUNITS, calendar=self.calendar) - self.secs_per_step = (t1-t0) * SECSPERDAY + self.secs_per_step = (t1 - t0) * SECSPERDAY else: raise ValueError('Need more than 1 forcing timestep in order to ' 'calculate timestep') @@ -177,9 +191,9 @@ def start(self, timestamp, rout_var): self.current_fhdl = Dataset(self.current_file, 'r') try: - self.current_tind = date2index(timestamp, - self.current_fhdl.variables[self.time_fld]) - except Exception, e: + self.current_tind = date2index( + timestamp, self.current_fhdl.variables[self.time_fld]) + except Exception as e: log.error(num2date(self.start_dates, self.time_units, calendar=self.calendar)) log.error('timestamp: %s', timestamp) @@ -197,11 +211,13 @@ def start(self, timestamp, rout_var): 'kg*m-2*s-1', 'kg s-1 m-2']: self.fld_mult[fld] = 1.0 elif units in ['mm', 'MM', 'milimeters', 'Milimeters']: - self.fld_mult[fld] = WATERDENSITY / MMPERMETER / self.secs_per_step + self.fld_mult[fld] = (WATERDENSITY / MMPERMETER / + self.secs_per_step) elif units in ['m', 'M', 'meters', 'Meters']: self.fld_mult[fld] = WATERDENSITY / self.secs_per_step elif units in ['cm', 'CM', 'centimeters', 'Centimeters']: - self.fld_mult[fld] = WATERDENSITY / CMPERMETER / self.secs_per_step + self.fld_mult[fld] = (WATERDENSITY / CMPERMETER / + self.secs_per_step) else: raise ValueError('unknown forcing units') # ------------------------------------------------------------ # @@ -213,7 +229,8 @@ def start(self, timestamp, rout_var): try: fill_val = getattr(self.current_fhdl.variables[fld], '_FillValue') - fmask = np.squeeze(self.current_fhdl.variables[fld][0]) == fill_val + fmask = np.squeeze( + self.current_fhdl.variables[fld][0]) == fill_val if np.any(fmask[rout_var.source_y_ind, rout_var.source_x_ind]): log.error('There are fill values in the routing domain') @@ -233,7 +250,8 @@ def read(self, timestamp): # ------------------------------------------------------------ # # Get the current data index location - if self.current_tind == len(self.current_fhdl.variables[self.time_fld]): + if self.current_tind == len( + self.current_fhdl.variables[self.time_fld]): # close file and open next self.current_fhdl.close() self.current_filenum += 1 @@ -244,18 +262,20 @@ def read(self, timestamp): # ------------------------------------------------------------ # # check that the timestamp is what was expected - expected_timestamp = ord_to_datetime(self.current_fhdl.variables[self.time_fld][self.current_tind], - self.time_units, - calendar=self.calendar) + expected_timestamp = ord_to_datetime( + self.current_fhdl.variables[self.time_fld][self.current_tind], + self.time_units, + calendar=self.calendar) if timestamp != expected_timestamp: log.warning('Timestamp is not what was expected') log.warning('Got timestamp %s', timestamp) log.warning('Expected timestamp %s', expected_timestamp) # Attempt to find the correct timestamp - self.ordtime = date2num(timestamp, - self.current_fhdl.variables[self.time_fld].units, - calendar=self.calendar) + self.ordtime = date2num( + timestamp, + self.current_fhdl.variables[self.time_fld].units, + calendar=self.calendar) # check to make sure the timestamp exists in input dataset if self.ordtime > max(self.end_ords): @@ -269,8 +289,8 @@ def read(self, timestamp): self.current_file = self.files[new_filenum] self.current_fhdl = Dataset(self.current_file, 'r') - self.current_tind = date2index(timestamp, - self.current_fhdl.variables[self.time_fld]) + self.current_tind = date2index( + timestamp, self.current_fhdl.variables[self.time_fld]) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -280,16 +300,24 @@ def read(self, timestamp): for i, fld in enumerate(self.liq_flds): self.current_fhdl.variables[fld].set_auto_maskandscale(False) if i == 0: - forcing['LIQ'] = self.current_fhdl.variables[fld][self.current_tind, :, :] * self.fld_mult[fld] + forcing['LIQ'] = \ + (self.current_fhdl.variables[fld][self.current_tind, :, :] + * self.fld_mult[fld]) else: - forcing['LIQ'] += self.current_fhdl.variables[fld][self.current_tind, :, :] * self.fld_mult[fld] + forcing['LIQ'] += \ + (self.current_fhdl.variables[fld][self.current_tind, :, :] + * self.fld_mult[fld]) for i, fld in enumerate(self.ice_flds): self.current_fhdl.variables[fld].set_auto_maskandscale(False) if i == 0: - forcing['ICE'] = self.current_fhdl.variables[fld][self.current_tind, :, :] * self.fld_mult[fld] + forcing['ICE'] = \ + (self.current_fhdl.variables[fld][self.current_tind, :, :] + * self.fld_mult[fld]) else: - forcing['ICE'] += self.current_fhdl.variables[fld][self.current_tind, :, :] * self.fld_mult[fld] + forcing['ICE'] += \ + (self.current_fhdl.variables[fld][self.current_tind, :, :] + * self.fld_mult[fld]) # move forward one step self.current_tind += 1 diff --git a/rvic/core/remap.py b/rvic/core/remap.py index 3f0aac8..05ea14b 100644 --- a/rvic/core/remap.py +++ b/rvic/core/remap.py @@ -6,7 +6,7 @@ from cdo import Cdo cdo = Cdo() from logging import getLogger -from log import LOG_NAME +from .log import LOG_NAME # -------------------------------------------------------------------- # # create logger diff --git a/rvic/core/share.py b/rvic/core/share.py index b735789..ad49e36 100644 --- a/rvic/core/share.py +++ b/rvic/core/share.py @@ -5,7 +5,7 @@ import socket import string import time as time_mod -from collections import OrderedDict +from .pycompat import OrderedDict, iteritems from netCDF4 import default_fillvals from getpass import getuser @@ -147,12 +147,13 @@ def __init__(self, self.atts['RvicDomainFile'] = RvicDomainFile def update(self): - self.atts['history'] = 'Created: {0}'.format(time_mod.ctime(time_mod.time())) + self.atts['history'] = \ + 'Created: {0}'.format(time_mod.ctime(time_mod.time())) class NcVar: def __init__(self, **kwargs): - for key, val in kwargs.iteritems(): + for key, val in iteritems(kwargs): setattr(self, key, val) # Coordinate Variables @@ -259,7 +260,8 @@ def __init__(self, **kwargs): streamflow = NcVar(long_name='Streamflow at outlet grid cell', units='kg m-2 s-1') -storage = NcVar(long_name='Mass storage in stream upstream of outlet grid cell', +storage = NcVar(long_name='Mass storage in stream upstream of outlet ' + 'grid cell', units='kg m-2 s-1') # valid values http://cf-pcmdi.llnl.gov/documents/cf-conventions/1.6/cf-conventions.html#calendar @@ -296,7 +298,8 @@ def __init__(self, **kwargs): valid_range='0, 86400') nhtfrq = NcVar(long_name='Frequency of history writes', - units='absolute value of negative is in hours, 0=monthly, positive is time-steps', + units='absolute value of negative is in hours, 0=monthly, ' + 'positive is time-steps', comment='Namelist item') mfilt = NcVar(long_name='Number of history time samples on a file', diff --git a/rvic/core/time_utility.py b/rvic/core/time_utility.py index d83015e..999554a 100644 --- a/rvic/core/time_utility.py +++ b/rvic/core/time_utility.py @@ -9,9 +9,10 @@ from netCDF4 import num2date, date2num from datetime import datetime from dateutil.relativedelta import relativedelta -from share import TIMEUNITS, SECSPERDAY, MINSPERDAY, HOURSPERDAY, TIMESTAMPFORM +from .share import TIMEUNITS, SECSPERDAY, MINSPERDAY, HOURSPERDAY, \ + TIMESTAMPFORM from logging import getLogger -from log import LOG_NAME +from .log import LOG_NAME # -------------------------------------------------------------------- # # create logger @@ -42,7 +43,7 @@ def __init__(self, start_date, stop_option, stop_n, stop_date, self.stop_option = stop_option self.stop_n = stop_n if self.stop_option == 'date': - date = map(int, stop_date.split('-')) + date = list(map(int, stop_date.split('-'))) self.stop_date = datetime(*date) else: self.stop_date = False @@ -51,7 +52,7 @@ def __init__(self, start_date, stop_option, stop_n, stop_date, self.rest_option = rest_option self.rest_n = rest_n if self.rest_option == 'date': - date = map(int, rest_date.split('-')) + date = list(map(int, rest_date.split('-'))) self.rest_date = datetime(*date) else: self.rest_date = False @@ -115,7 +116,8 @@ def __stop(self): elif self.stop_option == 'nmonths': temp = ord_to_datetime(self.time_ord + self.dt, TIMEUNITS, calendar=self.calendar) - if relativedelta(temp, self.start_date).months % self.stop_n == 0.0: + if relativedelta(temp, + self.start_date).months % self.stop_n == 0.0: flag = True elif self.stop_option == 'nmonth': temp = ord_to_datetime(self.time_ord + self.dt, TIMEUNITS, @@ -178,7 +180,8 @@ def __rest(self): elif self.rest_option == 'nmonths': temp = ord_to_datetime(self.time_ord + self.dt, TIMEUNITS, calendar=self.calendar) - if relativedelta(temp, self.start_date).months % self.rest_n == 0.0: + if relativedelta(temp, + self.start_date).months % self.rest_n == 0.0: flag = True elif self.rest_option == 'nmonth': temp = ord_to_datetime(self.time_ord + self.dt, TIMEUNITS, diff --git a/rvic/core/utilities.py b/rvic/core/utilities.py index 6ab3dfe..944d2f5 100644 --- a/rvic/core/utilities.py +++ b/rvic/core/utilities.py @@ -6,13 +6,13 @@ from scipy.spatial import cKDTree import numpy as np from shutil import rmtree, copyfile -from ConfigParser import SafeConfigParser +from .pycompat import iteritems, SafeConfigParser from netCDF4 import Dataset from logging import getLogger -from log import LOG_NAME -from share import TIMESTAMPFORM, RPOINTER, EARTHRADIUS, METERSPERMILE -from share import METERS2PERACRE, METERSPERKM, VALID_CHARS -from config import read_config +from .log import LOG_NAME +from .share import TIMESTAMPFORM, RPOINTER, EARTHRADIUS, METERSPERMILE +from .share import METERS2PERACRE, METERSPERKM, VALID_CHARS +from .config import read_config # -------------------------------------------------------------------- # # create logger @@ -84,7 +84,7 @@ def read_netcdf(nc_file, variables=None, coords=None): f = Dataset(nc_file, 'r') if not variables: - variables = f.variables.keys() + variables = list(f.variables.keys()) if not coords: coords = slice(None) @@ -114,7 +114,7 @@ def check_ncvars(config_section, nckeys): """ Make sure the variables listed in the config file are present in the netcdf """ - for key, value in config_section.iteritems(): + for key, value in iteritems(config_section): if key.endswith('var'): if value not in nckeys: log.error('%s (%s) not in %s', value, key, @@ -129,7 +129,7 @@ def check_ncvars(config_section, nckeys): # Find the index of the the nearest value def find_nearest(array, value): """ Find the index location in (array) with value nearest to (value)""" - return np.abs(array-value).argmin() + return np.abs(array - value).argmin() # -------------------------------------------------------------------- # @@ -179,7 +179,7 @@ def make_directories(rundir, subdir_names): # -------------------------------------------------------------------- # # Move all the input files to a central location -def copy_inputs(config_file, InputsDir): +def copy_inputs(config_file, inputs_dir): config_dict = read_config(config_file) @@ -187,20 +187,20 @@ def copy_inputs(config_file, InputsDir): config.optionxform = str config.read(config_file) - new_config = os.path.join(InputsDir, os.path.split(config_file)[1]) + new_config = os.path.join(inputs_dir, os.path.split(config_file)[1]) # ---------------------------------------------------------------- # # copy the inputs - for key, section in config_dict.iteritems(): - if 'FILE_NAME' in section.keys(): - new_file_name = os.path.join(InputsDir, - os.path.split(section['FILE_NAME'])[1]) + for key, section in iteritems(config_dict): + if 'FILE_NAME' in list(section.keys()): + new_file_name = os.path.join( + inputs_dir, os.path.split(section['FILE_NAME'])[1]) copyfile(section['FILE_NAME'], new_file_name) # update the config file for an easy restart config.set(key, 'FILE_NAME', - os.path.join(InputsDir, + os.path.join(inputs_dir, os.path.split(section['FILE_NAME'])[1])) # update the config_dict with the new value @@ -271,7 +271,7 @@ def read_domain(domain_dict, lat0_is_min=False): """ dom_data, dom_vatts, dom_gatts = read_netcdf(domain_dict['FILE_NAME']) - check_ncvars(domain_dict, dom_data.keys()) + check_ncvars(domain_dict, list(dom_data.keys())) # ---------------------------------------------------------------- # # Create the cell_ids variable @@ -294,7 +294,7 @@ def read_domain(domain_dict, lat0_is_min=False): if (dom_data[dom_lat][-1] > dom_data[dom_lat][0]) != lat0_is_min: log.debug('Domain Inputs came in upside down, flipping everything ' 'now.') - var_list = dom_data.keys() + var_list = list(dom_data.keys()) var_list.remove(dom_lon) for var in var_list: dom_data[var] = np.flipud(dom_data[var]) @@ -315,18 +315,18 @@ def read_domain(domain_dict, lat0_is_min=False): if area_units in ["rad2", "radians2", "radian2", "radian^2", "rad^2", "radians^2", "rads^2", "radians squared", "square-radians"]: - dom_data[dom_area] = dom_data[dom_area]*EARTHRADIUS*EARTHRADIUS + dom_data[dom_area] = dom_data[dom_area] * EARTHRADIUS * EARTHRADIUS elif area_units in ["m2", "m^2", "meters^2", "meters2", "square-meters", "meters squared"]: dom_data[dom_area] = dom_data[dom_area] elif area_units in ["km2", "km^2", "kilometers^2", "kilometers2", "square-kilometers", "kilometers squared"]: - dom_data[dom_area] = dom_data[dom_area]*METERSPERKM*METERSPERKM + dom_data[dom_area] = dom_data[dom_area] * METERSPERKM * METERSPERKM elif area_units in ["mi2", "mi^2", "miles^2", "miles", "square-miles", "miles squared"]: - dom_data[dom_area] = dom_data[dom_area]*METERSPERMILE*METERSPERMILE + dom_data[dom_area] = dom_data[dom_area] * METERSPERMILE * METERSPERMILE elif area_units in ["acres", "ac", "ac."]: - dom_data[dom_area] = dom_data[dom_area]*METERS2PERACRE + dom_data[dom_area] = dom_data[dom_area] * METERS2PERACRE else: log.warning("WARNING: UNKNOWN AREA units (%s), ASSUMING THEY ARE IN " "SQUARE METERS", diff --git a/rvic/core/variables.py b/rvic/core/variables.py index 8b9e3d1..65e2b94 100644 --- a/rvic/core/variables.py +++ b/rvic/core/variables.py @@ -5,13 +5,14 @@ import numpy as np from netCDF4 import Dataset, date2num, stringtochar from logging import getLogger -from log import LOG_NAME -from time_utility import ord_to_datetime -from share import TIMEUNITS, NC_INT, NC_DOUBLE, NC_CHAR -from share import RVIC_TRACERS, NcGlobals, SECSPERDAY, MAX_NC_CHARS -from share import CALENDAR_KEYS, REFERENCE_DATE, REFERENCE_TIME -from convolution_wrapper import rvic_convolve -import share +from .log import LOG_NAME +from .time_utility import ord_to_datetime +from .share import TIMEUNITS, NC_INT, NC_DOUBLE, NC_CHAR +from .share import RVIC_TRACERS, NcGlobals, SECSPERDAY, MAX_NC_CHARS +from .share import CALENDAR_KEYS, REFERENCE_DATE, REFERENCE_TIME +from .convolution_wrapper import rvic_convolve +from .pycompat import iteritems +from . import share # -------------------------------------------------------------------- # # create logger @@ -76,9 +77,9 @@ def __init__(self, param_file, case_name, calendar, out_dir, file_format, f = Dataset(param_file, 'r') self.n_sources = len(f.dimensions['sources']) self.n_outlets = len(f.dimensions['outlets']) - self.subset_length = f.variables['subset_length'][0] - self.full_time_length = f.variables['full_time_length'][0] - self.unit_hydrograph_dt = f.variables['unit_hydrograph_dt'][0] + self.subset_length = f.variables['subset_length'][:] + self.full_time_length = f.variables['full_time_length'][:] + self.unit_hydrograph_dt = f.variables['unit_hydrograph_dt'][:] self.source_lon = f.variables['source_lon'][:] self.source_lat = f.variables['source_lat'][:] self.source_x_ind = f.variables['source_x_ind'][:] @@ -99,7 +100,8 @@ def __init__(self, param_file, case_name, calendar, out_dir, file_format, except KeyError: log.warning('Could not find unit hydrograph var %s', tname) log.warning('trying var name unit_hydrograph') - self.unit_hydrograph[tracer] = f.variables['unit_hydrograph'][:] + self.unit_hydrograph[tracer] = \ + f.variables['unit_hydrograph'][:] except: raise ValueError('Cant find Unit Hydrograph Variable') self.outlet_name = f.variables['outlet_name'][:] @@ -132,12 +134,13 @@ def __init__(self, param_file, case_name, calendar, out_dir, file_format, # ------------------------------------------------------------ # self._calendar = calendar - self.__fname_format = os.path.join(out_dir, "%s.r.%%Y-%%m-%%d-%%H-%%M-%%S.nc" % (case_name)) + self.__fname_format = os.path.join( + out_dir, "%s.r.%%Y-%%m-%%d-%%H-%%M-%%S.nc" % (case_name)) # ------------------------------------------------------------ # # CESM calendar key (only NO_LEAP_C, GREGORIAN are supported in CESM) self._calendar_key = 0 - for key, cals in CALENDAR_KEYS.iteritems(): + for key, cals in iteritems(CALENDAR_KEYS): if self._calendar in cals: self._calendar_key = key break @@ -211,9 +214,10 @@ def init_state(self, state_file, run_type, timestamp): for tracer in RVIC_TRACERS: self.ring[tracer] = f.variables['{0}_ring'.format(tracer)][:] - file_timestamp = ord_to_datetime(f.variables['time'][:], - f.variables['time'].units, - calendar=f.variables['time'].calendar) + file_timestamp = ord_to_datetime( + f.variables['time'][:], + f.variables['time'].units, + calendar=f.variables['time'].calendar) if run_type == 'restart': self.timestamp = file_timestamp @@ -330,7 +334,7 @@ def get_time_mode(self, cpl_secs_per_step): if (self.unit_hydrograph_dt % cpl_secs_per_step == 0) and \ (self.unit_hydrograph_dt >= cpl_secs_per_step): - self.agg_tsteps = self.unit_hydrograph_dt/cpl_secs_per_step + self.agg_tsteps = self.unit_hydrograph_dt / cpl_secs_per_step else: log.error('unit_hydrograph_dt must be a multiple of the ' 'cpl_secs_per_step') @@ -382,7 +386,7 @@ def write_restart(self, current_history_files, history_restart_files): time = f.createVariable('time', NC_DOUBLE, ('time',), **self.ncvaropts) time[:] = date2num(self.timestamp, TIMEUNITS, calendar=self._calendar) - for key, val in share.time.__dict__.iteritems(): + for key, val in iteritems(share.time.__dict__): if val: setattr(time, key, val) time.calendar = self._calendar @@ -393,7 +397,7 @@ def write_restart(self, current_history_files, history_restart_files): **self.ncvaropts) timesteps[:] = np.arange(self.full_time_length) - for key, val in share.timesteps.__dict__.iteritems(): + for key, val in iteritems(share.timesteps.__dict__): if val: setattr(timesteps, key, val) timesteps.timestep_length = 'unit_hydrograph_dt' @@ -402,14 +406,14 @@ def write_restart(self, current_history_files, history_restart_files): unit_hydrograph_dt = f.createVariable('unit_hydrograph_dt', NC_DOUBLE, (), **self.ncvaropts) unit_hydrograph_dt[:] = self.unit_hydrograph_dt - for key, val in share.unit_hydrograph_dt.__dict__.iteritems(): + for key, val in iteritems(share.unit_hydrograph_dt.__dict__): if val: setattr(unit_hydrograph_dt, key, val) timemgr_rst_type = f.createVariable('timemgr_rst_type', NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_type[:] = self._calendar_key - for key, val in share.timemgr_rst_type.__dict__.iteritems(): + for key, val in iteritems(share.timemgr_rst_type.__dict__): if val: setattr(timemgr_rst_type, key, val) @@ -417,16 +421,16 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_step_sec[:] = self.unit_hydrograph_dt - for key, val in share.timemgr_rst_step_sec.__dict__.iteritems(): + for key, val in iteritems(share.timemgr_rst_step_sec.__dict__): if val: setattr(timemgr_rst_step_sec, key, val) timemgr_rst_start_ymd = f.createVariable('timemgr_rst_start_ymd', NC_DOUBLE, (), **self.ncvaropts) - timemgr_rst_start_ymd[:] = self._start_date.year*10000 \ - + self._start_date.month*100 + self._start_date.day - for key, val in share.timemgr_rst_start_ymd.__dict__.iteritems(): + timemgr_rst_start_ymd[:] = self._start_date.year * 10000 \ + + self._start_date.month * 100 + self._start_date.day + for key, val in iteritems(share.timemgr_rst_start_ymd.__dict__): if val: setattr(timemgr_rst_start_ymd, key, val) @@ -434,7 +438,7 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_start_tod[:] = (self._start_ord % 1) * SECSPERDAY - for key, val in share.timemgr_rst_start_tod.__dict__.iteritems(): + for key, val in iteritems(share.timemgr_rst_start_tod.__dict__): if val: setattr(timemgr_rst_start_tod, key, val) @@ -442,7 +446,7 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_ref_ymd[:] = REFERENCE_DATE - for key, val in share.timemgr_rst_ref_ymd.__dict__.iteritems(): + for key, val in iteritems(share.timemgr_rst_ref_ymd.__dict__): if val: setattr(timemgr_rst_ref_ymd, key, val) @@ -450,55 +454,62 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_ref_tod[:] = REFERENCE_TIME - for key, val in share.timemgr_rst_ref_tod.__dict__.iteritems(): + for key, val in iteritems(share.timemgr_rst_ref_tod.__dict__): if val: setattr(timemgr_rst_ref_tod, key, val) timemgr_rst_curr_ymd = f.createVariable('timemgr_rst_curr_ymd', NC_DOUBLE, (), **self.ncvaropts) - timemgr_rst_curr_ymd[:] = self.timestamp.year*10000 + \ - self.timestamp.month*100+self.timestamp.day - for key, val in share.timemgr_rst_curr_ymd.__dict__.iteritems(): + timemgr_rst_curr_ymd[:] = self.timestamp.year * 10000 + \ + self.timestamp.month * 100 + self.timestamp.day + for key, val in iteritems(share.timemgr_rst_curr_ymd.__dict__): if val: setattr(timemgr_rst_curr_ymd, key, val) timemgr_rst_curr_tod = f.createVariable('timemgr_rst_curr_tod', NC_DOUBLE, (), **self.ncvaropts) - timemgr_rst_curr_tod[:] = (self.time_ord % 1)*SECSPERDAY - for key, val in share.timemgr_rst_curr_tod.__dict__.iteritems(): + timemgr_rst_curr_tod[:] = (self.time_ord % 1) * SECSPERDAY + for key, val in iteritems(share.timemgr_rst_curr_tod.__dict__): if val: setattr(timemgr_rst_curr_tod, key, val) # ------------------------------------------------------------ # # Setup Tape Dimensions coords = ('tapes', 'max_chars') - ntapes = f.createDimension(coords[0], len(history_restart_files)) - ntapes = f.createDimension(coords[1], MAX_NC_CHARS) + f.createDimension(coords[0], len(history_restart_files)) + f.createDimension(coords[1], MAX_NC_CHARS) # ------------------------------------------------------------ # # ------------------------------------------------------------ # # Write Fields locfnh = f.createVariable('locfnh', NC_CHAR, coords, **self.ncvaropts) + print('locfnh.shape', locfnh.shape) for i, string in enumerate(current_history_files): - locfnh[i, :] = stringtochar(np.array(string.ljust(MAX_NC_CHARS))) + b_string = string.encode() + assert type(b_string) == bytes + locfnh[i, :] = stringtochar(np.array(b_string.ljust(MAX_NC_CHARS))) locfnh.long_name = 'History filename' - locfnh.comment = 'This variable is NOT needed for startup or branch simulations' + locfnh.comment = 'This variable is NOT needed for startup or branch '\ + 'simulations' locfnhr = f.createVariable('locfnhr', NC_CHAR, coords, **self.ncvaropts) for i, string in enumerate(history_restart_files): - locfnh[i, :] = stringtochar(np.array(string.ljust(MAX_NC_CHARS))) + b_string = string.encode() + assert type(b_string) == bytes + locfnh[i, :] = stringtochar(np.array(b_string.ljust(MAX_NC_CHARS))) locfnhr.long_name = 'History restart filename' - locfnhr.comment = 'This variable NOT needed for startup or branch simulations' + locfnhr.comment = 'This variable NOT needed for startup or branch '\ + 'simulations' # ------------------------------------------------------------ # # ------------------------------------------------------------ # # Setup Point Dimensions coords = ('outlets', ) - outlets = f.createDimension(coords[0], self.n_outlets) + f.createDimension(coords[0], self.n_outlets) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -506,21 +517,21 @@ def write_restart(self, current_history_files, history_restart_files): oyi = f.createVariable('outlet_y_ind', NC_INT, coords[0], **self.ncvaropts) oyi[:] = self.outlet_y_ind - for key, val in share.outlet_y_ind.__dict__.iteritems(): + for key, val in iteritems(share.outlet_y_ind.__dict__): if val: setattr(oyi, key, val) oxi = f.createVariable('outlet_x_ind', NC_INT, coords[0], **self.ncvaropts) oxi[:] = self.outlet_x_ind - for key, val in share.outlet_x_ind.__dict__.iteritems(): + for key, val in iteritems(share.outlet_x_ind.__dict__): if val: setattr(oxi, key, val) odi = f.createVariable('outlet_decomp_ind', NC_INT, coords[0], **self.ncvaropts) odi[:] = self.outlet_decomp_ind - for key, val in share.outlet_decomp_ind.__dict__.iteritems(): + for key, val in iteritems(share.outlet_decomp_ind.__dict__): if val: setattr(odi, key, val) @@ -531,7 +542,7 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, tcoords, **self.ncvaropts) ring[:, :] = self.ring[tracer][:, :] - for key, val in share.ring.__dict__.iteritems(): + for key, val in iteritems(share.ring.__dict__): if val: setattr(ring, key, val) # ------------------------------------------------------------ # @@ -540,7 +551,7 @@ def write_restart(self, current_history_files, history_restart_files): # write global attributes self.glob_atts.update() - for key, val in self.glob_atts.atts.iteritems(): + for key, val in iteritems(self.glob_atts.atts): if val: setattr(f, key, val) # ------------------------------------------------------------ # diff --git a/rvic/core/write.py b/rvic/core/write.py index 0853819..63b7777 100644 --- a/rvic/core/write.py +++ b/rvic/core/write.py @@ -5,10 +5,10 @@ import numpy as np from netCDF4 import Dataset, stringtochar from logging import getLogger -from log import LOG_NAME -from share import NC_DOUBLE, NC_INT, NC_CHAR, FILLVALUE_F, NcGlobals -import share - +from .log import LOG_NAME +from .share import NC_DOUBLE, NC_INT, NC_CHAR, FILLVALUE_F, NcGlobals +from .pycompat import iteritems +from . import share # -------------------------------------------------------------------- # # create logger log = getLogger(LOG_NAME) @@ -49,30 +49,30 @@ def write_agg_netcdf(file_name, agg_data, glob_atts, format, zlib=True, unit_hydrograph_dt = f.createVariable('unit_hydrograph_dt', NC_INT, (), **ncvaropts) unit_hydrograph_dt[:] = agg_data['unit_hydrograph_dt'] - for key, val in share.unit_hydrograph_dt.__dict__.iteritems(): + for key, val in iteritems(share.unit_hydrograph_dt.__dict__): if val: setattr(unit_hydrograph_dt, key, val) timesteps = f.createVariable('timesteps', NC_INT, ('timesteps',), **ncvaropts) timesteps[:] = agg_data['timesteps'] - for key, val in share.timesteps.__dict__.iteritems(): + for key, val in iteritems(share.timesteps.__dict__): if val: setattr(timesteps, key, val) lon = f.createVariable('lon', NC_DOUBLE, ('lon',), **ncvaropts) - for key, val in share.lon.__dict__.iteritems(): + for key, val in iteritems(share.lon.__dict__): if val: setattr(lon, key, val) lat = f.createVariable('lat', NC_DOUBLE, ('lat',), **ncvaropts) - for key, val in share.lat.__dict__.iteritems(): + for key, val in iteritems(share.lat.__dict__): if val: setattr(lat, key, val) fraction = f.createVariable('fraction', NC_DOUBLE, ('lat', 'lon',), **ncvaropts) - for key, val in share.fraction.__dict__.iteritems(): + for key, val in iteritems(share.fraction.__dict__): if val: setattr(fraction, key, val) @@ -80,7 +80,7 @@ def write_agg_netcdf(file_name, agg_data, glob_atts, format, zlib=True, ('timesteps', 'lat', 'lon',), fill_value=FILLVALUE_F, **ncvaropts) - for key, val in share.unit_hydrograph.__dict__.iteritems(): + for key, val in iteritems(share.unit_hydrograph.__dict__): if val: setattr(unit_hydrographs, key, val) # ---------------------------------------------------------------- # @@ -95,7 +95,7 @@ def write_agg_netcdf(file_name, agg_data, glob_atts, format, zlib=True, # ---------------------------------------------------------------- # # write globals - for key, val in glob_atts.atts.iteritems(): + for key, val in iteritems(glob_atts.atts): if val: setattr(f, key, val) # ---------------------------------------------------------------- # @@ -157,7 +157,7 @@ def write_param_file(file_name, timesteps = f.createVariable('timesteps', NC_DOUBLE, ('timesteps',), **ncvaropts) timesteps[:] = np.arange(subset_length) - for key, val in share.timesteps.__dict__.iteritems(): + for key, val in iteritems(share.timesteps.__dict__): if val: setattr(timesteps, key, val) timesteps.timestep_length = 'unit_hydrograph_dt' @@ -167,7 +167,7 @@ def write_param_file(file_name, # ---------------------------------------------------------------- # # write global attributes glob_atts.update() - for key, val in glob_atts.atts.iteritems(): + for key, val in iteritems(glob_atts.atts): if val: setattr(f, key, val) f.featureType = "timeSeries" @@ -179,21 +179,21 @@ def write_param_file(file_name, # Full time length (size of ring) ftl = f.createVariable('full_time_length', NC_INT, (), **ncvaropts) ftl[:] = full_time_length - for key, val in share.full_time_length.__dict__.iteritems(): + for key, val in iteritems(share.full_time_length.__dict__): if val: setattr(ftl, key, val) # Subset Length sl = f.createVariable('subset_length', NC_INT, (), **ncvaropts) sl[:] = subset_length - for key, val in share.subset_length.__dict__.iteritems(): + for key, val in iteritems(share.subset_length.__dict__): if val: setattr(sl, key, val) # UH timestep uh_dt = f.createVariable('unit_hydrograph_dt', NC_DOUBLE, (), **ncvaropts) uh_dt[:] = unit_hydrograph_dt - for key, val in share.unit_hydrograph_dt.__dict__.iteritems(): + for key, val in iteritems(share.unit_hydrograph_dt.__dict__): if val: setattr(uh_dt, key, val) # ---------------------------------------------------------------- # @@ -206,11 +206,11 @@ def write_param_file(file_name, else: numoutlets = len(outlet_lon) ocoords = ('outlets',) - outlets = f.createDimension(ocoords[0], numoutlets) + f.createDimension(ocoords[0], numoutlets) nocoords = ocoords + ('nc_chars',) char_names = stringtochar(outlet_name) - chars = f.createDimension(nocoords[1], char_names.shape[1]) + f.createDimension(nocoords[1], char_names.shape[1]) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -219,49 +219,49 @@ def write_param_file(file_name, # Outlet Cell Longitudes olon = f.createVariable('outlet_lon', NC_DOUBLE, ocoords, **ncvaropts) olon[:] = outlet_lon - for key, val in share.outlet_lon.__dict__.iteritems(): + for key, val in iteritems(share.outlet_lon.__dict__): if val: setattr(olon, key, val) # Outlet Cell Latitudes olat = f.createVariable('outlet_lat', NC_DOUBLE, ocoords, **ncvaropts) olat[:] = outlet_lat - for key, val in share.outlet_lat.__dict__.iteritems(): + for key, val in iteritems(share.outlet_lat.__dict__): if val: setattr(olat, key, val) # Outlet Cell X Indicies oxi = f.createVariable('outlet_x_ind', NC_INT, ocoords, **ncvaropts) oxi[:] = outlet_x_ind - for key, val in share.outlet_x_ind.__dict__.iteritems(): + for key, val in iteritems(share.outlet_x_ind.__dict__): if val: setattr(oxi, key, val) # Outlet Cell Y Indicies oyi = f.createVariable('outlet_y_ind', NC_INT, ocoords, **ncvaropts) oyi[:] = outlet_y_ind - for key, val in share.outlet_y_ind.__dict__.iteritems(): + for key, val in iteritems(share.outlet_y_ind.__dict__): if val: setattr(oyi, key, val) # Outlet Cell Decomp IDs odi = f.createVariable('outlet_decomp_ind', NC_INT, ocoords, **ncvaropts) odi[:] = outlet_decomp_ind - for key, val in share.outlet_decomp_ind.__dict__.iteritems(): + for key, val in iteritems(share.outlet_decomp_ind.__dict__): if val: setattr(odi, key, val) # Outlet Cell Number on = f.createVariable('outlet_number', NC_INT, ocoords, **ncvaropts) on[:] = outlet_number - for key, val in share.outlet_number.__dict__.iteritems(): + for key, val in iteritems(share.outlet_number.__dict__): if val: setattr(on, key, val) # Outlet Mask om = f.createVariable('outlet_mask', NC_INT, ocoords, **ncvaropts) om[:] = outlet_mask - for key, val in share.outlet_mask.__dict__.iteritems(): + for key, val in iteritems(share.outlet_mask.__dict__): if val: setattr(om, key, val) @@ -269,7 +269,7 @@ def write_param_file(file_name, oua = f.createVariable('outlet_upstream_area', NC_DOUBLE, ocoords, **ncvaropts) oua[:] = outlet_upstream_area - for key, val in share.outlet_upstream_area.__dict__.iteritems(): + for key, val in iteritems(share.outlet_upstream_area.__dict__): if val: setattr(oua, key, val) @@ -277,14 +277,14 @@ def write_param_file(file_name, oug = f.createVariable('outlet_upstream_gridcells', NC_INT, ocoords, **ncvaropts) oug[:] = outlet_upstream_gridcells - for key, val in share.outlet_upstream_gridcells.__dict__.iteritems(): + for key, val in iteritems(share.outlet_upstream_gridcells.__dict__): if val: setattr(oug, key, val) # Outlet Names onm = f.createVariable('outlet_name', NC_CHAR, nocoords) onm[:, :] = char_names - for key, val in share.outlet_name.__dict__.iteritems(): + for key, val in iteritems(share.outlet_name.__dict__): if val: setattr(onm, key, val) # ---------------------------------------------------------------- # @@ -292,7 +292,7 @@ def write_param_file(file_name, # ---------------------------------------------------------------- # # Source Dimension scoords = ('sources',) - sources = f.createDimension(scoords[0], len(source_lon)) + f.createDimension(scoords[0], len(source_lon)) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -301,49 +301,49 @@ def write_param_file(file_name, # Source Cell Longitudes slon = f.createVariable('source_lon', NC_DOUBLE, scoords, **ncvaropts) slon[:] = source_lon - for key, val in share.source_lon.__dict__.iteritems(): + for key, val in iteritems(share.source_lon.__dict__): if val: setattr(slon, key, val) # Source Cell Latitudes slat = f.createVariable('source_lat', NC_DOUBLE, scoords, **ncvaropts) slat[:] = source_lat - for key, val in share.source_lat.__dict__.iteritems(): + for key, val in iteritems(share.source_lat.__dict__): if val: setattr(slat, key, val) # Source Cell X Indicies sxi = f.createVariable('source_x_ind', NC_INT, scoords, **ncvaropts) sxi[:] = source_x_ind - for key, val in share.source_x_ind.__dict__.iteritems(): + for key, val in iteritems(share.source_x_ind.__dict__): if val: setattr(sxi, key, val) # Source Cell Y Indicies syi = f.createVariable('source_y_ind', NC_INT, scoords, **ncvaropts) syi[:] = source_y_ind - for key, val in share.source_y_ind.__dict__.iteritems(): + for key, val in iteritems(share.source_y_ind.__dict__): if val: setattr(syi, key, val) # Source Cell Decomp IDs sdi = f.createVariable('source_decomp_ind', NC_INT, scoords, **ncvaropts) sdi[:] = source_decomp_ind - for key, val in share.source_decomp_ind.__dict__.iteritems(): + for key, val in iteritems(share.source_decomp_ind.__dict__): if val: setattr(sdi, key, val) # Source Cell Time Offset sto = f.createVariable('source_time_offset', NC_INT, scoords, **ncvaropts) sto[:] = source_time_offset - for key, val in share.source_time_offset.__dict__.iteritems(): + for key, val in iteritems(share.source_time_offset.__dict__): if val: setattr(sto, key, val) # Source to Outlet Index Mapping s2o = f.createVariable('source2outlet_ind', NC_INT, scoords, **ncvaropts) s2o[:] = source2outlet_ind - for key, val in share.source2outlet_ind.__dict__.iteritems(): + for key, val in iteritems(share.source2outlet_ind.__dict__): if val: setattr(s2o, key, val) @@ -351,13 +351,13 @@ def write_param_file(file_name, # ---------------------------------------------------------------- # # 3-D Source Variables - uhcords = ('timesteps',) + scoords +('tracers',) - tracers = f.createDimension(uhcords[2], 1) + uhcords = ('timesteps',) + scoords + ('tracers',) + f.createDimension(uhcords[2], 1) # Unit Hydrographs uhs = f.createVariable('unit_hydrograph', NC_DOUBLE, uhcords, **ncvaropts) uhs[:, :] = unit_hydrograph - for key, val in share.unit_hydrograph.__dict__.iteritems(): + for key, val in iteritems(share.unit_hydrograph.__dict__): if val: setattr(uhs, key, val) # ---------------------------------------------------------------- # diff --git a/rvic/parameters.py b/rvic/parameters.py index 4588173..2621d2f 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -4,23 +4,23 @@ import os import numpy as np import pandas as pd -from collections import OrderedDict from logging import getLogger -from core.log import init_logger, close_logger, LOG_NAME -from core.mpi import LoggingPool -from core.utilities import make_directories, copy_inputs, strip_invalid_char -from core.utilities import read_netcdf, tar_inputs, latlon2yx -from core.utilities import check_ncvars, clean_file, read_domain -from core.aggregate import make_agg_pairs, aggregate -from core.make_uh import rout -from core.share import NcGlobals -from core.write import write_agg_netcdf -from core.variables import Point -from core.param_file import finish_params -from core.config import read_config +from .core.log import init_logger, close_logger, LOG_NAME +from .core.multi_proc import LoggingPool +from .core.utilities import make_directories, copy_inputs, strip_invalid_char +from .core.utilities import read_netcdf, tar_inputs, latlon2yx +from .core.utilities import check_ncvars, clean_file, read_domain +from .core.aggregate import make_agg_pairs, aggregate +from .core.make_uh import rout +from .core.share import NcGlobals +from .core.write import write_agg_netcdf +from .core.variables import Point +from .core.param_file import finish_params +from .core.config import read_config +from .core.pycompat import OrderedDict, iteritems, range try: - from core.remap import remap + from .core.remap import remap remap_available = True except: remap_available = False @@ -46,8 +46,8 @@ def parameters(config_file, numofproc=1): if numofproc > 1: pool = LoggingPool(processes=numofproc) - for i, (cell_id, outlet) in enumerate(outlets.iteritems()): - log.info('On Outlet #{0} of {1}'.format(i+1, len(outlets))) + for i, (cell_id, outlet) in enumerate(iteritems(outlets)): + log.info('On Outlet #{0} of {1}'.format(i + 1, len(outlets))) pool.apply_async(gen_uh_run, args=(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories), @@ -55,9 +55,10 @@ def parameters(config_file, numofproc=1): pool.close() pool.join() - outlets = OrderedDict(sorted(results.items(), key=lambda t: t[0])) + outlets = OrderedDict(sorted(list(iteritems(results)), + key=lambda t: t[0])) else: - for name, outlet in outlets.iteritems(): + for name, outlet in iteritems(outlets): outlet = gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories) @@ -116,8 +117,8 @@ def gen_uh_init(config_file): comment='#') log.info('Opened Pour Points File: ' '{0}'.format(config_dict['POUR_POINTS']['FILE_NAME'])) - if not (all(x in pour_points.keys() for x in ['lons', 'lats']) or - all(x in pour_points.keys() for x in ['x', 'y'])): + if not (all(x in list(pour_points.keys()) for x in ['lons', 'lats']) or + all(x in list(pour_points.keys()) for x in ['x', 'y'])): raise ValueError('Pour Points File must include ' 'variables (lons, lats) or (x, y)') if 'names' in pour_points: @@ -169,10 +170,11 @@ def gen_uh_init(config_file): if fdr_data[fdr_lat][-1] > fdr_data[fdr_lat][0]: log.debug('Flow Direction inputs came in upside down, flipping ' 'everything now.') - var_list = fdr_data.keys() + var_list = list(fdr_data.keys()) var_list.remove(fdr_lon) for var in var_list: - fdr_data[var] = np.flipud(fdr_data[var]) + if fdr_data[var].ndim >= 1: + fdr_data[var] = np.flipud(fdr_data[var]) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -190,7 +192,7 @@ def gen_uh_init(config_file): # ---------------------------------------------------------------- # fdr_data['resolution'] = np.abs(fdr_data[fdr_lon][1] - fdr_data[fdr_lon][0]) - check_ncvars(config_dict['ROUTING'], fdr_data.keys()) + check_ncvars(config_dict['ROUTING'], list(fdr_data.keys())) # ---------------------------------------------------------------- # log.info('Opened FDR File: {0}'.format(fdr_file)) @@ -227,12 +229,13 @@ def gen_uh_init(config_file): else: outlets = {} - if all(x in pour_points.keys() for x in ['x', 'y', 'lons', 'lats']): + if all(x in list(pour_points.keys()) for x in ['x', 'y', + 'lons', 'lats']): lats = pour_points['lats'].values lons = pour_points['lons'].values routys = pour_points['y'].values routxs = pour_points['x'].values - elif all(x in pour_points.keys() for x in ['x', 'y']): + elif all(x in list(pour_points.keys()) for x in ['x', 'y']): # use x and y (assume from routing inputs grid) # find lons and lats from xs and ys routys = pour_points['y'].values @@ -256,8 +259,8 @@ def gen_uh_init(config_file): glats=dom_data[domain['LATITUDE_VAR']], glons=dom_data[domain['LONGITUDE_VAR']]) - for i in xrange(len(lats)): - if 'names' in pour_points.keys(): + for i in range(len(lats)): + if 'names' in list(pour_points.keys()): name = pour_points['names'].values[i] name = name.replace("'", "").replace(" ", "_") else: @@ -319,7 +322,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, for j, pour_point in enumerate(outlet.pour_points): log.info('On pour_point #{0} of' - ' {1}'.format(j+1, len(outlet.pour_points))) + ' {1}'.format(j + 1, len(outlet.pour_points))) # -------------------------------------------------------- # # Make the Unit Hydrograph Grid @@ -339,7 +342,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, # -------------------------------------------------------- # # aggregate if options['AGGREGATE']: - if j != len(outlet.pour_points)-1: + if j != len(outlet.pour_points) - 1: agg_data = aggregate(rout_data, agg_data, res=fdr_data['resolution']) else: @@ -358,22 +361,26 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, # ------------------------------------------------------------ # # write temporary file #1 if options['REMAP']: - glob_atts = NcGlobals(title='RVIC Unit Hydrograph Grid File', - RvicPourPointsFile=os.path.split(config_dict['POUR_POINTS']['FILE_NAME'])[1], - RvicUHFile=os.path.split(config_dict['UH_BOX']['FILE_NAME'])[1], - RvicFdrFile=os.path.split(config_dict['ROUTING']['FILE_NAME'])[1], - RvicDomainFile=os.path.split(domain['FILE_NAME'])[1]) - - temp_file_1 = os.path.join(directories['aggregated'], - 'aggUH_{0}.nc'.format(outlet.name.replace(" ", "_"))) + glob_atts = NcGlobals( + title='RVIC Unit Hydrograph Grid File', + RvicPourPointsFile=os.path.split( + config_dict['POUR_POINTS']['FILE_NAME'])[1], + RvicUHFile=os.path.split(config_dict['UH_BOX']['FILE_NAME'])[1], + RvicFdrFile=os.path.split(config_dict['ROUTING']['FILE_NAME'])[1], + RvicDomainFile=os.path.split(domain['FILE_NAME'])[1]) + + temp_file_1 = os.path.join( + directories['aggregated'], + 'aggUH_{0}.nc'.format(outlet.name.replace(" ", "_"))) write_agg_netcdf(temp_file_1, agg_data, glob_atts, options['NETCDF_FORMAT'], **ncvaropts) # -------------------------------------------------------- # # Remap temporary file #1 to temporary file #2 - temp_file_2 = os.path.join(directories['remapped'], - 'remapUH_{0}.nc'.format(outlet.name.replace(" ", "_"))) + temp_file_2 = os.path.join( + directories['remapped'], + 'remapUH_{0}.nc'.format(outlet.name.replace(" ", "_"))) remap(domain['FILE_NAME'], temp_file_1, temp_file_2) @@ -388,7 +395,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, # Check latitude order, flip if necessary. if final_data[dom_lat].ndim == 1: if final_data[dom_lat][-1] > final_data[dom_lat][0]: - var_list = final_data.keys() + var_list = list(final_data.keys()) log.debug('Remapped inputs came in upside down, flipping {0}' ' now.'.format(", ".join(var_list))) @@ -396,7 +403,8 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, final_data[dom_lat] = final_data[dom_lat][::-1] final_data['fraction'] = final_data['fraction'][::-1, :] # flip unit hydrograph along y axis (axis 1) - final_data['unit_hydrograph'] = final_data['unit_hydrograph'][:, ::-1, :] + final_data['unit_hydrograph'] = \ + final_data['unit_hydrograph'][:, ::-1, :] assert dom_data['cord_lats'][0] == final_data[dom_lat][0] # -------------------------------------------------------- # diff --git a/setup.py b/setup.py index 23d4b22..fbdcb9b 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,6 @@ def write_version_py(filename=None): url='https://github.com/jhamman/RVIC', test_suite='pytest.collector', packages=['rvic', 'rvic.core'], - platform=['any'], py_modules=['rvic.parameters', 'rvic.convolution', 'rvic.convert'], scripts=['scripts/rvic', 'tools/find_pour_points.py', 'tools/fraction2domain.bash'], diff --git a/tests/run_tests.py b/tests/run_tests.py index 63d3ad2..6864d95 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -1,16 +1,16 @@ #!/usr/bin/env python """RVIC command line testing interface""" -from __future__ import print_function import os import textwrap import argparse import pytest import cProfile import pstats -import StringIO +import io from rvic import convert, convolution, parameters from rvic.core.config import read_config +from rvic.core.pycompat import PY3, iteritems if not os.environ.get('RVIC_TEST_DIR'): print('\n$RVIC_TEST_DIR not set.') @@ -62,11 +62,11 @@ def run_examples(config_file): # ---------------------------------------------------------------- # # run tests - num_tests = len(config_dict.keys()) + num_tests = len(list(config_dict.keys())) - for i, (test, test_dict) in enumerate(config_dict.iteritems()): + for i, (test, test_dict) in enumerate(iteritems(config_dict)): print("".center(100, '-')) - print("Starting Test #{0} of {1}: {2}".format(i+1, num_tests, + print("Starting Test #{0} of {1}: {2}".format(i + 1, num_tests, test).center(100)) desc = textwrap.fill(", ".join(test_dict['description']), 100) print("Description: {0}".format(desc)) @@ -92,13 +92,16 @@ def run_examples(config_file): '{0}'.format(test_dict['function'])) pr.disable() - s = StringIO.StringIO() + if PY3: + s = io.StringIO() + else: + s = io.BytesIO() sortby = 'cumulative' ps = pstats.Stats(pr, stream=s).sort_stats(sortby) ps.print_stats() print("".center(100, '-')) - print("Done With Test #{0} of {1}: {2}".format(i+1, num_tests, + print("Done With Test #{0} of {1}: {2}".format(i + 1, num_tests, test).center(100)) print(".....Printing Profile Information.....".center(100)) print("".center(100, '-')) diff --git a/tools/find_pour_points.py b/tools/find_pour_points.py index d93fac8..1aec097 100755 --- a/tools/find_pour_points.py +++ b/tools/find_pour_points.py @@ -17,10 +17,10 @@ def main(): input_file, output_file, which_points, verbose = process_command_line() # ---------------------------------------------------------------- # - #Load input Rasters (Basin Mask and Accumulated Upstream Area) - #Input rasters need to be the same size + # Load input Rasters (Basin Mask and Accumulated Upstream Area) + # Input rasters need to be the same size if verbose: - print 'Reading input file: %s' % input_file + print('Reading input file: %s' % input_file) f = Dataset(input_file, 'r') basin_id = f.variables['Basin_ID'][:] @@ -41,7 +41,7 @@ def main(): max_y = find_all(basin_id, source_area, land_mask, lons, lats, res, verbose) else: - print 'Returning all basin outlet grid cells as pour points' + print('Returning all basin outlet grid cells as pour points') basin, x_outlet, y_outlet, max_area, min_x, min_y, max_x, \ max_y = find_outlets(basin_id, source_area, lons, lats, res, verbose) @@ -50,7 +50,7 @@ def main(): # ---------------------------------------------------------------- # # Write output file if verbose: - print 'writing to outfile:', output_file + print('writing to outfile:', output_file) if os.path.splitext(output_file)[1] != '.nc': write_ascii_file(basin, x_outlet, y_outlet, max_area, min_x, min_y, @@ -84,8 +84,8 @@ def find_all(basin_id, source_area, land_mask, lons, lats, res, verbose): # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # - #Loop over every basin id, finding the maximum upstream area [and location] - #and record the basin#,longitude,latitude,area + # Loop over every basin id, finding the maximum upstream area, and location + # and record the basin#,longitude,latitude,area for i, (yi, xi, bi) in enumerate(zip(y, x, basin)): inds = np.nonzero(basin_id == bi) @@ -94,9 +94,9 @@ def find_all(basin_id, source_area, land_mask, lons, lats, res, verbose): x_outlet[i] = lon[y, x] y_outlet[i] = lat[y, x] min_x[i] = min(x_basin) - max_x[i] = max(x_basin)+res + max_x[i] = max(x_basin) + res min_y[i] = min(y_basin) - max_y[i] = max(y_basin)+res + max_y[i] = max(y_basin) + res # ---------------------------------------------------------------- # return basin, x_outlet, y_outlet, max_area, min_x, min_y, max_x, max_y @@ -107,12 +107,12 @@ def find_all(basin_id, source_area, land_mask, lons, lats, res, verbose): def find_outlets(basin_id, source_area, lons, lats, res, verbose): """ Find the outlet location of each basin """ # ---------------------------------------------------------------- # - #Make arrays of same dimensions as input arrays of lat/lon values + # Make arrays of same dimensions as input arrays of lat/lon values x, y = np.meshgrid(lons, lats) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # - #Setup basin in/out arrays + # Setup basin in/out arrays basin_ids = np.arange(np.min(basin_id), np.max(basin_id)) num_basins = len(basin_ids) @@ -127,8 +127,8 @@ def find_outlets(basin_id, source_area, lons, lats, res, verbose): # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # - #Loop over every basin id, finding the maximum upstream area [and location] - #and record the basin#,longitude,latitude,area + # Loop over every basin id, finding the maximum upstream area, and location + # and record the basin#,longitude,latitude,area if verbose: sys.stdout.write('Done reading input file...\n ') sys.stdout.write('Searching in %i basins for pour ' @@ -148,9 +148,9 @@ def find_outlets(basin_id, source_area, lons, lats, res, verbose): x_outlet[i] = x_basin[max_ind] y_outlet[i] = y_basin[max_ind] min_x[i] = min(x_basin) - max_x[i] = max(x_basin)+res + max_x[i] = max(x_basin) + res min_y[i] = min(y_basin) - max_y[i] = max(y_basin)+res + max_y[i] = max(y_basin) + res # ---------------------------------------------------------------- # return basin, x_outlet, y_outlet, max_area, min_x, min_y, max_x, max_y # -------------------------------------------------------------------- # @@ -173,7 +173,7 @@ def write_netcdf_file(basin, x_outlet, y_outlet, max_area, min_x, min_y, # ---------------------------------------------------------------- # # initialize variables - OIDs = f.createVariable('OID', 'i8', ('points', )) + oids = f.createVariable('OID', 'i8', ('points', )) x_outlets = f.createVariable('x_outlet', 'f8', ('points', )) y_outlets = f.createVariable('y_outlet', 'f8', ('points', )) max_areas = f.createVariable('max_area', 'i8', ('points', )) @@ -193,8 +193,8 @@ def write_netcdf_file(basin, x_outlet, y_outlet, max_area, min_x, min_y, # ---------------------------------------------------------------- # # write variable attributes - OIDs.long_name = 'Basin Identifier' - OIDs.standard_name = 'OID' + oids.long_name = 'Basin Identifier' + oids.standard_name = 'OID' x_outlets.long_name = 'longitude_coordinate_of_pour_point' x_outlets.standard_name = 'longitude' @@ -228,7 +228,7 @@ def write_netcdf_file(basin, x_outlet, y_outlet, max_area, min_x, min_y, # ---------------------------------------------------------------- # # fill vairables - OIDs[:] = basin + oids[:] = basin x_outlets[:] = x_outlet y_outlets[:] = y_outlet max_areas[:] = max_areas @@ -255,9 +255,10 @@ def write_ascii_file(basin, x_outlet, y_outlet, max_area, min_x, min_y, # ---------------------------------------------------------------- # # set format fmt = ['%i', '%.10f', '%.10f', '%i', '%.10f', '%.10f', '%.10f', '%.10f'] - out = np.column_stack((basin, x_outlet, y_outlet, max_area, min_x, min_y, + out = np.column_stack((basin, x_outlet, y_outlet, max_area, min_x, min_y, max_x, max_y)) - header = 'OID, longitude, latitude, basin_area, min_lon, min_lat, max_lon, max_lat\n' + header = 'OID, longitude, latitude, basin_area, min_lon, min_lat, ' \ + 'max_lon, max_lat\n' with file(out_file, 'w') as outfile: outfile.write(header) np.savetxt(outfile, out, fmt=fmt, delimiter=',') From 6c48ab4725fae9ab044a9aa9170e4d7da775d65f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 24 Mar 2015 19:57:09 -0700 Subject: [PATCH 02/47] update travis build for python3 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 660863e..0dc2a76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.7" + - "3.4" # Setup anaconda before_install: - wget http://repo.continuum.io/miniconda/Miniconda-2.2.2-Linux-x86_64.sh -O miniconda.sh From 42111da2c3b042a17b41a8fd939916142ccf6d52 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 24 Mar 2015 22:01:34 -0700 Subject: [PATCH 03/47] updates for python 3, now build ctypes suffix in platform independent way --- .travis.yml | 2 +- rvic/core/convolution_wrapper.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0dc2a76..58319f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ script: - conda list - pip freeze - echo $PATH - - python -c 'import rvic; print rvic.__file__' + - python -c 'import rvic; print(rvic.__file__)' - cd tests - pwd - python run_tests.py unit diff --git a/rvic/core/convolution_wrapper.py b/rvic/core/convolution_wrapper.py index 6711d5c..c9b4730 100644 --- a/rvic/core/convolution_wrapper.py +++ b/rvic/core/convolution_wrapper.py @@ -7,21 +7,22 @@ """ import os +import sysconfig import numpy as np import ctypes -SHAREDOBJECT = 'rvic_convolution.so' +SHAREDOBJECT = 'rvic_convolution' + sysconfig.get_config_var('SO') LIBPATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')) try: _convolution = np.ctypeslib.load_library(SHAREDOBJECT, LIBPATH) except ImportError as ie: - print(('looking for shared object {0} in {1}'.format(SHAREDOBJECT, - LIBPATH))) + print('error looking for shared object {0} in {1}'.format(SHAREDOBJECT, + LIBPATH)) raise ImportError(ie) except OSError as oe: - print(('looking for shared object {0} in {1}'.format(SHAREDOBJECT, - LIBPATH))) + print('error looking for shared object {0} in {1}'.format(SHAREDOBJECT, + LIBPATH)) raise ImportError(oe) _args = [ctypes.c_int, From 9ca96bad51e263b04be5343c2cea56cf7b9ac33f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 24 Mar 2015 22:38:23 -0700 Subject: [PATCH 04/47] simplify ctypes call by removing useless python function wrapper --- rvic/core/convolution_wrapper.py | 43 ++++++++++---------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/rvic/core/convolution_wrapper.py b/rvic/core/convolution_wrapper.py index c9b4730..d80bf6a 100644 --- a/rvic/core/convolution_wrapper.py +++ b/rvic/core/convolution_wrapper.py @@ -25,36 +25,19 @@ LIBPATH)) raise ImportError(oe) -_args = [ctypes.c_int, - ctypes.c_int, - ctypes.c_int, - ctypes.c_int, - np.ctypeslib.ndpointer(np.int32), - np.ctypeslib.ndpointer(np.int32), - np.ctypeslib.ndpointer(np.int32), - np.ctypeslib.ndpointer(np.int32), - np.ctypeslib.ndpointer(np.float64), - np.ctypeslib.ndpointer(np.float64), - np.ctypeslib.ndpointer(np.float64)] +_args = [ctypes.c_int, # nsources + ctypes.c_int, # noutlets + ctypes.c_int, # subset_length + ctypes.c_int, # xsize + np.ctypeslib.ndpointer(np.int32), # source2outlet_ind + np.ctypeslib.ndpointer(np.int32), # source_y_ind + np.ctypeslib.ndpointer(np.int32), # source_x_ind + np.ctypeslib.ndpointer(np.int32), # source_time_offset + np.ctypeslib.ndpointer(np.float64), # unit_hydrograph + np.ctypeslib.ndpointer(np.float64), # aggrunin + np.ctypeslib.ndpointer(np.float64) # ring + ] _convolution.convolve.argtypes = _args _convolution.convolve.restype = None - -def rvic_convolve(*args): - """args: - - nsources, - noutlets, - subset_length, - xsize, - source2outlet_ind, - source_y_ind, - source_x_ind, - source_time_offset, - unit_hydrograph, - aggrunin, - ring - """ - _convolution.convolve(*args) - - return +rvic_convolve = _convolution.convolve From fb43c3fca32c49d05c3bea19e0352fd88d0da830 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 25 Mar 2015 21:48:44 -0700 Subject: [PATCH 05/47] update docs for python3 --- docs/user-guide/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/install.md b/docs/user-guide/install.md index 04625b7..7866111 100644 --- a/docs/user-guide/install.md +++ b/docs/user-guide/install.md @@ -1,7 +1,7 @@ # Installing RVIC ## Dependencies -- [Python 2.7](http://www.python.org/) +- [Python 2.7 or later including Python3](http://www.python.org/) - [Numpy](http://www.numpy.org) - [Scipy](http://www.scipy.org/) - [netcdf4-python](https://code.google.com/p/netcdf4-python/) From afa49af069cf696d2b065cb1e9ecd5891d41f851 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 31 Mar 2015 08:20:03 -0700 Subject: [PATCH 06/47] landscape cleanup --- ci/requirements-2.7.txt | 2 +- ci/requirements-3.4.txt | 6 ++++ docs/user-guide/install.md | 1 + rvic/convert.py | 15 ++++----- rvic/convolution.py | 17 ++++------ rvic/core/aggregate.py | 24 +++++++------- rvic/core/config.py | 4 +-- rvic/core/convert.py | 4 +-- rvic/core/history.py | 4 +-- rvic/core/log.py | 2 +- rvic/core/make_uh.py | 27 ++++++++------- rvic/core/multi_proc.py | 13 ++++---- rvic/core/param_file.py | 10 +++--- rvic/core/plots.py | 39 +++++++++++----------- rvic/core/pycompat.py | 8 ++--- rvic/core/read_forcing.py | 22 ++++++------- rvic/core/share.py | 39 ++++++++++++---------- rvic/core/utilities.py | 27 +++++++-------- rvic/core/variables.py | 2 +- rvic/core/write.py | 9 +++-- rvic/parameters.py | 67 ++++++++++++++++++-------------------- scripts/rvic | 8 +++-- setup.py | 5 +-- tools/find_pour_points.py | 15 ++++----- 24 files changed, 186 insertions(+), 184 deletions(-) create mode 100644 ci/requirements-3.4.txt diff --git a/ci/requirements-2.7.txt b/ci/requirements-2.7.txt index cfe6cda..e6c604d 100644 --- a/ci/requirements-2.7.txt +++ b/ci/requirements-2.7.txt @@ -2,5 +2,5 @@ matplotlib==1.3.1 netCDF4==1.0.8 numpy==1.8.1 scipy==0.13.3 -pandas==0.13.1 +pandas==0.15.1 pytest==2.5.2 diff --git a/ci/requirements-3.4.txt b/ci/requirements-3.4.txt new file mode 100644 index 0000000..e6c604d --- /dev/null +++ b/ci/requirements-3.4.txt @@ -0,0 +1,6 @@ +matplotlib==1.3.1 +netCDF4==1.0.8 +numpy==1.8.1 +scipy==0.13.3 +pandas==0.15.1 +pytest==2.5.2 diff --git a/docs/user-guide/install.md b/docs/user-guide/install.md index 04625b7..7a0c5bb 100644 --- a/docs/user-guide/install.md +++ b/docs/user-guide/install.md @@ -5,6 +5,7 @@ - [Numpy](http://www.numpy.org) - [Scipy](http://www.scipy.org/) - [netcdf4-python](https://code.google.com/p/netcdf4-python/) +- [Pandas](http://pandas.pydata.org/) If using `REMAP=True`: diff --git a/rvic/convert.py b/rvic/convert.py index b31ad70..7814d98 100644 --- a/rvic/convert.py +++ b/rvic/convert.py @@ -13,7 +13,7 @@ # -------------------------------------------------------------------- # # Top level driver -def convert(config_file, numofproc): +def convert(config_file): # ---------------------------------------------------------------- # # Initilize @@ -29,7 +29,7 @@ def convert(config_file, numofproc): # ---------------------------------------------------------------- # # Run log.info('getting outlets now') - outlets = uhs2param_run(dom_data, outlets, config_dict, directories) + outlets = uhs2param_run(dom_data, outlets, config_dict) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -71,17 +71,16 @@ def uhs2param_init(config_file): options['VERBOSE']) for direc in directories: - log.info('%s directory is %s' % (direc, directories[direc])) + log.info('%s directory is %s', direc, directories[direc]) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # # Read domain file (if applicable) - dom_data, DomVats, DomGats = read_domain(config_dict['DOMAIN']) - log.info('Opened Domain File: %s' % config_dict['DOMAIN']['FILE_NAME']) + dom_data = read_domain(config_dict['DOMAIN'])[0] + log.info('Opened Domain File: %s', config_dict['DOMAIN']['FILE_NAME']) if 'NEW_DOMAIN' in config_dict: - new_dom_data, new_DomVats, \ - new_DomGats = read_domain(config_dict['NEW_DOMAIN']) + new_dom_data = read_domain(config_dict['NEW_DOMAIN'])[0] log.info('Opened New Domain File: %s', config_dict['NEW_DOMAIN']['FILE_NAME']) else: @@ -100,7 +99,7 @@ def uhs2param_init(config_file): # -------------------------------------------------------------------- # # run -def uhs2param_run(dom_data, outlets, config_dict, directories): +def uhs2param_run(dom_data, outlets, config_dict): # ---------------------------------------------------------------- # # Read uhs files diff --git a/rvic/convolution.py b/rvic/convolution.py index 854b1bd..51f1425 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -30,7 +30,7 @@ # -------------------------------------------------------------------- # # Top Level Driver -def convolution(config_file, numofproc=1): +def convolution(config_file): """ Top level driver for RVIC convolution model. """ @@ -50,8 +50,7 @@ def convolution(config_file, numofproc=1): # ---------------------------------------------------------------- # # Run time_handle, hist_tapes = convolution_run(hist_tapes, data_model, rout_var, - dom_data, time_handle, - directories, config_dict) + time_handle, directories) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -109,8 +108,7 @@ def convolution_init(config_file): # ---------------------------------------------------------------- # # Read Domain File domain = config_dict['DOMAIN'] - dom_data, dom_vatts, dom_gatts = read_domain( - domain, lat0_is_min=data_model.lat0_is_min) + dom_data = read_domain(domain, lat0_is_min=data_model.lat0_is_min)[0] # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -222,8 +220,8 @@ def convolution_init(config_file): # -------------------------------------------------------------------- # -def convolution_run(hist_tapes, data_model, rout_var, dom_data, time_handle, - directories, config_dict): +def convolution_run(hist_tapes, data_model, rout_var, time_handle, + directories): """ Main run loop for RVIC model. """ @@ -337,10 +335,9 @@ def convolution_final(time_handle, hist_tapes): # Write final log info log.info("-----------------------------------------------------------") log.info('Done with streamflow convolution') - log.info('Processed %i timesteps' % time_handle.timesteps) + log.info('Processed %i timesteps', time_handle.timesteps) for name, tape in iteritems(hist_tapes): - log.info('Wrote %i history files from %s' % (tape.files_count, name)) - # log.info('Routed to %i points' % time_handle.points) + log.info('Wrote %i history files from %s', tape.files_count, name) log.info("-----------------------------------------------------------") # ---------------------------------------------------------------- # diff --git a/rvic/core/aggregate.py b/rvic/core/aggregate.py index 4537a60..76b91cc 100644 --- a/rvic/core/aggregate.py +++ b/rvic/core/aggregate.py @@ -9,7 +9,7 @@ from .variables import Point from logging import getLogger from .log import LOG_NAME -from .pycompat import OrderedDict, iteritems, zip +from .pycompat import OrderedDict, iteritems, pyzip # -------------------------------------------------------------------- # # create logger @@ -50,7 +50,7 @@ def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): # Do the aggregation outlets = {} - for i, (lat, lon) in enumerate(zip(lats, lons)): + for i, (lat, lon) in enumerate(pyzip(lats, lons)): # Define pour point object (on ) pour_point = Point(lat=lat, lon=lon, @@ -99,11 +99,11 @@ def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): # ---------------------------------------------------------------- # # Print Debug Results log.info('\n------------------ SUMMARY OF MakeAggPairs ------------------') - log.info('NUMBER OF POUR POINTS IN INPUT LIST: %i' % num) - log.info('NUMBER OF POINTS TO AGGREGATE TO: %i' % key_count) - log.info('NUMBER OF POUR POINTS AGGREGATED: %i' % pp_count) - log.info('EFFECIENCY OF: %.2f %%' % (100. * pp_count / num)) - log.info('UNASSIGNED POUR POINTS: %i \n' % (num - pp_count)) + log.info('NUMBER OF POUR POINTS IN INPUT LIST: %i', num) + log.info('NUMBER OF POINTS TO AGGREGATE TO: %i', key_count) + log.info('NUMBER OF POUR POINTS AGGREGATED: %i', pp_count) + log.info('EFFECIENCY OF: %.2f %%', (100. * pp_count / num)) + log.info('UNASSIGNED POUR POINTS: %i \n', (num - pp_count)) log.info('-------------------------------------------------------------\n') # ---------------------------------------------------------------- # @@ -147,8 +147,8 @@ def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False): fraction = np.zeros((len(lats), len(lons)), dtype=np.float64) unit_hydrograph = np.zeros((tshape, len(lats), len(lons)), dtype=np.float64) - log.debug('fraction shape %s' % str(fraction.shape)) - log.debug('unit_hydrograph shape %s' % str(unit_hydrograph.shape)) + log.debug('fraction shape %s', fraction.shape) + log.debug('unit_hydrograph shape %s', unit_hydrograph.shape) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -159,8 +159,7 @@ def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False): ilon_min_ind = find_nearest(lons, np.min(in_data['lon'])) ilon_max_ind = find_nearest(lons, np.max(in_data['lon'])) + 1 - log.debug('in_data fraction shape: %s', - str(in_data['fraction'].shape)) + log.debug('in_data fraction shape: %s', in_data['fraction'].shape) if agg_data: alat_min_ind = find_nearest(lats, np.max(agg_data['lat'])) @@ -168,8 +167,7 @@ def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False): alon_min_ind = find_nearest(lons, np.min(agg_data['lon'])) alon_max_ind = find_nearest(lons, np.max(agg_data['lon'])) + 1 - log.debug('agg_data fraction shape: %s', - str(agg_data['fraction'].shape)) + log.debug('agg_data fraction shape: %s', agg_data['fraction'].shape) # ---------------------------------------------------------------- # diff --git a/rvic/core/config.py b/rvic/core/config.py index b263071..25e4eb2 100644 --- a/rvic/core/config.py +++ b/rvic/core/config.py @@ -70,11 +70,11 @@ def config_type(value): else: try: return list(map(float, val_list)) - except: + except (TypeError, ValueError): pass try: return list(map(int, val_list)) - except: + except (TypeError, ValueError): return val_list # -------------------------------------------------------------------- # diff --git a/rvic/core/convert.py b/rvic/core/convert.py index dc0496d..72618b5 100644 --- a/rvic/core/convert.py +++ b/rvic/core/convert.py @@ -8,7 +8,7 @@ from .log import LOG_NAME from .variables import Point from .share import FILLVALUE_I -from .pycompat import iteritems +from .pycompat import iteritems, pyrange # -------------------------------------------------------------------- # # create logger @@ -101,7 +101,7 @@ def read_uhs_files(outlets, dom_data, config_dict): uh = [] # loop over the source points - for j in range(num_sources): + for j in pyrange(num_sources): line = re.sub(' +', ' ', f.readline()) lon, lat, fracs, x, y = line.split() # move to zero based index diff --git a/rvic/core/history.py b/rvic/core/history.py index 22537e5..af88266 100644 --- a/rvic/core/history.py +++ b/rvic/core/history.py @@ -251,7 +251,7 @@ def update(self, data2tape, time_ord): # Update the fields for field in self._fincl: tracer = 'LIQ' - log.debug('updating {0}'.format(field)) + log.debug('updating %s', field) fdata = data2tape[field][tracer] if self._avgflag == 'A': self._temp_data[field] += fdata @@ -594,7 +594,7 @@ def __write_grid(self): setattr(f, key, val) # ------------------------------------------------------------ # f.close() - log.info('Finished writing %s' % self.filename) + log.info('Finished writing %s', self.filename) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # diff --git a/rvic/core/log.py b/rvic/core/log.py index d7a79b9..1b3de4d 100644 --- a/rvic/core/log.py +++ b/rvic/core/log.py @@ -71,7 +71,7 @@ def init_logger(log_dir='./', log_level='DEBUG', verbose=False): # ---------------------------------------------------------------- # logger.info('-------------------- INITIALIZED RVIC LOG ------------------') - logger.info('LOG LEVEL: %s' % log_level) + logger.info('LOG LEVEL: %s', log_level) logger.info('Logging To Console: %s', verbose) logger.info('LOG FILE: %s', log_file) logger.info('----------------------------------------------------------\n') diff --git a/rvic/core/make_uh.py b/rvic/core/make_uh.py index 8771cc6..21ab659 100644 --- a/rvic/core/make_uh.py +++ b/rvic/core/make_uh.py @@ -21,7 +21,7 @@ from .utilities import latlon2yx from .share import SECSPERDAY from .log import LOG_NAME -from .pycompat import zip, range +from .pycompat import pyzip, pyrange # -------------------------------------------------------------------- # # create logger @@ -52,8 +52,8 @@ def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): y_inds, x_inds = \ np.nonzero(fdr_data[rout_dict['BASIN_ID_VAR']] == basin_id) - y = np.arange(len(fdr_data[rout_dict['LATITUDE_VAR']])) - x = np.arange(len(fdr_data[rout_dict['LONGITUDE_VAR']])) + y = np.apyrange(len(fdr_data[rout_dict['LATITUDE_VAR']])) + x = np.apyrange(len(fdr_data[rout_dict['LONGITUDE_VAR']])) x_min = min(x[x_inds]) x_max = max(x[x_inds]) + 1 @@ -244,7 +244,7 @@ def search_catchment(to_y, to_x, pour_point, basin_ids, basin_id): cells = 0 - for yy, xx in zip(byinds, bxinds): + for yy, xx in pyzip(byinds, bxinds): if in_catch[yy, xx] >= 0: # set the old path to zero pathy[:cells + 1] = 0 @@ -293,7 +293,6 @@ def search_catchment(to_y, to_x, pour_point, basin_ids, basin_id): catch_fracs[cyinds, cxinds] = 1.0 count = len(cyinds) - tempy, tempx = np.nonzero(count_ds) log.debug("Found %i upstream grid cells from present station", count) log.debug("Expected at most %i upstream grid cells from present station", @@ -324,9 +323,9 @@ def make_uh(dt, t_cell, y_inds, x_inds, velocity, diffusion, xmask): log.debug('Making uh for each cell') uh = np.zeros((t_cell, xmask.shape[0], xmask.shape[1]), dtype=np.float64) - time = np.arange(dt, t_cell * dt + dt, dt, dtype=np.float64) + time = np.apyrange(dt, t_cell * dt + dt, dt, dtype=np.float64) - for y, x in zip(y_inds, x_inds): + for y, x in pyzip(y_inds, x_inds): xm = xmask[y, x] v = velocity[y, x] d = diffusion[y, x] @@ -354,7 +353,7 @@ def make_grid_uh_river(t_uh, t_cell, uh, to_y, to_x, pour_point, y_inds, uh_river = np.zeros((t_uh, uh.shape[1], uh.shape[2]), dtype=np.float64) - for (y, x, d) in zip(y_inds, x_inds, count_ds): + for (y, x, d) in pyzip(y_inds, x_inds, count_ds): if d > 0: yy = to_y[y, x] xx = to_x[y, x] @@ -386,7 +385,7 @@ def make_grid_uh(t_uh, t_cell, uh_river, uh_box, to_y, to_x, y_inds, x_inds, unit_hydrograph = np.zeros((t_uh, uh_river.shape[1], uh_river.shape[2])) irf_temp = np.zeros(t_uh + t_cell, dtype=np.float64) - for (y, x, d) in zip(y_inds, x_inds, count_ds): + for (y, x, d) in pyzip(y_inds, x_inds, count_ds): irf_temp[:] = 0.0 if d > 0: yy = to_y[y, x] @@ -415,17 +414,17 @@ def adjust_uh_timestep(unit_hydrograph, t_uh, input_interval, output_interval, log.debug('No need to aggregate in time (output_interval = ' 'input_interval) Skipping the adjust_uh_timestep step') uh_out = unit_hydrograph - ts_new = np.arange(t_uh) + ts_new = np.apyrange(t_uh) elif np.remainder(output_interval, input_interval) == 0: log.debug('Aggregating to %i from %i seconds', output_interval, input_interval) fac = int(output_interval / input_interval) t_uh_out = int(t_uh / fac) - ts_new = np.arange(t_uh_out) + ts_new = np.apyrange(t_uh_out) uh_out = np.zeros((t_uh_out, unit_hydrograph.shape[1], unit_hydrograph.shape[2]), dtype=np.float64) - for (y, x) in zip(y_inds, x_inds): - for t in range(t_uh_out): + for (y, x) in pyzip(y_inds, x_inds): + for t in pyrange(t_uh_out): uh_out[t, y, x] = unit_hydrograph[t * fac:t * fac + fac, y, x].sum() elif np.remainder(input_interval, output_interval): @@ -438,7 +437,7 @@ def adjust_uh_timestep(unit_hydrograph, t_uh, input_interval, output_interval, unit_hydrograph.shape[2]), dtype=np.float64) ts_orig = np.linspace(0, t_uh, t_uh) ts_new = np.linspace(0, t_uh, t_uh_out) - for (y, x) in zip(y_inds, x_inds): + for (y, x) in pyzip(y_inds, x_inds): f = interp1d(ts_orig, unit_hydrograph[:, y, x]) uh_out[:, y, x] = f(ts_new) return uh_out, ts_new diff --git a/rvic/core/multi_proc.py b/rvic/core/multi_proc.py index cb6e134..b44d945 100644 --- a/rvic/core/multi_proc.py +++ b/rvic/core/multi_proc.py @@ -8,15 +8,15 @@ import traceback -def error(msg, *args): +def error(*args): """ Error function""" - return multiprocessing.get_logger(LOG_NAME).error(msg, *args) + return multiprocessing.get_logger(LOG_NAME).error(*args) # -------------------------------------------------------------------- # class LogExceptions(object): - def __init__(self, callable): - self.__callable = callable + def __init__(self, func): + self.__callable = func return def __call__(self, *args, **kwargs): @@ -33,13 +33,12 @@ def __call__(self, *args, **kwargs): # It was fine, give a normal answer return result - pass # -------------------------------------------------------------------- # class LoggingPool(Pool): """Subclass of pool""" - def apply_async(self, func, args=(), kwds={}, callback=None): - return Pool.apply_async(self, LogExceptions(func), args, kwds, + def apply_async(self, func, callback=None, *args, **kwargs): + return Pool.apply_async(self, LogExceptions(func), args, kwargs, callback) # -------------------------------------------------------------------- # diff --git a/rvic/core/param_file.py b/rvic/core/param_file.py index a216f13..059c6fe 100644 --- a/rvic/core/param_file.py +++ b/rvic/core/param_file.py @@ -6,7 +6,7 @@ from .log import LOG_NAME from .write import write_param_file from .share import NcGlobals, SECSPERDAY, MAX_NC_CHARS -from .pycompat import iteritems, range, zip +from .pycompat import iteritems, pyrange, pyzip from . import plots import os from datetime import date @@ -147,7 +147,7 @@ def finish_params(outlets, dom_data, config_dict, directories): # ---------------------------------------------------------------- # # Make diagnostic plots sum_after = np.zeros(dom_data[domain['FRACTION_VAR']].shape) - for i, (y, x) in enumerate(zip(source_y_ind, source_x_ind)): + for i, (y, x) in enumerate(pyzip(source_y_ind, source_x_ind)): sum_after[y, x] += unit_hydrograph[:, i].sum() plot_dict['Sum UH Final'] = sum_after @@ -301,11 +301,13 @@ def subset(outlets, subset_length=None): log.info('subsetting unit-hydrographs now...') log.debug('Subset Length: %s', subset_length) - + log.debug(outlets) for i, (cell_id, outlet) in enumerate(iteritems(outlets)): + log.debug(i, cell_id, outlet, outlets) if i == 0: full_time_length = outlet.unit_hydrograph.shape[0] log.debug('Subset Length: %s', subset_length) + log.debug('full_time_length: %s', full_time_length) if not subset_length: subset_length = full_time_length log.debug('No subset_length provided, using full_time_length') @@ -322,7 +324,7 @@ def subset(outlets, subset_length=None): d_left = -1 * subset_length / 2 d_right = subset_length / 2 - for j in range(outlet.unit_hydrograph.shape[1]): + for j in pyrange(outlet.unit_hydrograph.shape[1]): # find index position of maximum maxind = np.argmax(outlet.unit_hydrograph[:, j]) diff --git a/rvic/core/plots.py b/rvic/core/plots.py index 6ecd408..b8cf707 100644 --- a/rvic/core/plots.py +++ b/rvic/core/plots.py @@ -14,9 +14,9 @@ try: from mpl_toolkits.basemap import Basemap basemap_available = False - except: + except ImportError: basemap_available = False -except: +except ImportError: matplotlib_available = False @@ -31,11 +31,7 @@ def uhs(data, title, case_id, plot_dir): """ Plot diagnostic plot showing all unit hydrographs """ - today = date.today().strftime('%Y%m%d') - file_name = "{0}_{1}_{2}.png".format(title.lower().replace(" ", "_"), - case_id.lower().replace(" ", "_"), - today) - pfname = os.path.join(plot_dir, file_name) + pfname = _make_filename(title, case_id, plot_dir) fig = plt.figure() plt.plot(data) @@ -55,11 +51,7 @@ def _fractions_grid(data, dom_x, dom_y, title, case_id, plot_dir): """ # ---------------------------------------------------------------- # # Plot Fractions - today = date.today().strftime('%Y%m%d') - file_name = "{0}_{1}_{2}.png".format(title.lower().replace(" ", "_"), - case_id.lower().replace(" ", "_"), - today) - pfname = os.path.join(plot_dir, file_name) + pfname = _make_filename(title, case_id, plot_dir) mask = data <= 0.0 data = np.ma.array(data, mask=mask) @@ -75,7 +67,8 @@ def _fractions_grid(data, dom_x, dom_y, title, case_id, plot_dir): plt.title(title) plt.xlabel('x') plt.ylabel('y') - # plt.gca().invert_yaxis() + plt.ylim([0, dom_y.shape[0]]) + plt.xlim([0, dom_x.shape[1]]) fig.savefig(pfname) # ---------------------------------------------------------------- # return pfname @@ -89,11 +82,7 @@ def _fractions_map(data, dom_x, dom_y, title, case_id, plot_dir): """ # ---------------------------------------------------------------- # # Plot Fractions - today = date.today().strftime('%Y%m%d') - file_name = "{0}_{1}_{2}.png".format(title.lower().replace(" ", "_"), - case_id.lower().replace(" ", "_"), - today) - pfname = os.path.join(plot_dir, file_name) + pfname = _make_filename(title, case_id, plot_dir) fig = plt.figure(figsize=(8, 8)) fig.add_axes([0.1, 0.1, 0.8, 0.8]) @@ -145,12 +134,22 @@ def _fractions_map(data, dom_x, dom_y, title, case_id, plot_dir): # -------------------------------------------------------------------- # -def _fractions_dummy(data, dom_x, dom_y, title, case_id, plot_dir): +def _make_filename(title, case_id, plot_dir): + today = date.today().strftime('%Y%m%d') + file_name = "{0}_{1}_{2}.png".format(title.lower().replace(" ", "_"), + case_id.lower().replace(" ", "_"), + today) + pfname = os.path.join(plot_dir, file_name) + return pfname +# -------------------------------------------------------------------- # + + +# -------------------------------------------------------------------- # +def _fractions_dummy(*args): """ Pass on plotting """ pfname = 'None <-- could not import matplotlib' - pass # ---------------------------------------------------------------- # return pfname # -------------------------------------------------------------------- # diff --git a/rvic/core/pycompat.py b/rvic/core/pycompat.py index a23dadb..17bc754 100644 --- a/rvic/core/pycompat.py +++ b/rvic/core/pycompat.py @@ -12,8 +12,8 @@ def iteritems(d): def itervalues(d): return iter(d.values()) - range = range - zip = zip + pyrange = range + pyzip = zip from configparser import SafeConfigParser else: # Python 2 @@ -26,8 +26,8 @@ def iteritems(d): def itervalues(d): return d.itervalues() - range = xrange - from itertools import izip as zip + pyrange = xrange + from itertools import izip as pyzip from ConfigParser import SafeConfigParser try: from cyordereddict import OrderedDict diff --git a/rvic/core/read_forcing.py b/rvic/core/read_forcing.py index 70bbb9f..b34589c 100644 --- a/rvic/core/read_forcing.py +++ b/rvic/core/read_forcing.py @@ -11,7 +11,7 @@ from .log import LOG_NAME from .time_utility import ord_to_datetime from .share import MMPERMETER, CMPERMETER, WATERDENSITY, TIMEUNITS, SECSPERDAY -from .pycompat import range +from .pycompat import pyrange # -------------------------------------------------------------------- # # create logger @@ -58,7 +58,7 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, # yearly files elif len(start) == 1: - for year in range(start[0], end[0] + 1): + for year in pyrange(start[0], end[0] + 1): self.files.append(os.path.join(self.path, file_str.replace('$YYYY', "{0:04d}".format(year)))) @@ -219,7 +219,7 @@ def start(self, timestamp, rout_var): self.fld_mult[fld] = (WATERDENSITY / CMPERMETER / self.secs_per_step) else: - raise ValueError('unknown forcing units') + raise ValueError('unknown forcing units: %s' % units) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -301,23 +301,23 @@ def read(self, timestamp): self.current_fhdl.variables[fld].set_auto_maskandscale(False) if i == 0: forcing['LIQ'] = \ - (self.current_fhdl.variables[fld][self.current_tind, :, :] - * self.fld_mult[fld]) + (self.fld_mult[fld] * + self.current_fhdl.variables[fld][self.current_tind, :, :]) else: forcing['LIQ'] += \ - (self.current_fhdl.variables[fld][self.current_tind, :, :] - * self.fld_mult[fld]) + (self.fld_mult[fld] * + self.current_fhdl.variables[fld][self.current_tind, :, :]) for i, fld in enumerate(self.ice_flds): self.current_fhdl.variables[fld].set_auto_maskandscale(False) if i == 0: forcing['ICE'] = \ - (self.current_fhdl.variables[fld][self.current_tind, :, :] - * self.fld_mult[fld]) + (self.fld_mult[fld] * + self.current_fhdl.variables[fld][self.current_tind, :, :]) else: forcing['ICE'] += \ - (self.current_fhdl.variables[fld][self.current_tind, :, :] - * self.fld_mult[fld]) + (self.fld_mult[fld] * + self.current_fhdl.variables[fld][self.current_tind, :, :]) # move forward one step self.current_tind += 1 diff --git a/rvic/core/share.py b/rvic/core/share.py index ad49e36..7cdb1f3 100644 --- a/rvic/core/share.py +++ b/rvic/core/share.py @@ -69,7 +69,7 @@ # ----------------------- NETCDF VARIABLES --------------------------------- # -class NcGlobals: +class NcGlobals(object): def __init__(self, title=None, casename=None, @@ -91,59 +91,62 @@ def __init__(self, self.atts = OrderedDict() - if title: + if title is not None: self.atts['title'] = title - if comment: + if comment is not None: self.atts['comment'] = comment - if Conventions: + if Conventions is not None: self.atts['Conventions'] = Conventions - if history: + if history is not None: self.atts['history'] = history - if source: + if source is not None: self.atts['source'] = source - if institution: + if institution is not None: self.atts['institution'] = institution - if hostname: + if hostname is not None: self.atts['hostname'] = hostname else: self.atts['hostname'] = socket.gethostname() - if username: + if username is not None: self.atts['username'] = username else: self.atts['username'] = getuser() - if casename: + if casename is not None: self.atts['casename'] = casename - if references: + if casestr is not None: + self.atts['casestr'] = casestr + + if references is not None: self.atts['references'] = references - if version: + if version is not None: self.atts['version'] = version else: try: from rvic import version self.atts['version'] = version.short_version - except: + except ImportError: self.atts['version'] = 'unknown' - if RvicPourPointsFile: + if RvicPourPointsFile is not None: self.atts['RvicPourPointsFile'] = RvicPourPointsFile - if RvicUHFile: + if RvicUHFile is not None: self.atts['RvicUHFile'] = RvicUHFile - if RvicFdrFile: + if RvicFdrFile is not None: self.atts['RvicFdrFile'] = RvicFdrFile - if RvicDomainFile: + if RvicDomainFile is not None: self.atts['RvicDomainFile'] = RvicDomainFile def update(self): @@ -151,7 +154,7 @@ def update(self): 'Created: {0}'.format(time_mod.ctime(time_mod.time())) -class NcVar: +class NcVar(object): def __init__(self, **kwargs): for key, val in iteritems(kwargs): setattr(self, key, val) diff --git a/rvic/core/utilities.py b/rvic/core/utilities.py index ad22773..4edea02 100644 --- a/rvic/core/utilities.py +++ b/rvic/core/utilities.py @@ -13,6 +13,7 @@ from .share import TIMESTAMPFORM, RPOINTER, EARTHRADIUS, METERSPERMILE from .share import METERS2PERACRE, METERSPERKM, VALID_CHARS from .config import read_config +from .pycompat import pyzip # -------------------------------------------------------------------- # # create logger @@ -43,7 +44,7 @@ def latlon2yx(plats, plons, glats, glons): points = list(np.vstack((np.array(plats), np.array(plons))).transpose()) mytree = cKDTree(combined) - dist, indexes = mytree.query(points, k=1) + indexes = mytree.query(points, k=1)[1] y, x = np.unravel_index(indexes, glons.shape) return y, x @@ -60,22 +61,22 @@ def search_for_channel(source_area, routys, routxs, search=2, tol=10): new_ys = np.empty_like(routys) new_xs = np.empty_like(routxs) - for i, (y, x) in enumerate(zip(routys, routxs)): + for i, (y, x) in enumerate(pyzip(routys, routxs)): area0 = source_area[y, x] - search_area = source_area[y-search:y+search+1, x-search:x+search+1] + search_area = source_area[y - search:y + search + 1, + x - search:x + search + 1] - if np.any(search_area > area0*tol): + if np.any(search_area > area0 * tol): sy, sx = np.unravel_index(search_area.argmax(), search_area.shape) new_ys[i] = y + sy - search new_xs[i] = x + sx - search - log.debug('Moving pour point to channel y: ' - '{0}->{1}, x: {2}->{3}'.format(y, new_ys[i], - x, new_xs[i])) - log.debug('Source Area has increased from {0}' - ' to {1}'.format(area0, source_area[new_ys[i], new_xs[i]])) + log.debug('Moving pour point to channel y: %s->%s, x: %s->%s', + y, new_ys[i], x, new_xs[i]) + log.debug('Source Area has increased from %s to %s', + area0, source_area[new_ys[i], new_xs[i]]) else: new_ys[i] = y new_xs[i] = x @@ -176,8 +177,8 @@ def clean_dir(directory): try: if os.path.isfile(file_path): os.unlink(file_path) - except: - log.exception('Error cleaning file: %s' % file_path) + except Exception: + log.exception('Error cleaning file: %s', file_path) return # -------------------------------------------------------------------- # @@ -189,8 +190,8 @@ def clean_file(file_name): try: if os.path.isfile(file_name): os.unlink(file_name) - except: - log.exception('Error cleaning file: %s' % file_name) + except Exception: + log.exception('Error cleaning file: %s', file_name) return # -------------------------------------------------------------------- # diff --git a/rvic/core/variables.py b/rvic/core/variables.py index 5c030e3..aee0cdf 100644 --- a/rvic/core/variables.py +++ b/rvic/core/variables.py @@ -112,7 +112,7 @@ def __init__(self, param_file, case_name, calendar, out_dir, file_format, self.file_format = file_format try: self.outlet_upstream_area = f.variables['outlet_upstream_area'][:] - except: + except KeyError: self.outlet_upstream_area = None self.glob_atts = NcGlobals(title='RVIC restart file', RvicPourPointsFile=f.RvicPourPointsFile, diff --git a/rvic/core/write.py b/rvic/core/write.py index 63b7777..6f76a27 100644 --- a/rvic/core/write.py +++ b/rvic/core/write.py @@ -17,7 +17,7 @@ # -------------------------------------------------------------------- # # Write the agg netcdf -def write_agg_netcdf(file_name, agg_data, glob_atts, format, zlib=True, +def write_agg_netcdf(file_name, agg_data, glob_atts, nc_format, zlib=True, complevel=4, least_significant_digit=None): """ Write output to netCDF. Writes out a netCDF4 data file containing @@ -33,8 +33,8 @@ def write_agg_netcdf(file_name, agg_data, glob_atts, format, zlib=True, # ---------------------------------------------------------------- # # Open file - f = Dataset(file_name, 'w', format=format) - log.info('writing aggregated netcdf: %s' % file_name) + f = Dataset(file_name, 'w', format=nc_format) + log.info('writing aggregated netcdf: %s', file_name) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -129,7 +129,6 @@ def write_param_file(file_name, source_decomp_ind=None, source_time_offset=None, source2outlet_ind=None, - source_tracer=None, unit_hydrograph=None, zlib=True, complevel=4, @@ -364,6 +363,6 @@ def write_param_file(file_name, f.close() - log.info('Finished writing %s' % file_name) + log.info('Finished writing %s', file_name) # ---------------------------------------------------------------- # # -------------------------------------------------------------------- # diff --git a/rvic/parameters.py b/rvic/parameters.py index 994753a..19c16be 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -18,12 +18,12 @@ from .core.variables import Point from .core.param_file import finish_params from .core.config import read_config -from .core.pycompat import OrderedDict, iteritems, range +from .core.pycompat import OrderedDict, iteritems, pyrange try: from .core.remap import remap remap_available = True -except: +except ImportError: remap_available = False @@ -47,8 +47,8 @@ def parameters(config_file, numofproc=1): if numofproc > 1: pool = LoggingPool(processes=numofproc) - for i, (cell_id, outlet) in enumerate(iteritems(outlets)): - log.info('On Outlet #{0} of {1}'.format(i + 1, len(outlets))) + for i, outlet in enumerate(outlets.values()): + log.info('On Outlet #%s of %s', i + 1, len(outlets)) pool.apply_async(gen_uh_run, args=(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories), @@ -59,7 +59,7 @@ def parameters(config_file, numofproc=1): outlets = OrderedDict(sorted(list(iteritems(results)), key=lambda t: t[0])) else: - for name, outlet in iteritems(outlets): + for outlet in outlets.values(): outlet = gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories) @@ -116,8 +116,8 @@ def gen_uh_init(config_file): try: pour_points = pd.read_csv(config_dict['POUR_POINTS']['FILE_NAME'], comment='#') - log.info('Opened Pour Points File: ' - '{0}'.format(config_dict['POUR_POINTS']['FILE_NAME'])) + log.info('Opened Pour Points File: %s', + config_dict['POUR_POINTS']['FILE_NAME']) if not (all(x in list(pour_points.keys()) for x in ['lons', 'lats']) or all(x in list(pour_points.keys()) for x in ['x', 'y'])): raise ValueError('Pour Points File must include ' @@ -130,8 +130,8 @@ def gen_uh_init(config_file): pour_points.drop_duplicates(inplace=True) pour_points.dropna() except Exception as e: - log.error('Error opening pour points file: ' - '{0}'.format(config_dict['POUR_POINTS']['FILE_NAME'])) + log.error('Error opening pour points file: %s', + config_dict['POUR_POINTS']['FILE_NAME']) log.exception(e) raise # ---------------------------------------------------------------- # @@ -146,11 +146,11 @@ def gen_uh_init(config_file): skip_header=uh_header, delimiter=',', unpack=True) - log.info('Opened UHbox File: ' - '{0}'.format(config_dict['UH_BOX']['FILE_NAME'])) + log.info('Opened UHbox File: %s', + config_dict['UH_BOX']['FILE_NAME']) except: - log.exception('Error opening uh_box file: ' - '{0}'.format(config_dict['POUR_POINTS']['FILE_NAME'])) + log.exception('Error opening uh_box file: %s', + config_dict['POUR_POINTS']['FILE_NAME']) raise # ---------------------------------------------------------------- # @@ -205,7 +205,7 @@ def gen_uh_init(config_file): check_ncvars(config_dict['ROUTING'], list(fdr_data.keys())) # ---------------------------------------------------------------- # - log.info('Opened FDR File: {0}'.format(fdr_file)) + log.info('Opened FDR File: %s', fdr_file) except: log.exception('Error opening FDR file') raise @@ -215,9 +215,8 @@ def gen_uh_init(config_file): # ---------------------------------------------------------------- # # Read domain file domain = config_dict['DOMAIN'] - dom_data, _, _ = read_domain(domain) - log.info('Opened Domain File: ' - '{0}'.format(domain['FILE_NAME'])) + dom_data = read_domain(domain)[0] + log.info('Opened Domain File: %s', domain['FILE_NAME']) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -277,7 +276,7 @@ def gen_uh_init(config_file): glats=dom_data[domain['LATITUDE_VAR']], glons=dom_data[domain['LONGITUDE_VAR']]) - for i in range(len(lats)): + for i in pyrange(len(lats)): if 'names' in list(pour_points.keys()): name = pour_points['names'].values[i] name = name.replace("'", '').replace(" ", "_") @@ -314,7 +313,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, """ log = getLogger(LOG_NAME) - log.info('Running outlet cell id {0}'.format(outlet.cell_id)) + log.info('Running outlet cell id %s', outlet.cell_id) agg_data = {} domain = config_dict['DOMAIN'] @@ -337,10 +336,10 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, # ------------------------------------------------------------ # # Loop over pour points + n_pour_points = len(outlet.pour_points) for j, pour_point in enumerate(outlet.pour_points): - log.info('On pour_point #{0} of' - ' {1}'.format(j + 1, len(outlet.pour_points))) + log.info('On pour_point #%s of %s', j + 1, n_pour_points) # -------------------------------------------------------- # # Make the Unit Hydrograph Grid @@ -348,12 +347,10 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, config_dict['ROUTING']) log.debug('Done routing to pour_point') - log.debug('rout_data: {0}, ' - '{1}'.format(rout_data['unit_hydrograph'].min(), - rout_data['unit_hydrograph'].max())) - log.debug('rout_data sum: {0}, ' - '{1}'.format(rout_data['unit_hydrograph'].sum(), - rout_data['fraction'].sum())) + log.debug('rout_data: %s, %s', rout_data['unit_hydrograph'].min(), + rout_data['unit_hydrograph'].max()) + log.debug('rout_data sum: %s, %s', rout_data['unit_hydrograph'].sum(), + rout_data['fraction'].sum()) # -------------------------------------------------------- # @@ -369,9 +366,9 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, pad=options['AGG_PAD'], maskandnorm=True) - log.debug('agg_data: {0}, ' - '{1}'.format(agg_data['unit_hydrograph'].min(), - agg_data['unit_hydrograph'].max())) + log.debug('agg_data: %s, %s', + agg_data['unit_hydrograph'].min(), + agg_data['unit_hydrograph'].max()) else: agg_data = rout_data # -------------------------------------------------------- # @@ -404,10 +401,8 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, # -------------------------------------------------------- # # Read temporary file #2 - final_data, _, _ = read_netcdf(temp_file_2, - variables=['unit_hydrograph', - 'fraction', - dom_lat]) + final_data = read_netcdf( + temp_file_2, variables=['unit_hydrograph', 'fraction', dom_lat])[0] # -------------------------------------------------------- # # Check latitude order, flip if necessary. @@ -415,8 +410,8 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, if final_data[dom_lat][-1] > final_data[dom_lat][0]: var_list = list(final_data.keys()) - log.debug('Remapped inputs came in upside down, flipping {0}' - ' now.'.format(", ".join(var_list))) + log.debug('Remapped inputs came in upside down, flipping %s', + ", ".join(var_list)) # flip lattiutude and fraction along y axis (axis 0) final_data[dom_lat] = final_data[dom_lat][::-1] final_data['fraction'] = final_data['fraction'][::-1, :] diff --git a/scripts/rvic b/scripts/rvic index b2d8f89..69ec3ea 100755 --- a/scripts/rvic +++ b/scripts/rvic @@ -41,12 +41,16 @@ def main(): if args.script == 'parameters': from rvic.parameters import parameters parameters(args.config_file, numofproc=args.numofproc) + else: + if args.numofproc > 1: + print('{0} processors were specified but script {1} only ' + 'excepts 1'.format(args.numofproc, args.script)) if args.script == 'convolution': from rvic.convolution import convolution - convolution(args.config_file, numofproc=args.numofproc) + convolution(args.config_file) if args.script == 'convert': from rvic.convert import convert - convert(args.config_file, numofproc=args.numofproc) + convert(args.config_file) else: parser.print_help() return diff --git a/setup.py b/setup.py index fbdcb9b..9d02d3c 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ - #!/usr/bin/env python +#!/usr/bin/env python import os import re import sys @@ -103,7 +103,8 @@ def write_version_py(filename=None): author='Joe Hamman', author_email='jhamman@hydro.washington.edu', install_requires=['scipy >= 0.13', 'numpy >= 1.8', - 'netCDF4 >= 1.0.6', 'matplotlib >= 1.3.1'], + 'netCDF4 >= 1.0.6', 'matplotlib >= 1.3.1', + 'pandas >= 0.15.1'], tests_require=['pytest >= 2.5.2'], url='https://github.com/jhamman/RVIC', test_suite='pytest.collector', diff --git a/tools/find_pour_points.py b/tools/find_pour_points.py index 1aec097..66b50d9 100755 --- a/tools/find_pour_points.py +++ b/tools/find_pour_points.py @@ -2,7 +2,7 @@ """ Find the outlet location of basins in a grid """ -from future import print_function +from __future__ import print_function import numpy as np from netCDF4 import Dataset import time as tm @@ -38,8 +38,7 @@ def main(): if which_points == 'all': print('Returning all land cells as pour pour points') basin, x_outlet, y_outlet, max_area, min_x, min_y, max_x, \ - max_y = find_all(basin_id, source_area, land_mask, lons, lats, res, - verbose) + max_y = find_all(basin_id, source_area, land_mask, lons, lats, res) else: print('Returning all basin outlet grid cells as pour points') basin, x_outlet, y_outlet, max_area, min_x, min_y, max_x, \ @@ -56,14 +55,14 @@ def main(): write_ascii_file(basin, x_outlet, y_outlet, max_area, min_x, min_y, max_x, max_y, output_file) else: - write_netcdf_file(basin, x_outlet, y_outlet, max_area, min_x, min_y, + write_netcdf_file(basin, x_outlet, y_outlet, min_x, min_y, max_x, max_y, output_file) # ---------------------------------------------------------------- # # -------------------------------------------------------------------- # # -------------------------------------------------------------------- # -def find_all(basin_id, source_area, land_mask, lons, lats, res, verbose): +def find_all(basin_id, source_area, land_mask, lons, lats, res): """Return the info for all land points """ lat, lon = np.meshgrid(lats, lons) @@ -87,7 +86,7 @@ def find_all(basin_id, source_area, land_mask, lons, lats, res, verbose): # Loop over every basin id, finding the maximum upstream area, and location # and record the basin#,longitude,latitude,area - for i, (yi, xi, bi) in enumerate(zip(y, x, basin)): + for i, bi in enumerate(basin): inds = np.nonzero(basin_id == bi) x_basin = lon[inds] y_basin = lat[inds] @@ -158,7 +157,7 @@ def find_outlets(basin_id, source_area, lons, lats, res, verbose): # -------------------------------------------------------------------- # # write output netcdf -def write_netcdf_file(basin, x_outlet, y_outlet, max_area, min_x, min_y, +def write_netcdf_file(basin, x_outlet, y_outlet, min_x, min_y, max_x, max_y, out_file): """ save the list of pour points as a comma seperated text file @@ -168,7 +167,7 @@ def write_netcdf_file(basin, x_outlet, y_outlet, max_area, min_x, min_y, # ---------------------------------------------------------------- # # set dimensions - points = f.createDimension('points', None) + f.createDimension('points', None) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # From ad6972f1d84d9d5b59195b9e56738e7c99fec3fa Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 31 Mar 2015 09:00:06 -0700 Subject: [PATCH 07/47] update travis build requirements --- ci/requirements-2.7.txt | 8 ++++---- ci/requirements-3.4.txt | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/requirements-2.7.txt b/ci/requirements-2.7.txt index e6c604d..18de97c 100644 --- a/ci/requirements-2.7.txt +++ b/ci/requirements-2.7.txt @@ -1,6 +1,6 @@ -matplotlib==1.3.1 -netCDF4==1.0.8 -numpy==1.8.1 -scipy==0.13.3 +matplotlib==1.4.3 +netCDF4==1.1.6 +numpy==1.9.2 +scipy==0.15.1 pandas==0.15.1 pytest==2.5.2 diff --git a/ci/requirements-3.4.txt b/ci/requirements-3.4.txt index e6c604d..18de97c 100644 --- a/ci/requirements-3.4.txt +++ b/ci/requirements-3.4.txt @@ -1,6 +1,6 @@ -matplotlib==1.3.1 -netCDF4==1.0.8 -numpy==1.8.1 -scipy==0.13.3 +matplotlib==1.4.3 +netCDF4==1.1.6 +numpy==1.9.2 +scipy==0.15.1 pandas==0.15.1 pytest==2.5.2 From 8fdf134bf2ed90b79986e772016de87868acee0d Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 31 Mar 2015 16:02:46 -0700 Subject: [PATCH 08/47] add unit tests for most modules, some fixes/changes to source files during development of tests --- rvic/clib/rvic_convolution.c | 2 +- rvic/core/aggregate.py | 1 - rvic/core/history.py | 8 +- rvic/core/make_uh.py | 12 +- rvic/core/read_forcing.py | 2 +- rvic/core/utilities.py | 20 +- rvic/core/variables.py | 4 +- tests/unit/rvic_unit_test.py | 66 ------- tests/unit/test_aggregate.py | 84 ++++++++ tests/unit/test_config.py | 25 +++ tests/unit/test_convolution_wrapper.py | 109 +++++++++++ tests/unit/test_history.py | 17 ++ tests/unit/test_log.py | 18 ++ tests/unit/test_make_uh.py | 183 ++++++++++++++++++ tests/unit/test_multi_proc.py | 7 + tests/unit/test_param_file.py | 0 tests/unit/test_plots.py | 0 tests/unit/test_pycompat.py | 0 tests/unit/test_read_forcing.py | 0 tests/unit/test_remap.py | 8 + tests/unit/test_share.py | 11 ++ tests/unit/test_time_utility.py | 28 +++ tests/unit/test_utilities.py | 74 +++++++ tests/unit/test_variables.py | 0 tests/unit/test_write.py | 0 ...n_parameters_01.rvic.prm.BLMSA.20150226.nc | Bin 0 -> 62977 bytes 26 files changed, 593 insertions(+), 86 deletions(-) delete mode 100644 tests/unit/rvic_unit_test.py create mode 100644 tests/unit/test_aggregate.py create mode 100644 tests/unit/test_config.py create mode 100644 tests/unit/test_convolution_wrapper.py create mode 100644 tests/unit/test_history.py create mode 100644 tests/unit/test_log.py create mode 100644 tests/unit/test_make_uh.py create mode 100644 tests/unit/test_multi_proc.py create mode 100644 tests/unit/test_param_file.py create mode 100644 tests/unit/test_plots.py create mode 100644 tests/unit/test_pycompat.py create mode 100644 tests/unit/test_read_forcing.py create mode 100644 tests/unit/test_remap.py create mode 100644 tests/unit/test_share.py create mode 100644 tests/unit/test_time_utility.py create mode 100644 tests/unit/test_utilities.py create mode 100644 tests/unit/test_variables.py create mode 100644 tests/unit/test_write.py create mode 100644 tests/unit/unit_test_data/gunnison_parameters_01.rvic.prm.BLMSA.20150226.nc diff --git a/rvic/clib/rvic_convolution.c b/rvic/clib/rvic_convolution.c index bc57087..69cf2b9 100644 --- a/rvic/clib/rvic_convolution.c +++ b/rvic/clib/rvic_convolution.c @@ -1,7 +1,7 @@ #include void -convolve(const int nsources, /*scalar - number of sources*/ +convolve(const int nsources, /*scalar - number of sources*/ const int noutlets, /*scalar - length of subset*/ const int subset_length, /*scalar - length of subset*/ const int x_size, diff --git a/rvic/core/aggregate.py b/rvic/core/aggregate.py index 4537a60..f0808d3 100644 --- a/rvic/core/aggregate.py +++ b/rvic/core/aggregate.py @@ -207,7 +207,6 @@ def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False): # ---------------------------------------------------------------- # # Put all the data into agg_data variable and return to main - agg_data['timesteps'] = in_data['timesteps'] agg_data['unit_hydrograph_dt'] = in_data['unit_hydrograph_dt'] agg_data['lon'] = lons agg_data['lat'] = lats diff --git a/rvic/core/history.py b/rvic/core/history.py index 22537e5..3388892 100644 --- a/rvic/core/history.py +++ b/rvic/core/history.py @@ -46,7 +46,7 @@ def __init__(self, time_ord, caseid, rvar, tape_num=0, avgflag='A', units='kg m-2 s-1', file_format='NETCDF4_CLASSIC', outtype='grid', grid_lons=False, grid_lats=False, grid_area=None, out_dir='.', - calendar=None, glob_ats=None, zlib=True, complevel=4, + calendar='standard', glob_ats=None, zlib=True, complevel=4, least_significant_digit=None): self._tape_num = tape_num self._time_ord = time_ord # Days since basetime @@ -88,7 +88,7 @@ def __init__(self, time_ord, caseid, rvar, tape_num=0, # ------------------------------------------------------------ # # Get Grid Lons/Lats if outtype is grid - if outtype == 'grid': + if outtype.lower() == 'grid': self._out_data_shape = self._grid_shape if type(grid_lons) == np.ndarray and type(grid_lats) == np.ndarray: self._grid_lons = grid_lons @@ -96,8 +96,10 @@ def __init__(self, time_ord, caseid, rvar, tape_num=0, else: raise ValueError('Must include grid lons / lats if ' 'outtype == grid') - else: + elif outtype.lower() == 'array': self._out_data_shape = (self._num_outlets, ) + else: + raise ValueError('Unknown value for outtype: {0}'.format(outtype)) # ------------------------------------------------------------ # # ------------------------------------------------------------ # diff --git a/rvic/core/make_uh.py b/rvic/core/make_uh.py index 8771cc6..54bfc9c 100644 --- a/rvic/core/make_uh.py +++ b/rvic/core/make_uh.py @@ -157,8 +157,7 @@ def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): # ---------------------------------------------------------------- # # Agregate to output timestep - rout_data['unit_hydrograph'], \ - rout_data['timesteps'] = adjust_uh_timestep( + rout_data['unit_hydrograph'] = adjust_uh_timestep( uh_s, t_uh, input_interval, rout_dict['OUTPUT_INTERVAL'], catchment['x_inds'], catchment['y_inds']) # ---------------------------------------------------------------- # @@ -415,20 +414,18 @@ def adjust_uh_timestep(unit_hydrograph, t_uh, input_interval, output_interval, log.debug('No need to aggregate in time (output_interval = ' 'input_interval) Skipping the adjust_uh_timestep step') uh_out = unit_hydrograph - ts_new = np.arange(t_uh) elif np.remainder(output_interval, input_interval) == 0: log.debug('Aggregating to %i from %i seconds', output_interval, input_interval) fac = int(output_interval / input_interval) t_uh_out = int(t_uh / fac) - ts_new = np.arange(t_uh_out) uh_out = np.zeros((t_uh_out, unit_hydrograph.shape[1], unit_hydrograph.shape[2]), dtype=np.float64) for (y, x) in zip(y_inds, x_inds): for t in range(t_uh_out): uh_out[t, y, x] = unit_hydrograph[t * fac:t * fac + fac, y, x].sum() - elif np.remainder(input_interval, output_interval): + else: log.debug('Interpolating unit hydrograph from input_interval: %i to ' 'output_interval: %i', input_interval, output_interval) fac = int(input_interval / output_interval) @@ -441,5 +438,8 @@ def adjust_uh_timestep(unit_hydrograph, t_uh, input_interval, output_interval, for (y, x) in zip(y_inds, x_inds): f = interp1d(ts_orig, unit_hydrograph[:, y, x]) uh_out[:, y, x] = f(ts_new) - return uh_out, ts_new + + # normalize + uh_out /= uh_out.sum(axis=0) + return uh_out # -------------------------------------------------------------------- # diff --git a/rvic/core/read_forcing.py b/rvic/core/read_forcing.py index 70bbb9f..87a2372 100644 --- a/rvic/core/read_forcing.py +++ b/rvic/core/read_forcing.py @@ -45,7 +45,7 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, self.fld_mult = {} if start: - if type(start) == float: + if type(start) in [float, int]: start = [int(start)] end = [int(end)] else: diff --git a/rvic/core/utilities.py b/rvic/core/utilities.py index ad22773..c8758e0 100644 --- a/rvic/core/utilities.py +++ b/rvic/core/utilities.py @@ -52,7 +52,7 @@ def latlon2yx(plats, plons, glats, glons): # -------------------------------------------------------------------- # # Search neighboring grid cells for channel -def search_for_channel(source_area, routys, routxs, search=2, tol=10): +def search_for_channel(source_area, routys, routxs, search=1, tol=10): """Search neighboring grid cells for channel""" log.debug('serching for channel') @@ -60,22 +60,30 @@ def search_for_channel(source_area, routys, routxs, search=2, tol=10): new_ys = np.empty_like(routys) new_xs = np.empty_like(routxs) + ysize, xsize = source_area.shape + for i, (y, x) in enumerate(zip(routys, routxs)): area0 = source_area[y, x] - search_area = source_area[y-search:y+search+1, x-search:x+search+1] + ymin = np.clip(y - search, 0, ysize) + ymax = np.clip(y + search + 1, 0, ysize) + xmin = np.clip(x - search, 0, xsize) + xmax = np.clip(x + search, 0, xsize) + + search_area = source_area[ymin:ymax, xmin:xmax] - if np.any(search_area > area0*tol): + if np.any(search_area / area0 > tol): sy, sx = np.unravel_index(search_area.argmax(), search_area.shape) - new_ys[i] = y + sy - search - new_xs[i] = x + sx - search + new_ys[i] = np.clip(y + sy - search, 0, ysize) + new_xs[i] = np.clip(x + sx - search, 0, xsize) log.debug('Moving pour point to channel y: ' '{0}->{1}, x: {2}->{3}'.format(y, new_ys[i], x, new_xs[i])) log.debug('Source Area has increased from {0}' - ' to {1}'.format(area0, source_area[new_ys[i], new_xs[i]])) + ' to {1}'.format(area0, source_area[new_ys[i], + new_xs[i]])) else: new_ys[i] = y new_xs[i] = x diff --git a/rvic/core/variables.py b/rvic/core/variables.py index 5c030e3..29d4ab7 100644 --- a/rvic/core/variables.py +++ b/rvic/core/variables.py @@ -305,9 +305,9 @@ def convolve(self, aggrunin, time_ord): self.source_y_ind, self.source_x_ind, self.source_time_offset, - self.unit_hydrograph[tracer][:, :], + self.unit_hydrograph[tracer], aggrunin[tracer], - self.ring[tracer][:, :]) + self.ring[tracer]) # -------------------------------------------------------- # # ------------------------------------------------------------ # diff --git a/tests/unit/rvic_unit_test.py b/tests/unit/rvic_unit_test.py deleted file mode 100644 index 624f316..0000000 --- a/tests/unit/rvic_unit_test.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/local/env python -""" -rvic_unit_test.py - -Set to run with pytest - -Usage: py.test (from RVIC or test directory) -""" -import numpy as np -import sys -sys.path.append("../") - -# -------------------------------------------------------------------- # -# Unit tests for utilities.py -from rvic.core.utilities import * -from rvic.core.config import * - - -def test_config_type_int(): - assert config_type('1') == 1 - - -def test_config_type_float(): - assert config_type('1.75') == 1.75 - - -def test_config_type_bool(): - assert config_type('True') - - -def test_find_nearest(): - assert find_nearest(np.array([8, 19, 39, 100, 399]), 20) == 1 - - -def test_find_nearest_max(): - assert find_nearest(np.array([8, 19, 39, 100, 399]), 500) == 4 - - -def test_find_nearest_min(): - assert find_nearest(np.array([8, 19, 39, 100, 399]), 1) == 0 -# -------------------------------------------------------------------- # - -# -------------------------------------------------------------------- # -# Unit tests for make_uh.py -from rvic.core.make_uh import * - - -def test_find_ts(): - assert find_ts(np.array([0, 86400, 172800])) == 86400 -# -------------------------------------------------------------------- # - -# -------------------------------------------------------------------- # -# Unit tests for make_uh.py -from rvic.core.time_utility import * -from rvic.core.share import TIMEUNITS -from netCDF4 import date2num -from datetime import datetime - - -def test_ord_to_datetime(): - # Independence day - date = datetime(1776, 7, 4, 12, 0, 0, 0) - ord_time = date2num(date, TIMEUNITS) - # Independence day (note that this fails if date has microseconds != 0) - assert ord_to_datetime(ord_time, TIMEUNITS) == date -# -------------------------------------------------------------------- # diff --git a/tests/unit/test_aggregate.py b/tests/unit/test_aggregate.py new file mode 100644 index 0000000..eadef67 --- /dev/null +++ b/tests/unit/test_aggregate.py @@ -0,0 +1,84 @@ +import pytest +import numpy as np +import pandas as pd +from rvic.core.aggregate import make_agg_pairs, aggregate + + +config_dict = {} +config_dict['DOMAIN'] = {'LONGITUDE_VAR': 'LONGITUDE', + 'LATITUDE_VAR': 'LATITUDE'} +config_dict['ROUTING'] = {'LONGITUDE_VAR': 'LONGITUDE', + 'LATITUDE_VAR': 'LATITUDE', + 'SOURCE_AREA_VAR': 'SOURCE_AREA'} + + +@pytest.fixture() +def dom_data(scope='function'): + dom = {} + lons, lats = np.meshgrid(np.linspace(-10, 0., 20), np.linspace(40, 50, 20)) + dom[config_dict['DOMAIN']['LONGITUDE_VAR']] = lons + dom[config_dict['DOMAIN']['LATITUDE_VAR']] = lats + dom['cell_ids'] = np.arange(lons.size).reshape(lons.shape) + return dom + + +@pytest.fixture() +def fdr_data(scope='function'): + fdr = {} + fdr['resolution'] = 1.0 + lons, lats = np.meshgrid(np.arange(-15, 15, fdr['resolution']), + np.arange(20, 60, fdr['resolution'])) + fdr[config_dict['ROUTING']['LONGITUDE_VAR']] = lons + fdr[config_dict['ROUTING']['LATITUDE_VAR']] = lats + fdr[config_dict['ROUTING']['SOURCE_AREA_VAR']] = np.ones_like(lons) + return fdr + + +@pytest.fixture() +def in_data(fdr_data, scope='function'): + rout_data = {} + rout_data['lat'] = fdr_data[config_dict['ROUTING']['LATITUDE_VAR']] + rout_data['lon'] = fdr_data[config_dict['ROUTING']['LONGITUDE_VAR']] + shape = fdr_data[config_dict['ROUTING']['LATITUDE_VAR']].shape + rout_data['unit_hydrograph'] = np.random.random((10, ) + shape) + rout_data['fraction'] = np.ones(shape) + rout_data['unit_hydrograph_dt'] = 3600. + return rout_data + + +@pytest.fixture() +def agg_data(fdr_data, scope='function'): + rout_data = {} + rout_data['lat'] = fdr_data[config_dict['ROUTING']['LATITUDE_VAR']][:4] + rout_data['lon'] = fdr_data[config_dict['ROUTING']['LONGITUDE_VAR']][:4] + shape = rout_data['lat'].shape + rout_data['unit_hydrograph'] = np.random.random((10, ) + shape) + rout_data['fraction'] = np.ones(shape) + rout_data['unit_hydrograph_dt'] = 3600. + return rout_data + + +def test_make_agg_pairs_all_unique(dom_data, fdr_data): + pour_points = pd.DataFrame({'lons': [-2.3, 0.3, 10.4, 17.8], + 'lats': [42.1, 40.5, 49.0, 45.2]}) + outlets = make_agg_pairs(pour_points, dom_data, fdr_data, config_dict) + assert len(outlets) == len(pour_points) + + +def test_make_agg_pairs_with_overlap(dom_data, fdr_data): + pour_points = pd.DataFrame({'lons': [-2.3, 0.3, 10.4, 17.8, -2.31], + 'lats': [42.1, 40.5, 49.0, 45.2, 42.11]}) + outlets = make_agg_pairs(pour_points, dom_data, fdr_data, config_dict) + assert len(outlets) == len(pour_points) - 1 + + +def test_aggregate_no_agg_data(in_data, fdr_data): + agg_data = aggregate(in_data, {}, res=fdr_data['resolution'], + pad=0, maskandnorm=False) + assert all([k in agg_data for k in in_data]) + + +def test_aggregate_with_aggdata(in_data, agg_data, fdr_data): + agg_data = aggregate(in_data, agg_data, res=fdr_data['resolution'], + pad=0, maskandnorm=False) + assert all([k in agg_data for k in in_data]) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..7da0bf8 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,25 @@ +from rvic.core.config import config_type, isfloat, isint + + +def test_config_type_int(): + assert config_type('1') == 1 + + +def test_config_type_float(): + assert config_type('1.75') == 1.75 + + +def test_config_type_bool(): + assert config_type('True') + + +def test_isfloat(): + assert isfloat(4.3) + assert isfloat('4.3') + + +def test_isint(): + assert isint(4) + assert isint('4') + assert not isint(4.3) + assert not isint('4.3') diff --git a/tests/unit/test_convolution_wrapper.py b/tests/unit/test_convolution_wrapper.py new file mode 100644 index 0000000..307f678 --- /dev/null +++ b/tests/unit/test_convolution_wrapper.py @@ -0,0 +1,109 @@ +import pytest +import numpy as np +from rvic.core.convolution_wrapper import rvic_convolve + + +def test_convolution_wraper(): + n_sources = 4 + n_outlets = 2 + time_length = 6 + ysize = 10 + xsize = 10 + source2outlet_ind = np.array([0, 0, 1, 1], dtype=np.int32) + source_y_ind = np.array([2, 4, 6, 8], dtype=np.int32) + source_x_ind = np.array([2, 6, 3, 5], dtype=np.int32) + source_time_offset = np.zeros(n_sources, dtype=np.int32) + unit_hydrograph = np.array([[0., 1., 0.5, 0.25], + [1., 0., 0.5, 0.25], + [0., 0., 0., 0.25], + [0., 0., 0., 0.25], + [0., 0., 0., 0.], + [0., 0., 0., 0.]]) + aggrunin = np.ones((ysize, xsize), dtype=np.float64) + ring = np.zeros((time_length, n_outlets)) + + rvic_convolve(n_sources, + n_outlets, + time_length, + xsize, + source2outlet_ind, + source_y_ind, + source_x_ind, + source_time_offset, + unit_hydrograph, + aggrunin, + ring) + assert ring.sum() == 4 + np.testing.assert_almost_equal(ring.sum(axis=1), + unit_hydrograph.sum(axis=1)) + np.testing.assert_almost_equal(ring.sum(axis=0), + [2, 2]) + + +def test_convolution_wraper_1_outlet(): + n_sources = 4 + n_outlets = 1 + time_length = 6 + ysize = 10 + xsize = 10 + source2outlet_ind = np.array([0, 0, 0, 0], dtype=np.int32) + source_y_ind = np.array([2, 4, 6, 8], dtype=np.int32) + source_x_ind = np.array([2, 6, 3, 5], dtype=np.int32) + source_time_offset = np.zeros(n_sources, dtype=np.int32) + unit_hydrograph = np.array([[0., 1., 0.5, 0.25], + [1., 0., 0.5, 0.25], + [0., 0., 0., 0.25], + [0., 0., 0., 0.25], + [0., 0., 0., 0.], + [0., 0., 0., 0.]]) + aggrunin = np.ones((ysize, xsize), dtype=np.float64) + ring = np.zeros((time_length, n_outlets)) + + rvic_convolve(n_sources, + n_outlets, + time_length, + xsize, + source2outlet_ind, + source_y_ind, + source_x_ind, + source_time_offset, + unit_hydrograph, + aggrunin, + ring) + assert ring.sum() == 4 + np.testing.assert_almost_equal(ring.sum(axis=1), + unit_hydrograph.sum(axis=1)) + np.testing.assert_almost_equal(ring.sum(axis=0), + [4]) + + +def test_convolution_wraper_1_source_1_outlet(): + n_sources = 1 + n_outlets = 1 + time_length = 6 + ysize = 10 + xsize = 10 + source2outlet_ind = np.array([0, 0, 1, 1], dtype=np.int32) + source_y_ind = np.array([2], dtype=np.int32) + source_x_ind = np.array([5], dtype=np.int32) + source_time_offset = np.zeros(n_sources, dtype=np.int32) + unit_hydrograph = np.array([[0., 0.25, 0.5, 0.25, 0., 0.]]).T + aggrunin = np.ones((ysize, xsize), dtype=np.float64) + ring = np.zeros((time_length, n_outlets)) + + rvic_convolve(n_sources, + n_outlets, + time_length, + xsize, + source2outlet_ind, + source_y_ind, + source_x_ind, + source_time_offset, + unit_hydrograph, + aggrunin, + ring) + assert ring.sum() == 1 + np.testing.assert_almost_equal(ring.sum(axis=1), + unit_hydrograph.sum(axis=1)) + np.testing.assert_almost_equal(ring.sum(axis=0), + [1]) diff --git a/tests/unit/test_history.py b/tests/unit/test_history.py new file mode 100644 index 0000000..063af70 --- /dev/null +++ b/tests/unit/test_history.py @@ -0,0 +1,17 @@ +import pytest +import numpy as np +from rvic.core.variables import Rvar +from rvic.core.history import Tape + + +@pytest.fixture() +def rvar(scope='function'): + rv = Rvar( + 'unit_test_data/gunnison_parameters_01.rvic.prm.BLMSA.20150226.nc', + 'test_case', 'noleap', './', 'NETCDF4') + return rv + + +def test_create_tape_instance(rvar): + history_tape = Tape(1.25, 'test', rvar, grid_area=np.zeros((10, 11)), + outtype='array') diff --git a/tests/unit/test_log.py b/tests/unit/test_log.py new file mode 100644 index 0000000..83c0e8e --- /dev/null +++ b/tests/unit/test_log.py @@ -0,0 +1,18 @@ +import pytest +from rvic.core.log import StreamToFile + + +def test_setup_stream(): + stream = StreamToFile() + stream.write('buf') + stream.flush() + + +def test_stream_raises_with_bad_log_level(): + with pytest.raises(TypeError): + stream = StreamToFile(log_level='junk') + stream.write('buf') + stream.flush() + + +# Cannot test logger in interactive session or using pytest diff --git a/tests/unit/test_make_uh.py b/tests/unit/test_make_uh.py new file mode 100644 index 0000000..aee9173 --- /dev/null +++ b/tests/unit/test_make_uh.py @@ -0,0 +1,183 @@ +import pytest +import numpy as np +from rvic.core.make_uh import (find_ts, read_direction, search_catchment, + make_uh, make_grid_uh_river, make_grid_uh, + adjust_uh_timestep) +from rvic.core.variables import Point + + +@pytest.fixture() +def fdr_vic(scope='function'): + a = [[0, 0, 0, 5, 5, 5, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 4, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 5, 3, 5, 5, 6, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 5, 4, 0, 5, 5, 5, 5, 6, 7, 5, 5, 5, 7, 5, 0, 0, 0, 0, 0, 0], + [0, 0, 5, 4, 0, 5, 4, 5, 5, 7, 7, 5, 6, 7, 5, 6, 0, 0, 0, 0, 0, 0], + [4, 0, 5, 7, 4, 0, 3, 5, 5, 7, 5, 5, 7, 4, 5, 6, 0, 0, 0, 0, 0, 0], + [4, 0, 3, 4, 0, 4, 0, 4, 3, 5, 6, 7, 7, 4, 5, 6, 0, 0, 0, 0, 0, 0], + [0, 4, 0, 4, 0, 3, 1, 2, 4, 0, 4, 5, 7, 5, 5, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 3, 4, 3, 1, 8, 4, 5, 4, 0, 5, 5, 3, 5, 6, 5, 0, 0, 0, 0, 0], + [0, 0, 0, 3, 4, 5, 5, 4, 5, 4, 3, 5, 6, 7, 7, 5, 5, 7, 0, 0, 0, 0], + [0, 0, 0, 4, 3, 5, 5, 3, 4, 0, 4, 3, 4, 3, 5, 5, 7, 7, 0, 0, 0, 0], + [0, 0, 0, 4, 0, 5, 4, 0, 4, 3, 4, 0, 4, 5, 3, 5, 7, 7, 0, 0, 0, 0], + [0, 0, 0, 0, 4, 0, 4, 0, 4, 0, 4, 5, 3, 6, 7, 7, 5, 5, 6, 0, 5, 7], + [0, 0, 0, 0, 0, 4, 0, 3, 4, 3, 5, 3, 6, 7, 6, 5, 7, 7, 7, 7, 7, 7], + [0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 5, 7, 5, 6, 7, 1, 7, 1, 7, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 5, 5, 6, 7, 7, 7, 6, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 5, 7, 1, 7, 7, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 4, 5, 7, 7, 1, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] + return np.array(a, dtype=np.int) + + +@pytest.fixture() +def fdr_vic_small(scope='function'): + a = [[0, 0, 0, 5, 5], + [0, 0, 0, 4, 0], + [0, 0, 5, 3, 5], + [0, 0, 5, 4, 0], + [0, 0, 5, 4, 0]] + return np.array(a, dtype=np.int) + + +@pytest.fixture() +def dy_vic(scope='module'): + dy = {1: -1, 2: -1, 3: 0, 4: 1, 5: 1, 6: 1, 7: 0, 8: -1} + return dy + + +@pytest.fixture() +def dx_vic(scope='module'): + dx = {1: 0, 2: 1, 3: 1, 4: 1, 5: 0, 6: -1, 7: -1, 8: - 1} + return dx + + +@pytest.fixture() +def pour_point(scope='module'): + p = Point() + p.basinx = 4 + p.basiny = 2 + return p + + +def test_find_ts(): + assert find_ts(np.array([0, 86400, 172800])) == 86400 + + +def test_find_ts_raises_when_scalar(): + with pytest.raises(TypeError): + find_ts(4) + + +def test_read_direction(fdr_vic, dy_vic, dx_vic): + to_y, to_x = read_direction(fdr_vic, dy_vic, dx_vic) + np.testing.assert_equal(to_y.shape, to_x.shape) + assert to_y.max() <= fdr_vic.shape[0] + 1 + assert to_x.max() <= fdr_vic.shape[1] + 1 + + +def test_search_catchment(fdr_vic_small, dy_vic, dx_vic, pour_point): + basin_ids = np.ones_like(fdr_vic_small, dtype=np.int) + basin_id = 1 + to_y, to_x = read_direction(fdr_vic_small, dy_vic, dx_vic) + catchment, catch_fracs = search_catchment(to_y, to_x, pour_point, + basin_ids, basin_id) + assert catch_fracs.min() <= 0. + assert catch_fracs.max() == 1. + assert type(catchment) == dict + assert all([k in catchment for k in ['count_ds', 'x_inds', 'y_inds']]) + assert len(catchment['count_ds']) > 0 + assert len(catchment['count_ds']) == len(catchment['x_inds']) + assert len(catchment['count_ds']) == len(catchment['y_inds']) + + +def test_make_uh(fdr_vic_small): + ndays = 4 + y_inds, x_inds = np.nonzero(fdr_vic_small) + velocity = np.zeros(fdr_vic_small.shape, dtype=np.float) + 2. + diffusion = np.zeros(fdr_vic_small.shape, dtype=np.float) + 3000 + xmask = np.ones(fdr_vic_small.shape, dtype=np.float) + uh = make_uh(86400, ndays, y_inds, x_inds, velocity, diffusion, xmask) + assert uh.shape[0] == ndays + assert uh.shape[1:] == fdr_vic_small.shape + assert uh.min() >= 0. + assert uh.max() <= 1. + np.testing.assert_almost_equal(uh.sum(axis=0)[y_inds, x_inds], 1) + + +def test_make_grid_uh_river(fdr_vic_small, dy_vic, dx_vic, pour_point): + ndays = 4 + t_uh = 40 + basin_ids = np.ones_like(fdr_vic_small, dtype=np.int) + basin_id = 1 + to_y, to_x = read_direction(fdr_vic_small, dy_vic, dx_vic) + uh = np.zeros((ndays, ) + fdr_vic_small.shape) + uh[0, :, :] = 1. + catchment, _ = search_catchment(to_y, to_x, pour_point, + basin_ids, basin_id) + uh_river = make_grid_uh_river(t_uh, ndays, uh, to_y, to_x, pour_point, + catchment['y_inds'], catchment['x_inds'], + catchment['count_ds']) + assert uh_river.shape[0] == t_uh + assert uh_river.max() <= 1. + assert uh_river.min() >= 0. + np.testing.assert_almost_equal(uh_river.sum(axis=0)[catchment['y_inds'], + catchment['x_inds']], + 1) + + +def test_make_grid_uh(fdr_vic_small, dy_vic, dx_vic, pour_point): + ndays = 4 + t_uh = 40 + basin_ids = np.ones_like(fdr_vic_small, dtype=np.int) + basin_id = 1 + to_y, to_x = read_direction(fdr_vic_small, dy_vic, dx_vic) + catchment, _ = search_catchment(to_y, to_x, pour_point, + basin_ids, basin_id) + uh_river = np.zeros((t_uh, ) + fdr_vic_small.shape) + uh_river[0] = 1. + uh_box = np.array([1., 0, 0, 0]) + unit_hydrograph = make_grid_uh(t_uh, ndays, uh_river, uh_box, to_y, to_x, + catchment['y_inds'], catchment['x_inds'], + catchment['count_ds']) + assert unit_hydrograph.shape[0] == t_uh + assert unit_hydrograph.max() <= 1. + assert unit_hydrograph.min() >= 0. + np.testing.assert_almost_equal( + unit_hydrograph.sum(axis=0)[catchment['y_inds'], catchment['x_inds']], + 1) + + +def test_adjust_uh_timestep_nochange(fdr_vic_small): + t_uh = 40 + y_inds, x_inds = np.nonzero(fdr_vic_small) + unit_hydrograph = np.zeros((t_uh, ) + fdr_vic_small.shape, dtype=np.float) + unit_hydrograph[0] = 1. + + uh = adjust_uh_timestep(unit_hydrograph, t_uh, 3600, 3600, x_inds, y_inds) + assert uh.shape[0] == t_uh + np.testing.assert_almost_equal( + uh.sum(axis=0)[y_inds, x_inds], 1) + + +def test_adjust_uh_timestep_downscale(fdr_vic_small): + t_uh = 40 + y_inds, x_inds = np.nonzero(fdr_vic_small) + unit_hydrograph = np.zeros((t_uh, ) + fdr_vic_small.shape, dtype=np.float) + unit_hydrograph[0] = 1. + + uh = adjust_uh_timestep(unit_hydrograph, t_uh, 3600, 1800, x_inds, y_inds) + np.testing.assert_almost_equal( + uh.sum(axis=0)[y_inds, x_inds], 1) + + +def test_adjust_uh_timestep_upscale(fdr_vic_small): + t_uh = 40 + y_inds, x_inds = np.nonzero(fdr_vic_small) + unit_hydrograph = np.zeros((t_uh, ) + fdr_vic_small.shape, dtype=np.float) + unit_hydrograph[0] = 1. + + uh = adjust_uh_timestep(unit_hydrograph, t_uh, 1800, 3600, x_inds, y_inds) + np.testing.assert_almost_equal( + uh.sum(axis=0)[y_inds, x_inds], 1) diff --git a/tests/unit/test_multi_proc.py b/tests/unit/test_multi_proc.py new file mode 100644 index 0000000..fcc8fdd --- /dev/null +++ b/tests/unit/test_multi_proc.py @@ -0,0 +1,7 @@ +from rvic.core.multi_proc import LogExceptions + + +def test_create_log_exceptions(): + LogExceptions(callable) + +# error and logging pool not easily testing in unit test. diff --git a/tests/unit/test_param_file.py b/tests/unit/test_param_file.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_plots.py b/tests/unit/test_plots.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_pycompat.py b/tests/unit/test_pycompat.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_read_forcing.py b/tests/unit/test_read_forcing.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_remap.py b/tests/unit/test_remap.py new file mode 100644 index 0000000..36d22a1 --- /dev/null +++ b/tests/unit/test_remap.py @@ -0,0 +1,8 @@ +import pytest +from cdo import CDOException +from rvic.core.remap import remap + + +def test_cdo_raises_exception(): + with pytest.raises(CDOException): + remap('grid_file', 'in_file', 'out_file') diff --git a/tests/unit/test_share.py b/tests/unit/test_share.py new file mode 100644 index 0000000..69fc345 --- /dev/null +++ b/tests/unit/test_share.py @@ -0,0 +1,11 @@ +from rvic.core.share import NcGlobals, NcVar + + +def test_nc_globals(): + ncg = NcGlobals() + ncg.update() + + +def test_nc_var(): + ncv = NcVar(test='test') + assert ncv.test == 'test' diff --git a/tests/unit/test_time_utility.py b/tests/unit/test_time_utility.py new file mode 100644 index 0000000..8758a64 --- /dev/null +++ b/tests/unit/test_time_utility.py @@ -0,0 +1,28 @@ +# -------------------------------------------------------------------- # +# Unit tests for make_uh.py +from rvic.core.time_utility import ord_to_datetime, Dtime +from rvic.core.share import TIMEUNITS +from netCDF4 import date2num +from datetime import datetime + + +def test_ord_to_datetime(): + # Independence day + date = datetime(1776, 7, 4, 12, 0, 0, 0) + ord_time = date2num(date, TIMEUNITS) + # Independence day (note that this fails if date has microseconds != 0) + assert ord_to_datetime(ord_time, TIMEUNITS) == date + + +def test_dtime(): + dt = Dtime('2014-12-01-00', 'ndays', 5, None, 'ndays', 5, None, 'noleap', + 3600.00001) + assert dt.timestamp.year == 2014 + assert dt.timestamp.month == 12 + assert dt.timestamp.day == 1 + assert dt.timestamp.hour == 0 + dt.advance_timestep() + assert dt.timestamp.year == 2014 + assert dt.timestamp.month == 12 + assert dt.timestamp.day == 1 + assert dt.timestamp.hour == 1 diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py new file mode 100644 index 0000000..a339b65 --- /dev/null +++ b/tests/unit/test_utilities.py @@ -0,0 +1,74 @@ +import numpy as np +import os +import datetime +from rvic.core.utilities import (latlon2yx, search_for_channel, find_nearest, + write_rpointer, strip_non_ascii, + strip_invalid_char) +from rvic.core.share import RPOINTER, VALID_CHARS + + +def test_latlon2yx(): + glats, glons = np.meshgrid(np.arange(0, 20, 2), np.arange(10, 30, 2)) + plats = np.array([0, 2, 17]) + plons = np.array([28, 17, 12]) + y, x = latlon2yx(plats, plons, glats, glons) + assert len(y) == len(x) == len(plats) + assert y[0] == 9 + assert x[0] == 0 + + +def test_search_for_channel(): + source_area = np.array([[0.00, 0.00, 0.00, 1.00, 0.00, 0.00, 0.00, 0.00], + [0.00, 0.00, 0.00, 2.00, 0.00, 0.00, 0.00, 0.00], + [0.00, 0.00, 0.00, 3.00, 0.00, 0.00, 8.00, 9.00], + [51.0, 52.0, 53.0, 0.00, 4.00, 0.00, 7.00, 10.0], + [50.0, 1.00, 54.0, 0.00, 0.00, 5.00, 6.00, 11.0], + [1.00, 77.0, 1.00, 0.00, 0.00, 0.00, 12.0, 0.00], + [78.0, 20.0, 19.0, 0.00, 0.00, 13.0, 0.00, 0.00], + [79.0, 21.0, 18.0, 0.00, 0.00, 14.0, 0.00, 0.00], + [80.0, 22.0, 0.00, 17.0, 16.0, 15.0, 0.00, 0.00]]) + + routys = np.array([4, 8, 0, 1]) + routxs = np.array([1, 0, 3, 7]) + + new_ys, new_xs = search_for_channel(source_area, routys, routxs, search=1) + + print(new_ys, new_xs) + + np.testing.assert_equal(new_ys, [5, 8, 0, 2]) + np.testing.assert_equal(new_xs, [1, 0, 3, 7]) + np.testing.assert_equal(source_area[new_ys, new_xs], [77., 80., 1., 9.]) + + +def test_write_rpointer(): + write_rpointer('./', 'test_restart_file.nc', datetime.datetime.now()) + testfile = os.path.join('./', RPOINTER) + assert os.path.isfile(testfile) + os.remove(testfile) + + +def test_find_nearest(): + assert find_nearest(np.array([8, 19, 39, 100, 399]), 20) == 1 + + +def test_find_nearest_max(): + assert find_nearest(np.array([8, 19, 39, 100, 399]), 500) == 4 + + +def test_find_nearest_min(): + assert find_nearest(np.array([8, 19, 39, 100, 399]), 1) == 0 + + +def test_strip_non_ascii(): + test = u'éáé123456tgreáé@€' + new = strip_non_ascii(test) + assert new == '123456tgre@' + test = 'standard' + new = strip_non_ascii(test) + assert new == test + + +def test_strip_invalid_char(): + test = '123$%^789' + new = strip_invalid_char(test) + assert new == '123789' diff --git a/tests/unit/test_variables.py b/tests/unit/test_variables.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_write.py b/tests/unit/test_write.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/unit_test_data/gunnison_parameters_01.rvic.prm.BLMSA.20150226.nc b/tests/unit/unit_test_data/gunnison_parameters_01.rvic.prm.BLMSA.20150226.nc new file mode 100644 index 0000000000000000000000000000000000000000..cb264821574c0fd65b07444f72f600760e58ebad GIT binary patch literal 62977 zcmeHQ33MFAnXaB0S+X>?d>acNjcj9MOM@gIU<0vrTEZjASTcTr;PF_~k_P6Gr+Z`z z1GWhy5WpA=0dqJ6HeQZwmINN*ahN0z3=SKI_3j2(AR*>h4u=rHELX_>e^vc0^;jcI zpw&ZG)sv>WtNyC4`l`D7tG}xLzOuTaYD9Kkwks=(k^6^q+Q%ZKNJcbnT6$@Hb;U}T z_d!=a4|C_qs7Z6g!HLU-@0WhIi)k1#elqib=gwtoyh7rA=}#CT&e?aQ@9;*Aa-7PO z?n8N`ye=N^%Ms$&3y)RSeo)4XKjK178;5}|P1CYvEcxznSmM$85+(py%I4>=`TuY; zmN61z+;cmD+O;fa0*1L1#wz96FroJxFP+sbjvA(Aw3!UE zc7fzb=EbZ7b%xaxYMx&aj)p^Q6jTfwi7}lg!y|7Emc-4Dg9HFs#0Z*JzznyD%su~(9+3AufYC9Qg0XoLmLla&I)R$K;7euSzEm6@OvcBFDq%&@o zEm%<`9%~82f>DE0BcxMEjT($Z*f8~Yvu=|l?{uJgkWSLCWGrl%JPoo9OoJg2!!*68 zFc5MBE#o|mG6+PBSc}!llOS)QB!SkBP$J%v2)4BbLKZ~abaT@rEPmD)phC>8?U7`f z$P&^zBx^?$WVp(Zw#W2a*S!+>BUc>=4dTPuXFULt(D$S z8$;Z4Py`OPh_~Nnyq&VOi+v02E4GGBE1u}!S3H44AaBbEE!G=blX{i0PA^%g7cW>`RJ^!o zkzP_%yuc05XMPbgo=h}}>fs|C^Y@*cNX7ydXnRawTZdhoaLfcKprr8_zt(Y5d{r#m zZY0dG)qzd2I%u{+U95P_4R0QPiPpGj*_;XAfq0QKAPM#2`x zcL)lf-x`cYgOFp6$T49w8wn%UWC&p3<-*0vf~FDD<1r3$7`j9lx<(X1#gh5st&m5o zKsPKs81WV8#fug#EYKT`NF-?%=<`pVU3yZ1eoE1)g{PDh+vxJD$r|H(5{pypEAoNF zSW%dj?ct^sfZ~dHIA)nu&>1*9bER+sq600-SS)PDV*%4{^#InJH!g?T*JQTa^}{Sz zRohwNOBb)Zs=70yR=8rML!^@Z1yfap5_U4=1ek1fGLR5GED(-CM>GRP3ybCz&-2Ba z+(1}(8)4dtcr+M}*~zEKAH^%ej4eUJ~8@O$yy6#q7X6F{^rbVgkwHjnL2Gcx1ExmLq|DnUL`C zrhf(FlcsZY)eBAkIikj?n*ODb4c@Yi@J7+$FQ~^1tGuF055YYgGvT?fuY=cEEDq04 zifRug!ohVB!>5xgh8(v3@ZH?5gVHh@p2$k!fUw&^Y3hJ*;wxv|)HS2MvbRt_^5&<> zh~6Bg(G-t`%>8R?4@0@ScQC}~Lk}K%+~K826xgM6&&p)(Pqe@CGgnSaCezy4V{jH) zyA~v2^asa5emst4pcFs$Fs+B<74IJ!^Xf~lXR*wH3)8IQX}rwMAFR9pL-PkQ+p=}; z>O1coIT^;E2i>pm5G`j!OG{=Z^E|G#KtL!2{Tx&eZa5Aac)XltZ(Q@hb9n4_vu2)% zw+?TGEuDvVoN@QY8}8WZYyb0`k0rgnyVrfZ^jB-IodcZe?P05jCg!Tv3ITxP z5IB;=p=yLX>GzJrF?KaqaWZzjOim=Ohf%>diNuvG0L?1%9slb^0+OXi=e7N|(&d++ zI!1S^@N5;7Cq4h|XBAYxPJhUbbM<|P-a{TM#yopwi^|o@I(lIYhjh5xAqb7({Gcqb zk|9HHipLWn(0W-0SAiu#J7wuDiEv170#ycNId$8d`}ky60u>|RG*Dc8jwR8cLxL8C zu}5aR?|(kVne|xg8Kf^STS()oPt5Ma57w~N|+q<%gFQ|B$FLaJjNK);c@nTiIm39 zzvzP{-9H#t@A$BSEi140W3$sNGe3q?DwQ@#T$-+t8jcTEbgAJ~o78Z4Dsz__j(UD{ zso{tty3}wwC!flR6Tj$E!%;PhE;T$y1RoKO&uDZczc*i;;~jvG!oRppoZ~zKy3}w~ z{i90_r=F1-j&tkiQp3lH;3L9uIR?5^dHxFN68={%1UMM#PW8ei59lbI z1V^b{9MR@;8JW{oYIfQh@Rm0tx|zfI>hapb$_9Chapb$_9ChxAj})Pt`Uv$r91CM&#T{Q`zNKc=&DFMPU^Jq4jN*tUxGe)cYx zJ^0>&T2=xpB=Xpb`(LkR-v@zdY~S0Dp2vOJJfPl6Z-*#PG#$^ zON5yPIm~3gy!65iY&TE#?yL_tv06Bs!Y*BM@s;d+kk#3uXYaj=jf5O#uy0?t=4v(z zLZ`7Q%l6&Es^D-2d-#pOLwwSIG8@0~FHf>Y$X91KHUGz3?0lYCURCYiSQbzAhY1(_ zjc>}&W4orE`vH3ZWRGWG-lY8(92}F5w?8dC5sM=x%Webp6>SSB2`8Wftc4RTMC)86 zZnV5bVrOS9Iu9*ok^O_#PDuD^>56PbTCpMLW@Ua52ZCD@}8ijBJvLS z?6bTdQsx8YJJ#vlE$!&Rz0mu0dL6x4r`PM{if7*Q^rxM5A8}IWF&=IG^QIEMZ@cSq zziG)g^2Mfi6d7(?Hlc+&ng{EHZd7wJ5(%JyG_{H+WT@s^PbyBJru5QM~ z&8B~wkmbMW#@|$yE~5PKoiyWbYOGXwb*>Ol2q**;0tx|zz~DzfY|NP7(`L4Q*qHHT zy+G;)2^j7D8>Ac9BstQq@Vwv{A-vNp>SvD-%6r(!GXrFiR~LuG5P?kJbhm-6*8_yh zZuz#{O~gw=bXOD$jKE2LOXp)3d~|1D8RN%_F%o>rR|qHs6aoqXg}{(OU|<;Io97BZ zcSu|ZIb)O>&a3Xx7C9K>>FY)PY{pp717k#uAu>jx3DkwUvDefvK6Y-kW${<~2`Al5jo={ObM+u1X6aoqXg@8gpA#e;LFfa`A zIH`=gNFq4M86-`FKU+K1!Qysp67{ngMnZ5Hnh_1JuMCW(`BNf z!g!Mr1Y6<7g^^$^RG^m>wj_fsMu9%BFcyy)6PNtpMJ$@P2$xo+vD}4A3a|wA(nJ?# zprdW`Gk5L~GvZ5LTYO$$8RbaDC;?OD6aoqXg@8gpAuzNM7#K!*PMbjWRtfGPXOwdF zwtEz%;}Dk8;NUVMnpGz_#iB|z~lxaaHEi_Rn*j0)-}}B*9oeI_M(ug5ktc=q6Q8f z`FQ!gb9fLpIM?+YsgEex;Wi=hE#X1w>41IoFSiWqj!hB|@_vDq?68Ra#U{xT5&17p zw+q7$$l?)&9&OJz*)Jk_vR6dsWTA-8$rKTtJIoD1AhBDDKkK0>2b@-@0Q}mIU)ra6 zaAF=E8AQT$pOjqlkn%HAkXVyG8K-&30$H|*K_gJPyF^x+hwPErkwI(9-4u!451m)0 zB;r6rQ&CM`a`1j{X zNZP<+GNTe{%L%MWO@E1G$og5Pl?XN&36s8zr+LT_T5gJUt()HzX}@{D;%SlD#i|6| zfaJ?>x1KOf4#RXdF^f&y3{}Z+@h3$#-ilf5iMK@UT>QZSCio`v1n%tgpFW<8DcoQ# zG7Ck^{iO{J{4s?63oqaK*k~v(^;a@47)8MK${8X-U1@D4mtQL78(YJs4u3&CW?1DF zRk}U&(_vkU9*e_L7m8{RCc?pW5yPjGD}h{owtDBMM@}FS6y&G>oNdNP4$0h3b^oqq z&aa_j41Nn$+ zUa_(HHv2^)|hVsO>)+5Kssx1QY^80Rh1dyBg&s{Sf)?elW@#zDHnrgM@UDGeSAJklQ}2#K8yy zdqn;E#0b0nY{yq-gpZtEEtddcQE2)>!p`!?zaCA;5YsKkG14$Znuemhm9G#`2q**; z0tx|zz_EkCz%ay34+zAL5r&8h0Kh$W*DqdiFvK4}BO+{u_*FmK@RbSTjr;Si6J_AP zsSJ@;q>#vKKYi=+zB0!3iZK#=%2x;|1QY@a0foSjL117Q@;geD**rh&6<(<0c57=KG{AhB#y*yUv-kH zjFLhapb$_93>gFj!|ZB^h^)Af-#bG@j5WP1 zK)YUIM=Y`jYh~XoLqx?;@H_Ko?G}(+p=tB6CN8(8k!3L=as*AfaA97ke3L-pY~iuK z8Y#*;UToR+{Q7N?16~e`fR{%*IE3)P?NA3xz-S1n_vYo?P)u~>W6{aH9NPTqdXDzJ zx}K9&uSX7!F&(Pmq-+Po193nmpUI2WUnb?S80g3+CNDx6J2c0)zPqsnSts(sy)e9A zOGG)7pK4>NzqQq=V+%rn5b@^v=cgNMoIo%g#s5*t*dyKTM0Gw&b}CvSpb$_9Cy)5cmco@PGGXU3dTh literal 0 HcmV?d00001 From 8740bf771dd270ae9db4792d17467847f7fe861c Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 31 Mar 2015 22:22:00 -0700 Subject: [PATCH 09/47] updates to tests and a few minor code updates --- rvic/core/param_file.py | 4 +++- rvic/core/utilities.py | 12 +++++------- rvic/core/write.py | 8 ++++---- rvic/parameters.py | 17 +++++++++++++---- tests/run_tests.py | 30 ++++++++++++++++++++++++++---- 5 files changed, 51 insertions(+), 20 deletions(-) diff --git a/rvic/core/param_file.py b/rvic/core/param_file.py index 059c6fe..1dd533a 100644 --- a/rvic/core/param_file.py +++ b/rvic/core/param_file.py @@ -30,6 +30,9 @@ def finish_params(outlets, dom_data, config_dict, directories): dom_area = domain['AREA_VAR'] dom_frac = domain['FRACTION_VAR'] + if not len(outlets.keys()) > 0: + raise ValueError('outlets in finish_params are empty') + # ------------------------------------------------------------ # # netCDF variable options ncvaropts = {} @@ -303,7 +306,6 @@ def subset(outlets, subset_length=None): log.debug('Subset Length: %s', subset_length) log.debug(outlets) for i, (cell_id, outlet) in enumerate(iteritems(outlets)): - log.debug(i, cell_id, outlet, outlets) if i == 0: full_time_length = outlet.unit_hydrograph.shape[0] log.debug('Subset Length: %s', subset_length) diff --git a/rvic/core/utilities.py b/rvic/core/utilities.py index e3ccedc..db2866c 100644 --- a/rvic/core/utilities.py +++ b/rvic/core/utilities.py @@ -63,7 +63,7 @@ def search_for_channel(source_area, routys, routxs, search=1, tol=10): ysize, xsize = source_area.shape - for i, (y, x) in enumerate(zip(routys, routxs)): + for i, (y, x) in enumerate(pyzip(routys, routxs)): area0 = source_area[y, x] ymin = np.clip(y - search, 0, ysize) @@ -79,12 +79,10 @@ def search_for_channel(source_area, routys, routxs, search=1, tol=10): new_ys[i] = np.clip(y + sy - search, 0, ysize) new_xs[i] = np.clip(x + sx - search, 0, xsize) - log.debug('Moving pour point to channel y: ' - '{0}->{1}, x: {2}->{3}'.format(y, new_ys[i], - x, new_xs[i])) - log.debug('Source Area has increased from {0}' - ' to {1}'.format(area0, source_area[new_ys[i], - new_xs[i]])) + log.debug('Moving pour point to channel y: %s->%s, x: %s->%s', + y, new_ys[i], x, new_xs[i]) + log.debug('Source Area has increased from %s to %s', + area0, source_area[new_ys[i], new_xs[i]]) else: new_ys[i] = y new_xs[i] = x diff --git a/rvic/core/write.py b/rvic/core/write.py index 6f76a27..16b9e6b 100644 --- a/rvic/core/write.py +++ b/rvic/core/write.py @@ -39,9 +39,9 @@ def write_agg_netcdf(file_name, agg_data, glob_atts, nc_format, zlib=True, # ---------------------------------------------------------------- # # set dimensions - timesteps = f.createDimension('timesteps', None) - lon = f.createDimension('lon', (len(agg_data['lon']))) - lat = f.createDimension('lat', (len(agg_data['lat']))) + f.createDimension('timesteps', None) + f.createDimension('lon', (len(agg_data['lon']))) + f.createDimension('lat', (len(agg_data['lat']))) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -55,7 +55,7 @@ def write_agg_netcdf(file_name, agg_data, glob_atts, nc_format, zlib=True, timesteps = f.createVariable('timesteps', NC_INT, ('timesteps',), **ncvaropts) - timesteps[:] = agg_data['timesteps'] + timesteps[:] = np.arange(agg_data['unit_hydrograph'].shape[0]) for key, val in iteritems(share.timesteps.__dict__): if val: setattr(timesteps, key, val) diff --git a/rvic/parameters.py b/rvic/parameters.py index afb42ab..168a631 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -49,10 +49,12 @@ def parameters(config_file, numofproc=1): for i, outlet in enumerate(outlets.values()): log.info('On Outlet #%s of %s', i + 1, len(outlets)) - pool.apply_async(gen_uh_run, - args=(uh_box, fdr_data, fdr_vatts, dom_data, - outlet, config_dict, directories), - callback=store_result) + status = pool.apply_async(gen_uh_run, + args=(uh_box, fdr_data, fdr_vatts, + dom_data, outlet, config_dict, + directories), + callback=store_result, + error_callback=pool.terminate) pool.close() pool.join() @@ -63,6 +65,8 @@ def parameters(config_file, numofproc=1): outlet = gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories) + if not len(outlets.keys()) > 0: + raise ValueError('outlets in parameters are empty') # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -485,6 +489,9 @@ def gen_uh_final(outlets, dom_data, config_dict, directories): log.info('In gen_uh_final') + if not len(outlets.keys()) > 0: + raise ValueError('outlets in gen_uh_final are empty') + # ---------------------------------------------------------------- # # Write the parameter file param_file, today = finish_params(outlets, dom_data, config_dict, @@ -512,6 +519,8 @@ def gen_uh_final(outlets, dom_data, config_dict, directories): def store_result(result): # This is called whenever foo_pool(i) returns a result. # result_list is modified only by the main process, not the pool workers. + print('in store_result') + print('storing result for %s' % result.cell_id) results[result.cell_id] = result # -------------------------------------------------------------------- # results = {} diff --git a/tests/run_tests.py b/tests/run_tests.py index 6864d95..a3b0e61 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -8,6 +8,7 @@ import cProfile import pstats import io +from rvic.core.pycompat import OrderedDict from rvic import convert, convolution, parameters from rvic.core.config import read_config from rvic.core.pycompat import PY3, iteritems @@ -64,6 +65,8 @@ def run_examples(config_file): # run tests num_tests = len(list(config_dict.keys())) + test_outcomes = OrderedDict() + for i, (test, test_dict) in enumerate(iteritems(config_dict)): print("".center(100, '-')) print("Starting Test #{0} of {1}: {2}".format(i + 1, num_tests, @@ -81,12 +84,27 @@ def run_examples(config_file): pr.enable() if test_dict['function'] == 'convert': - convert.convert(test_dict['config_file']) + try: + convert.convert(test_dict['config_file']) + test_outcomes[test] = 'Passed' + except Exception as e: + print('Error in conversion example: {0}'.format(test)) + test_outcomes[test] = 'Failed: {0}'.format(e) elif test_dict['function'] == 'convolution': - convolution.convolution(test_dict['config_file']) + try: + convolution.convolution(test_dict['config_file']) + test_outcomes[test] = 'Passed' + except Exception as e: + print('Error in convolution example: {0}'.format(test)) + test_outcomes[test] = 'Failed: {0}'.format(e) elif test_dict['function'] == 'parameters': - parameters.parameters(test_dict['config_file'], - numofproc=numofproc) + try: + parameters.parameters(test_dict['config_file'], + numofproc=numofproc) + test_outcomes[test] = 'Passed' + except Exception as e: + print('Error in parameters example: {0}'.format(test)) + test_outcomes[test] = 'Failed: {0}'.format(e) else: raise ValueError('Unknow function variable: ' '{0}'.format(test_dict['function'])) @@ -108,6 +126,10 @@ def run_examples(config_file): print(s.getvalue()) print("".center(100, '-')) + print('Done with examples...printing summary') + for test, outcome in iteritems(test_outcomes): + print('\t{0}: {1}'.format(test, outcome)) + return From d7480a3b8aa9dd1b37aa37cf9f4a0dd80f447eac Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 3 Apr 2015 07:58:26 -0700 Subject: [PATCH 10/47] install cdo and cdo in travis build --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 58319f3..199db89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,10 +14,12 @@ before_install: # The next couple lines fix a crash with multiprocessing on Travis and are not specific to using Miniconda - sudo rm -rf /dev/shm - sudo ln -s /run/shm /dev/shm + - sudo apt-get cdo # Install packages install: - conda create --yes --name=rvic_test_env python=$TRAVIS_PYTHON_VERSION --file=ci/requirements-2.7.txt - source activate rvic_test_env + - pip install cdo - python setup.py install # Run test script: From cc991e5842fae9f32b7368e49ee2aa3cf58a8080 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 3 Apr 2015 08:04:02 -0700 Subject: [PATCH 11/47] update to travis apt-get intall cdo command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 199db89..72673c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: # The next couple lines fix a crash with multiprocessing on Travis and are not specific to using Miniconda - sudo rm -rf /dev/shm - sudo ln -s /run/shm /dev/shm - - sudo apt-get cdo + - sudo apt-get install cdo # Install packages install: - conda create --yes --name=rvic_test_env python=$TRAVIS_PYTHON_VERSION --file=ci/requirements-2.7.txt From 74304ab06fdb1d8bd80e2a9bb29f5b7123955973 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 3 Apr 2015 08:56:07 -0700 Subject: [PATCH 12/47] updates to file headers to include utf-8 character coding and a few new tests for pycompat --- rvic/convert.py | 1 + rvic/convolution.py | 1 + rvic/core/aggregate.py | 4 +- rvic/core/config.py | 1 + rvic/core/convert.py | 1 + rvic/core/convolution_wrapper.py | 1 + rvic/core/history.py | 1 + rvic/core/log.py | 1 + rvic/core/make_uh.py | 1 + rvic/core/multi_proc.py | 1 + rvic/core/param_file.py | 1 + rvic/core/plots.py | 1 + rvic/core/pycompat.py | 1 + rvic/core/read_forcing.py | 1 + rvic/core/remap.py | 1 + rvic/core/share.py | 1 + rvic/core/time_utility.py | 1 + rvic/core/utilities.py | 1 + rvic/core/variables.py | 1 + rvic/core/write.py | 1 + rvic/parameters.py | 1 + tests/unit/test_pycompat.py | 83 ++++++++++++++++++++++++++++++++ tests/unit/test_utilities.py | 3 +- 23 files changed, 107 insertions(+), 3 deletions(-) diff --git a/rvic/convert.py b/rvic/convert.py index 7814d98..05815b9 100644 --- a/rvic/convert.py +++ b/rvic/convert.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Read a set of uhs files and write an RVIC parameter file """ diff --git a/rvic/convolution.py b/rvic/convolution.py index 51f1425..3d4d8ce 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ This is the convolution routine developed in preparation in coupling RVIC to CESM. Eventually, this will be the offline RVIC model. diff --git a/rvic/core/aggregate.py b/rvic/core/aggregate.py index 4e73533..f8e08ea 100644 --- a/rvic/core/aggregate.py +++ b/rvic/core/aggregate.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python +# -*- coding: utf-8 -*- """ aggregate.py """ @@ -50,7 +50,7 @@ def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): # Do the aggregation outlets = {} - for i, (lat, lon) in enumerate(zip(lats, lons)): + for i, (lat, lon) in enumerate(pyzip(lats, lons)): # Define pour point object pour_point = Point(lat=lat, lon=lon, diff --git a/rvic/core/config.py b/rvic/core/config.py index 25e4eb2..8d8a183 100644 --- a/rvic/core/config.py +++ b/rvic/core/config.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ config.py diff --git a/rvic/core/convert.py b/rvic/core/convert.py index 72618b5..c86f202 100644 --- a/rvic/core/convert.py +++ b/rvic/core/convert.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ convert.py """ diff --git a/rvic/core/convolution_wrapper.py b/rvic/core/convolution_wrapper.py index d80bf6a..ad62fbf 100644 --- a/rvic/core/convolution_wrapper.py +++ b/rvic/core/convolution_wrapper.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ convolution_wrapper.py diff --git a/rvic/core/history.py b/rvic/core/history.py index 4ea823e..e619eeb 100644 --- a/rvic/core/history.py +++ b/rvic/core/history.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ history.py diff --git a/rvic/core/log.py b/rvic/core/log.py index 1b3de4d..f0cce2c 100644 --- a/rvic/core/log.py +++ b/rvic/core/log.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ log.py """ diff --git a/rvic/core/make_uh.py b/rvic/core/make_uh.py index 4544c9d..fe9013d 100644 --- a/rvic/core/make_uh.py +++ b/rvic/core/make_uh.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ make_uh.py diff --git a/rvic/core/multi_proc.py b/rvic/core/multi_proc.py index b44d945..29640b2 100644 --- a/rvic/core/multi_proc.py +++ b/rvic/core/multi_proc.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ multi_proc.py """ diff --git a/rvic/core/param_file.py b/rvic/core/param_file.py index 1dd533a..eb6df3c 100644 --- a/rvic/core/param_file.py +++ b/rvic/core/param_file.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ param_file.py """ diff --git a/rvic/core/plots.py b/rvic/core/plots.py index b8cf707..248dfd0 100644 --- a/rvic/core/plots.py +++ b/rvic/core/plots.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ plots.py """ diff --git a/rvic/core/pycompat.py b/rvic/core/pycompat.py index 17bc754..ad3eabb 100644 --- a/rvic/core/pycompat.py +++ b/rvic/core/pycompat.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import sys PY3 = sys.version_info[0] >= 3 diff --git a/rvic/core/read_forcing.py b/rvic/core/read_forcing.py index 68214a0..48d9332 100644 --- a/rvic/core/read_forcing.py +++ b/rvic/core/read_forcing.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ read_forcings.py """ diff --git a/rvic/core/remap.py b/rvic/core/remap.py index 05ea14b..850c364 100644 --- a/rvic/core/remap.py +++ b/rvic/core/remap.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ remap.py """ diff --git a/rvic/core/share.py b/rvic/core/share.py index 7cdb1f3..583fed2 100644 --- a/rvic/core/share.py +++ b/rvic/core/share.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ share.py """ diff --git a/rvic/core/time_utility.py b/rvic/core/time_utility.py index 999554a..f10084a 100644 --- a/rvic/core/time_utility.py +++ b/rvic/core/time_utility.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ time_utility.py diff --git a/rvic/core/utilities.py b/rvic/core/utilities.py index db2866c..082bd36 100644 --- a/rvic/core/utilities.py +++ b/rvic/core/utilities.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ utilities.py """ diff --git a/rvic/core/variables.py b/rvic/core/variables.py index bc0a8b4..1744aa1 100644 --- a/rvic/core/variables.py +++ b/rvic/core/variables.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ variables.py """ diff --git a/rvic/core/write.py b/rvic/core/write.py index 16b9e6b..ec1c996 100644 --- a/rvic/core/write.py +++ b/rvic/core/write.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ write.py """ diff --git a/rvic/parameters.py b/rvic/parameters.py index 168a631..5cd58ed 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ RVIC parameter file development driver """ diff --git a/tests/unit/test_pycompat.py b/tests/unit/test_pycompat.py index e69de29..d2d3b7b 100644 --- a/tests/unit/test_pycompat.py +++ b/tests/unit/test_pycompat.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +import pytest +from rvic.core.pycompat import (pyrange, pyzip, iteritems, itervalues, + OrderedDict, SafeConfigParser) + + +def test_pyrange(): + x = pyrange(10) + assert hasattr(x, '__iter__') + assert len(x) == 10 + assert x[9] == 9 + + +def test_pyzip(): + a = [1, 2, 3] + b = ['a', 'b', 'c'] + x = pyzip(a, b) + assert hasattr(x, '__iter__') + for i, (aa, bb) in enumerate(x): + assert aa == a[i] + assert bb == b[i] + with pytest.raises(TypeError): + len(x) + x[1] + + +def test_iteritems(): + a = [1, 2, 3] + b = ['a', 'b', 'c'] + d = dict(pyzip(a, b)) + for k, v in iteritems(d): + assert k in a + assert v in b + assert d[k] == v + + +def test_iteritems_ordered_dict(): + a = [1, 2, 3] + b = ['a', 'b', 'c'] + d = OrderedDict(pyzip(a, b)) + for i, (k, v) in enumerate(iteritems(d)): + assert k in a + assert v in b + assert d[k] == v + assert b[i] == v + + +def test_itervalues(): + a = [1, 2, 3] + b = ['a', 'b', 'c'] + d = dict(pyzip(a, b)) + for v in itervalues(d): + assert v in b + + +def test_itervalues_ordered_dict(): + a = [1, 2, 3] + b = ['a', 'b', 'c'] + d = OrderedDict(pyzip(a, b)) + for i, v in enumerate(itervalues(d)): + assert v in b + assert b[i] == v + + +def test_ordered_dict_order(): + a = [1, 2, 3] + b = ['a', 'b', 'c'] + d = OrderedDict(pyzip(a, b)) + assert list(d.keys()) == a + assert list(d.values()) == b + + +def test_ordered_dict_append(): + a = [1, 2, 3] + b = ['a', 'b', 'c'] + d = OrderedDict(pyzip(a, b)) + d[4] = 'd' + assert len(d) == 4 + assert list(d.keys())[-1] == 4 + + +def test_init_safe_config_parser(): + SafeConfigParser() diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index a339b65..17921bd 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -1,10 +1,11 @@ +# -*- coding: utf-8 -*- import numpy as np import os import datetime from rvic.core.utilities import (latlon2yx, search_for_channel, find_nearest, write_rpointer, strip_non_ascii, strip_invalid_char) -from rvic.core.share import RPOINTER, VALID_CHARS +from rvic.core.share import RPOINTER def test_latlon2yx(): From 532eaa8659453fa38249570494bb1041ed9e2ed8 Mon Sep 17 00:00:00 2001 From: ymao Date: Mon, 6 Apr 2015 16:34:58 -0700 Subject: [PATCH 13/47] Change mask reading from 'mask==1' to 'mask>np.finfo(np.float).resolution', so that the model can handle imperfect input mask file (i.e., if the mask values are not exactly 1) --- rvic/parameters.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rvic/parameters.py b/rvic/parameters.py index 5cd58ed..5fc7ef3 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -332,7 +332,6 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, dom_lat = domain['LATITUDE_VAR'] dom_lon = domain['LONGITUDE_VAR'] dom_mask = domain['LAND_MASK_VAR'] - options = config_dict['OPTIONS'] # ------------------------------------------------------------ # @@ -462,7 +461,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, # ------------------------------------------------------------ # # Add to "adjust fractions structure" - y, x = np.nonzero((final_fracs > 0.0) * (dom_data[dom_mask] == 1)) + y, x = np.nonzero((final_fracs > 0.0) * (dom_data[dom_mask] > np.finfo(np.float).resolution)) yi = y - y0 xi = x - x0 From e4ca0e0ac26be309482df6e6cb9f6d17d8585276 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 9 Apr 2015 10:20:36 -0700 Subject: [PATCH 14/47] replace forcing fill_values with numpy.nan, then check for nans before writing history files, if nans are present at that stage, RVIC exists gracefully --- rvic/core/history.py | 6 ++++++ rvic/core/read_forcing.py | 35 +++++++++++++++-------------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/rvic/core/history.py b/rvic/core/history.py index e619eeb..aa54481 100644 --- a/rvic/core/history.py +++ b/rvic/core/history.py @@ -385,6 +385,12 @@ def __update_out_data(self): else: self._out_data[field][self._out_data_i, :] = \ self._temp_data[field] * self._units_mult + + # Check that all values are valid, if not, exit gracefully + if np.isnan(self._out_data[field][self._out_data_i].sum()): + raise ValueError('nan found in output field: {0}, most likely ' + 'there is a nan/missing/fill value in the' + 'input forcings') # ------------------------------------------------------------ # self._out_times[self._out_data_i] = self._write_ord diff --git a/rvic/core/read_forcing.py b/rvic/core/read_forcing.py index 5cdfd42..f017522 100644 --- a/rvic/core/read_forcing.py +++ b/rvic/core/read_forcing.py @@ -298,27 +298,22 @@ def read(self, timestamp): # Get the liquid fluxes log.info('getting fluxes for %s (%s)', timestamp, self.current_tind) forcing = {} - for i, fld in enumerate(self.liq_flds): - self.current_fhdl.variables[fld].set_auto_maskandscale(False) - if i == 0: - forcing['LIQ'] = \ - (self.fld_mult[fld] * - self.current_fhdl.variables[fld][self.current_tind, :, :]) - else: - forcing['LIQ'] += \ - (self.fld_mult[fld] * - self.current_fhdl.variables[fld][self.current_tind, :, :]) + for flux, fields in (('LIQ', self.liq_flds), ('ICE', self.ice_flds)): + for i, fld in enumerate(fields): + # set auto_maskandscale to False to ovoid slowdown from numpy + # masked array + self.current_fhdl.variables[fld].set_auto_maskandscale(False) + temp = self.current_fhdl.variables[fld][self.current_tind] + fill_val = getattr(self.current_fhdl.variables[fld], + '_FillValue', None) + # replace fill values with numpy.nan + if fill_val is not None: + temp[temp == fill_val] = np.nan - for i, fld in enumerate(self.ice_flds): - self.current_fhdl.variables[fld].set_auto_maskandscale(False) - if i == 0: - forcing['ICE'] = \ - (self.fld_mult[fld] * - self.current_fhdl.variables[fld][self.current_tind, :, :]) - else: - forcing['ICE'] += \ - (self.fld_mult[fld] * - self.current_fhdl.variables[fld][self.current_tind, :, :]) + if i == 0: + forcing[flux] = self.fld_mult[fld] * temp + else: + forcing[flux] += self.fld_mult[fld] * temp # move forward one step self.current_tind += 1 From 0d8ddcc5a6261a499838837d76f39415729d3b07 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 9 Apr 2015 14:36:23 -0700 Subject: [PATCH 15/47] fix a few spelling errors --- rvic/core/read_forcing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rvic/core/read_forcing.py b/rvic/core/read_forcing.py index f017522..6d45472 100644 --- a/rvic/core/read_forcing.py +++ b/rvic/core/read_forcing.py @@ -205,13 +205,14 @@ def start(self, timestamp, rout_var): # ------------------------------------------------------------ # # find multiplier for units all fields will be in liquid flux # (kg m-2 s-1) + # Todo: use udunits to convert units for fld in self.liq_flds: units = self.current_fhdl.variables[fld].units if units in ['kg/m2*s', 'kg m-2 s-1', 'kg m^-2 s^-1', 'kg*m-2*s-1', 'kg s-1 m-2']: self.fld_mult[fld] = 1.0 - elif units in ['mm', 'MM', 'milimeters', 'Milimeters', 'mm.d-1']: + elif units in ['mm', 'MM', 'millimeters', 'Millimeters']: self.fld_mult[fld] = (WATERDENSITY / MMPERMETER / self.secs_per_step) elif units in ['m', 'M', 'meters', 'Meters']: @@ -300,7 +301,7 @@ def read(self, timestamp): forcing = {} for flux, fields in (('LIQ', self.liq_flds), ('ICE', self.ice_flds)): for i, fld in enumerate(fields): - # set auto_maskandscale to False to ovoid slowdown from numpy + # set auto_maskandscale to False to avoid slowdown from numpy # masked array self.current_fhdl.variables[fld].set_auto_maskandscale(False) temp = self.current_fhdl.variables[fld][self.current_tind] From 0da8b15ccbdde01e8e9b6cd27b2faf284382bba9 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 14 Apr 2015 09:51:15 -0700 Subject: [PATCH 16/47] allow remap==False if grid resolution matches but domain grid and routing grid are not identical, raises error if resolutions do not match --- rvic/core/make_uh.py | 20 ++++++++------ rvic/parameters.py | 64 +++++++++++++++++++++++++++----------------- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/rvic/core/make_uh.py b/rvic/core/make_uh.py index fe9013d..dbba2b8 100644 --- a/rvic/core/make_uh.py +++ b/rvic/core/make_uh.py @@ -87,13 +87,17 @@ def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): # ---------------------------------------------------------------- # # Create the rout_data Dictionary rout_data = {'lat': basin['lat'], 'lon': basin['lon'], - 'dom_x_min': x_min, 'dom_y_min': y_min, - 'dom_x_max': x_max, 'dom_y_max': y_max} + 'basiny': pour_point.basiny[0], + 'basinx': pour_point.basinx[0]} log.debug('Clipping indicies:') - log.debug('dom_x_min: %s', rout_data['dom_x_min']) - log.debug('dom_x_max: %s', rout_data['dom_x_max']) - log.debug('dom_y_min: %s', rout_data['dom_y_min']) - log.debug('dom_y_max: %s', rout_data['dom_y_max']) + log.debug('rout_x_min: %s', x_min) + log.debug('rout_x_max: %s', x_max) + log.debug('rout_y_min: %s', y_min) + log.debug('rout_y_max: %s', y_max) + log.debug('pour_point.basiny: %s', pour_point.basiny) + log.debug('pour_point.basinx: %s', pour_point.basinx) + # ---------------------------------------------------------------- # + # ---------------------------------------------------------------- # # Determine/Set flow direction syntax # Flow directions {north, northeast, east, southeast, @@ -159,8 +163,8 @@ def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): # ---------------------------------------------------------------- # # Agregate to output timestep rout_data['unit_hydrograph'] = adjust_uh_timestep( - uh_s, t_uh, input_interval, rout_dict['OUTPUT_INTERVAL'], - catchment['x_inds'], catchment['y_inds']) + uh_s, t_uh, input_interval, rout_dict['OUTPUT_INTERVAL'], + catchment['x_inds'], catchment['y_inds']) # ---------------------------------------------------------------- # return rout_data # -------------------------------------------------------------------- # diff --git a/rvic/parameters.py b/rvic/parameters.py index 5fc7ef3..b9e7885 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -50,12 +50,12 @@ def parameters(config_file, numofproc=1): for i, outlet in enumerate(outlets.values()): log.info('On Outlet #%s of %s', i + 1, len(outlets)) - status = pool.apply_async(gen_uh_run, - args=(uh_box, fdr_data, fdr_vatts, - dom_data, outlet, config_dict, - directories), - callback=store_result, - error_callback=pool.terminate) + pool.apply_async(gen_uh_run, + args=(uh_box, fdr_data, fdr_vatts, + dom_data, outlet, config_dict, + directories), + callback=store_result, + error_callback=pool.terminate) pool.close() pool.join() @@ -237,6 +237,16 @@ def gen_uh_init(config_file): log.error('RVIC parameter generation requires REMAP option to be True' ' if AGGREGATE is True') raise ValueError('Invalid option') + + # If remap is False, then the resolution needs to match the routing data + if not options['REMAP']: + domain_res = np.abs(dom_data[domain['LONGITUDE_VAR']][0, 1] - + dom_data[domain['LONGITUDE_VAR']][0, 0]) + if not np.isclose(fdr_data['resolution'], domain_res): + log.error('routing grid resolution: %s', fdr_data['resolution']) + log.error('domain grid resolution: %s', domain_res) + raise ValueError('If remap is false, domain and routing grid ' + 'resolutions must match.') # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -439,36 +449,40 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, clean_file(temp_file_2) # -------------------------------------------------------- # - # -------------------------------------------------------- # - # Set the domain index offest to zero and use the remapped fraction - # as the final fractions - y0, x0 = 0, 0 - final_fracs = final_data['fraction'] - # -------------------------------------------------------- # else: # -------------------------------------------------------- # # Put the agg data back onto the original grid - final_data = agg_data - final_fracs = np.zeros_like(fdr_data['velocity'], - dtype=np.float64) - x0 = final_data['dom_x_min'] - x1 = final_data['dom_x_max'] - y0 = final_data['dom_y_min'] - y1 = final_data['dom_y_max'] - final_fracs[y0:y1, x0:x1] = final_data['fraction'] + uh_shape = (agg_data['unit_hydrograph'].shape[0], ) + \ + dom_data[dom_mask].shape + final_data = {} + final_data['unit_hydrograph'] = np.zeros(uh_shape, dtype=np.float64) + final_data['fraction'] = np.zeros(dom_data[dom_mask].shape, + dtype=np.float64) + + bys, bxs = np.nonzero(agg_data['fraction']) + + ys = bys + pour_point.domy - agg_data['basiny'] + xs = bxs + pour_point.domx - agg_data['basinx'] + + if (ys < 0).any() or (xs < 0).any(): + raise ValueError('Negative indicies found when mapping ' + '`non-remapped` rout_data to domain grid.') + + final_data['unit_hydrograph'][:, ys, xs] = \ + agg_data['unit_hydrograph'][:, bys, bxs] + final_data['fraction'][ys, xs] = agg_data['fraction'][bys, bxs] # -------------------------------------------------------- # # ------------------------------------------------------------ # # ------------------------------------------------------------ # # Add to "adjust fractions structure" - y, x = np.nonzero((final_fracs > 0.0) * (dom_data[dom_mask] > np.finfo(np.float).resolution)) - yi = y - y0 - xi = x - x0 + y, x = np.nonzero((final_data['fraction'] > 0.) * + (dom_data[dom_mask] > np.finfo(np.float).resolution)) # From final data outlet.time = np.arange(final_data['unit_hydrograph'].shape[0]) - outlet.fractions = final_data['fraction'][yi, xi] - outlet.unit_hydrograph = final_data['unit_hydrograph'][:, yi, xi] + outlet.fractions = final_data['fraction'][y, x] + outlet.unit_hydrograph = final_data['unit_hydrograph'][:, y, x] # From domain data outlet.lon_source = dom_data[dom_lon][y, x] From 3cd16e4f9bca6da322cd87e9743b8e3f1db51e10 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 14 Apr 2015 10:21:32 -0700 Subject: [PATCH 17/47] update documentation for remap==False option --- docs/user-guide/parameters.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/user-guide/parameters.md b/docs/user-guide/parameters.md index cc14ff3..8be1c98 100644 --- a/docs/user-guide/parameters.md +++ b/docs/user-guide/parameters.md @@ -58,6 +58,8 @@ For irregular grids, you need to come up with a netCDF file that contains these - area (in m2 or rad2 or similar) - coordinate vars (lon and lat or similar) +*Note: If `REMAP==False`, the resolution of the routing data grid and domain grid must be the same. Furthermore, the domain grid must be a subset of the routing grid. RVIC will raise an error if either of these conditions are not met.* + ## Pour Points File A pour points CSV is needed to develop the parameter file. This is simply a csv file containing a list of the pour point locations with 2 or 3 columns (required: lons, lats; optional: names/contributing area/etc). One header row is required. From 51a775d9a00f454aa825baf355419e6a29bb0830 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 14 Apr 2015 10:30:41 -0700 Subject: [PATCH 18/47] minor fix to source of basinx and basiny in gen_uh_run --- rvic/parameters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rvic/parameters.py b/rvic/parameters.py index b9e7885..f0bd883 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -461,8 +461,8 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, bys, bxs = np.nonzero(agg_data['fraction']) - ys = bys + pour_point.domy - agg_data['basiny'] - xs = bxs + pour_point.domx - agg_data['basinx'] + ys = bys + outlet.domy - agg_data['basiny'] + xs = bxs + outlet.domx - agg_data['basinx'] if (ys < 0).any() or (xs < 0).any(): raise ValueError('Negative indicies found when mapping ' From 3cb4048ff01ce0a01528c1e9e605507f6237db9a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 14 Apr 2015 12:17:09 -0700 Subject: [PATCH 19/47] fix typo in parameters docs --- docs/user-guide/parameters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/parameters.md b/docs/user-guide/parameters.md index 8be1c98..ef8526d 100644 --- a/docs/user-guide/parameters.md +++ b/docs/user-guide/parameters.md @@ -58,7 +58,7 @@ For irregular grids, you need to come up with a netCDF file that contains these - area (in m2 or rad2 or similar) - coordinate vars (lon and lat or similar) -*Note: If `REMAP==False`, the resolution of the routing data grid and domain grid must be the same. Furthermore, the domain grid must be a subset of the routing grid. RVIC will raise an error if either of these conditions are not met.* +*Note: If `REMAP==False`, the resolution of the routing data grid and domain grid must be the same. Furthermore, the domain grid must be a subset of the routing grid. RVIC will raise an error if either of these conditions is not met.* ## Pour Points File A pour points CSV is needed to develop the parameter file. This is simply a csv file containing a list of the pour point locations with 2 or 3 columns (required: lons, lats; optional: names/contributing area/etc). One header row is required. From a46a934d91b30b10140eecea9a4c764202f86f66 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 12 May 2015 08:49:05 -0700 Subject: [PATCH 20/47] tests and cleanup for multiprocessing, two tests still failing! --- rvic/convolution.py | 4 +- rvic/core/history.py | 2 +- rvic/core/log.py | 1 + rvic/core/multi_proc.py | 13 ++++-- rvic/core/param_file.py | 40 ++++++++-------- rvic/parameters.py | 34 +++++++------- tests/unit/test_multi_proc.py | 87 ++++++++++++++++++++++++++++++++++- 7 files changed, 135 insertions(+), 46 deletions(-) diff --git a/rvic/convolution.py b/rvic/convolution.py index 3d4d8ce..39f5a6d 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -38,8 +38,8 @@ def convolution(config_file): # ---------------------------------------------------------------- # # Initilize - hist_tapes, data_model, rout_var, dom_data,\ - time_handle, directories, config_dict = convolution_init(config_file) + hist_tapes, data_model, rout_var, \ + time_handle, directories = convolution_init(config_file) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # diff --git a/rvic/core/history.py b/rvic/core/history.py index aa54481..7bd2bef 100644 --- a/rvic/core/history.py +++ b/rvic/core/history.py @@ -43,7 +43,7 @@ class Tape(object): # ---------------------------------------------------------------- # # Init def __init__(self, time_ord, caseid, rvar, tape_num=0, - fincl=['streamflow'], mfilt=1, ndens=2, nhtfrq=0, + fincl=('streamflow'), mfilt=1, ndens=2, nhtfrq=0, avgflag='A', units='kg m-2 s-1', file_format='NETCDF4_CLASSIC', outtype='grid', grid_lons=False, grid_lats=False, grid_area=None, out_dir='.', diff --git a/rvic/core/log.py b/rvic/core/log.py index f0cce2c..28d143c 100644 --- a/rvic/core/log.py +++ b/rvic/core/log.py @@ -91,3 +91,4 @@ def close_logger(): i.close() sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ +# -------------------------------------------------------------------- # diff --git a/rvic/core/multi_proc.py b/rvic/core/multi_proc.py index 29640b2..b535eb3 100644 --- a/rvic/core/multi_proc.py +++ b/rvic/core/multi_proc.py @@ -9,9 +9,9 @@ import traceback -def error(*args): +def error(msg, *args): """ Error function""" - return multiprocessing.get_logger(LOG_NAME).error(*args) + return multiprocessing.get_logger(LOG_NAME).error(msg, *args) # -------------------------------------------------------------------- # @@ -34,12 +34,17 @@ def __call__(self, *args, **kwargs): # It was fine, give a normal answer return result + pass # -------------------------------------------------------------------- # class LoggingPool(Pool): """Subclass of pool""" - def apply_async(self, func, callback=None, *args, **kwargs): - return Pool.apply_async(self, LogExceptions(func), args, kwargs, + def apply_async(self, func, args=(), kwds={}, callback=None): + return Pool.apply_async(self, LogExceptions(func), args, kwds, callback) + + # def map_async(self, func, callback=None, *args, **kwargs): + # return Pool.map_async(self, LogExceptions(func), args, kwargs, + # callback) # -------------------------------------------------------------------- # diff --git a/rvic/core/param_file.py b/rvic/core/param_file.py index eb6df3c..d9d0b69 100644 --- a/rvic/core/param_file.py +++ b/rvic/core/param_file.py @@ -31,7 +31,7 @@ def finish_params(outlets, dom_data, config_dict, directories): dom_area = domain['AREA_VAR'] dom_frac = domain['FRACTION_VAR'] - if not len(outlets.keys()) > 0: + if not len(outlets) > 0: raise ValueError('outlets in finish_params are empty') # ------------------------------------------------------------ # @@ -71,7 +71,7 @@ def finish_params(outlets, dom_data, config_dict, directories): else: log.info('Not subsetting because either SUBSET_DAYS is null or ' 'SUBSET_DAYS 1: pool = LoggingPool(processes=numofproc) - for i, outlet in enumerate(outlets.values()): + for i, outlet in enumerate(outlets): log.info('On Outlet #%s of %s', i + 1, len(outlets)) pool.apply_async(gen_uh_run, - args=(uh_box, fdr_data, fdr_vatts, - dom_data, outlet, config_dict, - directories), - callback=store_result, - error_callback=pool.terminate) + args=(uh_box, fdr_data, fdr_vatts, dom_data, + outlet, config_dict, directories), + callback=store_result) pool.close() pool.join() - outlets = OrderedDict(sorted(list(iteritems(results)), - key=lambda t: t[0])) + outlets = sorted(results, key=lambda x: x.cell_id, reverse=True) else: - for outlet in outlets.values(): + for outlet in outlets: outlet = gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories) - if not len(outlets.keys()) > 0: + if not len(outlets) > 0: raise ValueError('outlets in parameters are empty') # ---------------------------------------------------------------- # @@ -258,7 +258,7 @@ def gen_uh_init(config_file): 'pour points and outlet grid cells') else: - outlets = {} + outlets = [] if all(x in list(pour_points.keys()) for x in ['x', 'y', 'lons', 'lats']): lats = pour_points['lats'].values @@ -503,7 +503,7 @@ def gen_uh_final(outlets, dom_data, config_dict, directories): log.info('In gen_uh_final') - if not len(outlets.keys()) > 0: + if not len(outlets) > 0: raise ValueError('outlets in gen_uh_final are empty') # ---------------------------------------------------------------- # @@ -531,10 +531,8 @@ def gen_uh_final(outlets, dom_data, config_dict, directories): # -------------------------------------------------------------------- # # store_result helper function def store_result(result): - # This is called whenever foo_pool(i) returns a result. + # This is called whenever gen_uh_run() returns a result. # result_list is modified only by the main process, not the pool workers. - print('in store_result') - print('storing result for %s' % result.cell_id) - results[result.cell_id] = result + global results + results.append(result) # -------------------------------------------------------------------- # -results = {} diff --git a/tests/unit/test_multi_proc.py b/tests/unit/test_multi_proc.py index fcc8fdd..428dc7f 100644 --- a/tests/unit/test_multi_proc.py +++ b/tests/unit/test_multi_proc.py @@ -1,7 +1,92 @@ +import pytest +from time import sleep +import multiprocessing +from rvic.core.multi_proc import LoggingPool as Pool from rvic.core.multi_proc import LogExceptions +from rvic.core.pycompat import pyrange, bytes_type +from rvic.core.variables import Point +import pickle +from rvic.parameters import store_result, results + +multiprocessing.log_to_stderr() + + +@pytest.fixture() +def pool(scope='function'): + return Pool() + + +@pytest.fixture() +def point(scope='function'): + return Point() def test_create_log_exceptions(): LogExceptions(callable) -# error and logging pool not easily testing in unit test. + +def test_pool_raises(pool): + def go_exception(): + print(1) + sleep(0.5) + raise Exception() + print(2) + return 3 + + def go_value_error(): + print(1) + sleep(0.5) + raise ValueError() + print(2) + return 3 + + with pytest.raises(Exception): + for i in pyrange(3): + pool.apply_async(go_exception) + pool.close() + pool.join() + + with pytest.raises(ValueError): + for i in pyrange(3): + pool.apply_async(go_value_error) + pool.close() + pool.join() + + +def test_pool_callback(pool): + + class Foo(object): + '''simple object to hold a few attributes''' + + def __init__(self, i): + self.cell_id = i + self.cube = i ** 3 + + def __repr__(self): + return "%s: %s" % (self.cell_id, self.cube) + + def __str__(self): + return "%s: %s" % (self.cell_id, self.cube) + + def go_foo(i): + '''create a foo object''' + sleep(0.5) + return Foo(i) + + n = 5 + + for i in pyrange(n): + pool.apply_async(go_foo, store_result, i) + pool.close() + pool.join() + + assert type(results) == list + assert len(results) == n + for foo in results: + assert type(foo) == Foo + assert foo.cube == foo.cell_id ** 3 + + +def test_pickle_point(point): + pickle_string = pickle.dumps(point) + assert type(pickle_string) == bytes_type From 0b45852e0492fb91b6c36bde2384b6ed4ca4cf39 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 12 May 2015 09:09:07 -0700 Subject: [PATCH 21/47] two small fixes to problems indicated by landsape build --- rvic/convolution.py | 3 +-- rvic/core/multi_proc.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/rvic/convolution.py b/rvic/convolution.py index 39f5a6d..806944f 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -215,8 +215,7 @@ def convolution_init(config_file): tape.write_initial() # ---------------------------------------------------------------- # - return (hist_tapes, data_model, rout_var, dom_data, time_handle, - directories, config_dict) + return hist_tapes, data_model, rout_var, time_handle, directories # -------------------------------------------------------------------- # diff --git a/rvic/core/multi_proc.py b/rvic/core/multi_proc.py index b535eb3..40bca17 100644 --- a/rvic/core/multi_proc.py +++ b/rvic/core/multi_proc.py @@ -34,7 +34,6 @@ def __call__(self, *args, **kwargs): # It was fine, give a normal answer return result - pass # -------------------------------------------------------------------- # From 8a46aee189191d13406d097a74116edb62ed1eb3 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 12 May 2015 12:16:49 -0700 Subject: [PATCH 22/47] add exit code to run_tests.py so travis tests look like they failed when they fail --- tests/run_tests.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/run_tests.py b/tests/run_tests.py index a3b0e61..533e3f9 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -2,6 +2,7 @@ """RVIC command line testing interface""" import os +import sys import textwrap import argparse import pytest @@ -30,6 +31,7 @@ def main(): """ Run RVIC tests """ + exit_code = 0 # Parse arguments parser = argparse.ArgumentParser(description='Test script for RVIC') @@ -46,9 +48,10 @@ def main(): if any(i in ['all', 'unit'] for i in args.test_set): # run unit tests - pytest.main('-x unit') + exit_code = pytest.main('-x unit') if any(i in ['all', 'examples'] for i in args.test_set): - run_examples(args.examples) + exit_code = run_examples(args.examples) + sys.exit(exit_code) return # -------------------------------------------------------------------- # @@ -130,7 +133,7 @@ def run_examples(config_file): for test, outcome in iteritems(test_outcomes): print('\t{0}: {1}'.format(test, outcome)) - return + return 0 # -------------------------------------------------------------------- # From a135b6561b62e9c1141983e7f0d8fc90dbf452f6 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 15 Jun 2015 14:30:16 -0700 Subject: [PATCH 23/47] add numpy style doc strings to rvic.parameters and rvic.convolution routines --- rvic/convolution.py | 72 +++++++++++++++++++++++++++++-- rvic/parameters.py | 102 ++++++++++++++++++++++++++++++++++++++++++-- setup.py | 4 +- 3 files changed, 168 insertions(+), 10 deletions(-) diff --git a/rvic/convolution.py b/rvic/convolution.py index 3d4d8ce..0846a26 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -34,6 +34,11 @@ def convolution(config_file): """ Top level driver for RVIC convolution model. + + Parameters + ---------- + config_file : str + Path to RVIC convolution configuration file. """ # ---------------------------------------------------------------- # @@ -66,9 +71,40 @@ def convolution(config_file): # Initialize RVIC def convolution_init(config_file): """ - - Read Grid File - - Load the unit hydrograph files (put into point_dict) - - Load the initial state file and put it in convolution rings + Initialize the RVIC convolution routine + + This function performs these main tasks: + - Reads the configuration file + - Sets up the RVIC case directories + - Copies all input files to the case directory + - Initializes the logging + - Read domain file + - Load the RVIC parameter file + - Load the initial state file and put it in convolution rings + - Setup time and history file objects + + Parameters + ---------- + config_file : str + Path to RVIC convolution configuration file. + + Returns + ---------- + hist_tapes : OrderedDict + Ordered dictionary of History objects + data_model : DataModel + DataModel instance containing the forcings + rout_var : Rvar + Rvar instance containing the RVIC parameters / unit hydrographs + dom_data : dict + Dictionary of arrays of mask, fraction, lats, lons, etc. + This dictionary includes all the variables from the domain netCDF file. + time_handle : Dtime + Dtime instance containing information about run length, time + directories : dict + Dictionary of directories created by this function. + config_dict : dict + Dictionary of values from the configuration file. """ # ---------------------------------------------------------------- # @@ -225,6 +261,26 @@ def convolution_run(hist_tapes, data_model, rout_var, time_handle, directories): """ Main run loop for RVIC model. + + Parameters + ---------- + hist_tapes : OrderedDict + Ordered dictionary of History objects + data_model : DataModel + DataModel instance containing the forcings + rout_var : Rvar + Rvar instance containing the RVIC parameters / unit hydrographs + time_handle : Dtime + Dtime instance containing information about run length, time + directories : dict + Dictionary of directories created by this function. + + Returns + ---------- + time_handle : Dtime + Dtime instance containing information about run length, time + hist_tapes : OrderedDict + Ordered dictionary of History objects """ data2tape = {} @@ -325,7 +381,15 @@ def convolution_run(hist_tapes, data_model, rout_var, time_handle, # -------------------------------------------------------------------- # # Final def convolution_final(time_handle, hist_tapes): - """ Finalize RVIC Convolution""" + """ Finalize RVIC Convolution + + Parameters + ---------- + time_handle : Dtime + Dtime instance containing information about run length, time + hist_tapes : OrderedDict + Ordered dictionary of History objects + """ # ---------------------------------------------------------------- # # Start log log = getLogger(LOG_NAME) diff --git a/rvic/parameters.py b/rvic/parameters.py index f0bd883..deb8da5 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -31,6 +31,16 @@ # -------------------------------------------------------------------- # # Top level driver def parameters(config_file, numofproc=1): + ''' + Top level function for RVIC parameter generation function. + + Parameters + ---------- + config_file : str + Path to RVIC parameters configuration file. + numofproc : int + Number of processors to use when developing RVIC parameters. + ''' # ---------------------------------------------------------------- # # Initilize @@ -79,7 +89,40 @@ def parameters(config_file, numofproc=1): def gen_uh_init(config_file): - """Initialize RVIC parameter""" + """Initialize RVIC parameters scrpt. + + This function: + - Reads the configuration file + - Sets up the RVIC case directories + - Copies all input files to the case directory + - Initializes the logging + - Reads the pour-points, uh-box, FDR, and domain files + - Aggregates pour points into outlet grid cells + + Parameters + ---------- + config_file : str + Path to RVIC parameters configuration file. + + Returns + ---------- + uh_box : numpy.ndarray + UH-box array. + fdr_data : dict + Dictionary of arrays of flow direction, velocity, diffusion, etc. + This dictionary includes all the variables from the FDR netCDF file. + fdr_vatts : dict + Dictionary of attributes from the FDR netCDF file. + dom_data : dict + Dictionary of arrays of mask, fraction, lats, lons, etc. + This dictionary includes all the variables from the domain netCDF file. + outlets : dict + Dictionary of outlet `Point` objects. + config_dict : dict + Dictionary of values from the configuration file. + directories : dict + Dictionary of directories created by this function. + """ # ---------------------------------------------------------------- # # Read Configuration files @@ -331,7 +374,31 @@ def gen_uh_init(config_file): def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories): """ - Run Genuh_run + Develop unit hydrographs for one outlet `Point`. + + Parameters + ---------- + uh_box : numpy.ndarray + UH-box array. + fdr_data : dict + Dictionary of arrays of flow direction, velocity, diffusion, etc. + This dictionary includes all the variables from the FDR netCDF file. + fdr_vatts : dict + Dictionary of attributes from the FDR netCDF file. + dom_data : dict + Dictionary of arrays of mask, fraction, lats, lons, etc. + This dictionary includes all the variables from the domain netCDF file. + outlet : Point + Outlet Point + config_dict : dict + Dictionary of values from the configuration file. + directories : dict + Dictionary of directories created by gen_uh_init. + + Returns + ---------- + outlet: Point + Point object with unit hydrographs. """ log = getLogger(LOG_NAME) @@ -498,6 +565,18 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, def gen_uh_final(outlets, dom_data, config_dict, directories): """ Make the RVIC Parameter File + + Parameters + ---------- + outlets : dict + Dictionary of outlet `Point` objects. + dom_data : dict + Dictionary of arrays of mask, fraction, lats, lons, etc. + This dictionary includes all the variables from the domain netCDF file. + config_dict : dict + Dictionary of values from the configuration file. + directories : dict + Dictionary of directories created by gen_uh_init. """ log = getLogger(LOG_NAME) @@ -531,8 +610,23 @@ def gen_uh_final(outlets, dom_data, config_dict, directories): # -------------------------------------------------------------------- # # store_result helper function def store_result(result): - # This is called whenever foo_pool(i) returns a result. - # result_list is modified only by the main process, not the pool workers. + ''' + Store values returned by a multiprocessing.pool member. + + This is called whenever foo_pool(i) returns a result. + result_list is modified only by the main process, not the pool workers. + + Parameters + ---------- + result : object + Result to append to the global `results` list + + Globals + ---------- + results : list + Global results container for multiprocessing results to be appended to. + ''' + print('in store_result') print('storing result for %s' % result.cell_id) results[result.cell_id] = result diff --git a/setup.py b/setup.py index 9d02d3c..6633266 100644 --- a/setup.py +++ b/setup.py @@ -101,12 +101,12 @@ def write_version_py(filename=None): 'original model of Lohmann, et al., 1996, Tellus, ' '48(A), 708-721', author='Joe Hamman', - author_email='jhamman@hydro.washington.edu', + author_email='jhamman1@uw.edu', install_requires=['scipy >= 0.13', 'numpy >= 1.8', 'netCDF4 >= 1.0.6', 'matplotlib >= 1.3.1', 'pandas >= 0.15.1'], tests_require=['pytest >= 2.5.2'], - url='https://github.com/jhamman/RVIC', + url='https://github.com/UW-Hydro/RVIC', test_suite='pytest.collector', packages=['rvic', 'rvic.core'], py_modules=['rvic.parameters', 'rvic.convolution', 'rvic.convert'], From 631dfd0ec1ea1aef1fa11bcc6c1ba392bb4d88a2 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sun, 28 Jun 2015 11:59:55 -0700 Subject: [PATCH 24/47] standardize string character --- rvic/convert.py | 8 ++--- rvic/convolution.py | 20 ++++++------ rvic/core/aggregate.py | 12 +++---- rvic/core/config.py | 16 ++++----- rvic/core/convert.py | 12 +++---- rvic/core/convolution_wrapper.py | 4 +-- rvic/core/history.py | 36 ++++++++++---------- rvic/core/log.py | 14 ++++---- rvic/core/make_uh.py | 46 +++++++++++++------------- rvic/core/multi_proc.py | 8 ++--- rvic/core/param_file.py | 20 ++++++------ rvic/core/plots.py | 30 ++++++++--------- rvic/core/pycompat.py | 7 ++-- rvic/core/read_forcing.py | 22 ++++++------- rvic/core/remap.py | 6 ++-- rvic/core/share.py | 8 ++--- rvic/core/time_utility.py | 10 +++--- rvic/core/utilities.py | 56 ++++++++++++++++---------------- rvic/core/variables.py | 54 +++++++++++++++--------------- rvic/core/write.py | 12 +++---- rvic/parameters.py | 16 ++++----- setup.py | 4 +-- 22 files changed, 211 insertions(+), 210 deletions(-) diff --git a/rvic/convert.py b/rvic/convert.py index 05815b9..48ebe60 100644 --- a/rvic/convert.py +++ b/rvic/convert.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' Read a set of uhs files and write an RVIC parameter file -""" +''' from logging import getLogger from .core.log import init_logger, close_logger, LOG_NAME @@ -114,9 +114,9 @@ def uhs2param_run(dom_data, outlets, config_dict): # -------------------------------------------------------------------- # # def uhs2param_final(outlets, dom_data, new_dom_data, config_dict, directories): - """ + ''' Make the RVIC Parameter File - """ + ''' log = getLogger(LOG_NAME) diff --git a/rvic/convolution.py b/rvic/convolution.py index 0846a26..716edc1 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -""" +''' This is the convolution routine developed in preparation in coupling RVIC to CESM. Eventually, this will be the offline RVIC model. @@ -13,7 +13,7 @@ Made necessary changes to run routines to accept the new parameter file structure. Major updates to the... -""" +''' import os from collections import OrderedDict from logging import getLogger @@ -32,14 +32,14 @@ # -------------------------------------------------------------------- # # Top Level Driver def convolution(config_file): - """ + ''' Top level driver for RVIC convolution model. Parameters ---------- config_file : str Path to RVIC convolution configuration file. - """ + ''' # ---------------------------------------------------------------- # # Initilize @@ -70,7 +70,7 @@ def convolution(config_file): # -------------------------------------------------------------------- # # Initialize RVIC def convolution_init(config_file): - """ + ''' Initialize the RVIC convolution routine This function performs these main tasks: @@ -105,7 +105,7 @@ def convolution_init(config_file): Dictionary of directories created by this function. config_dict : dict Dictionary of values from the configuration file. - """ + ''' # ---------------------------------------------------------------- # # Read Configuration files @@ -259,7 +259,7 @@ def convolution_init(config_file): # -------------------------------------------------------------------- # def convolution_run(hist_tapes, data_model, rout_var, time_handle, directories): - """ + ''' Main run loop for RVIC model. Parameters @@ -281,7 +281,7 @@ def convolution_run(hist_tapes, data_model, rout_var, time_handle, Dtime instance containing information about run length, time hist_tapes : OrderedDict Ordered dictionary of History objects - """ + ''' data2tape = {} aggrunin = {} @@ -381,7 +381,7 @@ def convolution_run(hist_tapes, data_model, rout_var, time_handle, # -------------------------------------------------------------------- # # Final def convolution_final(time_handle, hist_tapes): - """ Finalize RVIC Convolution + '''Finalize RVIC Convolution Parameters ---------- @@ -389,7 +389,7 @@ def convolution_final(time_handle, hist_tapes): Dtime instance containing information about run length, time hist_tapes : OrderedDict Ordered dictionary of History objects - """ + ''' # ---------------------------------------------------------------- # # Start log log = getLogger(LOG_NAME) diff --git a/rvic/core/aggregate.py b/rvic/core/aggregate.py index f8e08ea..d61086c 100644 --- a/rvic/core/aggregate.py +++ b/rvic/core/aggregate.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' aggregate.py -""" +''' import numpy as np from .share import FILLVALUE_F @@ -20,9 +20,9 @@ # -------------------------------------------------------------------- # # Find target cells for pour points def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): - """ + ''' Group pour points by domain grid outlet cell - """ + ''' lons = pour_points['lons'] lats = pour_points['lats'] dom_lon = dom_data[config_dict['DOMAIN']['LONGITUDE_VAR']] @@ -114,11 +114,11 @@ def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): # -------------------------------------------------------------------- # # Aggregate the UH grids def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False): - """ + ''' Add the two data sets together and return the combined arrays. Expand the horizontal dimensions as necessary to fit in_data with agg_data. The two data sets must include the coordinate variables lon,lat, and time. - """ + ''' # ---------------------------------------------------------------- # # find range of coordinates diff --git a/rvic/core/config.py b/rvic/core/config.py index 8d8a183..14e2ce5 100644 --- a/rvic/core/config.py +++ b/rvic/core/config.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -""" +''' config.py -""" +''' import os from .pycompat import OrderedDict, SafeConfigParser @@ -28,9 +28,9 @@ class ParametersConfig(Config): # -------------------------------------------------------------------- # # Read the Configuration File def read_config(config_file): - """ + ''' Return a dictionary with subdictionaries of all configFile options/values - """ + ''' config = SafeConfigParser() config.optionxform = str config.read(config_file) @@ -49,10 +49,10 @@ def read_config(config_file): # -------------------------------------------------------------------- # # Find the type of the config options def config_type(value): - """ + ''' Parse the type of the configuration file option. First see the value is a bool, then try float, finally return a string. - """ + ''' val_list = [x.strip() for x in value.split(',')] if len(val_list) == 1: value = val_list[0] @@ -82,7 +82,7 @@ def config_type(value): # -------------------------------------------------------------------- # def isfloat(x): - """Test of value is a float""" + '''Test of value is a float''' try: float(x) except ValueError: @@ -94,7 +94,7 @@ def isfloat(x): # -------------------------------------------------------------------- # def isint(x): - """Test if value is an integer""" + '''Test if value is an integer''' try: a = float(x) b = int(a) diff --git a/rvic/core/convert.py b/rvic/core/convert.py index c86f202..03bf540 100644 --- a/rvic/core/convert.py +++ b/rvic/core/convert.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' convert.py -""" +''' import os import re import numpy as np @@ -20,10 +20,10 @@ # -------------------------------------------------------------------- # # read station file def read_station_file(file_name, dom_data, config_dict): - """ + ''' Read a standard routing station file http://www.hydro.washington.edu/Lettenmaier/Models/VIC/Documentation/Routing/StationLocation.shtml - """ + ''' outlets = {} f = open(file_name, 'r') @@ -76,7 +76,7 @@ def read_station_file(file_name, dom_data, config_dict): # -------------------------------------------------------------------- # # Read uhs files def read_uhs_files(outlets, dom_data, config_dict): - """ + ''' Read a standard routing uh_s file Format: line0: num_sources @@ -84,7 +84,7 @@ def read_uhs_files(outlets, dom_data, config_dict): line2: unit_hydrograph_time_series line1: ... line2: ... - """ + ''' if config_dict['UHS_FILES']['ROUT_PROGRAM'] == 'C': for cell_id, outlet in iteritems(outlets): log.info('Reading outlet %i: %s', cell_id, outlet.name) diff --git a/rvic/core/convolution_wrapper.py b/rvic/core/convolution_wrapper.py index ad62fbf..1900c74 100644 --- a/rvic/core/convolution_wrapper.py +++ b/rvic/core/convolution_wrapper.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -""" +''' convolution_wrapper.py ctypes wrapper for rvic_convolution.c gcc -shared -o rvic_convolution.so rvic_convolution.c -""" +''' import os import sysconfig diff --git a/rvic/core/history.py b/rvic/core/history.py index aa54481..dc05516 100644 --- a/rvic/core/history.py +++ b/rvic/core/history.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -""" +''' history.py Summary: @@ -13,7 +13,7 @@ - __next_write_out_data: method to determine when to write the out_data container - finish: method to close all remaining history tapes. -""" +''' import os import numpy as np @@ -38,7 +38,7 @@ # -------------------------------------------------------------------- # # RVIC History File Object class Tape(object): - """ History Tape Object""" + ''' History Tape Object''' # ---------------------------------------------------------------- # # Init @@ -166,27 +166,27 @@ def __init__(self, time_ord, caseid, rvar, tape_num=0, # Determine the format of the output filename if self._avgflag == 'I': self._fname_format = os.path.join( - out_dir, "%s.rvic.h%s%s.%%Y-%%m-%%d-%%H-%%M-%%S.nc" % + out_dir, '%s.rvic.h%s%s.%%Y-%%m-%%d-%%H-%%M-%%S.nc' % (self._caseid, self._tape_num, self._avgflag.lower())) else: if self._nhtfrq == 0: self._fname_format = os.path.join( out_dir, - "%s.rvic.h%s%s.%%Y-%%m.nc" % + '%s.rvic.h%s%s.%%Y-%%m.nc' % (self._caseid, self._tape_num, self._avgflag.lower())) elif (self._nhtfrq == -24) or (nhtfrq * self._dt == SECSPERDAY): self._fname_format = os.path.join( out_dir, - "%s.rvic.h%s%s.%%Y-%%m-%%d.nc" % + '%s.rvic.h%s%s.%%Y-%%m-%%d.nc' % (self._caseid, self._tape_num, self._avgflag.lower())) else: self._fname_format = os.path.join( out_dir, - "%s.rvic.h%s%s.%%Y-%%m-%%d-%%H.nc" % + '%s.rvic.h%s%s.%%Y-%%m-%%d-%%H.nc' % (self._caseid, self._tape_num, self._avgflag.lower())) self._rest_fname_format = os.path.join( out_dir, - "%s.rvic.rh%s.%%Y-%%m-%%d-%%H-%%M-%%S.nc" % + '%s.rvic.rh%s.%%Y-%%m-%%d-%%H-%%M-%%S.nc' % (self._caseid, self._tape_num)) # ------------------------------------------------------------ # @@ -229,7 +229,7 @@ def __repr__(self): # ---------------------------------------------------------------- # # Update the history tapes with new fluxes def update(self, data2tape, time_ord): - """ Update the tape with new data""" + ''' Update the tape with new data''' # ------------------------------------------------------------ # # Check that the time_ord is in sync @@ -296,7 +296,7 @@ def write_initial(self): # ---------------------------------------------------------------- # def __next_write_out_data(self): - """determine the maximum size of out_data""" + '''determine the maximum size of out_data''' log.debug('determining size of out_data') @@ -414,7 +414,7 @@ def __update_out_data(self): # ---------------------------------------------------------------- # def finish(self): - """write out_data""" + '''write out_data''' log.debug('finishing tape %s', self._tape_num) if self._out_data_has_values: if self._outtype == 'grid': @@ -428,7 +428,7 @@ def finish(self): # ---------------------------------------------------------------- # # Get import rvar fields def __get_rvar(self, rvar): - """ Get the rvar Fields that are useful for writing output """ + ''' Get the rvar Fields that are useful for writing output ''' self._dt = rvar.unit_hydrograph_dt self._num_outlets = rvar.n_outlets self._outlet_decomp_ind = rvar.outlet_decomp_ind @@ -443,7 +443,7 @@ def __get_rvar(self, rvar): # ---------------------------------------------------------------- # # Determine next write time def __next_update_out_data(self): - """ Determine the count for when the next write should occur """ + ''' Determine the count for when the next write should occur ''' # ------------------------------------------------------------ # # If monthly, write at (YYYY,MM,1,0,0) # b0 is first timestep of next period @@ -498,7 +498,7 @@ def __next_update_out_data(self): # ---------------------------------------------------------------- # # Average fields def __average(self): - """ Take the average based on the number of accumulated timesteps """ + ''' Take the average based on the number of accumulated timesteps ''' for field in self._fincl: self._temp_data[field] /= self._count # ---------------------------------------------------------------- # @@ -506,7 +506,7 @@ def __average(self): # ---------------------------------------------------------------- # # Write grid style history file def __write_grid(self): - """ Write history file """ + ''' Write history file ''' # ------------------------------------------------------------ # # Open file @@ -592,7 +592,7 @@ def __write_grid(self): setattr(var, key, val) var.units = self._units if self._grid_lons.ndim > 1: - var.coordinates = " ".join(coords) + var.coordinates = ' '.join(coords) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -609,7 +609,7 @@ def __write_grid(self): # ---------------------------------------------------------------- # # Write array style history file def __write_array(self): - """ Write history file """ + ''' Write history file ''' # ------------------------------------------------------------ # # Open file @@ -717,7 +717,7 @@ def __write_array(self): for key, val in iteritems(self._glob_ats.atts): if val: setattr(f, key, val) - f.featureType = "timeSeries" + f.featureType = 'timeSeries' # ------------------------------------------------------------ # f.close() log.info('Finished writing %s', self.filename) diff --git a/rvic/core/log.py b/rvic/core/log.py index f0cce2c..8f18f56 100644 --- a/rvic/core/log.py +++ b/rvic/core/log.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' log.py -""" +''' import os import sys from time import gmtime, strftime @@ -16,10 +16,10 @@ # -------------------------------------------------------------------- # # Fake stream file to handle stdin/stdout class StreamToFile(object): - """ + ''' Fake file-like stream object that redirects writes to a logger instance. http://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ - """ + ''' def __init__(self, logger_name=LOG_NAME, log_level=logging.INFO): self.logger = logging.getLogger(logger_name) self.log_level = log_level @@ -35,7 +35,7 @@ def flush(self): def init_logger(log_dir='./', log_level='DEBUG', verbose=False): - """ Setup the logger """ + ''' Setup the logger ''' logger = logging.getLogger(LOG_NAME) logger.setLevel(log_level) @@ -44,7 +44,7 @@ def init_logger(log_dir='./', log_level='DEBUG', verbose=False): # ---------------------------------------------------------------- # # create log file handler if log_dir: - log_file = os.path.join(log_dir, 'RVIC-' + strftime("%Y%m%d-%H%M%S", + log_file = os.path.join(log_dir, 'RVIC-' + strftime('%Y%m%d-%H%M%S', gmtime()) + '.log') fh = logging.FileHandler(log_file) fh.setLevel(log_level) @@ -82,7 +82,7 @@ def init_logger(log_dir='./', log_level='DEBUG', verbose=False): def close_logger(): - """Close the handlers of the logger""" + '''Close the handlers of the logger''' log = logging.getLogger(LOG_NAME) x = list(log.handlers) for i in x: diff --git a/rvic/core/make_uh.py b/rvic/core/make_uh.py index dbba2b8..aa8ba46 100644 --- a/rvic/core/make_uh.py +++ b/rvic/core/make_uh.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -""" +''' make_uh.py PROGRAM rout, Python-Version, written by Joe Hamman winter 2012/2013 @@ -14,7 +14,7 @@ July 2013, Joe Hamman Removed read and write functions to make more modular. Now called from make_parameters.py -""" +''' import numpy as np import logging @@ -32,10 +32,10 @@ # -------------------------------------------------------------------- # def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): - """ + ''' Make the Unit Hydrograph Grid - """ - log.info("Starting routing program for point: %s", pour_point) + ''' + log.info('Starting routing program for point: %s', pour_point) # ---------------------------------------------------------------- # # Unpack a few structures uh_t = uh_box['time'] @@ -173,9 +173,9 @@ def rout(pour_point, uh_box, fdr_data, fdr_atts, rout_dict): # -------------------------------------------------------------------- # # Find the Timestep from the uh box def find_ts(uh_t): - """ + ''' Determines the (input_interval) based on the timestep given in uhfile - """ + ''' input_interval = uh_t[1] - uh_t[0] log.debug('Input Timestep = %i seconds', input_interval) return input_interval @@ -185,11 +185,11 @@ def find_ts(uh_t): # -------------------------------------------------------------------- # # Read the flow direction file def read_direction(fdr, dy, dx): - """ + ''' Reads the direction file and makes two grids (to_x) and (to_y). The input grids follow the 1-8 or 1-128 grid directions as shown below. val = direction [to_y][to_x] - """ + ''' log.debug('Reading direction input and finding target row/columns') to_y = np.zeros_like(fdr, dtype=np.int16) @@ -212,7 +212,7 @@ def read_direction(fdr, dy, dx): # -------------------------------------------------------------------- # # Search the catchment def search_catchment(to_y, to_x, pour_point, basin_ids, basin_id): - """ + ''' Find all cells upstream of pour point. Retrun a dictionary with x_inds, yinds, and #of cell to downstream pour point. All are sorted the by the latter. For each x,y pair, the flow path is followed until either the @@ -220,7 +220,7 @@ def search_catchment(to_y, to_x, pour_point, basin_ids, basin_id): (if (yy==pour_point.basiny and xx==pour_point.basinx):) or the flowpath leads outside of grid. *** Does not handle wrapped coordinates. *** - """ + ''' log.debug('Searching catchment') (len_y, len_x) = to_x.shape @@ -298,8 +298,8 @@ def search_catchment(to_y, to_x, pour_point, basin_ids, basin_id): count = len(cyinds) - log.debug("Found %i upstream grid cells from present station", count) - log.debug("Expected at most %i upstream grid cells from present station", + log.debug('Found %i upstream grid cells from present station', count) + log.debug('Expected at most %i upstream grid cells from present station', bsize) if count > bsize: @@ -320,10 +320,10 @@ def search_catchment(to_y, to_x, pour_point, basin_ids, basin_id): # -------------------------------------------------------------------- # # Make uh Grid def make_uh(dt, t_cell, y_inds, x_inds, velocity, diffusion, xmask): - """ + ''' Calculate the impulse response function for grid cells using equation 15 from Lohmann, et al. (1996) Tellus article. Return 3d uh grid. - """ + ''' log.debug('Making uh for each cell') uh = np.zeros((t_cell, xmask.shape[0], xmask.shape[1]), dtype=np.float64) @@ -347,11 +347,11 @@ def make_uh(dt, t_cell, y_inds, x_inds, velocity, diffusion, xmask): # Make uh river def make_grid_uh_river(t_uh, t_cell, uh, to_y, to_x, pour_point, y_inds, x_inds, count_ds): - """ + ''' Calculate impulse response function for river routing. Starts at downstream point incrementally moves upstream. - """ - log.debug("Making uh_river grid....") + ''' + log.debug('Making uh_river grid....') y_ind = pour_point.basiny x_ind = pour_point.basinx @@ -380,11 +380,11 @@ def make_grid_uh_river(t_uh, t_cell, uh, to_y, to_x, pour_point, y_inds, # Make grid uh def make_grid_uh(t_uh, t_cell, uh_river, uh_box, to_y, to_x, y_inds, x_inds, count_ds): - """ + ''' Combines the uh_box with downstream cell uh_river. Cell [0] is given the uh_box without river routing - """ - log.debug("Making unit_hydrograph grid") + ''' + log.debug('Making unit_hydrograph grid') unit_hydrograph = np.zeros((t_uh, uh_river.shape[1], uh_river.shape[2])) irf_temp = np.zeros(t_uh + t_cell, dtype=np.float64) @@ -409,11 +409,11 @@ def make_grid_uh(t_uh, t_cell, uh_river, uh_box, to_y, to_x, y_inds, x_inds, # Adjust the timestep def adjust_uh_timestep(unit_hydrograph, t_uh, input_interval, output_interval, x_inds, y_inds): - """ + ''' Aggregates to timestep (output_interval). output_interval must be a multiple of input_interval. This function is not setup to disaggregate the Unit Hydrographs to a output_interval= 3 +py3 = sys.version_info[0] >= 3 -if PY3: +if py3: basestring = str unicode_type = str bytes_type = bytes diff --git a/rvic/core/read_forcing.py b/rvic/core/read_forcing.py index 6d45472..84a96e4 100644 --- a/rvic/core/read_forcing.py +++ b/rvic/core/read_forcing.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' read_forcings.py -""" +''' import os import numpy as np @@ -23,7 +23,7 @@ # -------------------------------------------------------------------- # # Data Model class DataModel(object): - """ RVIC Forcing Data Model Class""" + '''RVIC Forcing Data Model Class''' # ---------------------------------------------------------------- # # Initialize @@ -62,7 +62,7 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, for year in pyrange(start[0], end[0] + 1): self.files.append(os.path.join(self.path, file_str.replace('$YYYY', - "{0:04d}".format(year)))) + '{0:04d}'.format(year)))) # Monthly elif len(start) == 2: @@ -74,8 +74,8 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, self.path, file_str.replace( '$YYYY', - "{0:04d}".format(year)).replace( - '$MM', "{0:02d}".format(month)))) + '{0:04d}'.format(year)).replace( + '$MM', '{0:02d}'.format(month)))) if year == end[0] and month == end[1]: break else: @@ -96,11 +96,11 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, self.path, file_str.replace( '$YYYY', - "{0:04d}".format(year)).replace( + '{0:04d}'.format(year)).replace( '$MM', - "{0:02d}".format(month)).replace( + '{0:02d}'.format(month)).replace( '$DD', - "{0:02d}".format(day)))) + '{0:02d}'.format(day)))) if year == end[0] and month == end[1] and day == end[2]: break else: @@ -174,7 +174,7 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, # ---------------------------------------------------------------- # def start(self, timestamp, rout_var): - """ Initialize the first files inputs""" + '''Initialize the first files inputs''' # ------------------------------------------------------------ # # find and open first file self.ordtime = date2num(timestamp, self.time_units, @@ -248,7 +248,7 @@ def start(self, timestamp, rout_var): # ---------------------------------------------------------------- # def read(self, timestamp): - """ Read the current timestamp from the data stream """ + '''Read the current timestamp from the data stream ''' # ------------------------------------------------------------ # # Get the current data index location diff --git a/rvic/core/remap.py b/rvic/core/remap.py index 850c364..1b9aec1 100644 --- a/rvic/core/remap.py +++ b/rvic/core/remap.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' remap.py -""" +''' import os from cdo import Cdo @@ -19,7 +19,7 @@ # Remap a file using CDO def remap(grid_file, in_file, out_file, operator='remapcon', remap_options=None): - """Remap infile using cdo""" + '''Remap infile using cdo''' log.info('Remapping %s to %s', in_file, out_file) diff --git a/rvic/core/share.py b/rvic/core/share.py index 583fed2..aacdf03 100644 --- a/rvic/core/share.py +++ b/rvic/core/share.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' share.py -""" +''' import sys import socket import string @@ -66,7 +66,7 @@ 5: ['360_day'], 6: ['julian']} -VALID_CHARS = "-_. %s%s" % (string.ascii_letters, string.digits) +VALID_CHARS = '-_. %s%s' % (string.ascii_letters, string.digits) # ----------------------- NETCDF VARIABLES --------------------------------- # @@ -340,7 +340,7 @@ def __init__(self, **kwargs): units='units') avgflag = NcVar(long_name='Averaging flag', - units='"A=Average, X=Maximum, M=Minimum, I=Instantaneous') + units='A=Average, X=Maximum, M=Minimum, I=Instantaneous') name = NcVar(long_name='Fieldnames') diff --git a/rvic/core/time_utility.py b/rvic/core/time_utility.py index f10084a..a68c9af 100644 --- a/rvic/core/time_utility.py +++ b/rvic/core/time_utility.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -""" +''' time_utility.py time units conventions: - Timesteps are in seconds (unit_hydrograph_dt) - Time ordinal is in days (time_ord) -""" +''' import numpy as np from netCDF4 import num2date, date2num from datetime import datetime @@ -24,7 +24,7 @@ # -------------------------------------------------------------------- # # RVIC Time Class class Dtime(object): - """ A Time Module for handling flags and timesteps """ + '''A Time Class for handling flags and timesteps ''' # ---------------------------------------------------------------- # # Setup Dtim object @@ -215,10 +215,10 @@ def __rest(self): # -------------------------------------------------------------------- # def ord_to_datetime(time, units, calendar='standard'): - """ + ''' netCDF4.num2date yields a fake datetime object, this function converts converts that back to a real datetime object - """ + ''' # this is the netCDF4.datetime object # if the calendar is standard, t is already a real datetime object if type(time) == np.ndarray: diff --git a/rvic/core/utilities.py b/rvic/core/utilities.py index 082bd36..0310955 100644 --- a/rvic/core/utilities.py +++ b/rvic/core/utilities.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' utilities.py -""" +''' import os import tarfile from scipy.spatial import cKDTree @@ -25,7 +25,7 @@ # -------------------------------------------------------------------- # # find x y coordinates def latlon2yx(plats, plons, glats, glons): - """find y x coordinates """ + '''find y x coordinates ''' # use astronomical conventions for longitude # (i.e. negative longitudes to the east of 0) @@ -55,7 +55,7 @@ def latlon2yx(plats, plons, glats, glons): # -------------------------------------------------------------------- # # Search neighboring grid cells for channel def search_for_channel(source_area, routys, routxs, search=1, tol=10): - """Search neighboring grid cells for channel""" + '''Search neighboring grid cells for channel''' log.debug('serching for channel') @@ -95,7 +95,7 @@ def search_for_channel(source_area, routys, routxs, search=1, tol=10): # -------------------------------------------------------------------- # # Write rpointer file def write_rpointer(restart_dir, restart_file, timestamp): - """ Write a configuration file with restart file and time """ + ''' Write a configuration file with restart file and time ''' rpointer_file = os.path.join(restart_dir, RPOINTER) config = SafeConfigParser() @@ -116,12 +116,12 @@ def write_rpointer(restart_dir, restart_file, timestamp): # -------------------------------------------------------------------- # # A helper function to read a netcdf file def read_netcdf(nc_file, variables=None, coords=None): - """ + ''' Read data from input netCDF. Will read all variables if none provided. Will also return all variable attributes. Both variables (data and attributes) are returned as dictionaries named by variable - """ + ''' f = Dataset(nc_file, 'r') @@ -153,9 +153,9 @@ def read_netcdf(nc_file, variables=None, coords=None): # -------------------------------------------------------------------- # # Check to make sure all the expected variables are present in the dictionary def check_ncvars(config_section, nckeys): - """ + ''' Make sure the variables listed in the config file are present in the netcdf - """ + ''' for key, value in iteritems(config_section): if key.endswith('var'): if value not in nckeys: @@ -170,7 +170,7 @@ def check_ncvars(config_section, nckeys): # -------------------------------------------------------------------- # # Find the index of the the nearest value def find_nearest(array, value): - """ Find the index location in (array) with value nearest to (value)""" + ''' Find the index location in (array) with value nearest to (value)''' return np.abs(array - value).argmin() # -------------------------------------------------------------------- # @@ -178,7 +178,7 @@ def find_nearest(array, value): # -------------------------------------------------------------------- # # Delete all the files in a directory def clean_dir(directory): - """ Clean all files in a directory""" + ''' Clean all files in a directory''' for file_name in os.listdir(directory): file_path = os.path.join(directory, file_name) try: @@ -193,7 +193,7 @@ def clean_dir(directory): # -------------------------------------------------------------------- # # Delete a particular file def clean_file(file_name): - """ Delete the file""" + ''' Delete the file''' try: if os.path.isfile(file_name): os.unlink(file_name) @@ -206,7 +206,7 @@ def clean_file(file_name): # -------------------------------------------------------------------- # # Make a set of directories def make_directories(rundir, subdir_names): - """Make rvic directory structure""" + '''Make rvic directory structure''' if not os.path.exists(rundir): os.makedirs(rundir) @@ -261,7 +261,7 @@ def copy_inputs(config_file, inputs_dir): # -------------------------------------------------------------------- # def tar_inputs(inputs, suffix='', tar_type='tar'): - """ Tar the inputss directory or file at the end of a run""" + ''' Tar the inputss directory or file at the end of a run''' # ---------------------------------------------------------------- # # Make the TarFile if tar_type == 'tar': @@ -307,10 +307,10 @@ def tar_inputs(inputs, suffix='', tar_type='tar'): # -------------------------------------------------------------------- # # Read the domain def read_domain(domain_dict, lat0_is_min=False): - """ + ''' Read the domain file and return all the variables and attributes. Area is returned in m2 - """ + ''' dom_data, dom_vatts, dom_gatts = read_netcdf(domain_dict['FILE_NAME']) check_ncvars(domain_dict, list(dom_data.keys())) @@ -354,24 +354,24 @@ def read_domain(domain_dict, lat0_is_min=False): dom_area = domain_dict['AREA_VAR'] area_units = dom_vatts[dom_area]['units'] - if area_units in ["rad2", "radians2", "radian2", "radian^2", "rad^2", - "radians^2", "rads^2", "radians squared", - "square-radians"]: + if area_units in ['rad2', 'radians2', 'radian2', 'radian^2', 'rad^2', + 'radians^2', 'rads^2', 'radians squared', + 'square-radians']: dom_data[dom_area] = dom_data[dom_area] * EARTHRADIUS * EARTHRADIUS - elif area_units in ["m2", "m^2", "meters^2", "meters2", "square-meters", - "meters squared"]: + elif area_units in ['m2', 'm^2', 'meters^2', 'meters2', 'square-meters', + 'meters squared']: dom_data[dom_area] = dom_data[dom_area] - elif area_units in ["km2", "km^2", "kilometers^2", "kilometers2", - "square-kilometers", "kilometers squared"]: + elif area_units in ['km2', 'km^2', 'kilometers^2', 'kilometers2', + 'square-kilometers', 'kilometers squared']: dom_data[dom_area] = dom_data[dom_area] * METERSPERKM * METERSPERKM - elif area_units in ["mi2", "mi^2", "miles^2", "miles", "square-miles", - "miles squared"]: + elif area_units in ['mi2', 'mi^2', 'miles^2', 'miles', 'square-miles', + 'miles squared']: dom_data[dom_area] = dom_data[dom_area] * METERSPERMILE * METERSPERMILE - elif area_units in ["acres", "ac", "ac."]: + elif area_units in ['acres', 'ac', 'ac.']: dom_data[dom_area] = dom_data[dom_area] * METERS2PERACRE else: - log.warning("WARNING: UNKNOWN AREA units (%s), ASSUMING THEY ARE IN " - "SQUARE METERS", + log.warning('WARNING: UNKNOWN AREA units (%s), ASSUMING THEY ARE IN ' + 'SQUARE METERS', dom_data[domain_dict['AREA_VAR']]['units']) # ---------------------------------------------------------------- # return dom_data, dom_vatts, dom_gatts diff --git a/rvic/core/variables.py b/rvic/core/variables.py index 1744aa1..7ec8124 100644 --- a/rvic/core/variables.py +++ b/rvic/core/variables.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' variables.py -""" +''' import os import numpy as np from netCDF4 import Dataset, date2num, stringtochar @@ -47,19 +47,19 @@ def __init__(self, lat=-9999.9, lon=-9999.9, domx=-9999, domy=-9999, return def __str__(self): - return ("Point({0}, lat:{1:3.4f}, lon:{2:3.4f}, y:{3:d}, " - "x:{4:d})".format(self.name, self.lat, self.lon, self.domy, + return ('Point({0}, lat:{1:3.4f}, lon:{2:3.4f}, y:{3:d}, ' + 'x:{4:d})'.format(self.name, self.lat, self.lon, self.domy, self.domx)) def __repr__(self): - return (" -- Point -- \n" - "name:\t{0}\n" - "lat:\t{1:3.4f}\n" - "lon:\t{2:3.4f}\n" - "domy:\t{3:d}\n" - "domx:\t{4:d}\n" - "routy:\t{5:d}\n" - "routx:\t{6:d}\n".format(self.name, self.lat, self.lon, + return (' -- Point -- \n' + 'name:\t{0}\n' + 'lat:\t{1:3.4f}\n' + 'lon:\t{2:3.4f}\n' + 'domy:\t{3:d}\n' + 'domx:\t{4:d}\n' + 'routy:\t{5:d}\n' + 'routx:\t{6:d}\n'.format(self.name, self.lat, self.lon, self.domy, self.domx, self.routy, self.routx)) # -------------------------------------------------------------------- # @@ -68,7 +68,7 @@ def __repr__(self): # -------------------------------------------------------------------- # # Rvar Object class Rvar(object): - """ Creates a RVIC structure """ + ''' Creates a RVIC structure ''' # ---------------------------------------------------------------- # # Initialize @@ -136,7 +136,7 @@ def __init__(self, param_file, case_name, calendar, out_dir, file_format, self._calendar = calendar self.__fname_format = os.path.join( - out_dir, "%s.r.%%Y-%%m-%%d-%%H-%%M-%%S.nc" % (case_name)) + out_dir, '%s.r.%%Y-%%m-%%d-%%H-%%M-%%S.nc' % (case_name)) # ------------------------------------------------------------ # # CESM calendar key (only NO_LEAP_C, GREGORIAN are supported in CESM) @@ -159,9 +159,9 @@ def __init__(self, param_file, case_name, calendar, out_dir, file_format, # ---------------------------------------------------------------- # # Check that dom file matches def _check_domain_file(self, domain_file): - """ + ''' Confirm that the dom files match in the parameter and domain files - """ + ''' input_file = os.path.split(domain_file)[1] log.info('domain_file: %s', input_file) log.info('Parameter RvicDomainFile: %s', self.RvicDomainFile) @@ -175,7 +175,7 @@ def _check_domain_file(self, domain_file): # ---------------------------------------------------------------- # def set_domain(self, dom_data, domain, lat0_is_min): - """ Set the domain size """ + ''' Set the domain size ''' self._check_domain_file(domain['FILE_NAME']) self.domain_shape = dom_data[domain['LAND_MASK_VAR']].shape @@ -199,9 +199,9 @@ def set_domain(self, dom_data, domain, lat0_is_min): # ---------------------------------------------------------------- # # Flip the y index order def _flip_y_inds(self): - """ + ''' Flip the y index order - """ + ''' self.source_y_ind = self.ysize - self.source_y_ind - 1 self.outlet_y_ind = self.ysize - self.outlet_y_ind - 1 # ---------------------------------------------------------------- # @@ -262,13 +262,13 @@ def init_state(self, state_file, run_type, timestamp): # ---------------------------------------------------------------- # # Convolve def convolve(self, aggrunin, time_ord): - """ + ''' This convoluition funciton works by looping over all points and doing the convolution one timestep at a time. This is accomplished by creating a convolution ring. Contributing flow from each timestep is added to the convolution ring. The convolution ring is saved as the state. The first column of values in the ring are the current runoff. - """ + ''' # ------------------------------------------------------------ # # Check that the time_ord is in sync # This is the time at the start of the current step (end of last step) @@ -324,11 +324,11 @@ def convolve(self, aggrunin, time_ord): # ---------------------------------------------------------------- # def get_time_mode(self, cpl_secs_per_step): - """ + ''' Determine the relationship between the coupling period and the unit- hydrograph period. In cases where they do not match, the model will aggregate the appropriate quantities before/after the confolution step. - """ + ''' log.info('Coupling Timestep is (seconds): %s', cpl_secs_per_step) log.info('RVIC Timestep is (seconds): %s', self.unit_hydrograph_dt) @@ -339,7 +339,7 @@ def get_time_mode(self, cpl_secs_per_step): else: log.error('unit_hydrograph_dt must be a multiple of the ' 'cpl_secs_per_step') - raise ValueError("Stopped due to error in determining agg_tsteps") + raise ValueError('Stopped due to error in determining agg_tsteps') log.info('RVIC will run 1 time for every %i coupling periods', self.agg_tsteps) @@ -347,7 +347,7 @@ def get_time_mode(self, cpl_secs_per_step): # ---------------------------------------------------------------- # def get_rof(self): - """Extract the current rof""" + '''Extract the current rof''' rof = {} for tracer in RVIC_TRACERS: rof[tracer] = self.ring[tracer][0, :] @@ -356,7 +356,7 @@ def get_rof(self): # ---------------------------------------------------------------- # def get_storage(self): - """Extract the current storage""" + '''Extract the current storage''' storage = {} for tracer in RVIC_TRACERS: storage[tracer] = self.ring[tracer].sum(axis=1) @@ -365,7 +365,7 @@ def get_storage(self): # ---------------------------------------------------------------- # def write_initial(self): - """write initial flux""" + '''write initial flux''' pass # ---------------------------------------------------------------- # diff --git a/rvic/core/write.py b/rvic/core/write.py index ec1c996..52d3057 100644 --- a/rvic/core/write.py +++ b/rvic/core/write.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' write.py -""" +''' import numpy as np from netCDF4 import Dataset, stringtochar @@ -20,11 +20,11 @@ # Write the agg netcdf def write_agg_netcdf(file_name, agg_data, glob_atts, nc_format, zlib=True, complevel=4, least_significant_digit=None): - """ + ''' Write output to netCDF. Writes out a netCDF4 data file containing the UH_S and fractions and a full set of history and description attributes. - """ + ''' # ---------------------------------------------------------------- # # netCDF variable options ncvaropts = {'zlib': zlib, @@ -135,7 +135,7 @@ def write_param_file(file_name, complevel=4, least_significant_digit=None): - """ Write a standard RVIC Parameter file """ + '''Write a standard RVIC Parameter file ''' # ---------------------------------------------------------------- # # netCDF variable options @@ -170,7 +170,7 @@ def write_param_file(file_name, for key, val in iteritems(glob_atts.atts): if val: setattr(f, key, val) - f.featureType = "timeSeries" + f.featureType = 'timeSeries' # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # diff --git a/rvic/parameters.py b/rvic/parameters.py index deb8da5..229d337 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' RVIC parameter file development driver -""" +''' import os import numpy as np import pandas as pd @@ -89,7 +89,7 @@ def parameters(config_file, numofproc=1): def gen_uh_init(config_file): - """Initialize RVIC parameters scrpt. + '''Initialize RVIC parameters scrpt. This function: - Reads the configuration file @@ -122,7 +122,7 @@ def gen_uh_init(config_file): Dictionary of values from the configuration file. directories : dict Dictionary of directories created by this function. - """ + ''' # ---------------------------------------------------------------- # # Read Configuration files @@ -373,7 +373,7 @@ def gen_uh_init(config_file): def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories): - """ + ''' Develop unit hydrographs for one outlet `Point`. Parameters @@ -399,7 +399,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, ---------- outlet: Point Point object with unit hydrographs. - """ + ''' log = getLogger(LOG_NAME) log.info('Running outlet cell id %s', outlet.cell_id) @@ -563,7 +563,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, def gen_uh_final(outlets, dom_data, config_dict, directories): - """ + ''' Make the RVIC Parameter File Parameters @@ -577,7 +577,7 @@ def gen_uh_final(outlets, dom_data, config_dict, directories): Dictionary of values from the configuration file. directories : dict Dictionary of directories created by gen_uh_init. - """ + ''' log = getLogger(LOG_NAME) log.info('In gen_uh_final') diff --git a/setup.py b/setup.py index 6633266..da89401 100644 --- a/setup.py +++ b/setup.py @@ -23,10 +23,10 @@ # -------------------------------------------------------------------- # def write_version_py(filename=None): - cnt = """\ + cnt = '''\ version = '%s' short_version = '%s' -""" +''' if not filename: filename = os.path.join( os.path.dirname(__file__), 'rvic', 'version.py') From a8260cd29823f363f8ac97a800b16ad74d366121 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sun, 28 Jun 2015 12:05:07 -0700 Subject: [PATCH 25/47] fix minor typo and a few last string character updates --- rvic/convolution.py | 4 ++-- rvic/parameters.py | 12 ++++++------ setup.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/rvic/convolution.py b/rvic/convolution.py index 716edc1..b86b08c 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -398,12 +398,12 @@ def convolution_final(time_handle, hist_tapes): # ---------------------------------------------------------------- # # Write final log info - log.info("-----------------------------------------------------------") + log.info('-----------------------------------------------------------') log.info('Done with streamflow convolution') log.info('Processed %i timesteps', time_handle.timesteps) for name, tape in iteritems(hist_tapes): log.info('Wrote %i history files from %s', tape.files_count, name) - log.info("-----------------------------------------------------------") + log.info('-----------------------------------------------------------') # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # diff --git a/rvic/parameters.py b/rvic/parameters.py index 229d337..6fa2746 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -89,7 +89,7 @@ def parameters(config_file, numofproc=1): def gen_uh_init(config_file): - '''Initialize RVIC parameters scrpt. + '''Initialize RVIC parameters script. This function: - Reads the configuration file @@ -344,7 +344,7 @@ def gen_uh_init(config_file): for i in pyrange(len(lats)): if 'names' in list(pour_points.keys()): name = pour_points['names'].values[i] - name = name.replace("'", '').replace(" ", "_") + name = name.replace("'", '').replace(' ', '_') else: # fill name filed with p-outlet_num name = 'p-{0}'.format(i) @@ -474,7 +474,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, temp_file_1 = os.path.join( directories['aggregated'], - 'aggUH_{0}.nc'.format(outlet.name.replace(" ", "_"))) + 'aggUH_{0}.nc'.format(outlet.name.replace(' ', '_'))) write_agg_netcdf(temp_file_1, agg_data, glob_atts, options['NETCDF_FORMAT'], **ncvaropts) @@ -483,7 +483,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, # Remap temporary file #1 to temporary file #2 temp_file_2 = os.path.join( directories['remapped'], - 'remapUH_{0}.nc'.format(outlet.name.replace(" ", "_"))) + 'remapUH_{0}.nc'.format(outlet.name.replace(' ', '_'))) remap(domain['FILE_NAME'], temp_file_1, temp_file_2) @@ -499,7 +499,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, var_list = list(final_data.keys()) log.debug('Remapped inputs came in upside down, flipping %s', - ", ".join(var_list)) + ', '.join(var_list)) # flip lattiutude and fraction along y axis (axis 0) final_data[dom_lat] = final_data[dom_lat][::-1] final_data['fraction'] = final_data['fraction'][::-1, :] @@ -542,7 +542,7 @@ def gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, # ------------------------------------------------------------ # # ------------------------------------------------------------ # - # Add to "adjust fractions structure" + # Add to 'adjust fractions structure' y, x = np.nonzero((final_data['fraction'] > 0.) * (dom_data[dom_mask] > np.finfo(np.float).resolution)) diff --git a/setup.py b/setup.py index da89401..97c8236 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def write_version_py(filename=None): for cmd in ['git', 'git.cmd']: try: pipe = subprocess.Popen( - [cmd, "describe", "--always", "--match", "v[0-9]*"], + [cmd, 'describe', '--always', '--match', 'v[0-9]*'], stdout=subprocess.PIPE) (so, serr) = pipe.communicate() if pipe.returncode == 0: @@ -77,11 +77,11 @@ def write_version_py(filename=None): if sys.version_info[0] >= 3: rev = rev.decode('ascii') - if not rev.startswith('v') and re.match("[a-zA-Z0-9]{7,9}", rev): + if not rev.startswith('v') and re.match('[a-zA-Z0-9]{7,9}', rev): # partial clone, manually construct version string # this is the format before we started using git-describe # to get an ordering on dev version strings. - rev = "v%s.dev-%s" % (VERSION, rev) + rev = 'v%s.dev-%s' % (VERSION, rev) # Strip leading v from tags format "vx.y.z" to get th version string FULLVERSION = rev.lstrip('v') From 9c2e47ad86d7f7e6b91c4fc9381de3bccd4d097d Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sun, 28 Jun 2015 12:10:23 -0700 Subject: [PATCH 26/47] update run_tests.py --- tests/run_tests.py | 50 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/run_tests.py b/tests/run_tests.py index a3b0e61..de42740 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""RVIC command line testing interface""" +'''RVIC command line testing interface''' import os import textwrap @@ -11,34 +11,34 @@ from rvic.core.pycompat import OrderedDict from rvic import convert, convolution, parameters from rvic.core.config import read_config -from rvic.core.pycompat import PY3, iteritems +from rvic.core.pycompat import py3, iteritems if not os.environ.get('RVIC_TEST_DIR'): print('\n$RVIC_TEST_DIR not set.') - os.environ["RVIC_TEST_DIR"] = os.path.abspath(os.path.dirname(__file__)) + os.environ['RVIC_TEST_DIR'] = os.path.abspath(os.path.dirname(__file__)) print('Setting to run_tests.py dir: ' - '{0}\n'.format(os.environ["RVIC_TEST_DIR"])) + '{0}\n'.format(os.environ['RVIC_TEST_DIR'])) if not os.environ.get('WORKDIR'): print('\n$WORKDIR not set.') - os.environ["WORKDIR"] = os.environ["RVIC_TEST_DIR"] + os.environ['WORKDIR'] = os.environ['RVIC_TEST_DIR'] print('Setting to output run_tests.py dir to $WORKDIR: ' - '{0}\n'.format(os.environ["WORKDIR"])) + '{0}\n'.format(os.environ['WORKDIR'])) # -------------------------------------------------------------------- # def main(): - """ + ''' Run RVIC tests - """ + ''' # Parse arguments parser = argparse.ArgumentParser(description='Test script for RVIC') - parser.add_argument("test_set", type=str, - help="Test set to run", + parser.add_argument('test_set', type=str, + help='Test set to run', choices=['all', 'unit', 'examples'], default=['all'], nargs='+') - parser.add_argument("--examples", type=str, - help="examples configuration file", + parser.add_argument('--examples', type=str, + help='examples configuration file', default='examples/examples.cfg') args = parser.parse_args() @@ -55,7 +55,7 @@ def main(): # -------------------------------------------------------------------- # def run_examples(config_file): - """ Run examples from config file """ + ''' Run examples from config file ''' # ---------------------------------------------------------------- # # Read Configuration files config_dict = read_config(config_file) @@ -68,12 +68,12 @@ def run_examples(config_file): test_outcomes = OrderedDict() for i, (test, test_dict) in enumerate(iteritems(config_dict)): - print("".center(100, '-')) - print("Starting Test #{0} of {1}: {2}".format(i + 1, num_tests, + print(''.center(100, '-')) + print('Starting Test #{0} of {1}: {2}'.format(i + 1, num_tests, test).center(100)) - desc = textwrap.fill(", ".join(test_dict['description']), 100) - print("Description: {0}".format(desc)) - print("".center(100, '-')) + desc = textwrap.fill(', '.join(test_dict['description']), 100) + print('Description: {0}'.format(desc)) + print(''.center(100, '-')) if 'processors' in test_dict: numofproc = test_dict['processors'] @@ -110,7 +110,7 @@ def run_examples(config_file): '{0}'.format(test_dict['function'])) pr.disable() - if PY3: + if py3: s = io.StringIO() else: s = io.BytesIO() @@ -118,13 +118,13 @@ def run_examples(config_file): ps = pstats.Stats(pr, stream=s).sort_stats(sortby) ps.print_stats() - print("".center(100, '-')) - print("Done With Test #{0} of {1}: {2}".format(i + 1, num_tests, + print(''.center(100, '-')) + print('Done With Test #{0} of {1}: {2}'.format(i + 1, num_tests, test).center(100)) - print(".....Printing Profile Information.....".center(100)) - print("".center(100, '-')) + print('.....Printing Profile Information.....'.center(100)) + print(''.center(100, '-')) print(s.getvalue()) - print("".center(100, '-')) + print(''.center(100, '-')) print('Done with examples...printing summary') for test, outcome in iteritems(test_outcomes): @@ -134,6 +134,6 @@ def run_examples(config_file): # -------------------------------------------------------------------- # -if __name__ == "__main__": +if __name__ == '__main__': main() # -------------------------------------------------------------------- # From ff96ba24830e2776ce6bdb25c0dbc3accb21f09b Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 10 Jul 2015 10:57:12 -0700 Subject: [PATCH 27/47] minor fixes from code review session, mostly typos and documentation issues --- docs/user-guide/install.md | 12 ++++++------ rvic/convolution.py | 3 +-- rvic/core/config.py | 2 +- tests/run_tests.py | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/user-guide/install.md b/docs/user-guide/install.md index 87c4ba7..9afc84d 100644 --- a/docs/user-guide/install.md +++ b/docs/user-guide/install.md @@ -1,15 +1,15 @@ # Installing RVIC ## Dependencies -- [Python 2.7 or later including Python3](http://www.python.org/) -- [Numpy](http://www.numpy.org) -- [Scipy](http://www.scipy.org/) -- [netcdf4-python](https://code.google.com/p/netcdf4-python/) -- [Pandas](http://pandas.pydata.org/) +- [Python 2.7 or later including Python3](http://www.python.org) +- [NumPy](http://www.numpy.org) +- [SciPy](http://www.scipy.org) +- [netcdf4-python](https://code.google.com/p/netcdf4-python) +- [pandas](http://pandas.pydata.org) If using `REMAP=True`: -- [cdo](https://code.zmaw.de/projects/cdo) +- [CDO](https://code.zmaw.de/projects/cdo) - [cdo.py](https://github.com/Try2Code/cdo-bindings) ## Building RVIC diff --git a/rvic/convolution.py b/rvic/convolution.py index 3d4d8ce..aa3a4a5 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -83,8 +83,7 @@ def convolution_init(config_file): # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # - # Copy Inputs to $case_dir/inputs and update configuration - # config_dict = copy_inputs(config_file, directories['inputs']) + # unpack options options = config_dict['OPTIONS'] # ---------------------------------------------------------------- # diff --git a/rvic/core/config.py b/rvic/core/config.py index 8d8a183..b3d1896 100644 --- a/rvic/core/config.py +++ b/rvic/core/config.py @@ -82,7 +82,7 @@ def config_type(value): # -------------------------------------------------------------------- # def isfloat(x): - """Test of value is a float""" + """Test if value is a float""" try: float(x) except ValueError: diff --git a/tests/run_tests.py b/tests/run_tests.py index a3b0e61..720901b 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -106,7 +106,7 @@ def run_examples(config_file): print('Error in parameters example: {0}'.format(test)) test_outcomes[test] = 'Failed: {0}'.format(e) else: - raise ValueError('Unknow function variable: ' + raise ValueError('Unknown function variable: ' '{0}'.format(test_dict['function'])) pr.disable() From 909bba4b88675fa471df68a1e8923ec2f0153a84 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 19 Aug 2015 08:12:00 -0700 Subject: [PATCH 28/47] temporary checkin of fix_multiproc branch --- rvic/core/multi_proc.py | 6 +-- rvic/parameters.py | 5 ++- tests/unit/test_multi_proc.py | 75 +++++++++++++++++++---------------- 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/rvic/core/multi_proc.py b/rvic/core/multi_proc.py index 40bca17..0efe773 100644 --- a/rvic/core/multi_proc.py +++ b/rvic/core/multi_proc.py @@ -38,12 +38,8 @@ def __call__(self, *args, **kwargs): class LoggingPool(Pool): - """Subclass of pool""" + """Subclass of multiprocessing.pool.Pool""" def apply_async(self, func, args=(), kwds={}, callback=None): return Pool.apply_async(self, LogExceptions(func), args, kwds, callback) - - # def map_async(self, func, callback=None, *args, **kwargs): - # return Pool.map_async(self, LogExceptions(func), args, kwargs, - # callback) # -------------------------------------------------------------------- # diff --git a/rvic/parameters.py b/rvic/parameters.py index 026fe2e..53b1985 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -534,5 +534,8 @@ def store_result(result): # This is called whenever gen_uh_run() returns a result. # result_list is modified only by the main process, not the pool workers. global results - results.append(result) + if result[0]: + results.append(result[1]) + else: + raise(result[1]) # -------------------------------------------------------------------- # diff --git a/tests/unit/test_multi_proc.py b/tests/unit/test_multi_proc.py index 428dc7f..83fcf3a 100644 --- a/tests/unit/test_multi_proc.py +++ b/tests/unit/test_multi_proc.py @@ -21,62 +21,67 @@ def point(scope='function'): return Point() +class Foo(object): + '''simple object to hold a few attributes''' + + def __init__(self, i): + self.cell_id = i + self.cube = i ** 3 + + def __repr__(self): + return "%s: %s" % (self.cell_id, self.cube) + + def __str__(self): + return "%s: %s" % (self.cell_id, self.cube) + + +def go_foo(i): + '''create a foo object''' + sleep(0.5) + return Foo(i) + + +def go_exception(i): + print('go_exception', i, 0) + sleep(0.5) + raise Exception('go_exception raised an Exception') + print('go_exception', i, 1) + return ('go_exception', i, 2) + + +def go_value_error(i): + print('go_value_error', i, 3) + sleep(0.5) + raise ValueError('go_value_error raised an ValueError') + print('go_value_error', i, 4) + return ('go_value_error', i, 5) + + def test_create_log_exceptions(): LogExceptions(callable) def test_pool_raises(pool): - def go_exception(): - print(1) - sleep(0.5) - raise Exception() - print(2) - return 3 - - def go_value_error(): - print(1) - sleep(0.5) - raise ValueError() - print(2) - return 3 with pytest.raises(Exception): for i in pyrange(3): - pool.apply_async(go_exception) + status = pool.apply_async(go_exception, [i]) + status.get() pool.close() pool.join() with pytest.raises(ValueError): for i in pyrange(3): - pool.apply_async(go_value_error) + pool.apply_async(go_value_error, [i]) pool.close() pool.join() def test_pool_callback(pool): - - class Foo(object): - '''simple object to hold a few attributes''' - - def __init__(self, i): - self.cell_id = i - self.cube = i ** 3 - - def __repr__(self): - return "%s: %s" % (self.cell_id, self.cube) - - def __str__(self): - return "%s: %s" % (self.cell_id, self.cube) - - def go_foo(i): - '''create a foo object''' - sleep(0.5) - return Foo(i) - n = 5 for i in pyrange(n): - pool.apply_async(go_foo, store_result, i) + pool.apply_async(go_foo, [i], callback=store_result) pool.close() pool.join() From caa4c09d12eb2d87da94c7440d2050570732a445 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 20 Oct 2015 20:43:04 -0400 Subject: [PATCH 29/47] rework search for channel --- rvic/core/param_file.py | 4 +++- rvic/core/utilities.py | 40 +++++++++++++++++++++------------------- rvic/parameters.py | 20 +++++++++++--------- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/rvic/core/param_file.py b/rvic/core/param_file.py index eb6df3c..8522020 100644 --- a/rvic/core/param_file.py +++ b/rvic/core/param_file.py @@ -199,6 +199,8 @@ def finish_params(outlets, dom_data, config_dict, directories): RvicFdrFile=os.path.split(routing['FILE_NAME'])[1], RvicDomainFile=os.path.split(dom_file_name)[1]) + log.debug('UH Range: (%f %f)', unit_hydrograph.min(), unit_hydrograph.max()) + write_param_file(param_file, nc_format=options['NETCDF_FORMAT'], glob_atts=glob_atts, @@ -425,7 +427,7 @@ def group(outlets, subset_length): a = 0 for i, (cell_id, outlet) in enumerate(iteritems(outlets)): b = a + len(outlet.y_source) - log.debug('outlet.unit_hydrograph.shape %s', + log.debug('%s unit_hydrograph.shape %s', outlet.name, outlet.unit_hydrograph.shape) # -------------------------------------------------------- # # Point specific values diff --git a/rvic/core/utilities.py b/rvic/core/utilities.py index 082bd36..f756e84 100644 --- a/rvic/core/utilities.py +++ b/rvic/core/utilities.py @@ -57,36 +57,38 @@ def latlon2yx(plats, plons, glats, glons): def search_for_channel(source_area, routys, routxs, search=1, tol=10): """Search neighboring grid cells for channel""" - log.debug('serching for channel') + log.debug('serching for channel, tol: %f, search: %i', tol, search) - new_ys = np.empty_like(routys) - new_xs = np.empty_like(routxs) + new_ys = np.copy(routys) + new_xs = np.copy(routxs) ysize, xsize = source_area.shape for i, (y, x) in enumerate(pyzip(routys, routxs)): area0 = source_area[y, x] - ymin = np.clip(y - search, 0, ysize) - ymax = np.clip(y + search + 1, 0, ysize) - xmin = np.clip(x - search, 0, xsize) - xmax = np.clip(x + search, 0, xsize) + for j in range(search + 1): + ymin = np.clip(y - j, 0, ysize) + ymax = np.clip(y + j + 1, 0, ysize) + xmin = np.clip(x - j, 0, xsize) + xmax = np.clip(x + j + 1, 0, xsize) - search_area = source_area[ymin:ymax, xmin:xmax] + search_area = source_area[ymin:ymax, xmin:xmax] - if np.any(search_area / area0 > tol): - sy, sx = np.unravel_index(search_area.argmax(), search_area.shape) + if np.any(search_area / area0 > tol): + sy, sx = np.unravel_index(search_area.argmax(), + search_area.shape) - new_ys[i] = np.clip(y + sy - search, 0, ysize) - new_xs[i] = np.clip(x + sx - search, 0, xsize) + new_ys[i] = np.clip(y + sy - j, 0, ysize) + new_xs[i] = np.clip(x + sx - j, 0, xsize) + + log.debug('Moving pour point to channel y: %s->%s, x: %s->%s', + y, new_ys[i], x, new_xs[i]) + log.debug('Source Area has increased from %s to %s', + area0, source_area[new_ys[i], new_xs[i]]) + + break - log.debug('Moving pour point to channel y: %s->%s, x: %s->%s', - y, new_ys[i], x, new_xs[i]) - log.debug('Source Area has increased from %s to %s', - area0, source_area[new_ys[i], new_xs[i]]) - else: - new_ys[i] = y - new_xs[i] = x return new_ys, new_xs # -------------------------------------------------------------------- # diff --git a/rvic/parameters.py b/rvic/parameters.py index f0bd883..8f12bc3 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -19,7 +19,7 @@ from .core.variables import Point from .core.param_file import finish_params from .core.config import read_config -from .core.pycompat import OrderedDict, iteritems, pyrange +from .core.pycompat import OrderedDict, iteritems, pyrange, basestring try: from .core.remap import remap @@ -130,7 +130,7 @@ def gen_uh_init(config_file): if 'names' in pour_points: pour_points.fillna(inplace=True, value='unknown') for i, name in enumerate(pour_points.names): - pour_points.names[i] = strip_invalid_char(name) + pour_points.ix[i, 'names'] = strip_invalid_char(name) pour_points.drop_duplicates(inplace=True) pour_points.dropna() @@ -193,12 +193,14 @@ def gen_uh_init(config_file): # ---------------------------------------------------------------- # # Add velocity and/or diffusion grids if not present yet - if not type(fdr_vel) == str: - fdr_data['velocity'] = np.zeros(fdr_shape) + fdr_vel + if not isinstance(fdr_vel, basestring): + fdr_data['velocity'] = \ + np.zeros(fdr_shape, dtype=np.float64) + fdr_vel config_dict['ROUTING']['VELOCITY'] = 'velocity' log.info('Added velocity grid to fdr_data') - if not type(fdr_dif) == str: - fdr_data['diffusion'] = np.zeros(fdr_shape) + fdr_dif + if not isinstance(fdr_dif, basestring): + fdr_data['diffusion'] = \ + np.zeros(fdr_shape, dtype=np.float64) + fdr_dif config_dict['ROUTING']['DIFFUSION'] = 'diffusion' log.info('Added diffusion grid to fdr_data') if ('SOURCE_AREA_VAR' not in config_dict['ROUTING'] or @@ -207,7 +209,7 @@ def gen_uh_init(config_file): 'source area will be zero.') config_dict['ROUTING']['SOURCE_AREA_VAR'] = 'src_area' fdr_data[config_dict['ROUTING']['SOURCE_AREA_VAR']] = \ - np.zeros(fdr_shape) + np.zeros(fdr_shape, dtype=np.float64) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -273,7 +275,7 @@ def gen_uh_init(config_file): lats = fdr_data[fdr_lat][routys] lons = fdr_data[fdr_lon][routxs] else: - # use lons and lats to find xs and ys + # use and lats to find xs and ys lats = pour_points['lats'].values lons = pour_points['lons'].values @@ -286,7 +288,7 @@ def gen_uh_init(config_file): if options['SEARCH_FOR_CHANNEL']: routys, routxs = search_for_channel( fdr_data[config_dict['ROUTING']['SOURCE_AREA_VAR']], - routys, routxs, tol=10, search=2) + routys, routxs, tol=10, search=5) # update lats and lons lats = fdr_data[fdr_lat][routys] From e3e1770d3c1225d81ba70c051684b2e71d3b0f40 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 20 Oct 2015 21:32:00 -0400 Subject: [PATCH 30/47] fix failing multiproc test for python 2.7, still failing on 3.4 --- rvic/core/multi_proc.py | 20 +++++++++++++++----- tests/unit/test_multi_proc.py | 26 ++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/rvic/core/multi_proc.py b/rvic/core/multi_proc.py index 29640b2..0a74f4b 100644 --- a/rvic/core/multi_proc.py +++ b/rvic/core/multi_proc.py @@ -10,7 +10,7 @@ def error(*args): - """ Error function""" + """Error function""" return multiprocessing.get_logger(LOG_NAME).error(*args) # -------------------------------------------------------------------- # @@ -38,8 +38,18 @@ def __call__(self, *args, **kwargs): class LoggingPool(Pool): - """Subclass of pool""" - def apply_async(self, func, callback=None, *args, **kwargs): - return Pool.apply_async(self, LogExceptions(func), args, kwargs, - callback) + __doc__ = 'RVIC Logging Subclass of pool\n' + Pool.__doc__ + + @property + def has_logging(self): + # just for testing + return True + + def apply_async(self, func, callback=None, error_callback=None, + *args, **kwds): + """Overloaded Pool.apply_async to support Logging""" + return Pool.apply_async(self, func, args=args, kwds=kwds, + callback=callback, + error_callback=error_callback) + # -------------------------------------------------------------------- # diff --git a/tests/unit/test_multi_proc.py b/tests/unit/test_multi_proc.py index fcc8fdd..1055f82 100644 --- a/tests/unit/test_multi_proc.py +++ b/tests/unit/test_multi_proc.py @@ -1,7 +1,29 @@ -from rvic.core.multi_proc import LogExceptions +from rvic.core.multi_proc import LogExceptions, LoggingPool +import pytest + + +def go_and_raise(): + print(1) + raise Exception() + print(2) def test_create_log_exceptions(): LogExceptions(callable) -# error and logging pool not easily testing in unit test. + +def test_create_logging_pool(): + n = 2 + p = LoggingPool(processes=n) + assert hasattr(p, 'apply_async') + assert p._processes == n + assert p.has_logging + + +def test_logging_pool_raises(): + p = LoggingPool(processes=2) + + with pytest.raises(Exception): + p.apply_async(go_and_raise) + p.close() + p.join() From 907df7266e2ac6639e01324155b863a886a7b29b Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 21 Oct 2015 11:01:21 -0700 Subject: [PATCH 31/47] update travis build and fix for python3 multiproc --- .travis.yml | 41 ++++++++++++++++------------------- ci/requirements-2.7-dev.txt | 7 ------ ci/requirements-2.7.txt | 6 ----- ci/requirements-3.4.txt | 6 ----- tests/unit/test_multi_proc.py | 3 ++- 5 files changed, 21 insertions(+), 42 deletions(-) delete mode 100644 ci/requirements-2.7-dev.txt delete mode 100644 ci/requirements-2.7.txt delete mode 100644 ci/requirements-3.4.txt diff --git a/.travis.yml b/.travis.yml index 72673c0..5845944 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,34 +1,31 @@ language: python - +sudo: false # use container based build +notifications: + email: false python: - "2.7" - "3.4" + - "3.5" # Setup anaconda before_install: - - wget http://repo.continuum.io/miniconda/Miniconda-2.2.2-Linux-x86_64.sh -O miniconda.sh - - chmod +x miniconda.sh - - ./miniconda.sh -b - - export PATH=/home/travis/anaconda/bin:$PATH - # Update conda itself - - conda update --yes conda - # The next couple lines fix a crash with multiprocessing on Travis and are not specific to using Miniconda - - sudo rm -rf /dev/shm - - sudo ln -s /run/shm /dev/shm - - sudo apt-get install cdo + - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + wget http://repo.continuum.io/miniconda/Miniconda-3.16.0-Linux-x86_64.sh -O miniconda.sh; + else + wget http://repo.continuum.io/miniconda/Miniconda3-3.16.0-Linux-x86_64.sh -O miniconda.sh; + fi + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + - conda info -a # Install packages install: - - conda create --yes --name=rvic_test_env python=$TRAVIS_PYTHON_VERSION --file=ci/requirements-2.7.txt - - source activate rvic_test_env + - conda create --yes --name=test python=$TRAVIS_PYTHON_VERSION numpy scipy pandas netcdf4 matplotlib pytest + - source activate test + - conda install -c https://conda.anaconda.org/ioos cdo - pip install cdo - python setup.py install # Run test script: - - which python - - python --version - - conda list - - pip freeze - - echo $PATH - - python -c 'import rvic; print(rvic.__file__)' - - cd tests - - pwd - - python run_tests.py unit + - py.test diff --git a/ci/requirements-2.7-dev.txt b/ci/requirements-2.7-dev.txt deleted file mode 100644 index c1220c8..0000000 --- a/ci/requirements-2.7-dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -cdo==1.2.3 -matplotlib==1.3.1 -netCDF4==1.0.8 -numpy==1.8.1 -scipy==0.13.3 -pandas==0.13.1 -pytest==2.5.2 diff --git a/ci/requirements-2.7.txt b/ci/requirements-2.7.txt deleted file mode 100644 index 18de97c..0000000 --- a/ci/requirements-2.7.txt +++ /dev/null @@ -1,6 +0,0 @@ -matplotlib==1.4.3 -netCDF4==1.1.6 -numpy==1.9.2 -scipy==0.15.1 -pandas==0.15.1 -pytest==2.5.2 diff --git a/ci/requirements-3.4.txt b/ci/requirements-3.4.txt deleted file mode 100644 index 18de97c..0000000 --- a/ci/requirements-3.4.txt +++ /dev/null @@ -1,6 +0,0 @@ -matplotlib==1.4.3 -netCDF4==1.1.6 -numpy==1.9.2 -scipy==0.15.1 -pandas==0.15.1 -pytest==2.5.2 diff --git a/tests/unit/test_multi_proc.py b/tests/unit/test_multi_proc.py index 1055f82..1ae6b4f 100644 --- a/tests/unit/test_multi_proc.py +++ b/tests/unit/test_multi_proc.py @@ -24,6 +24,7 @@ def test_logging_pool_raises(): p = LoggingPool(processes=2) with pytest.raises(Exception): - p.apply_async(go_and_raise) + x = p.apply_async(go_and_raise) + x.get() p.close() p.join() From 6354f92ed614ed386f412919de4ecdc09df91532 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 21 Oct 2015 11:09:11 -0700 Subject: [PATCH 32/47] dont install cdo --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5845944..7fa72ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,8 @@ before_install: install: - conda create --yes --name=test python=$TRAVIS_PYTHON_VERSION numpy scipy pandas netcdf4 matplotlib pytest - source activate test - - conda install -c https://conda.anaconda.org/ioos cdo - - pip install cdo + # - conda install -c https://conda.anaconda.org/ioos cdo + # - pip install cdo - python setup.py install # Run test script: From a83d75ccdc1f6966660e7dd67485fc9cd799e945 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 21 Oct 2015 11:11:59 -0700 Subject: [PATCH 33/47] travis matrix build with fast finish --- .travis.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7fa72ff..5e08005 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,14 @@ language: python sudo: false # use container based build notifications: email: false -python: - - "2.7" - - "3.4" - - "3.5" + +matrix: + fast_finish: true + include: + - python: 2.7 + - python: 3.4 + - python: 3.5 + # Setup anaconda before_install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then From bfd4092a8e7af005bbee079e9f4c9f67ad8a77a8 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 21 Oct 2015 11:21:46 -0700 Subject: [PATCH 34/47] skip cdo test on travis --- tests/unit/test_remap.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_remap.py b/tests/unit/test_remap.py index 36d22a1..5bf80b6 100644 --- a/tests/unit/test_remap.py +++ b/tests/unit/test_remap.py @@ -1,8 +1,13 @@ import pytest -from cdo import CDOException +try: + from cdo import CDOException + cdo_unavailable = False +except ImportError: + cdo_unavailable = True from rvic.core.remap import remap +@pytest.mark.skipif(cdo_unavailable, reason='cdo not installed') def test_cdo_raises_exception(): with pytest.raises(CDOException): remap('grid_file', 'in_file', 'out_file') From 04858e4f58bc5eba4cf6821e1c871bb52ec4c06c Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 21 Oct 2015 11:27:50 -0700 Subject: [PATCH 35/47] check apply_async results after pool completes --- rvic/parameters.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rvic/parameters.py b/rvic/parameters.py index 8f12bc3..2fff2b4 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -47,15 +47,23 @@ def parameters(config_file, numofproc=1): # Run if numofproc > 1: pool = LoggingPool(processes=numofproc) + results = [] for i, outlet in enumerate(outlets.values()): log.info('On Outlet #%s of %s', i + 1, len(outlets)) - pool.apply_async(gen_uh_run, - args=(uh_box, fdr_data, fdr_vatts, - dom_data, outlet, config_dict, - directories), - callback=store_result, - error_callback=pool.terminate) + result = pool.apply_async(gen_uh_run, + args=(uh_box, fdr_data, fdr_vatts, + dom_data, outlet, config_dict, + directories), + callback=store_result, + error_callback=pool.terminate) + # Store the result + results.append(result) + + # Check that everything worked + [result.get() for result in results] + + # Close the pool pool.close() pool.join() From 775fd7e3f41125de191a94940d22a54c0da1cd6a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 21 Oct 2015 11:29:40 -0700 Subject: [PATCH 36/47] fix cdo skip in tests --- tests/unit/test_remap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_remap.py b/tests/unit/test_remap.py index 5bf80b6..455f380 100644 --- a/tests/unit/test_remap.py +++ b/tests/unit/test_remap.py @@ -1,10 +1,10 @@ import pytest try: + from rvic.core.remap import remap from cdo import CDOException cdo_unavailable = False except ImportError: cdo_unavailable = True -from rvic.core.remap import remap @pytest.mark.skipif(cdo_unavailable, reason='cdo not installed') From 0ecd9999b60c13249db84750f90fa63ec0779ebe Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 21 Oct 2015 11:38:04 -0700 Subject: [PATCH 37/47] simplify check for missing outlets --- rvic/parameters.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rvic/parameters.py b/rvic/parameters.py index 2fff2b4..7305a3d 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -60,11 +60,12 @@ def parameters(config_file, numofproc=1): # Store the result results.append(result) + # Close the pool + pool.close() + # Check that everything worked [result.get() for result in results] - # Close the pool - pool.close() pool.join() outlets = OrderedDict(sorted(list(iteritems(results)), @@ -74,7 +75,7 @@ def parameters(config_file, numofproc=1): outlet = gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories) - if not len(outlets.keys()) > 0: + if not outlets: raise ValueError('outlets in parameters are empty') # ---------------------------------------------------------------- # From c998cc2151df4478d8e29dd5081bb0ebe607a8c9 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 21 Oct 2015 12:06:50 -0700 Subject: [PATCH 38/47] fix multiproc tests --- tests/unit/test_multi_proc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_multi_proc.py b/tests/unit/test_multi_proc.py index ad80ff0..c1e70bf 100644 --- a/tests/unit/test_multi_proc.py +++ b/tests/unit/test_multi_proc.py @@ -1,4 +1,5 @@ import pytest +from rvic.core.multi_proc import LoggingPool, LogExceptions def go_and_raise(): From 51ee8e3d9f8683d2921dd8e772637f24c6ed6dd0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 24 Oct 2015 10:51:56 -0700 Subject: [PATCH 39/47] simplify multiproc, use dictionary containers, multiprocessing now works on python 3.4 and 3.5 --- rvic/core/aggregate.py | 2 +- rvic/core/multi_proc.py | 46 +------------------------------- rvic/core/param_file.py | 14 +++++----- rvic/parameters.py | 50 ++++++++++++++++++++++------------- tests/unit/test_multi_proc.py | 30 --------------------- 5 files changed, 40 insertions(+), 102 deletions(-) delete mode 100644 tests/unit/test_multi_proc.py diff --git a/rvic/core/aggregate.py b/rvic/core/aggregate.py index d61086c..cc0c069 100644 --- a/rvic/core/aggregate.py +++ b/rvic/core/aggregate.py @@ -48,7 +48,7 @@ def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): # ---------------------------------------------------------------- # # Do the aggregation - outlets = {} + outlets = OrderedDict() for i, (lat, lon) in enumerate(pyzip(lats, lons)): # Define pour point object diff --git a/rvic/core/multi_proc.py b/rvic/core/multi_proc.py index 8247b27..e6ff6de 100644 --- a/rvic/core/multi_proc.py +++ b/rvic/core/multi_proc.py @@ -3,53 +3,9 @@ multi_proc.py ''' -from .log import LOG_NAME import multiprocessing -from multiprocessing.pool import Pool -import traceback def error(*args): """Error function""" - return multiprocessing.get_logger(LOG_NAME).error(*args) -# -------------------------------------------------------------------- # - - -class LogExceptions(object): - def __init__(self, func): - self.__callable = func - return - - def __call__(self, *args, **kwargs): - try: - result = self.__callable(*args, **kwargs) - - except Exception: - # Here we add some debugging help. If multiprocessing's - # debugging is on, it will arrange to log the traceback - error(traceback.format_exc()) - # Re-raise the original exception so the Pool worker can - # clean up - raise - - # It was fine, give a normal answer - return result -# -------------------------------------------------------------------- # - - -class LoggingPool(Pool): - __doc__ = 'RVIC Logging Subclass of pool\n' + Pool.__doc__ - - @property - def has_logging(self): - # just for testing - return True - - def apply_async(self, func, callback=None, error_callback=None, - *args, **kwds): - """Overloaded Pool.apply_async to support Logging""" - return Pool.apply_async(self, func, args=args, kwds=kwds, - callback=callback, - error_callback=error_callback) - -# -------------------------------------------------------------------- # + return multiprocessing.get_logger().error(*args) diff --git a/rvic/core/param_file.py b/rvic/core/param_file.py index aabf709..742e927 100644 --- a/rvic/core/param_file.py +++ b/rvic/core/param_file.py @@ -71,7 +71,7 @@ def finish_params(outlets, dom_data, config_dict, directories): else: log.info('Not subsetting because either SUBSET_DAYS is null or ' 'SUBSET_DAYS 1: - pool = LoggingPool(processes=numofproc) - results = [] + from multiprocessing import Pool + pool = Pool(processes=numofproc) + status = [] - for i, outlet in enumerate(outlets): + for i, (key, outlet) in enumerate(iteritems(outlets)): log.info('On Outlet #%s of %s', i + 1, len(outlets)) - result = pool.apply_async(gen_uh_run, - args=(uh_box, fdr_data, fdr_vatts, - dom_data, outlet, config_dict, - directories), - callback=store_result, - error_callback=pool.terminate) + stat = pool.apply_async(gen_uh_run, + (uh_box, fdr_data, fdr_vatts, + dom_data, outlet, config_dict, + directories), + callback=store_result, + error_callback=error) # Store the result - results.append(result) + status.append(stat) # Close the pool pool.close() # Check that everything worked - [result.get() for result in results] + [stat.get() for stat in status] pool.join() - outlets = sorted(results, key=lambda x: x.cell_id, reverse=True) + outlets = OrderedDict(sorted(results.items(), reverse=True)) else: - for outlet in outlets: + for i, (key, outlet) in enumerate(iteritems(outlets)): outlet = gen_uh_run(uh_box, fdr_data, fdr_vatts, dom_data, outlet, config_dict, directories) @@ -314,7 +315,7 @@ def gen_uh_init(config_file): 'pour points and outlet grid cells') else: - outlets = [] + outlets = OrderedDict() if all(x in list(pour_points.keys()) for x in ['x', 'y', 'lons', 'lats']): lats = pour_points['lats'].values @@ -362,6 +363,18 @@ def gen_uh_init(config_file): # fill name filed with p-outlet_num name = 'p-{0}'.format(i) + print(dict(lat=lats[i])) + print(dict(lon=lons[i])) + print(dict(domx=domxs[i])) + print(dict(domy=domys[i])) + print(dict(routx=routxs[i])) + print(dict(routy=routys[i])) + print(dict(name=name)) + print(dom_data['cell_ids'].shape) + print([domys[i], domxs[i]]) + + print(dict(cell_id=dom_data['cell_ids'][domys[i], domxs[i]])) + outlets[i] = Point(lat=lats[i], lon=lons[i], domx=domxs[i], @@ -369,8 +382,7 @@ def gen_uh_init(config_file): routx=routxs[i], routy=routys[i], name=name, - cell_id=dom_data['cell_ids'][domys[i], - domxs[i]]) + cell_id=dom_data['cell_ids'][domys[i], domxs[i]]) outlets[i].pour_points = [outlets[i]] # ---------------------------------------------------------------- # @@ -636,7 +648,7 @@ def store_result(result): Globals ---------- - results : list + results : dict Global results container for multiprocessing results to be appended to. ''' results[result.cell_id] = result diff --git a/tests/unit/test_multi_proc.py b/tests/unit/test_multi_proc.py deleted file mode 100644 index c1e70bf..0000000 --- a/tests/unit/test_multi_proc.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest -from rvic.core.multi_proc import LoggingPool, LogExceptions - - -def go_and_raise(): - print(1) - raise Exception() - print(2) - - -def test_create_log_exceptions(): - LogExceptions(callable) - - -def test_create_logging_pool(): - n = 2 - p = LoggingPool(processes=n) - assert hasattr(p, 'apply_async') - assert p._processes == n - assert p.has_logging - - -def test_logging_pool_raises(): - p = LoggingPool(processes=2) - - with pytest.raises(Exception): - x = p.apply_async(go_and_raise) - x.get() - p.close() - p.join() From e316f3fda978f312f41ab63950620fbe2f6af8b4 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 24 Oct 2015 11:49:03 -0700 Subject: [PATCH 40/47] remove extra timestep in output files, replace NcVar with simple dictionary --- rvic/core/history.py | 78 ++++++------- rvic/core/share.py | 266 +++++++++++++++++++++---------------------- rvic/core/write.py | 120 +++++++++---------- 3 files changed, 230 insertions(+), 234 deletions(-) diff --git a/rvic/core/history.py b/rvic/core/history.py index 41ec3bd..9629edf 100644 --- a/rvic/core/history.py +++ b/rvic/core/history.py @@ -519,10 +519,10 @@ def __write_grid(self): time = f.createVariable('time', self._ncprec, ('time',)) time[:] = self._out_times[:self._out_data_i + 1] - for key, val in iteritems(share.time.__dict__): + for key, val in iteritems(share.time): if val: - setattr(time, key, val) - time.calendar = self._calendar + setattr(time, key, val.encode()) + time.calendar = self._calendar.encode() if self._avgflag != 'I': f.createDimension('nv', 2) @@ -548,13 +548,13 @@ def __write_grid(self): xc[:, :] = self._grid_lons yc[:, :] = self._grid_lats - for key, val in iteritems(share.xc.__dict__): + for key, val in iteritems(share.xc): if val: - setattr(xc, key, val) + setattr(xc, key, val.encode()) - for key, val in iteritems(share.yc.__dict__): + for key, val in iteritems(share.yc): if val: - setattr(yc, key, val) + setattr(yc, key, val.encode()) else: coords = ('lat', 'lon',) @@ -569,13 +569,13 @@ def __write_grid(self): lon[:] = self._grid_lons lat[:] = self._grid_lats - for key, val in iteritems(share.lon.__dict__): + for key, val in iteritems(share.lon): if val: - setattr(lon, key, val) + setattr(lon, key, val.encode()) - for key, val in iteritems(share.lat.__dict__): + for key, val in iteritems(share.lat): if val: - setattr(lat, key, val) + setattr(lat, key, val.encode()) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -587,12 +587,12 @@ def __write_grid(self): **self.ncvaropts) var[:, :] = self._out_data[field][:self._out_data_i + 1] - for key, val in iteritems(getattr(share, field).__dict__): + for key, val in iteritems(getattr(share, field)): if val: - setattr(var, key, val) - var.units = self._units + setattr(var, key, val.encode()) + var.units = self._units.encode() if self._grid_lons.ndim > 1: - var.coordinates = ' '.join(coords) + var.coordinates = ' '.join(coords).encode() # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -600,7 +600,7 @@ def __write_grid(self): self._glob_ats.update() for key, val in iteritems(self._glob_ats.atts): if val: - setattr(f, key, val) + setattr(f, key, val.encode()) # ------------------------------------------------------------ # f.close() log.info('Finished writing %s', self.filename) @@ -622,11 +622,11 @@ def __write_array(self): time = f.createVariable('time', self._ncprec, ('time',), **self.ncvaropts) - time[:] = self._out_times[:self._out_data_i + 1] - for key, val in iteritems(share.time.__dict__): + time[:] = self._out_times[:self._out_data_i] + for key, val in iteritems(share.time): if val: - setattr(time, key, val) - time.calendar = self._calendar + setattr(time, key, val.encode()) + time.calendar = self._calendar.encode() if self._avgflag != 'I': f.createDimension('nv', 2) @@ -635,7 +635,7 @@ def __write_array(self): time_bnds = f.createVariable('time_bnds', self._ncprec, ('time', 'nv',), **self.ncvaropts) - time_bnds[:, :] = self._out_time_bnds[:self._out_data_i + 1] + time_bnds[:, :] = self._out_time_bnds[:self._out_data_i] # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -671,29 +671,29 @@ def __write_array(self): outlet_decomp_ind[:] = self._outlet_decomp_ind onm[:, :] = char_names - for key, val in iteritems(share.outlet_lon.__dict__): + for key, val in iteritems(share.outlet_lon): if val: - setattr(outlet_lon, key, val) + setattr(outlet_lon, key, val.encode()) - for key, val in iteritems(share.outlet_lat.__dict__): + for key, val in iteritems(share.outlet_lat): if val: - setattr(outlet_lat, key, val) + setattr(outlet_lat, key, val.encode()) - for key, val in iteritems(share.outlet_y_ind.__dict__): + for key, val in iteritems(share.outlet_y_ind): if val: - setattr(outlet_y_ind, key, val) + setattr(outlet_y_ind, key, val.encode()) - for key, val in iteritems(share.outlet_x_ind.__dict__): + for key, val in iteritems(share.outlet_x_ind): if val: - setattr(outlet_x_ind, key, val) + setattr(outlet_x_ind, key, val.encode()) - for key, val in iteritems(share.outlet_decomp_ind.__dict__): + for key, val in iteritems(share.outlet_decomp_ind): if val: - setattr(outlet_decomp_ind, key, val) + setattr(outlet_decomp_ind, key, val.encode()) - for key, val in iteritems(share.outlet_name.__dict__): + for key, val in iteritems(share.outlet_name): if val: - setattr(onm, key, val) + setattr(onm, key, val.encode()) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -703,12 +703,12 @@ def __write_array(self): for field in self._fincl: var = f.createVariable(field, self._ncprec, tcoords, **self.ncvaropts) - var[:, :] = self._out_data[field][:self._out_data_i + 1] + var[:, :] = self._out_data[field][:self._out_data_i] - for key, val in iteritems(getattr(share, field).__dict__): + for key, val in iteritems(getattr(share, field)): if val: - setattr(var, key, val) - var.units = self._units + setattr(var, key, val.encode()) + var.units = self._units.encode() # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -716,8 +716,8 @@ def __write_array(self): self._glob_ats.update() for key, val in iteritems(self._glob_ats.atts): if val: - setattr(f, key, val) - f.featureType = 'timeSeries' + setattr(f, key, val.encode()) + f.featureType = 'timeSeries'.encode() # ------------------------------------------------------------ # f.close() log.info('Finished writing %s', self.filename) diff --git a/rvic/core/share.py b/rvic/core/share.py index aacdf03..a5378a4 100644 --- a/rvic/core/share.py +++ b/rvic/core/share.py @@ -23,8 +23,8 @@ # time # reference time REFERENCE_STRING = '0001-1-1 0:0:0' -REFERENCE_DATE = 10101 # i.e. REFERENCE_STRING -REFERENCE_TIME = 0 # i.e. REFERENCE_STRING +REFERENCE_DATE = 10101 # i.e. REFERENCE_STRING +REFERENCE_TIME = 0 # i.e. REFERENCE_STRING TIMEUNITS = 'days since ' + REFERENCE_STRING # do not change (MUST BE DAYS)! TIMESTAMPFORM = '%Y-%m-%d-%H' CALENDAR = 'noleap' @@ -155,195 +155,191 @@ def update(self): 'Created: {0}'.format(time_mod.ctime(time_mod.time())) -class NcVar(object): - def __init__(self, **kwargs): - for key, val in iteritems(kwargs): - setattr(self, key, val) - # Coordinate Variables -time = NcVar(long_name='time', - units=TIMEUNITS) +time = dict(long_name='time', + units=TIMEUNITS) -time_bnds = NcVar() +time_bnds = dict() -timesteps = NcVar(long_name='Series of timesteps', - units='unitless') +timesteps = dict(long_name='Series of timesteps', + nits='unitless') -lon = NcVar(long_name='longitude', - units='degrees_east') +lon = dict(long_name='longitude', + nits='degrees_east') -lat = NcVar(long_name='latitude', - units='degrees_north') +lat = dict(long_name='latitude', + units='degrees_north') -xc = NcVar(long_name='longitude', - units='degrees_east') +xc = dict(long_name='longitude', + units='degrees_east') -yc = NcVar(long_name='latitude', - units='degrees_north') +yc = dict(long_name='latitude', + units='degrees_north') # Data Variables -fraction = NcVar(long_name='fraction of grid cell that is active', - units='unitless') +fraction = dict(long_name='fraction of grid cell that is active', + units='unitless') -unit_hydrograph = NcVar(long_name='Unit Hydrograph', - units='unitless') +unit_hydrograph = dict(long_name='Unit Hydrograph', + units='unitless') -avg_velocity = NcVar(long_name='Flow Velocity Parameter', - units='m s-1') +avg_velocity = dict(long_name='Flow Velocity Parameter', + units='m s-1') -avg_diffusion = NcVar(long_name='Diffusion Parameter', - units='m2 s-1') +avg_diffusion = dict(long_name='Diffusion Parameter', + units='m2 s-1') -global_basin_id = NcVar(long_name='Global Basin ID from RvicFdrFile', - units='unitless') +global_basin_id = dict(long_name='Global Basin ID from RvicFdrFile', + units='unitless') -full_time_length = NcVar(long_name='Length of original unit hydrograph', - units='timesteps') +full_time_length = dict(long_name='Length of original unit hydrograph', + units='timesteps') -subset_length = NcVar(long_name='Shortened length of the unit hydrograph', - units='timesteps') +subset_length = dict(long_name='Shortened length of the unit hydrograph', + units='timesteps') -unit_hydrograph_dt = NcVar(long_name='Unit hydrograph timestep', - units='seconds') +unit_hydrograph_dt = dict(long_name='Unit hydrograph timestep', + units='seconds') -outlet_x_ind = NcVar(long_name='x grid coordinate of outlet grid cell', - units='unitless') +outlet_x_ind = dict(long_name='x grid coordinate of outlet grid cell', + units='unitless') -outlet_y_ind = NcVar(long_name='y grid coordinate of outlet grid cell', - units='unitless') +outlet_y_ind = dict(long_name='y grid coordinate of outlet grid cell', + units='unitless') -outlet_lon = NcVar(long_name='Longitude coordinate of outlet grid cell', - units='degrees_east') +outlet_lon = dict(long_name='Longitude coordinate of outlet grid cell', + units='degrees_east') -outlet_lat = NcVar(long_name='Latitude coordinate of outlet grid cell', - units='degrees_north') +outlet_lat = dict(long_name='Latitude coordinate of outlet grid cell', + units='degrees_north') -outlet_decomp_ind = NcVar(long_name='1d grid location of outlet grid cell', - units='unitless') +outlet_decomp_ind = dict(long_name='1d grid location of outlet grid cell', + units='unitless') -outlet_number = NcVar(long_name='outlet number', - units='unitless') +outlet_number = dict(long_name='outlet number', + units='unitless') -outlet_mask = NcVar(long_name='type of outlet point', - units='0-ocean, 1-land, 2-guage, 3-none') +outlet_mask = dict(long_name='type of outlet point', + units='0-ocean, 1-land, 2-guage, 3-none') -outlet_name = NcVar(long_name='Outlet guage name', - units='unitless') +outlet_name = dict(long_name='Outlet guage name', + units='unitless') -outlet_upstream_area = NcVar(long_name='Upstream catchment area contributing ' - 'to outlet', - units='m2') +outlet_upstream_area = dict(long_name='Upstream catchment area contributing ' + 'to outlet', + units='m2') -outlet_upstream_gridcells = NcVar(long_name='Number of upstream grid cells ' - 'contributing to outlet', - units='number of grid cells') +outlet_upstream_gridcells = dict(long_name='Number of upstream grid cells ' + 'contributing to outlet', + units='number of grid cells') -source_x_ind = NcVar(long_name='x grid coordinate of source grid cell', - units='unitless') +source_x_ind = dict(long_name='x grid coordinate of source grid cell', + units='unitless') -source_y_ind = NcVar(long_name='y grid coordinate of source grid cell', - units='unitless') +source_y_ind = dict(long_name='y grid coordinate of source grid cell', + units='unitless') -source_lon = NcVar(long_name='Longitude coordinate of source grid cell', - units='degrees_east') +source_lon = dict(long_name='Longitude coordinate of source grid cell', + units='degrees_east') -source_lat = NcVar(long_name='Latitude coordinate of source grid cell', - units='degrees_north') +source_lat = dict(long_name='Latitude coordinate of source grid cell', + units='degrees_north') -source_decomp_ind = NcVar(long_name='1d grid location of source grid cell', - units='unitless') -source_time_offset = NcVar(long_name='Number of leading timesteps ommited', - units='timesteps') +source_decomp_ind = dict(long_name='1d grid location of source grid cell', + units='unitless') +source_time_offset = dict(long_name='Number of leading timesteps ommited', + units='timesteps') -source2outlet_ind = NcVar(long_name='source to outlet index mapping', - units='unitless') +source2outlet_ind = dict(long_name='source to outlet index mapping', + units='unitless') -ring = NcVar(long_name='Convolution Ring', - units='kg m-2 s-1') +ring = dict(long_name='Convolution Ring', + units='kg m-2 s-1') -streamflow = NcVar(long_name='Streamflow at outlet grid cell', - units='kg m-2 s-1') +streamflow = dict(long_name='Streamflow at outlet grid cell', + units='kg m-2 s-1') -storage = NcVar(long_name='Mass storage in stream upstream of outlet ' - 'grid cell', - units='kg m-2 s-1') +storage = dict(long_name='Mass storage in stream upstream of outlet ' + 'grid cell', + units='kg m-2 s-1') -# valid values http://cf-pcmdi.llnl.gov/documents/cf-conventions/1.6/cf-conventions.html#calendar -timemgr_rst_type = NcVar(long_name='calendar type', - units='unitless', - flag_values='0, 1, 2, 3, 4, 5, 6', - flag_meanings='NONE, NO_LEAP_C, GREGORIAN, ' - 'PROLEPTIC_GREGORIAN, ALL_LEAP, ' - '360_DAY, JULIAN') +# valid values http://cf-pcmdi.llnl.gov/documents/cf-conventions/1.6/\ +# cf-conventions.html#calendar +timemgr_rst_type = dict(long_name='calendar type', + units='unitless', + flag_values='0, 1, 2, 3, 4, 5, 6', + flag_meanings='NONE, NO_LEAP_C, GREGORIAN, ' + 'PROLEPTIC_GREGORIAN, ALL_LEAP, ' + '360_DAY, JULIAN') -timemgr_rst_step_sec = NcVar(long_name='seconds component of timestep size', +timemgr_rst_step_sec = dict(long_name='seconds component of timestep size', + units='sec', + valid_range='0, 86400') + +timemgr_rst_start_ymd = dict(long_name='start date', + units='YYYYMMDD') + +timemgr_rst_start_tod = dict(long_name='start time of day', units='sec', valid_range='0, 86400') -timemgr_rst_start_ymd = NcVar(long_name='start date', - units='YYYYMMDD') +timemgr_rst_ref_ymd = dict(long_name='reference date', + units='YYYYMMDD') -timemgr_rst_start_tod = NcVar(long_name='start time of day', - units='sec', - valid_range='0, 86400') +timemgr_rst_ref_tod = dict(long_name='reference time of day', + units='sec', + valid_range='0, 86400') -timemgr_rst_ref_ymd = NcVar(long_name='reference date', +timemgr_rst_curr_ymd = dict(long_name='current date', units='YYYYMMDD') -timemgr_rst_ref_tod = NcVar(long_name='reference time of day', +timemgr_rst_curr_tod = dict(long_name='current time of day', units='sec', valid_range='0, 86400') -timemgr_rst_curr_ymd = NcVar(long_name='current date', - units='YYYYMMDD') - -timemgr_rst_curr_tod = NcVar(long_name='current time of day', - units='sec', - valid_range='0, 86400') - -nhtfrq = NcVar(long_name='Frequency of history writes', - units='absolute value of negative is in hours, 0=monthly, ' - 'positive is time-steps', - comment='Namelist item') - -mfilt = NcVar(long_name='Number of history time samples on a file', - units='initless', +nhtfrq = dict(long_name='Frequency of history writes', + units='absolute value of negative is in hours, 0=monthly, ' + 'positive is time-steps', comment='Namelist item') -ncprec = NcVar(long_name='Flag for data precision', - flag_values='1, 2', - flag_meanings='single-precision double-precision', - comment='Namelist item', - valid_range='1, 2') +mfilt = dict(long_name='Number of history time samples on a file', + units='initless', + comment='Namelist item') -fincl = NcVar(long_name='Fieldnames to include', - comment='Namelist item') +ncprec = dict(long_name='Flag for data precision', + flag_values='1, 2', + flag_meanings='single-precision double-precision', + comment='Namelist item', + valid_range='1, 2') -fexcl = NcVar(long_name='Fieldnames to exclude', - comment='Namelist item') +fincl = dict(long_name='Fieldnames to include', + comment='Namelist item') + +fexcl = dict(long_name='Fieldnames to exclude', + comment='Namelist item') -nflds = NcVar(long_name='Number of fields on file', - units='unitless') +nflds = dict(long_name='Number of fields on file', + units='unitless') -ntimes = NcVar(long_name='Number of time steps on file', - units='time-step') -is_endhist = NcVar(long_name='End of history file', - flag_values='0, 1', - flag_meanings='FALSE TRUE', - comment='Namelist item', - valid_range='0, 1') +ntimes = dict(long_name='Number of time steps on file', + units='time-step') +is_endhist = dict(long_name='End of history file', + flag_values='0, 1', + flag_meanings='FALSE TRUE', + comment='Namelist item', + valid_range='0, 1') -begtime = NcVar(long_name='Beginning time', - units='time units') +begtime = dict(long_name='Beginning time', + units='time units') -hpindex = NcVar(long_name='History pointer index', - units='units') +hpindex = dict(long_name='History pointer index', + units='units') -avgflag = NcVar(long_name='Averaging flag', - units='A=Average, X=Maximum, M=Minimum, I=Instantaneous') +avgflag = dict(long_name='Averaging flag', + units='A=Average, X=Maximum, M=Minimum, I=Instantaneous') -name = NcVar(long_name='Fieldnames') +name = dict(long_name='Fieldnames') -long_name = NcVar(long_name='Long descriptive names for fields') +long_name = dict(long_name='Long descriptive names for fields') -units = NcVar(long_name='Units for each history field output') +units = dict(long_name='Units for each history field output') diff --git a/rvic/core/write.py b/rvic/core/write.py index 52d3057..1913452 100644 --- a/rvic/core/write.py +++ b/rvic/core/write.py @@ -50,40 +50,40 @@ def write_agg_netcdf(file_name, agg_data, glob_atts, nc_format, zlib=True, unit_hydrograph_dt = f.createVariable('unit_hydrograph_dt', NC_INT, (), **ncvaropts) unit_hydrograph_dt[:] = agg_data['unit_hydrograph_dt'] - for key, val in iteritems(share.unit_hydrograph_dt.__dict__): + for key, val in iteritems(share.unit_hydrograph_dt): if val: - setattr(unit_hydrograph_dt, key, val) + setattr(unit_hydrograph_dt, key, val.encode()) timesteps = f.createVariable('timesteps', NC_INT, ('timesteps',), **ncvaropts) timesteps[:] = np.arange(agg_data['unit_hydrograph'].shape[0]) - for key, val in iteritems(share.timesteps.__dict__): + for key, val in iteritems(share.timesteps): if val: - setattr(timesteps, key, val) + setattr(timesteps, key, val.encode()) lon = f.createVariable('lon', NC_DOUBLE, ('lon',), **ncvaropts) - for key, val in iteritems(share.lon.__dict__): + for key, val in iteritems(share.lon): if val: - setattr(lon, key, val) + setattr(lon, key, val.encode()) lat = f.createVariable('lat', NC_DOUBLE, ('lat',), **ncvaropts) - for key, val in iteritems(share.lat.__dict__): + for key, val in iteritems(share.lat): if val: - setattr(lat, key, val) + setattr(lat, key, val.encode()) fraction = f.createVariable('fraction', NC_DOUBLE, ('lat', 'lon',), **ncvaropts) - for key, val in iteritems(share.fraction.__dict__): + for key, val in iteritems(share.fraction): if val: - setattr(fraction, key, val) + setattr(fraction, key, val.encode()) unit_hydrographs = f.createVariable('unit_hydrograph', NC_DOUBLE, ('timesteps', 'lat', 'lon',), fill_value=FILLVALUE_F, **ncvaropts) - for key, val in iteritems(share.unit_hydrograph.__dict__): + for key, val in iteritems(share.unit_hydrograph): if val: - setattr(unit_hydrographs, key, val) + setattr(unit_hydrographs, key, val.encode()) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -98,7 +98,7 @@ def write_agg_netcdf(file_name, agg_data, glob_atts, nc_format, zlib=True, # write globals for key, val in iteritems(glob_atts.atts): if val: - setattr(f, key, val) + setattr(f, key, val.encode()) # ---------------------------------------------------------------- # f.close() @@ -157,10 +157,10 @@ def write_param_file(file_name, timesteps = f.createVariable('timesteps', NC_DOUBLE, ('timesteps',), **ncvaropts) timesteps[:] = np.arange(subset_length) - for key, val in iteritems(share.timesteps.__dict__): + for key, val in iteritems(share.timesteps): if val: - setattr(timesteps, key, val) - timesteps.timestep_length = 'unit_hydrograph_dt' + setattr(timesteps, key, val.encode()) + timesteps.timestep_length = b'unit_hydrograph_dt' # ---------------------------------------------------------------- # @@ -169,8 +169,8 @@ def write_param_file(file_name, glob_atts.update() for key, val in iteritems(glob_atts.atts): if val: - setattr(f, key, val) - f.featureType = 'timeSeries' + setattr(f, key, val.encode()) + f.featureType = b'timeSeries' # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -179,23 +179,23 @@ def write_param_file(file_name, # Full time length (size of ring) ftl = f.createVariable('full_time_length', NC_INT, (), **ncvaropts) ftl[:] = full_time_length - for key, val in iteritems(share.full_time_length.__dict__): + for key, val in iteritems(share.full_time_length): if val: - setattr(ftl, key, val) + setattr(ftl, key, val.encode()) # Subset Length sl = f.createVariable('subset_length', NC_INT, (), **ncvaropts) sl[:] = subset_length - for key, val in iteritems(share.subset_length.__dict__): + for key, val in iteritems(share.subset_length): if val: - setattr(sl, key, val) + setattr(sl, key, val.encode()) # UH timestep uh_dt = f.createVariable('unit_hydrograph_dt', NC_DOUBLE, (), **ncvaropts) uh_dt[:] = unit_hydrograph_dt - for key, val in iteritems(share.unit_hydrograph_dt.__dict__): + for key, val in iteritems(share.unit_hydrograph_dt): if val: - setattr(uh_dt, key, val) + setattr(uh_dt, key, val.encode()) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -219,74 +219,74 @@ def write_param_file(file_name, # Outlet Cell Longitudes olon = f.createVariable('outlet_lon', NC_DOUBLE, ocoords, **ncvaropts) olon[:] = outlet_lon - for key, val in iteritems(share.outlet_lon.__dict__): + for key, val in iteritems(share.outlet_lon): if val: - setattr(olon, key, val) + setattr(olon, key, val.encode()) # Outlet Cell Latitudes olat = f.createVariable('outlet_lat', NC_DOUBLE, ocoords, **ncvaropts) olat[:] = outlet_lat - for key, val in iteritems(share.outlet_lat.__dict__): + for key, val in iteritems(share.outlet_lat): if val: - setattr(olat, key, val) + setattr(olat, key, val.encode()) # Outlet Cell X Indicies oxi = f.createVariable('outlet_x_ind', NC_INT, ocoords, **ncvaropts) oxi[:] = outlet_x_ind - for key, val in iteritems(share.outlet_x_ind.__dict__): + for key, val in iteritems(share.outlet_x_ind): if val: - setattr(oxi, key, val) + setattr(oxi, key, val.encode()) # Outlet Cell Y Indicies oyi = f.createVariable('outlet_y_ind', NC_INT, ocoords, **ncvaropts) oyi[:] = outlet_y_ind - for key, val in iteritems(share.outlet_y_ind.__dict__): + for key, val in iteritems(share.outlet_y_ind): if val: - setattr(oyi, key, val) + setattr(oyi, key, val.encode()) # Outlet Cell Decomp IDs odi = f.createVariable('outlet_decomp_ind', NC_INT, ocoords, **ncvaropts) odi[:] = outlet_decomp_ind - for key, val in iteritems(share.outlet_decomp_ind.__dict__): + for key, val in iteritems(share.outlet_decomp_ind): if val: - setattr(odi, key, val) + setattr(odi, key, val.encode()) # Outlet Cell Number on = f.createVariable('outlet_number', NC_INT, ocoords, **ncvaropts) on[:] = outlet_number - for key, val in iteritems(share.outlet_number.__dict__): + for key, val in iteritems(share.outlet_number): if val: - setattr(on, key, val) + setattr(on, key, val.encode()) # Outlet Mask om = f.createVariable('outlet_mask', NC_INT, ocoords, **ncvaropts) om[:] = outlet_mask - for key, val in iteritems(share.outlet_mask.__dict__): + for key, val in iteritems(share.outlet_mask): if val: - setattr(om, key, val) + setattr(om, key, val.encode()) # Outlet Upstream area oua = f.createVariable('outlet_upstream_area', NC_DOUBLE, ocoords, **ncvaropts) oua[:] = outlet_upstream_area - for key, val in iteritems(share.outlet_upstream_area.__dict__): + for key, val in iteritems(share.outlet_upstream_area): if val: - setattr(oua, key, val) + setattr(oua, key, val.encode()) # Outlet Upstream grid cells oug = f.createVariable('outlet_upstream_gridcells', NC_INT, ocoords, **ncvaropts) oug[:] = outlet_upstream_gridcells - for key, val in iteritems(share.outlet_upstream_gridcells.__dict__): + for key, val in iteritems(share.outlet_upstream_gridcells): if val: - setattr(oug, key, val) + setattr(oug, key, val.encode()) # Outlet Names onm = f.createVariable('outlet_name', NC_CHAR, nocoords) onm[:, :] = char_names - for key, val in iteritems(share.outlet_name.__dict__): + for key, val in iteritems(share.outlet_name): if val: - setattr(onm, key, val) + setattr(onm, key, val.encode()) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -301,51 +301,51 @@ def write_param_file(file_name, # Source Cell Longitudes slon = f.createVariable('source_lon', NC_DOUBLE, scoords, **ncvaropts) slon[:] = source_lon - for key, val in iteritems(share.source_lon.__dict__): + for key, val in iteritems(share.source_lon): if val: - setattr(slon, key, val) + setattr(slon, key, val.encode()) # Source Cell Latitudes slat = f.createVariable('source_lat', NC_DOUBLE, scoords, **ncvaropts) slat[:] = source_lat - for key, val in iteritems(share.source_lat.__dict__): + for key, val in iteritems(share.source_lat): if val: - setattr(slat, key, val) + setattr(slat, key, val.encode()) # Source Cell X Indicies sxi = f.createVariable('source_x_ind', NC_INT, scoords, **ncvaropts) sxi[:] = source_x_ind - for key, val in iteritems(share.source_x_ind.__dict__): + for key, val in iteritems(share.source_x_ind): if val: - setattr(sxi, key, val) + setattr(sxi, key, val.encode()) # Source Cell Y Indicies syi = f.createVariable('source_y_ind', NC_INT, scoords, **ncvaropts) syi[:] = source_y_ind - for key, val in iteritems(share.source_y_ind.__dict__): + for key, val in iteritems(share.source_y_ind): if val: - setattr(syi, key, val) + setattr(syi, key, val.encode()) # Source Cell Decomp IDs sdi = f.createVariable('source_decomp_ind', NC_INT, scoords, **ncvaropts) sdi[:] = source_decomp_ind - for key, val in iteritems(share.source_decomp_ind.__dict__): + for key, val in iteritems(share.source_decomp_ind): if val: - setattr(sdi, key, val) + setattr(sdi, key, val.encode()) # Source Cell Time Offset sto = f.createVariable('source_time_offset', NC_INT, scoords, **ncvaropts) sto[:] = source_time_offset - for key, val in iteritems(share.source_time_offset.__dict__): + for key, val in iteritems(share.source_time_offset): if val: - setattr(sto, key, val) + setattr(sto, key, val.encode()) # Source to Outlet Index Mapping s2o = f.createVariable('source2outlet_ind', NC_INT, scoords, **ncvaropts) s2o[:] = source2outlet_ind - for key, val in iteritems(share.source2outlet_ind.__dict__): + for key, val in iteritems(share.source2outlet_ind): if val: - setattr(s2o, key, val) + setattr(s2o, key, val.encode()) # ---------------------------------------------------------------- # @@ -357,9 +357,9 @@ def write_param_file(file_name, # Unit Hydrographs uhs = f.createVariable('unit_hydrograph', NC_DOUBLE, uhcords, **ncvaropts) uhs[:, :] = unit_hydrograph - for key, val in iteritems(share.unit_hydrograph.__dict__): + for key, val in iteritems(share.unit_hydrograph): if val: - setattr(uhs, key, val) + setattr(uhs, key, val.encode()) # ---------------------------------------------------------------- # f.close() From addd50c0ddbe7db03e0c5e3f09096ff281b040e7 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 24 Oct 2015 11:51:35 -0700 Subject: [PATCH 41/47] remove test for ncvar --- tests/unit/test_share.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/unit/test_share.py b/tests/unit/test_share.py index 69fc345..67382e9 100644 --- a/tests/unit/test_share.py +++ b/tests/unit/test_share.py @@ -1,11 +1,6 @@ -from rvic.core.share import NcGlobals, NcVar +from rvic.core.share import NcGlobals def test_nc_globals(): ncg = NcGlobals() ncg.update() - - -def test_nc_var(): - ncv = NcVar(test='test') - assert ncv.test == 'test' From 16471eee50d029a8f6f5f5ec3efa2f579bc471bb Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 24 Oct 2015 20:03:00 -0700 Subject: [PATCH 42/47] remove print statements in parameters.py --- rvic/parameters.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/rvic/parameters.py b/rvic/parameters.py index 155c72a..3123bc9 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -363,18 +363,6 @@ def gen_uh_init(config_file): # fill name filed with p-outlet_num name = 'p-{0}'.format(i) - print(dict(lat=lats[i])) - print(dict(lon=lons[i])) - print(dict(domx=domxs[i])) - print(dict(domy=domys[i])) - print(dict(routx=routxs[i])) - print(dict(routy=routys[i])) - print(dict(name=name)) - print(dom_data['cell_ids'].shape) - print([domys[i], domxs[i]]) - - print(dict(cell_id=dom_data['cell_ids'][domys[i], domxs[i]])) - outlets[i] = Point(lat=lats[i], lon=lons[i], domx=domxs[i], From 59dc2159ca90bd39051fd86cefc37fdb5b1e5078 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 24 Oct 2015 20:29:59 -0700 Subject: [PATCH 43/47] update version string for 1.1.0 release --- rvic/__init__.py | 1 + rvic/version.py | 2 ++ setup.py | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 rvic/version.py diff --git a/rvic/__init__.py b/rvic/__init__.py index e69de29..8f32b37 100644 --- a/rvic/__init__.py +++ b/rvic/__init__.py @@ -0,0 +1 @@ +from .version import short_version as __version__ diff --git a/rvic/version.py b/rvic/version.py new file mode 100644 index 0000000..e284ae3 --- /dev/null +++ b/rvic/version.py @@ -0,0 +1,2 @@ +version = '1.1.0' +short_version = '1.1.0' diff --git a/setup.py b/setup.py index 97c8236..fe870a6 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,9 @@ from distutils.extension import Extension MAJOR = 1 -MINOR = 0 -MICRO = 1 -ISRELEASED = False +MINOR = 1 +MICRO = 0 +ISRELEASED = True VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) QUALIFIER = '' From d6b7d2870d945e1f0a179ed81c967db6a4a8dbca Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 24 Oct 2015 21:21:02 -0700 Subject: [PATCH 44/47] update docs for 1.1 release --- docs/development/whats_new.md | 16 ++++++++++++++++ docs/support/faq.md | 4 ++-- docs/user-guide/install.md | 14 ++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 docs/development/whats_new.md diff --git a/docs/development/whats_new.md b/docs/development/whats_new.md new file mode 100644 index 0000000..adba2d5 --- /dev/null +++ b/docs/development/whats_new.md @@ -0,0 +1,16 @@ +# What's New + +v1.1.0 (25 October 2015) + +This release contains a number of bug and compatibility fixes. + +### Enhancements + +- Compatibility with Python 3.4 and 3.5 ([GH38](https://github.com/UW-Hydro/RVIC/pull/38)). +- Simplified multiprocessing in `parameters` which should improve stability ([GH60](https://github.com/UW-Hydro/RVIC/pull/60)). + +### Bug Fixes + +- Fixed bug that caused `REMAP=False` to fail ([GH60](https://github.com/UW-Hydro/RVIC/pull/60)). +- Improvements were made to the `SEARCH_FOR_CHANNEL` option in `parameters`. +- Fixed bug where last timestep of history output was not populated ([GH71](https://github.com/UW-Hydro/RVIC/pull/71)). diff --git a/docs/support/faq.md b/docs/support/faq.md index 8ac6a32..67705df 100644 --- a/docs/support/faq.md +++ b/docs/support/faq.md @@ -4,7 +4,7 @@ At the moment, no. However, many of the pieces needed to support this feature are already included in RVIC. This is a feature that has been identified for a future version of RVIC. ### Can route flows to every grid cell in my domain using RVIC? -Yes, it is possible to route to every grid cell in the RVIC domain. That said, it may not be the most efficent way to use RVIC. RVIC is a source to sink routing model which means it doesn't track where streamflow is along its flow path. When you route to every grid cell in the domain, you are duplicating a lot of calculations. There are other routing models that route flow from grid cell to grid cell. +Yes, it is possible to route to every grid cell in the RVIC domain. That said, it may not be the most efficient way to use RVIC. RVIC is a source to sink routing model which means it doesn't track where streamflow is along its flow path. When you route to every grid cell in the domain, you are duplicating a lot of calculations. There are other routing models that route flow from grid cell to grid cell. ### Is there a C or Fortran Version of RVIC? -RVIC has been coupled in the Community Earth System Model (CESM) as the streamflow routing model. For that project, a Fortran version of the "convolution" step was written. At this time, there is not a C version of this routing model. In the future, a C binding may be created to coupled with the stand-alone VIC model version 5. +RVIC has been coupled in the Community Earth System Model (CESM) as the streamflow routing model. For that project, a Fortran version of the "convolution" step was written. Work is currently underway to couple RVIC to the stand-alone image driver in VIC 5.0 ([VIC231](https://github.com/UW-Hydro/VIC/pull/231)). diff --git a/docs/user-guide/install.md b/docs/user-guide/install.md index 9afc84d..0409470 100644 --- a/docs/user-guide/install.md +++ b/docs/user-guide/install.md @@ -12,6 +12,20 @@ If using `REMAP=True`: - [CDO](https://code.zmaw.de/projects/cdo) - [cdo.py](https://github.com/Try2Code/cdo-bindings) +## Installing using a package manager + +RVIC is available via [PyPi](https://pypi.python.org/pypi/rvic): + +``` +pip install rvic +``` + +or Anaconda via the UW-Hydro channel: + +``` +conda install --channel https://conda.anaconda.org/UW-Hydro rvic +``` + ## Building RVIC ### Option 1: Using Anaconda From d89e622b1b35b85592a6240c9b7e5c56ba46e64a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 24 Oct 2015 21:22:15 -0700 Subject: [PATCH 45/47] update setup.py for pypi upload --- setup.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fe870a6..5fec1e1 100644 --- a/setup.py +++ b/setup.py @@ -102,12 +102,22 @@ def write_version_py(filename=None): '48(A), 708-721', author='Joe Hamman', author_email='jhamman1@uw.edu', + classifiers=['Development Status :: 4 - Beta', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Operating System :: OS Independent', + 'Intended Audience :: Science/Research', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Topic :: Scientific/Engineering'], install_requires=['scipy >= 0.13', 'numpy >= 1.8', 'netCDF4 >= 1.0.6', 'matplotlib >= 1.3.1', 'pandas >= 0.15.1'], tests_require=['pytest >= 2.5.2'], url='https://github.com/UW-Hydro/RVIC', - test_suite='pytest.collector', packages=['rvic', 'rvic.core'], py_modules=['rvic.parameters', 'rvic.convolution', 'rvic.convert'], scripts=['scripts/rvic', 'tools/find_pour_points.py', From 2cb7922a6d511bc1cbc77f45fd31c58a448bca01 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 24 Oct 2015 23:01:17 -0700 Subject: [PATCH 46/47] add rasm and pnw sample configurations using git lfs --- .gitattributes | 2 + .gitignore | 1 + config/rvic_convert_example.cfg | 95 ------------- .../configs/rvic.convolution.rasm.cfg | 27 ++-- samples/configs/rvic.parameters.pnw.cfg | 133 ++++++++++++++++++ .../configs/rvic.parameters.rasm.cfg | 24 ++-- samples/domains/domain.lnd.bpa304.20140311.nc | 3 + .../domains/domain.lnd.wr50a_ar9v4.100920.nc | 3 + .../Wu_routing_inputs_060313.nc | 3 + .../pnw.RVIC.input_20140218.nc | 3 + samples/forcings/rasm_sample_runoff.nc | 3 + .../columbia_sample_pour_points.csv | 3 + .../pour_points/rasm_sample_pour_points.csv | 3 + samples/uh_box/UH_Columbia_hourly.csv | 3 + samples/uh_box/UH_RASM_hourly.csv | 3 + 15 files changed, 188 insertions(+), 121 deletions(-) create mode 100644 .gitattributes delete mode 100644 config/rvic_convert_example.cfg rename config/rvic_convolution_example.cfg => samples/configs/rvic.convolution.rasm.cfg (90%) create mode 100644 samples/configs/rvic.parameters.pnw.cfg rename config/rvic_parameters_example.cfg => samples/configs/rvic.parameters.rasm.cfg (89%) create mode 100644 samples/domains/domain.lnd.bpa304.20140311.nc create mode 100644 samples/domains/domain.lnd.wr50a_ar9v4.100920.nc create mode 100644 samples/flow_directions/Wu_routing_inputs_060313.nc create mode 100644 samples/flow_directions/pnw.RVIC.input_20140218.nc create mode 100644 samples/forcings/rasm_sample_runoff.nc create mode 100644 samples/pour_points/columbia_sample_pour_points.csv create mode 100644 samples/pour_points/rasm_sample_pour_points.csv create mode 100644 samples/uh_box/UH_Columbia_hourly.csv create mode 100644 samples/uh_box/UH_RASM_hourly.csv diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a0e751d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.nc filter=lfs diff=lfs merge=lfs -text +*.csv filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 3652051..484bd40 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ rvic.egg-info # Configuration Files # ####################### *cfg +samples/cases/* # Packages # ############ diff --git a/config/rvic_convert_example.cfg b/config/rvic_convert_example.cfg deleted file mode 100644 index ffeddd8..0000000 --- a/config/rvic_convert_example.cfg +++ /dev/null @@ -1,95 +0,0 @@ -#-- ========================================================================== --# -#-- --# -#-- This RVIC namelist contains options and paths for the --# -#-- development of the RVIC parameter file based on existing --# -#-- UHS files from either the C or Fortran Version of the model. --# -#-- --# -#-- ========================================================================== --# - -# Note: namelist is parsed by the python ConfigParser module. %(Interploation) is -# supported inside [sections] only. - -[OPTIONS] -#-- ====================================== --# -#--Level to log output at (char) --# -# valid values: DEBUG, INFO, WARNING, ERROR, CRITICAL -LOG_LEVEL:INFO - -#--Print output to console in addition to the log file (bool) --# -# valid values: True, False -VERBOSE:True - -#--case description (char) --# -CASEID:example_case_name - -#--routing domain grid shortname (char) --# -GRIDID: example_grid_id - -#--case run directory (char) --# -CASE_DIR:/path/to/%(CASEID)s/ - -#-- Output parameter file format (char) --# -# Valid Values: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, and NETCDF4 -# For use with CESM, NETCDF3_CLASSIC is reccomended. -NETCDF_FORMAT:NETCDF3_CLASSIC - -#-- Output parameter file compression options --# -# Descriptions of these options can be found in -NETCDF_ZLIB: False -NETCDF_COMPLEVEL: 4 -NETCDF_SIGFIGS: None - -#-- Length of unit hydrograph subset in days (int) --# -SUBSET_DAYS:10 - -#-- Constrain the final unit hydrographs sum to be less than or equal to the domain fractions --# -# True when routing to coastal grid cells, else False -CONSTRAIN_FRACTIONS:False -#-- ====================================== --# - -[UHS_FILES] -#-- ====================================== --# -#-- Routing program UHS files were created in (char) --# -# Valid Values: C, Fortran -ROUT_PROGRAM:C - -#-- Location of UHS files (char) --# -ROUT_DIR: /path/to/inputUHS/ - -#-- Path to stations file (char) --# -STATION_FILE: %(ROUT_DIR)s/stations.route_c -#-- ====================================== --# - -[ROUTING] -#-- ====================================== --# -#-- Output Interval --# -# Timestep of unit hydrographs. -OUTPUT_INTERVAL: 86400 -#-- ====================================== --# - -[DOMAIN] -#-- ====================================== --# -#-- Path to cesm complient domain file, this needs to match the dimensions of -# the grid the UHS files were developed on (char) --# -FILE_NAME: /path/to/domain.lnd.nc - -#-- netCDF Variable Names --# -LONGITUDE_VAR: lon -LATITUDE_VAR: lat -LAND_MASK_VAR: mask -FRACTION_VAR: frac -AREA_VAR: area -#-- ====================================== --# - -[NEW_DOMAIN] -#-- ====================================== --# -#-- Path to cesm complient domain file final routing to be done on (char) --# -FILE_NAME: /path/to/domain.rvic.nc - -#-- netCDF Variable Names --# -LONGITUDE_VAR: lon -LATITUDE_VAR: lat -LAND_MASK_VAR: mask -FRACTION_VAR: frac -AREA_VAR: area -#-- ====================================== --# diff --git a/config/rvic_convolution_example.cfg b/samples/configs/rvic.convolution.rasm.cfg similarity index 90% rename from config/rvic_convolution_example.cfg rename to samples/configs/rvic.convolution.rasm.cfg index e3a3ca8..ac2d35e 100644 --- a/config/rvic_convolution_example.cfg +++ b/samples/configs/rvic.convolution.rasm.cfg @@ -19,13 +19,10 @@ VERBOSE:True #-- ====================================== --# #--case run directory (char) --# -CASE_DIR : /path/to/%(CASEID)s/ - -#--RVIC tag (char) --# -RVIC_TAG : 1.0beta +CASE_DIR : ./samples/cases/%(CASEID)s/ #--case description (char) --# -CASEID : testing +CASEID : sample_rasm_convolution #--case description (char) --# CASESTR : a simple test run @@ -39,7 +36,7 @@ CALENDAR : noleap RUN_TYPE : drystart #--Run start date (yyyy-mm-dd-hh). Only used for startup and drystart runs (char) --# -RUN_STARTDATE : 1989-01-01-01 +RUN_STARTDATE : 1979-09-01-00 #-- ====================================== --# @@ -106,7 +103,7 @@ RVICHIST_OUTTYPE: grid, array RVICHIST_NCFORM: NETCDF4_CLASSIC, NETCDF4_CLASSIC #-- Output parameter file compression options --# -# Descriptions of these options can be found in +# Descriptions of these options can be found in RVICHIST_NETCDF_ZLIB: False RVICHIST_NETCDF_COMPLEVEL: 4 RVICHIST_NETCDF_SIGFIGS: None @@ -119,7 +116,7 @@ RVICHIST_UNITS: m3/s, m3/s [DOMAIN] #--rof domain file (char) --> -FILE_NAME: /path/to/domain.lnd.nc +FILE_NAME: ./samples/domains/domain.lnd.wr50a_ar9v4.100920.nc LONGITUDE_VAR: xc LATITUDE_VAR: yc AREA_VAR: area @@ -136,24 +133,24 @@ FILE_NAME: None [PARAM_FILE] #--rvic parameter file file (char) --> -FILE_NAME:/path/to/rvic.prm.nc +FILE_NAME: ./samples/cases/sample_rasm_parameters/params/sample_rasm_parameters.rvic.prm.wr50a.20151024.nc #-- ====================================== --# [INPUT_FORCINGS] -DATL_PATH:/path/to/ +DATL_PATH: ./samples/forcings/ # prfix.$YYYY[-$MM-[$DD[-$HH]]].nc -DATL_FILE: inputs.lnd.h.$YYYY-$MM.nc +DATL_FILE: rasm_sample_runoff.nc #--variable names (char) --> TIME_VAR: time -LATITUDE_VAR: lat -DATL_LIQ_FLDS: l2x_Flrl_rofliq +LATITUDE_VAR: yc +DATL_LIQ_FLDS: Runoff, Baseflow #--start date, date formate YYYY[-MM[-DD]] (char) --> -START:1989-01 +START: #--end date, date formate YYYY[-MM[-DD]] (char) --> -END:1989-12 +END: #-- ====================================== --# diff --git a/samples/configs/rvic.parameters.pnw.cfg b/samples/configs/rvic.parameters.pnw.cfg new file mode 100644 index 0000000..a61483a --- /dev/null +++ b/samples/configs/rvic.parameters.pnw.cfg @@ -0,0 +1,133 @@ +#-- ========================================================================== --# +#-- --# +#-- This RVIC namelist contains options and paths for the --# +#-- development of the RVIC parameter file. --# +#-- --# +#-- --# +#-- ========================================================================== --# + +# Note: namelist is parsed by the python ConfigParser module. %(Interploation) is +# supported inside [sections] only. + +[OPTIONS] +#-- ====================================== --# +#--Level to log output at (char) --# +# valid values: DEBUG, INFO, WARNING, ERROR, CRITICAL +LOG_LEVEL:DEBUG + +#--Print output to console in addition to the log file (bool) --# +# valid values: True, False +VERBOSE:True + +#--Delete temporary files, only used if REMAP=True (bool) --# +# valid values: True, False +CLEAN: False + +#--case description (char) --# +CASEID: sample_pnw_parameters + +#--routing domain grid shortname (char) --# +GRIDID: pnw + +#--case run directory (char) --# +CASE_DIR: ./samples/cases/%(CASEID)s + +#--Directory to use for temporary read/write operations (char) --# +TEMP_DIR:%(CASE_DIR)s/temp/ + +#--Remap Unit Hydrographs from [ROUTING] grid to [DOMAIN] grid (bool) --# +# valid values: True, False +REMAP:False + +#--Aggregate all [POUR_POINTS] inside each [DOMAIN] grid cell (bool) --# +# This should only be used when routing to coastal grid cells for CESM +AGGREGATE:False + +#--Size of pad to add to aggregated files prior to remapping (int) --# +AGG_PAD:25 + +#-- Output parameter file format (char) --# +# Valid Values: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, and NETCDF4 +# For use with CESM, NETCDF3_CLASSIC is recommended. +NETCDF_FORMAT:NETCDF3_CLASSIC + +#-- Output parameter file compression options --# +# Descriptions of these options can be found in +NETCDF_ZLIB: False +NETCDF_COMPLEVEL: 4 +NETCDF_SIGFIGS: None + +#-- Length of unit hydrograph subset in days (int) --# +SUBSET_DAYS:10 + +#-- Constrain the final unit hydrographs sum to be less than or equal to the domain fractions --# +# True when routing to coastal grid cells, else False +CONSTRAIN_FRACTIONS: False + +SEARCH_FOR_CHANNEL: True + +#-- ====================================== --# + +[POUR_POINTS] +#-- ====================================== --# +#-- Path to Pour Points File (char) --# +# A comma separated file of outlets to route to [lons, lats] - one coordinate pair per line (order not important) +# May optionally include a column [names] - which will (if not aggregating) be included in param file +FILE_NAME: ./samples/pour_points/columbia_sample_pour_points.csv + +#-- ====================================== --# + +[UH_BOX] +#-- ====================================== --# +#-- Path to UH Box File (char) --# +# This defines the unit hydrograph to rout flow to the edge of each grid cell. +# A comma separated file of [time in seconds, unit hydrograph ordinate] - one timestep per line +# The timestep should be 1hr (3600 sec) or less. +FILE_NAME: ./samples/uh_box/UH_Columbia_hourly.csv + +#-- Number of Header lines to ignore in [UH_BOX]FILE_NAME (INT) --# +HEADER_LINES = 1 +#-- ====================================== --# + +[ROUTING] +#-- ====================================== --# +#-- Path to routing inputs netcdf (char) --# +FILE_NAME: ./samples/flow_directions/pnw.RVIC.input_20140218.nc + +#-- netCDF Variable Names --# +LONGITUDE_VAR: lon +LATITUDE_VAR: lat +FLOW_DISTANCE_VAR: Flow_Distance +FLOW_DIRECTION_VAR: Flow_Direction +BASIN_ID_VAR: Basin_ID +SOURCE_AREA_VAR: Source_Area + +#-- Velocity and diffusion --# +# The velocity and diffusion parameters may either be specified as variables in +# the routing netcdf (char) or as a single value (float or int) +VELOCITY: 1 +DIFFUSION: 2000 + +#-- Output Interval --# +# Timestep of output unit hydrographs. Must be a multiple of the timestep in the UH_BOX +OUTPUT_INTERVAL:86400 + +#-- Maximum time for runoff to reach outlet (days) --# +BASIN_FLOWDAYS:50 + +#-- Maximum time for runoff to pass through a grid cell (days) --# +CELL_FLOWDAYS:2 +#-- ====================================== --# + +[DOMAIN] +#-- ====================================== --# +#-- Path to cesm complient domain file (char) --# +FILE_NAME: ./samples/domains/domain.lnd.bpa304.20140311.nc + +#-- netCDF Variable Names --# +LONGITUDE_VAR: lon +LATITUDE_VAR: lat +LAND_MASK_VAR: mask +FRACTION_VAR: frac +AREA_VAR: area +#-- ====================================== --# diff --git a/config/rvic_parameters_example.cfg b/samples/configs/rvic.parameters.rasm.cfg similarity index 89% rename from config/rvic_parameters_example.cfg rename to samples/configs/rvic.parameters.rasm.cfg index 01f514f..3877a19 100644 --- a/config/rvic_parameters_example.cfg +++ b/samples/configs/rvic.parameters.rasm.cfg @@ -24,13 +24,13 @@ VERBOSE:True CLEAN:False #--case description (char) --# -CASEID:example2 +CASEID: sample_rasm_parameters #--routing domain grid shortname (char) --# -GRIDID: some grid +GRIDID: wr50a #--case run directory (char) --# -CASE_DIR:/path/to/%(CASEID)s/ +CASE_DIR: ./samples/cases/%(CASEID)s #--Directory to use for temporary read/write operations (char) --# TEMP_DIR:%(CASE_DIR)s/temp/ @@ -48,11 +48,11 @@ AGG_PAD:25 #-- Output parameter file format (char) --# # Valid Values: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, and NETCDF4 -# For use with CESM, NETCDF3_CLASSIC is reccomended. +# For use with CESM, NETCDF3_CLASSIC is recommended. NETCDF_FORMAT:NETCDF3_CLASSIC #-- Output parameter file compression options --# -# Descriptions of these options can be found in +# Descriptions of these options can be found in NETCDF_ZLIB: False NETCDF_COMPLEVEL: 4 NETCDF_SIGFIGS: None @@ -62,7 +62,9 @@ SUBSET_DAYS:10 #-- Constrain the final unit hydrographs sum to be less than or equal to the domain fractions --# # True when routing to coastal grid cells, else False -CONSTRAIN_FRACTIONS:False +CONSTRAIN_FRACTIONS: True + +SEARCH_FOR_CHANNEL: False #-- ====================================== --# @@ -71,7 +73,7 @@ CONSTRAIN_FRACTIONS:False #-- Path to Pour Points File (char) --# # A comma separated file of outlets to route to [lons, lats] - one coordinate pair per line (order not important) # May optionally include a column [names] - which will (if not aggregating) be included in param file -FILE_NAME: /path/to/rout2points.csv +FILE_NAME: ./samples/pour_points/rasm_sample_pour_points.csv #-- ====================================== --# @@ -81,7 +83,7 @@ FILE_NAME: /path/to/rout2points.csv # This defines the unit hydrograph to rout flow to the edge of each grid cell. # A comma separated file of [time in seconds, unit hydrograph ordinate] - one timestep per line # The timestep should be 1hr (3600 sec) or less. -FILE_NAME: /path/to/UH_file.csv +FILE_NAME: ./samples/uh_box/UH_RASM_hourly.csv #-- Number of Header lines to ignore in [UH_BOX]FILE_NAME (INT) --# HEADER_LINES = 1 @@ -90,7 +92,7 @@ HEADER_LINES = 1 [ROUTING] #-- ====================================== --# #-- Path to routing inputs netcdf (char) --# -FILE_NAME: /path/to/routing_inputs.nc +FILE_NAME: ./samples/flow_directions/Wu_routing_inputs_060313.nc #-- netCDF Variable Names --# LONGITUDE_VAR: lon @@ -108,7 +110,7 @@ DIFFUSION: 2000 #-- Output Interval --# # Timestep of output unit hydrographs. Must be a multiple of the timestep in the UH_BOX -OUTPUT_INTERVAL:3600 +OUTPUT_INTERVAL:86400 #-- Maximum time for runoff to reach outlet (days) --# BASIN_FLOWDAYS:50 @@ -120,7 +122,7 @@ CELL_FLOWDAYS:2 [DOMAIN] #-- ====================================== --# #-- Path to cesm complient domain file (char) --# -FILE_NAME: /path/to/domain.lnd.nc +FILE_NAME: ./samples/domains/domain.lnd.wr50a_ar9v4.100920.nc #-- netCDF Variable Names --# LONGITUDE_VAR: xc diff --git a/samples/domains/domain.lnd.bpa304.20140311.nc b/samples/domains/domain.lnd.bpa304.20140311.nc new file mode 100644 index 0000000..0b656b2 --- /dev/null +++ b/samples/domains/domain.lnd.bpa304.20140311.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d80e82382b3b63e5ce85439b4f09ad6d37a8e0e9286055e45048e95420529a0e +size 1429444 diff --git a/samples/domains/domain.lnd.wr50a_ar9v4.100920.nc b/samples/domains/domain.lnd.wr50a_ar9v4.100920.nc new file mode 100644 index 0000000..650b14f --- /dev/null +++ b/samples/domains/domain.lnd.wr50a_ar9v4.100920.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8da151a4973ba9f5da754d6f9ac2f7ec0b3d3a00bfe48bcbe5903feb94c7203f +size 5639788 diff --git a/samples/flow_directions/Wu_routing_inputs_060313.nc b/samples/flow_directions/Wu_routing_inputs_060313.nc new file mode 100644 index 0000000..732480f --- /dev/null +++ b/samples/flow_directions/Wu_routing_inputs_060313.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3aaa36c73babae9f658040b52b53ca7a1785c7a1fef562731f51259ec6d65c2 +size 9176972 diff --git a/samples/flow_directions/pnw.RVIC.input_20140218.nc b/samples/flow_directions/pnw.RVIC.input_20140218.nc new file mode 100644 index 0000000..0ae1a14 --- /dev/null +++ b/samples/flow_directions/pnw.RVIC.input_20140218.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d542a17c7699a003324df0ac59b878e0b9ff6b8f132d23dc035bf446e6155d61 +size 578248 diff --git a/samples/forcings/rasm_sample_runoff.nc b/samples/forcings/rasm_sample_runoff.nc new file mode 100644 index 0000000..20acbd3 --- /dev/null +++ b/samples/forcings/rasm_sample_runoff.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:391d25ede08bdfb9dac4375ae37d7a3f4dc02fb62bc72078a9ce52a2ebf61c91 +size 28862969 diff --git a/samples/pour_points/columbia_sample_pour_points.csv b/samples/pour_points/columbia_sample_pour_points.csv new file mode 100644 index 0000000..c76fda6 --- /dev/null +++ b/samples/pour_points/columbia_sample_pour_points.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce13e2314839c6c2a560db0381b32df6e09300f35a0f4591dc256e0690f9610e +size 518 diff --git a/samples/pour_points/rasm_sample_pour_points.csv b/samples/pour_points/rasm_sample_pour_points.csv new file mode 100644 index 0000000..2acaaab --- /dev/null +++ b/samples/pour_points/rasm_sample_pour_points.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16458c8ced7dd66fb6116a81adb42ce2742cfc4b97b5d6f6788b90ce6668e735 +size 289 diff --git a/samples/uh_box/UH_Columbia_hourly.csv b/samples/uh_box/UH_Columbia_hourly.csv new file mode 100644 index 0000000..a63327a --- /dev/null +++ b/samples/uh_box/UH_Columbia_hourly.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d71cfae383a339a5f3168ef304fc571bf9f81b49417392be3f1881c3b030def +size 881 diff --git a/samples/uh_box/UH_RASM_hourly.csv b/samples/uh_box/UH_RASM_hourly.csv new file mode 100644 index 0000000..a63327a --- /dev/null +++ b/samples/uh_box/UH_RASM_hourly.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d71cfae383a339a5f3168ef304fc571bf9f81b49417392be3f1881c3b030def +size 881 From cda05347252de0bbe9bc1503903cc0469fcdb4bc Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 24 Oct 2015 23:19:02 -0700 Subject: [PATCH 47/47] updates for rvic 1.1.0 --- docs/user-guide/convolution.md | 3 --- rvic/core/variables.py | 30 +++++++++++++++--------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docs/user-guide/convolution.md b/docs/user-guide/convolution.md index 7d7a3d1..8349d34 100644 --- a/docs/user-guide/convolution.md +++ b/docs/user-guide/convolution.md @@ -28,9 +28,6 @@ The flux file(s) must be in netCDF format and have a `time` dimension as well as 3. **CASE_DIR** - Description: case run directory - Type: char -4. **RVIC_TAG** - - Description: RVIC Tag - - Type: char 5. **CASEID** - Description: Case ID - Type: char diff --git a/rvic/core/variables.py b/rvic/core/variables.py index 7ec8124..fc960df 100644 --- a/rvic/core/variables.py +++ b/rvic/core/variables.py @@ -387,7 +387,7 @@ def write_restart(self, current_history_files, history_restart_files): time = f.createVariable('time', NC_DOUBLE, ('time',), **self.ncvaropts) time[:] = date2num(self.timestamp, TIMEUNITS, calendar=self._calendar) - for key, val in iteritems(share.time.__dict__): + for key, val in iteritems(share.time): if val: setattr(time, key, val) time.calendar = self._calendar @@ -398,7 +398,7 @@ def write_restart(self, current_history_files, history_restart_files): **self.ncvaropts) timesteps[:] = np.arange(self.full_time_length) - for key, val in iteritems(share.timesteps.__dict__): + for key, val in iteritems(share.timesteps): if val: setattr(timesteps, key, val) timesteps.timestep_length = 'unit_hydrograph_dt' @@ -407,14 +407,14 @@ def write_restart(self, current_history_files, history_restart_files): unit_hydrograph_dt = f.createVariable('unit_hydrograph_dt', NC_DOUBLE, (), **self.ncvaropts) unit_hydrograph_dt[:] = self.unit_hydrograph_dt - for key, val in iteritems(share.unit_hydrograph_dt.__dict__): + for key, val in iteritems(share.unit_hydrograph_dt): if val: setattr(unit_hydrograph_dt, key, val) timemgr_rst_type = f.createVariable('timemgr_rst_type', NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_type[:] = self._calendar_key - for key, val in iteritems(share.timemgr_rst_type.__dict__): + for key, val in iteritems(share.timemgr_rst_type): if val: setattr(timemgr_rst_type, key, val) @@ -422,7 +422,7 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_step_sec[:] = self.unit_hydrograph_dt - for key, val in iteritems(share.timemgr_rst_step_sec.__dict__): + for key, val in iteritems(share.timemgr_rst_step_sec): if val: setattr(timemgr_rst_step_sec, key, val) @@ -431,7 +431,7 @@ def write_restart(self, current_history_files, history_restart_files): **self.ncvaropts) timemgr_rst_start_ymd[:] = self._start_date.year * 10000 \ + self._start_date.month * 100 + self._start_date.day - for key, val in iteritems(share.timemgr_rst_start_ymd.__dict__): + for key, val in iteritems(share.timemgr_rst_start_ymd): if val: setattr(timemgr_rst_start_ymd, key, val) @@ -439,7 +439,7 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_start_tod[:] = (self._start_ord % 1) * SECSPERDAY - for key, val in iteritems(share.timemgr_rst_start_tod.__dict__): + for key, val in iteritems(share.timemgr_rst_start_tod): if val: setattr(timemgr_rst_start_tod, key, val) @@ -447,7 +447,7 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_ref_ymd[:] = REFERENCE_DATE - for key, val in iteritems(share.timemgr_rst_ref_ymd.__dict__): + for key, val in iteritems(share.timemgr_rst_ref_ymd): if val: setattr(timemgr_rst_ref_ymd, key, val) @@ -455,7 +455,7 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_ref_tod[:] = REFERENCE_TIME - for key, val in iteritems(share.timemgr_rst_ref_tod.__dict__): + for key, val in iteritems(share.timemgr_rst_ref_tod): if val: setattr(timemgr_rst_ref_tod, key, val) @@ -464,7 +464,7 @@ def write_restart(self, current_history_files, history_restart_files): **self.ncvaropts) timemgr_rst_curr_ymd[:] = self.timestamp.year * 10000 + \ self.timestamp.month * 100 + self.timestamp.day - for key, val in iteritems(share.timemgr_rst_curr_ymd.__dict__): + for key, val in iteritems(share.timemgr_rst_curr_ymd): if val: setattr(timemgr_rst_curr_ymd, key, val) @@ -472,7 +472,7 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, (), **self.ncvaropts) timemgr_rst_curr_tod[:] = (self.time_ord % 1) * SECSPERDAY - for key, val in iteritems(share.timemgr_rst_curr_tod.__dict__): + for key, val in iteritems(share.timemgr_rst_curr_tod): if val: setattr(timemgr_rst_curr_tod, key, val) @@ -518,21 +518,21 @@ def write_restart(self, current_history_files, history_restart_files): oyi = f.createVariable('outlet_y_ind', NC_INT, coords[0], **self.ncvaropts) oyi[:] = self.outlet_y_ind - for key, val in iteritems(share.outlet_y_ind.__dict__): + for key, val in iteritems(share.outlet_y_ind): if val: setattr(oyi, key, val) oxi = f.createVariable('outlet_x_ind', NC_INT, coords[0], **self.ncvaropts) oxi[:] = self.outlet_x_ind - for key, val in iteritems(share.outlet_x_ind.__dict__): + for key, val in iteritems(share.outlet_x_ind): if val: setattr(oxi, key, val) odi = f.createVariable('outlet_decomp_ind', NC_INT, coords[0], **self.ncvaropts) odi[:] = self.outlet_decomp_ind - for key, val in iteritems(share.outlet_decomp_ind.__dict__): + for key, val in iteritems(share.outlet_decomp_ind): if val: setattr(odi, key, val) @@ -543,7 +543,7 @@ def write_restart(self, current_history_files, history_restart_files): NC_DOUBLE, tcoords, **self.ncvaropts) ring[:, :] = self.ring[tracer][:, :] - for key, val in iteritems(share.ring.__dict__): + for key, val in iteritems(share.ring): if val: setattr(ring, key, val) # ------------------------------------------------------------ #