From fd5cc6313418781a65694e63d15370ef1de2bdd1 Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Mon, 9 Nov 2020 13:28:55 +1100 Subject: [PATCH 1/9] Merge commit 'd36378c42cc3bf9c3c9e770270562c14c49f8b9c' --- conf.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf.py b/conf.py index fbfbe413..a0743520 100644 --- a/conf.py +++ b/conf.py @@ -53,7 +53,7 @@ # built documents. # # The short X.Y version. -version = '3.1.3' +version = '3.1' # The full version, including alpha/beta/rc tags. release = latest_tag.name diff --git a/setup.py b/setup.py index 64bd0a6b..ba3b06e5 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name = "TCRM", - version = '3.1.3', + version = '3.1.4', packages=find_packages(), scripts=['tcrm.py', 'tcevent.py'], include_package_data=True, From ea8e3b816cf01089a304e121669880ef193c4b23 Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Mon, 9 Nov 2020 14:50:53 +1100 Subject: [PATCH 2/9] Make wind field plotting configurable This means we don't need to download the cartopy data each time we stand up a new compute instance --- Utilities/config.py | 3 ++- tcevent.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Utilities/config.py b/Utilities/config.py index 25ea922b..facc37b5 100644 --- a/Utilities/config.py +++ b/Utilities/config.py @@ -137,7 +137,8 @@ def formatList(lst): 'WindfieldInterface_thetamax': float, 'WindfieldInterface_trackfile': str, 'WindfieldInterface_trackpath': str, - 'WindfieldInterface_windfieldtype': str} + 'WindfieldInterface_windfieldtype': str, + 'WindfieldInterface_plotoutput': parseBool} DEFAULTS = """ [Actions] diff --git a/tcevent.py b/tcevent.py index d6c48eba..f48962d3 100755 --- a/tcevent.py +++ b/tcevent.py @@ -36,7 +36,6 @@ from Utilities.version import version from Utilities.progressbar import SimpleProgressBar as ProgressBar from Evaluate import interpolateTracks -from PlotInterface.maps import saveWindfieldMap __version__ = version() @@ -119,6 +118,8 @@ def doWindfieldPlotting(configFile): """ from netCDF4 import Dataset import numpy as np + from PlotInterface.maps import saveWindfieldMap + config = ConfigParser() config.read(configFile) outputPath = config.get('Output', 'Path') @@ -205,7 +206,9 @@ def status(done, total): import impact impact.run_optional(config) - doWindfieldPlotting(configFile) + if config.getboolean('WindfieldInterface', 'PlotOutput'): + doWindfieldPlotting(configFile) + if config.getboolean('Timeseries', 'Extract'): doTimeseriesPlotting(configFile) From 726b51fc9eb6fec2eb5bab4bad406902e2087886 Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Wed, 11 Nov 2020 14:30:51 +1100 Subject: [PATCH 3/9] Read single track file retrieved from SST --- Utilities/track.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Utilities/track.py b/Utilities/track.py index 1e6d94e1..2faa35ea 100644 --- a/Utilities/track.py +++ b/Utilities/track.py @@ -199,6 +199,24 @@ def ncReadTrackData(trackfile): raise IOError("Cannot open {0}".format(trackfile)) g = ncobj.groups + if not bool(g): + # We have a track file that stores data in separate variables + log.debug(f"Reading data from a single track file") + dt = ncobj.variables['Datetime'] + units = ncobj.getncattr('time_units') + calendar = ncobj.getncattr('calendar') + dtt = num2date(dt[:], units, calendar) + newtd = np.zeros(len(dtt), dtype=track_dtype) + for f in ncobj.variables.keys(): + if f != 'Datetime' and f in track_dtype.names: + newtd[f] = ncobj.variables[f][:] + newtd['Datetime'] = dtt + track = Track(newtd) + track.trackfile = trackfile + track.trackId = eval(ncobj.trackId) + + return [track] + tracks = [] if 'tracks' in g: tgroup = g['tracks'].groups From 66f8a162e91ff1941318dfa82b4d338e3d75f93b Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Thu, 12 Nov 2020 17:36:00 +1100 Subject: [PATCH 4/9] Fix test_system.Yasi_example --- tests/test_system.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_system.py b/tests/test_system.py index 157b6223..9c5b5447 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -38,7 +38,7 @@ def setUp(self): self.addCleanup(Utilities.config.reset) self.tmpdir = tempfile.TemporaryDirectory() - self.addCleanup(self.tmpdir.cleanup) + #self.addCleanup(self.tmpdir.cleanup) self.configFile = os.path.join( Utilities.pathLocator.getRootDirectory(), @@ -48,6 +48,7 @@ def setUp(self): config = Utilities.config.ConfigParser() config.read(self.configFile) config['Output']['Path'] = self.tmpdir.name + config['WindfieldInterface']['PlotOutput'] = 'True' @decimate(100) def test_scenario(self): @@ -71,5 +72,8 @@ def test_scenario(self): self.assertGreater(white.sum() / pixels, 0.2) # substantial space self.assertGreater(color.sum() / pixels, 0.05) # significant color + from time import sleep + sleep(1) + if __name__ == '__main__': unittest.main() From ba55a3f68356e6ce6c8d49548008669cd386676c Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Thu, 12 Nov 2020 18:01:37 +1100 Subject: [PATCH 5/9] Fix wind.writer doctests --- wind/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wind/writer.py b/wind/writer.py index d06b9a36..405d6888 100644 --- a/wind/writer.py +++ b/wind/writer.py @@ -111,7 +111,7 @@ def __call__(self, time, gust, Ux, Uy, P, lon, lat): t = len(self.time) if not t: - self.time.units = "days since " + time.strftime() + self.time.units = "days since " + time.strftime("%Y-%m-%d %H:%M") # convert window extent to slice indices origin = np.rint(self.affine * (lon[0], lat[0])).astype(int) From e036c317084eb2dbdee102502dff66b5f8073197 Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Thu, 12 Nov 2020 18:18:30 +1100 Subject: [PATCH 6/9] Resolve xlabels deprecation warning in maps.py --- PlotInterface/maps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PlotInterface/maps.py b/PlotInterface/maps.py index 3fe9362c..53d4b3e1 100644 --- a/PlotInterface/maps.py +++ b/PlotInterface/maps.py @@ -196,8 +196,8 @@ def addGraticule(self, axes, mapobj): draw_labels=True) gl.xformatter = LONGITUDE_FORMATTER gl.yformatter = LATITUDE_FORMATTER - gl.xlabels_top = False - gl.ylabels_right = False + gl.top_labels = False + gl.right_labels = False def addCoastline(self, mapobj): """ From 68b65a3eace9ad33597da8397209b8ca904c08bd Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Sun, 22 Nov 2020 12:32:41 +1100 Subject: [PATCH 7/9] Travis CI migration (#109) * Get Windows environment working on Travis * Ensure order of unit test execution doesn't cause failues --- .travis.yml | 37 +++++++++++++------------ postinstall.sh | 46 ++++++++++++++++++++++++++++++++ preinstall.sh | 24 +++++++++++++++++ tests/test_processMultipliers.py | 7 ++--- 4 files changed, 92 insertions(+), 22 deletions(-) create mode 100644 postinstall.sh create mode 100644 preinstall.sh diff --git a/.travis.yml b/.travis.yml index b2da90c2..aef70720 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,29 @@ -language: python -python: - - '3.6' - - '3.7' - - '3.8' +language: shell + +env: + - PYTHON_VERSION=3.6 + - PYTHON_VERSION=3.7 + - PYTHON_VERSION=3.8 + +os: + - linux + - windows + +before_install: + - source ./preinstall.sh + install: - - sudo apt-get update - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - # install in batch mode i.e. no input - - bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/conda - - export PATH="$HOME/conda/bin:$PATH" - # clear bash cache table - - hash -r - - conda config --set always_yes yes - - conda config --set changeps1 no - - conda update -q conda - - conda config --add channels conda-forge - - conda config --set channel_priority strict - - conda env create -q -f tcrmenv.yml python=$TRAVIS_PYTHON_VERSION - - source activate tcrm + - source ./postinstall.sh + branches: except: - config - notebooks + script: - python installer/setup.py build_ext -i - nosetests -v --with-coverage --cover-package=. + after_success: coveralls notifications: slack: diff --git a/postinstall.sh b/postinstall.sh new file mode 100644 index 00000000..3bd82652 --- /dev/null +++ b/postinstall.sh @@ -0,0 +1,46 @@ +# begin installing miniconda +if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then + echo "installing miniconda for posix"; + bash $HOME/download/miniconda.sh -b -u -p $MINICONDA_PATH; +elif [[ "$TRAVIS_OS_NAME" == "windows" ]]; then + echo "folder $MINICONDA_SUB_PATH does not exist" + echo "installing miniconda for windows"; + choco install miniconda3 --params="'/JustMe /AddToPath:1 /D:$MINICONDA_PATH_WIN'"; +fi; +# end installing miniconda + +export PATH="$MINICONDA_PATH:$MINICONDA_SUB_PATH:$MINICONDA_LIB_BIN_PATH:$PATH"; + +# begin checking miniconda existance +echo "checking if folder $MINICONDA_SUB_PATH exists" +if [[ -d $MINICONDA_SUB_PATH ]]; then + echo "folder $MINICONDA_SUB_PATH exists" +else + echo "folder $MINICONDA_SUB_PATH does not exist" +fi; +# end checking miniconda existance + +source $MINICONDA_PATH/etc/profile.d/conda.sh; +hash -r; +echo $TRAVIS_OS_NAME +echo $PYTHON_VERSION +python --version + +if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then + echo "Removing mpi4py from environment for windows build" + echo "Package not available in conda channels" + sed -i '/mpi4py/d' ./tcrmenv.yml +fi + +conda config --set always_yes yes --set changeps1 no; +conda update -q conda; +conda config --add channels conda-forge; +conda config --set channel_priority strict; +# Useful for debugging any issues with conda +conda info -a + +echo "Create TCRM environment" +conda env create -q -f tcrmenv.yml python=$PYTHON_VERSION; +conda activate tcrm +python --version +conda list diff --git a/preinstall.sh b/preinstall.sh new file mode 100644 index 00000000..339b2536 --- /dev/null +++ b/preinstall.sh @@ -0,0 +1,24 @@ +if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then + export MINICONDA_PATH=$HOME/miniconda; + export MINICONDA_SUB_PATH=$MINICONDA_PATH/bin; +elif [[ "$TRAVIS_OS_NAME" == "windows" ]]; then + export MINICONDA_PATH=$HOME/miniconda; + export MINICONDA_PATH_WIN=`cygpath --windows $MINICONDA_PATH`; + export MINICONDA_SUB_PATH=$MINICONDA_PATH/Scripts; +fi; +export MINICONDA_LIB_BIN_PATH=$MINICONDA_PATH/Library/bin; + # Obtain miniconda installer +if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then + mkdir -p $HOME/download; + if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + echo "downloading miniconda.sh for linux"; + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O $HOME/download/miniconda.sh; + elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + echo "downloading miniconda.sh for osx"; + wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O $HOME/download/miniconda.sh; + fi; +fi; + # Install openssl for Windows +if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then + choco install openssl.light; +fi; \ No newline at end of file diff --git a/tests/test_processMultipliers.py b/tests/test_processMultipliers.py index f090e906..5f967177 100755 --- a/tests/test_processMultipliers.py +++ b/tests/test_processMultipliers.py @@ -21,7 +21,7 @@ try: from . import pathLocate except: - from unittests import pathLocate + from tests import pathLocate # Add parent folder to python path unittest_dir = pathLocate.getUnitTestDirectory() @@ -250,8 +250,8 @@ def test_generate_syn_mult_img(self): shutil.rmtree(dir_path) - def test_computeOutputExtentIfInvalid(self): - """Test createRaster returns a gdal dataset""" + def test_resetOutputExtentIfInvalid(self): + """Test computation of output extent""" # Write a .nc file to test with dummy data. This is the gust file f_nc = tempfile.NamedTemporaryFile(suffix='.nc', prefix='test_processMultipliers', delete=False) @@ -335,6 +335,7 @@ def test_xprocessMult_A(self): if __name__ == "__main__": # Suite = unittest.makeSuite(TestProcessMultipliers, 'test_x') + unittest.TestLoader.sortTestMethodsUsing = None Suite = unittest.makeSuite(TestProcessMultipliers, 'test') Runner = unittest.TextTestRunner() Runner.run(Suite) From 838da34a71f8245c3cdf061e413464f51b3ff264 Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Tue, 2 Feb 2021 12:03:59 +1100 Subject: [PATCH 8/9] Confirm units of speed --- DataProcess/DataProcess.py | 10 +-- TrackGenerator/TrackGenerator.py | 12 ++-- Utilities/track.py | 2 +- docs/install.rst | 113 ++++++------------------------- 4 files changed, 33 insertions(+), 104 deletions(-) diff --git a/DataProcess/DataProcess.py b/DataProcess/DataProcess.py index 8c949139..91ea5ec4 100644 --- a/DataProcess/DataProcess.py +++ b/DataProcess/DataProcess.py @@ -465,8 +465,8 @@ def _speed(self, dist, dt, indicator, initIndex): """ Extract speeds for all obs, initial obs and TC origins Input: dist - array of distances between consecutive TC - observations - dt - array of times between consecutive TC observations + observations (km) + dt - array of times between consecutive TC observations (hours) indicator - array of ones/zeros representing initial TC observations (including TCs with a single observation) @@ -499,17 +499,17 @@ def _speed(self, dist, dt, indicator, initIndex): speed_no_init = pjoin(self.processPath, 'speed_no_init') # Extract all speeds self.logger.debug('Outputting data into {0}'.format(all_speed)) - header = 'all cyclone speed in m/s' + header = 'all cyclone speed in km/h' flSaveFile(all_speed, speed, header, fmt='%6.2f') # Extract initial speeds self.logger.debug('Outputting data into {0}'.format(init_speed)) - header = 'initial cyclone speed in m/s' + header = 'initial cyclone speed in km/h' flSaveFile(init_speed, initSpeed, header, fmt='%f') # Extract speeds, excluding initial speeds self.logger.debug('Outputting data into {0}'.format(speed_no_init)) - header = 'cyclone speed without initial ones in m/s' + header = 'cyclone speed without initial ones in km/h' flSaveFile(speed_no_init, speedNoInit, header, fmt='%6.2f') def _pressure(self, pressure, indicator): diff --git a/TrackGenerator/TrackGenerator.py b/TrackGenerator/TrackGenerator.py index 4fd552f3..87af2e82 100644 --- a/TrackGenerator/TrackGenerator.py +++ b/TrackGenerator/TrackGenerator.py @@ -1293,7 +1293,7 @@ def dumpAllCellCoefficients(self): 'dtype': 'f', 'atts': { 'long_name': 'Mean forward speed', - 'units': 'm/s' + 'units': 'km/h' } }, 1: { @@ -1314,7 +1314,7 @@ def dumpAllCellCoefficients(self): 'dtype': 'f', 'atts': { 'long_name': 'Standard deviation forward speed', - 'units': 'm/s' + 'units': 'km/h' } }, 3: { @@ -1324,7 +1324,7 @@ def dumpAllCellCoefficients(self): 'dtype': 'f', 'atts': { 'long_name': 'Minimum forward speed', - 'units': 'm/s' + 'units': 'km/h' } }, 4: { @@ -1334,7 +1334,7 @@ def dumpAllCellCoefficients(self): 'dtype': 'f', 'atts': { 'long_name': 'Mean forward speed (over land)', - 'units': 'm/s' + 'units': 'km/h' } }, 5: { @@ -1356,7 +1356,7 @@ def dumpAllCellCoefficients(self): 'atts': { 'long_name': 'Standard deviation of forward' + ' speed (over land)', - 'units': 'm/s' + 'units': 'km/h' } }, 7: { @@ -1366,7 +1366,7 @@ def dumpAllCellCoefficients(self): 'dtype': 'f', 'atts': { 'long_name': 'Minimum forward speed (over land)', - 'units': 'm/s' + 'units': 'km/h' } }, 8: { diff --git a/Utilities/track.py b/Utilities/track.py index 2faa35ea..bdca9e97 100644 --- a/Utilities/track.py +++ b/Utilities/track.py @@ -324,7 +324,7 @@ def ncSaveTracks(trackfile, tracks, tvar.lon_units = 'degrees east' tvar.lat_units = 'degrees north' tvar.pressure_units = 'hPa' - tvar.speed_units = 'm/s' + tvar.speed_units = 'km/h' tvar.length_units = 'km' tvar.trackId = repr(t.trackId) diff --git a/docs/install.rst b/docs/install.rst index 1ff10d1d..880902e5 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -19,10 +19,13 @@ For users wanting to only run the code, a zip file or gzipped tar file of the latest releases can be downloaded from the `Releases page `_. +To have access to the latest updates, users should clone the repository, and +then regularly pull from the repository as updates are made. + Those wanting to contribute to development can `fork `_ the repository. Submit a pull request to have your changes integrated into -TCRM. +TCRM. Read more about contribting_ to the TCRM code. .. _dependencies: @@ -30,7 +33,7 @@ Dependencies ------------ TCRM relies on a number of additional libraries that are not part of -the standard library. There are several ways to obtain the required +the standard Pyhton library. There are several ways to obtain the required libraries -- using Python's recommended tool `pip `_, installing a distribution such as `Python(x,y) package `_ @@ -59,23 +62,6 @@ please see the documentation for each individual library. * Parallel execution in multi-processor environments (with MPI installed) requires `mpi4py `_ -Using pip -~~~~~~~~~ - -If you have `pip `_ installed, -the required modules can be installed using the following command, -executed in the main TCRM directory - -.. code-block:: bash - - pip -v install -r requirements.txt - -This will automatically build the required libraries (listed in the -``requirements.txt`` file) and any dependencies. ``pip`` must be on -the ``$PATH`` for this to work. - -.. _compilation: - Using Anaconda ~~~~~~~~~~~~~~ @@ -98,6 +84,22 @@ The bash promt will look like (tcrm) user@server:~/tcrm$ +Using pip +~~~~~~~~~ + +If you have `pip `_ installed, +the required modules can be installed using the following command, +executed in the main TCRM directory + +.. code-block:: bash + + pip -v install -r requirements.txt + +This will automatically build the required libraries (listed in the +``requirements.txt`` file) and any dependencies. ``pip`` must be on +the ``$PATH`` for this to work. + + .. _environment: @@ -140,79 +142,6 @@ CSH/TCSH shell setenv PYTHONPATH $PYTHONPATH:/path/to/tcrm:/path/to/tcrm/Utilities -.. _dependencies: - -Dependencies ------------- - -TCRM relies on a number of additional libraries that are not part of -the standard library. There are several ways to obtain the required -libraries -- using Python's recommended tool `pip -`_, installing a distribution -such as `Python(x,y) package `_ -(for Windows environments) or `Anaconda -`_ (cross-platform), or -installing the libraries from source or binary installers -(pre-compiled binary Windows installer versions for all the libraries -(both 32-bit and 64-bit) can be obtained `here -`_). - -For detailed instructions on installation of these dependencies, -please see the documentation for each individual library. - -* `Python `_ - v3.5 or later -* `Numpy `_ - v1.6 or later -* `Scipy `_ - v0.12 or later -* `Matplotlib `_ v1.2 or later. -* `Basemap `_ -* `netcdf4-python `_ - - version 1.0.8 or later -* `Shapely `_ - v1.2.15 or later -* `statsmodels `_ -* `seaborn `_ -* `pandas `_ -* `gitpython `_ -* Parallel execution in multi-processor environments (with MPI - installed) requires `mpi4py `_ - -Using pip -~~~~~~~~~ - -If you have `pip `_ installed, -the required modules can be installed using the following command, -executed in the main TCRM directory - -.. code-block:: bash - - pip -v install -r requirements.txt - -This will automatically build the required libraries (listed in the -``requirements.txt`` file) and any dependencies. ``pip`` must be on -the ``$PATH`` for this to work. - - -Using Anaconda -~~~~~~~~~~~~~~ - -To install ``tcrm``, make a new environment: - -.. code-block:: bash - - conda env create -f tcrmenv.yml - -After creating the environment the user needs to move to that environment using the command - -.. code-block:: bash - - conda activate tcrm - -The bash promt will look like - -.. code-block:: - - (tcrm) user@server:~/tcrm$ - - .. _compilation: Windows From d93f1e3bd407e5d17441565bb7e7cfbc8f90d6e8 Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Fri, 16 Apr 2021 14:05:42 +1000 Subject: [PATCH 9/9] HOTFIX: interpolation of tracks `interpolateTracks.interpolate()` function had unhandled situation of raw track with three points unhandled, which led to interpolated track having central pressure values set to missing values. --- Evaluate/interpolateTracks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Evaluate/interpolateTracks.py b/Evaluate/interpolateTracks.py index ef2e41ea..c1b551d6 100644 --- a/Evaluate/interpolateTracks.py +++ b/Evaluate/interpolateTracks.py @@ -114,7 +114,7 @@ def interpolate(track, delta, interpolation_type=None): nLon = interp1d(timestep, track.Longitude, kind='linear')(newtime) nLat = interp1d(timestep, track.Latitude, kind='linear')(newtime) - if len(validIdx) == 2: + if len(validIdx) >= 2: npCentre = interp1d(timestep, track.CentralPressure, kind='linear')(newtime)