Merge pull request #37 from USDA-ARS-NWRC/12_docker

Updated docker image

.dockerignore

@@ -3,12 +3,16 @@ npm-debug.log
 Dockerfile*
 docker-compose*
 .dockerignore
-.git
 .gitignore
 .env
 .venv
 */bin
 */obj
 # README.md
 LICENSE
-.vscode
+.vscode
+build/
+dist/
+htmlcov/
+*.egg-info
+.github

Dockerfile

@@ -1,68 +1,38 @@
-# The purpose of this Docker image is to facilitate the download and
-# conversion of the HRRR grib2 files. 
-FROM python:3.6.8-alpine3.9
+# Multistage build to build wgrib2 first
+FROM python:3.8.10-buster as builder
 
-# Install and make wgrib2
+WORKDIR /build/wgrib2
+
+# build wgrib2, this takes a while
 ENV CC gcc
 ENV FC gfortran
-ENV CLASSPATH ".:/usr/local/bin/antlr.jar:$CLASSPATH"
 
+RUN apt-get update \
+    && apt-get install -y gfortran \
+    && curl ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz | tar xvz \
+    && cd /build/wgrib2/grib2 \
+    && wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.gz \
+    && wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-c-4.7.3.tar.gz \
+    && sed -i "s/USE_NETCDF4=0/USE_NETCDF4=1/" makefile \
+    && sed -i "s/USE_NETCDF3=1/USE_NETCDF3=0/" makefile \
+    && make \
+    && cp wgrib2/wgrib2 /usr/local/bin/wgrib2
+
+##############################################
+# main image
+##############################################
+FROM python:3.8.10-slim-buster
+
+COPY . /code
 WORKDIR /code
-RUN apk --no-cache --virtual .build-dependencies add build-base curl gfortran cmake zlib-dev perl diffutils bash curl-dev m4 && \
-    apk --no-cache add libgfortran libgomp libstdc++ && \
-    curl ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz | tar xvz && \
-    cd /code/grib2 && \
-    wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.gz && \
-    wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.6.1.tar.gz && \
-    sed -i "s/USE_NETCDF4=0/USE_NETCDF4=1/" makefile && \
-    sed -i "s/USE_NETCDF3=1/USE_NETCDF3=0/" makefile && \
-    make && \
-    cp wgrib2/wgrib2 /usr/local/bin/wgrib2 && \
-    make deep-clean && \
-    rm *.tar.gz
-
-# ncap2 needs antlr which is a java program
-RUN apk add openjdk8-jre-base && \
-    curl http://dust.ess.uci.edu/nco/antlr-2.7.7.tar.gz | tar xvz && \
-    cd /code/antlr-2.7.7 && \
-    export CLASSPATH=".:/usr/local/bin/antlr.jar:$CLASSPATH" && \
-    ./configure --prefix=/usr/local --disable-examples && \
-    make && \
-    make test && \
-    make install && \
-    mv antlr.jar /usr/local/bin && \
-    rm -rf /code/antlr-2.7.7
 
-# install NCO
-# There is something wrong with either a dependency or with nco as the hdf5 headers are
-# compiled with 1.10.4 but installed version is 1.10.5, go figure, just suppress the
-# warnings and hope for the best!
-ENV HDF5_DISABLE_VERSION_CHECK 2
-RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-    apk --no-cache --virtual .nco-dependencies add flex byacc && \
-    apk --no-cache add netcdf netcdf-dev && \
-    cd /code && \
-    wget https://github.com/nco/nco/archive/4.7.2.tar.gz && \
-    tar xvzf 4.7.2.tar.gz && \
-    rm 4.7.2.tar.gz && \
-    cd nco-4.7.2 && \
-    ./configure --prefix=/usr/local && \
-    make && \
-    make install && \
-    apk del .nco-dependencies && \
-    rm -rf /code/nco-4.7.2
-
-# Add the weather code
-ADD . /code/weather_forecast_retrieval
+COPY --from=builder /usr/local/bin/wgrib2 /usr/local/bin/wgrib2
 
 # Add and build weather forecast retrival
-RUN cd /code/weather_forecast_retrieval && \
-    apk --no-cache add netcdf-utils hdf5 hdf5-dev libffi-dev && \
-    CFLAGS="-g0 -Wl,--strip-all" \
-    python3 -m pip install --no-cache-dir --compile --global-option=build_ext -r requirements.txt && \
-    python3 setup.py install && \
-    apk del .build-dependencies
-
-VOLUME /data
+RUN apt-get update -y \
+    && apt-get install -y git libeccodes-tools libgfortran5 libgomp1 \
+    && python3 -m pip install --no-cache-dir -r requirements.txt \
+    && python3 setup.py install \
+    && rm -rf /var/lib/apt/lists/*
 
-# ENTRYPOINT ["/code/weather_forecast_retrieval/docker-entrypoint.sh"]
+VOLUME /data

README.md

@@ -35,17 +35,16 @@ ln wgrib2/wgrib2 ~/bin/wgrib2
 
 ## Docker
 
-The retrieval aspect of `weather_forecast_retieval` has been built into a Docker image based on the Python 3 Alpine linux image. This allows for a docker deployment to run and retrieve HRRR data and convert to netcdf if needed. To use, first build the image
+The retrieval aspect of `weather_forecast_retieval` has been built into a Docker image based Python 3.8. This allows for a docker deployment to run and retrieve HRRR data. The docker image can call any of the command line programs in `weather_forecast_retrieval`.
+
+For example, to run `hrrr_nomads` with docker:
 
 ```
-docker build -t usdaarsnwrc/weather_forecast_retieval .
+docker run --rm usdaarsnwrs/weather_forecast_retrieval hrrr_nomads -l 3 -f 0,1,2 --bbox="-119,-118,37,38" -o /path/to/output -p /path/to/crop/output
 ```
 
-Grab a coffee as this has to compile `pandas` from source (10+ minutes of compile time). Once completed, modify or create a new `docker-compose.yml` and modify the volume attachments as necessary. There are 2 volumes to attach, a `data` drive mounted to `/data` and the config file folders at `/code/config`. To setup the download, the config file is passed to `docker-compose`:
+The paths to the output directories are internal to the docker image and the necessary volume mounts are needed.
 
-```
-docker-compose run weather_forecast_retrieval /code/config/hrrr.ini
-```
 
 # Command line usage
 
@@ -170,7 +169,3 @@ optional arguments:
   --verbose             increase logging verbosity
   --overwrite           Download and overwrite existing HRRR files
 ```
-
-## convert_grib2nc
-
-## run_hrrr_retrieval