Merge branch 'develop'

.github/workflows/openstudio-server-tests.yml

@@ -10,8 +10,8 @@ on: [push, pull_request]
 
 env:
   USE_TESTING_TIMEOUTS: "true"
-  OPENSTUDIO_VERSION: 3.6.1
-  OPENSTUDIO_VERSION_SHA: bb9481519e
+  OPENSTUDIO_VERSION: 3.7.0
+  OPENSTUDIO_VERSION_SHA: d5269793f1
   OPENSTUDIO_VERSION_EXT: ""
   DOCKER_COMPOSE_VERSION: 1.21.1
   BUNDLE_WITHOUT: native_ext
@@ -97,7 +97,7 @@ jobs:
     - name: docker
       shell: bash 
       run: |
-          export OPENSTUDIO_TAG=3.6.1
+          export OPENSTUDIO_TAG=3.7.0
           sed -i -E "s/.git//g" .dockerignore
           docker volume create --name=osdata
           docker images --all

.github/workflows/security-scan.yml

@@ -10,8 +10,8 @@ on: [push, pull_request]
 
 env:
   USE_TESTING_TIMEOUTS: "true"
-  OPENSTUDIO_VERSION: 3.6.1
-  OPENSTUDIO_VERSION_SHA: bb9481519e
+  OPENSTUDIO_VERSION: 3.7.0
+  OPENSTUDIO_VERSION_SHA: d5269793f1
   OPENSTUDIO_VERSION_EXT: ""
   DOCKER_COMPOSE_VERSION: 1.21.1
   BUNDLE_WITHOUT: native_ext
@@ -31,7 +31,7 @@ jobs:
     - name: docker
       shell: bash 
       run: |
-          export OPENSTUDIO_TAG=3.6.1
+          export OPENSTUDIO_TAG=3.7.0
           sed -i -E "s/.git//g" .dockerignore
           docker volume create --name=osdata
           docker images --all

CHANGELOG.md

@@ -1,6 +1,15 @@
 OpenStudio Server
 =================
 
+Version 3.7.0
+-------------
+* update Analysis-gem to 1.3.5 in https://github.com/NREL/OpenStudio-server/pull/722
+* update Mongo to 6.0.9 in https://github.com/NREL/OpenStudio-server/pull/709
+* update Passenger to 6.0.18 in https://github.com/NREL/OpenStudio-server/pull/714
+* Fix indicies in Mongo Database in https://github.com/NREL/OpenStudio-server/pull/713
+* add XML and MAT files as valid downloadable Report Types https://github.com/NREL/OpenStudio-server/pull/720
+
+
 Version 3.5.1
 -------------
 * update urbanopt to 0.9.0

Dockerfile

@@ -4,8 +4,8 @@
 # NOTES:            Currently this is one big dockerfile and non-optimal.
 
 #may include suffix
-ARG OPENSTUDIO_VERSION=3.6.1
-FROM nrel/openstudio:3.6.1 as base
+ARG OPENSTUDIO_VERSION=3.7.0
+FROM nrel/openstudio:3.7.0 as base
 MAINTAINER Nicholas Long [email protected]
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -59,7 +59,7 @@ RUN apt-get update && apt-get install -y wget gnupg \
     && rm -rf /var/lib/apt/lists/*
 
 # Install passenger (this also installs nginx)
-ENV PASSENGER_VERSION 6.0.2
+ENV PASSENGER_VERSION 6.0.18
 
 RUN gem install passenger -v $PASSENGER_VERSION
 RUN passenger-install-nginx-module
@@ -104,7 +104,7 @@ RUN bundle exec rake assets:precompile
 # Bundle app source
 ADD /server /opt/openstudio/server
 # Add in /spec for testing 
-#ADD /spec /opt/openstudio/spec
+#ADD /server/spec /opt/openstudio/server/spec
 ADD .rubocop.yml /opt/openstudio/.rubocop.yml
 # Run bundle again, because if the user has a local Gemfile.lock it will have been overriden
 RUN rm Gemfile.lock

README.md

@@ -3,20 +3,26 @@
 [![Build Status][gh-img]][gh-url] 
 [![Coverage Status][coveralls-img]][coveralls-url]
 
-Windows Build (Under Development): [![Windows Build Status][appveyor-img]][appveyor-url]
+Please refer to the [wiki](https://github.com/NREL/OpenStudio-server/wiki) for additional documentation.
 
-The below documentation has not been recently reviewed. Please refer to the [wiki](https://github.com/NREL/OpenStudio-server/wiki) for current documentation.
+<img src="https://github.com/NREL/OpenStudio-server/assets/2235296/fe91da7d-9e3f-4459-ac5d-579444e6125e" width=50% height=50%>
+<img src="https://github.com/NREL/OpenStudio-server/assets/2235296/c4a75731-72f7-42e6-afc7-d24b9e835a2e" width=50% height=50%>
+
+## About
+
+OpenStudio Server is a web application and distributed computing tool, which is the backbone of the OpenStudio Analysis Framework (OSAF).
+It is intended to make parametric analysis of building energy models accessible to architects, engineers, and designers via the [OpenStudio PAT](http://nrel.github.io/OpenStudio-user-documentation/reference/parametric_studies/) GUI or the [OpenStudio Analysis Gem](https://github.com/NREL/OpenStudio-analysis-gem). 
+OpenStudio Server analyses are defined by PAT projects or OSA's.  Each analysis may include many OpenStudio simulations, as determined by project configuration.
+
+Journal of Building Performance Simulation article: [An open source analysis framework for large-scale building energy modeling](https://www.tandfonline.com/doi/full/10.1080/19401493.2020.1778788)
 
 ## Application Development and Deployment
 
-There are primarily three ways to utilize and deploy this codebase.
+There are primarily two ways to utilize and deploy this codebase.
 
-* [openstudio_meta](./bin/openstudio_meta) CLI: Allows for the server to be deployed on a local 
-desktop without docker through a pre-compilation process of all required gem dependencies. Additionally, it allows for 
-cloud instances to be created and analyses run on them. 
-* [Docker Compose](https://docs.docker.com/compose/): This is the preferred environment for application development, as 
-it is allows for rapid iteration and does not encumber developers with deployment configuration details. 
-* [Docker Swarm](https://docs.docker.com/engine/swarm/): This is the recommended deployment pathway. Swarm is an 
+* [openstudio-server-helm](https://github.com/NREL/openstudio-server-helm) This helm chart installs a OpenStudio-server instance deployment on a AWS, Azure, or Google Kubernetes cluster using the Helm package manager. You can interface with the OpenStudio-server cluster using the Parametric Analysis Tool or the [openstudio_meta](./bin/openstudio_meta) CLI.
+
+* [Docker Swarm](https://docs.docker.com/engine/swarm/): This is the recommended local deployment pathway. Swarm is an 
 orchestration engine which allows for multi-node clusters and provides significant benefits in the forms of 
 customization and hardening of network and storage 
 fundamentals.
@@ -25,42 +31,33 @@ fundamentals.
 
 The [openstudio_meta](./bin/openstudio_meta) file is a ruby script which provides access to packaging and execution 
 commands which allow for this codebase to be embedded in applications deployed to computers without docker. Deployment 
-requires that [MongoDB v3.2](https://www.mongodb.com/download-center#previous) and [Ruby v2.2](https://www.ruby-lang.org/en/news/2014/12/25/ruby-2-2-0-released/) 
-are additionally packaged. For an example of cross-platform deployment please see the OpenStudio build guide for the 
-[2.X releases](https://github.com/NREL/OpenStudio/wiki/Configuring-OpenStudio-Build-Environments) and the [CMake lists](https://github.com/NREL/OpenStudio/blob/develop/openstudiocore/CMakeLists.txt). 
+requires that [MongoDB 6.0.7](https://www.mongodb.com/download-center/community/releases/archive) and [Ruby v2.7](https://www.ruby-lang.org/en/downloads/) 
+are additionally packaged. 
 
 The openstudio_meta deployment relies on the `install_gems` command, which uses local system libraries to build all 
 required gem dependencies of the server. Additionally, the export flag allows for the resulting package to be 
 automatically assembled and zipped for deployment. It is important to note that when used on OSX and Linux systems, 
 it is critical to not specify the export path with home (`~`) substitution. Instead, pass a fully specified path to the 
 desired output directory. 
 
-Once compiled or unpacked, the openstudio_meta file can be used for starting and stopping local and remote server, and 
-submitting analyses to both. Assembling the required files for the analysis is left to either the OpenStudio Analysis 
-Spreadsheet (the Spreadsheet) or the Parametric Assessment Tool (PAT). The Spreadsheet has a similar interface for 
-submitting analyses to servers, and PAT makes complete use of the openstudio_meta features. For more details, please 
-refer to the [wiki](https://github.com/NREL/OpenStudio-server/wiki/CLI).
+Once compiled or unpacked, the openstudio_meta file can be used for starting and stopping the local server for the [Parametric Analysis Tool (PAT)](https://github.com/NREL/OpenStudio-PAT) and 
+submitting analyses to it. Assembling the required files for the analysis is done with the [Analysis-gem](https://github.com/NREL/OpenStudio-analysis-gem) or the export OSA function in PAT. For more details, please 
+refer to the [wiki](https://github.com/NREL/OpenStudio-server/wiki/CLI).  For examples, please refer to [OSAF notebooks](https://github.com/NREL/docker-openstudio-jupyter/tree/master).
 
 ### Local Docker Development
 
 To develop locally the following dependency stack is recommended. 
 
-* Install Docker (Version 17.09.0 or greater is required)
+* Install Docker (Version 20.10.5 or greater is required)
     * OSX Users: [install Docker CE for Mac](https://docs.docker.com/docker-for-mac/install/). Please refer to [this guide](https://docs.docker.com/docker-for-mac/install/)
-    * Windows 10 Users: [Docker CE for Windows](https://docs.docker.com/docker-for-windows/install/). More information 
-    can be found in [this guide](https://docs.docker.com/docker-for-windows/).
-    * Pre Windows 10 Users: Use Docker Toolbox. You will need to install and configure dependencies, including [VirtualBox](https://docs.docker.com/toolbox/toolbox_install_windows/#next-steps). 
+    * Windows 10 Users: [Docker Desktop](https://www.docker.com/products/docker-desktop/).
     * Linux Users: Follow the instructions in the [appropriate guide](https://www.docker.com/community-edition)
 
     *Note: Although generally newer versions of docker will behave as expected, certain CLI interactions change between
     releases, leading to scripts breaking and default behaviours, particularly regarding persistence, changing. The 
     docker version installed and running can be found by typing `docker info` on the command line.*
 
-* Install Docker Compose (Version 1.17.0 or greater is required)
-    * Docker compose will be installed on Mac and Windows by default
-    * Linux Users: See instructions [here](https://docs.docker.com/compose/install/)
-
-#### Run Docker Compose 
+#### Docker Compose 
 
 ```bash
 docker-compose build
@@ -105,23 +102,13 @@ git checkout -- .dockerignore && git checkout -- Dockerfile
 docker-compose rm -f
 ```
 
-
-
-### Docker Deployment
+### Local Docker Swarm Deployment
 
 To deploy the OpenStudio Server in a docker-based production environment one or more machines need to be running Docker 
-Server version 17.9.01. If using docker on a Linux machine it is recommended that significant storage be available to 
-the `/var` folder. This is where Docker reads and writes all data to by default. In addition, advanced users may wish 
-to consider using specialized [storage drivers](https://docs.docker.com/engine/userguide/storagedriver/). Please refer 
-to the [wiki](https://github.com/NREL/OpenStudio-server/wiki) page for additional details and 
-a [configuration and reset guide](). Deploying a production docker swarm system outside of AWS (where complications 
-are managed and support by NREL) can be a non-trivial problem that may require significant systems administration 
-experience. Those embarking on this process are encouraged to refer to the scripts used by Packer to configure 
-[Ubuntu](https://github.com/NREL/OpenStudio-server/blob/develop/docker/deployment/scripts/aws_system_init.sh) and 
-[docker](https://github.com/NREL/OpenStudio-server/blob/develop/docker/deployment/scripts/aws_osserver_init.sh) in the 
-base AMI images, as well as the scripts used to provision the [server](https://github.com/NREL/OpenStudio-server/blob/develop/docker/deployment/scripts/server_provision.sh) 
-and [worker](https://github.com/NREL/OpenStudio-server/blob/develop/docker/deployment/scripts/worker_provision.sh) 
-nodes upon instantiation in a cluster.
+Server version 20.10.05. If using docker on a Linux machine it is recommended that significant storage be available to 
+the `/var` folder. This is where Docker reads and writes all data to by default unless changed in the docker-compose.yml file. 
+There are scripts to help with docker swarm deployment [here](https://github.com/NREL/OpenStudio-server/tree/develop/local_setup_scripts).
+Make sure to change the defaults to be applicable to your hardware requirements.
 
 ## Testing procedure
 

appveyor.yml

@@ -4,8 +4,8 @@ image: Visual Studio 2019
 
 environment:
   USE_TESTING_TIMEOUTS: "true"
-  OPENSTUDIO_VERSION: 3.6.1
-  OPENSTUDIO_VERSION_SHA: bb9481519e
+  OPENSTUDIO_VERSION: 3.7.0
+  OPENSTUDIO_VERSION_SHA: d5269793f1
   OPENSTUDIO_VERSION_EXT: ""
   OPENSTUDIO_TEST_EXE: C:\projects\openstudio\bin\openstudio.exe
   BUILD_TYPE: "test"

ci/appveyor/setup.cmd

@@ -10,7 +10,7 @@ set OS_INSTALL_NAME=OpenStudio-%OPENSTUDIO_VERSION%%OPENSTUDIO_VERSION_EXT%%%2B%
 echo Install name is %OS_INSTALL_NAME%
 
 REM curl -SLO --insecure https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/develop/%OS_INSTALL_NAME%
-curl -SLO --insecure  https://github.com/NREL/OpenStudio/releases/download/v3.6.1/%OS_INSTALL_NAME%
+curl -SLO --insecure  https://github.com/NREL/OpenStudio/releases/download/v3.7.0/%OS_INSTALL_NAME%
 dir .
 REM Install OpenStudio
 %OS_INSTALL_NAME% --script ci/appveyor/install-windows.qs

ci/github-actions/install_openstudio.sh

@@ -11,7 +11,7 @@ if [ ! -z ${OPENSTUDIO_VERSION} ] && [ ! -z ${OPENSTUDIO_SHA} ]; then
     OPENSTUDIO_DOWNLOAD_FILENAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}%2B${OPENSTUDIO_SHA}-Ubuntu-20.04-x86_64.deb
     echo "Installing OpenStudio ${OPENSTUDIO_DOWNLOAD_FILENAME}"
     #OPENSTUDIO_DOWNLOAD_BASE_URL=https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/develop
-    OPENSTUDIO_DOWNLOAD_BASE_URL=https://github.com/NREL/OpenStudio/releases/download/v3.6.1
+    OPENSTUDIO_DOWNLOAD_BASE_URL=https://github.com/NREL/OpenStudio/releases/download/v3.7.0
     OPENSTUDIO_DOWNLOAD_URL=$OPENSTUDIO_DOWNLOAD_BASE_URL/$OPENSTUDIO_DOWNLOAD_FILENAME
 
     # copying this from the docker-openstudio dockerfile

ci/github-actions/setup.sh

@@ -37,7 +37,7 @@ else
         export OS_NAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}%2B${OPENSTUDIO_VERSION_SHA}-Darwin-x86_64
         export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-x86_64
         #curl -SL --insecure https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/develop/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
-        curl -SL --insecure https://github.com/NREL/OpenStudio/releases/download/v3.6.1/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
+        curl -SL --insecure https://github.com/NREL/OpenStudio/releases/download/v3.7.0/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
         # OSX downloads with %2B but installs with + sign. These are the encoded chars in url strings.
         #hdiutil attach ${OS_NAME}.dmg
         #sed -i -e "s|REPLACEME|$HOME/openstudio|" ci/github-actions/install-mac.qs

docker/server/nginx.conf

@@ -1,14 +1,14 @@
 daemon off;
 user nobody nogroup;
-worker_processes 1;
+worker_processes auto;
 error_log /dev/stderr;
 
 events {
     worker_connections 1024;
 }
 
 http {
-    passenger_root /usr/local/lib/ruby/gems/2.7.0/gems/passenger-6.0.2;
+    passenger_root /usr/local/lib/ruby/gems/2.7.0/gems/passenger-6.0.18;
     passenger_ruby /usr/local/bin/ruby;
     passenger_friendly_error_pages on;
 

server/Gemfile

@@ -20,13 +20,8 @@ gem 'tzinfo-data', '~>1.2021.1'
 gem 'loofah',  '2.20.0'
 
 # database modules
-gem 'mongoid', '7.4.3'
-#gem 'mongoid-paperclip' 
-# forked gem is neccessary as mongoid-paperclip relies on mimemagic which now requires freedesktop.org.xml at run time"
-gem 'mongoid-paperclip', :github => 'tijcolem/mongoid-paperclip', :ref => 'master' 
-# Once this PR is merged (https://github.com/kreeti/kt-paperclip/pull/54) and released change to gem version
-gem 'kt-paperclip', :github => 'tijcolem/kt-paperclip', :ref => 'use-marcel'
-#gem 'paperclip'
+gem 'mongoid', '7.5.4'
+gem 'mongoid-paperclip'
 
 # R
 gem 'rserve-simpler', '~> 0.0.6'
@@ -69,7 +64,7 @@ gem 'sassc', '~> 2.4.0'
 
 gem 'openstudio-workflow', '= 2.3.1'
 #gem 'openstudio-analysis', :github => 'NREL/OpenStudio-analysis-gem', :ref => 'develop'
-gem 'openstudio-analysis', '= 1.3.4'
+gem 'openstudio-analysis', '= 1.3.5'
 
 # Remove urbanopt cli gems for minor for releases as they use different versions of ext gems
 gem 'urbanopt-cli', '= 0.9.2'

server/app/jobs/dj_jobs/run_simulate_data_point.rb

@@ -180,9 +180,9 @@ def perform
             else  #OS CLI workflow
               @sim_logger.info "analysis is configured with #{@data_point.analysis.to_json}"
               if @data_point.analysis.gemfile
-                cmd = "#{Utility::Oss.oscli_cmd_bundle_args(@sim_logger, analysis_dir)} #{@data_point.analysis.cli_verbose} run --workflow #{osw_path} #{@data_point.analysis.cli_debug}"
+                cmd = "#{Utility::Oss.oscli_cmd_bundle_args(@sim_logger, analysis_dir)} classic #{@data_point.analysis.cli_verbose} run --workflow #{osw_path} #{@data_point.analysis.cli_debug}"
               else
-                cmd = "#{Utility::Oss.oscli_cmd_no_bundle_args(@sim_logger)} #{@data_point.analysis.cli_verbose} run --workflow #{osw_path} #{@data_point.analysis.cli_debug}"
+                cmd = "#{Utility::Oss.oscli_cmd_no_bundle_args(@sim_logger)} classic #{@data_point.analysis.cli_verbose} run --workflow #{osw_path} #{@data_point.analysis.cli_debug}"
               end
               process_log = File.join(simulation_dir, 'oscli_simulation.log')
               @sim_logger.info "Running workflow using cmd #{cmd} and writing log to #{process_log}"
@@ -289,10 +289,10 @@ def perform
           # Post the reports back to the server
           uploads_successful = []
           if @data_point.analysis.download_reports
-            @sim_logger.info 'downloading reports/*.{html,json,csv}'
-            Dir["#{simulation_dir}/reports/*.{html,json,csv}"].each { |rep| uploads_successful << upload_file(rep, 'Report') }
+            @sim_logger.info 'downloading reports/*.{html,json,csv,xml,mat}'
+            Dir["#{simulation_dir}/reports/*.{html,json,csv,xml,mat}"].each { |rep| uploads_successful << upload_file(rep, 'Report') }
           else
-            @sim_logger.info "NOT downloading /reports/*.{html,json,csv} since download_reports value is: #{@data_point.analysis.download_reports}"
+            @sim_logger.info "NOT downloading /reports/*.{html,json,csv,xml,mat} since download_reports value is: #{@data_point.analysis.download_reports}"
           end
           report_file = "#{run_dir}/objectives.json"
           uploads_successful << upload_file(report_file, 'Report', 'objectives', 'application/json') if File.exist?(report_file)

server/app/lib/openstudio_server/version.rb

@@ -4,9 +4,9 @@
 # *******************************************************************************
 
 module OpenstudioServer
-  VERSION = '3.6.1'.freeze
+  VERSION = '3.7.0'.freeze
   # format should be ^.*\-{1}[a-z]+[0-9]+
   # for example: -rc1, -beta6, -customusecase0
   VERSION_EXT = ''.freeze # with preceding - or +
-  OS_SHA = 'bb9481519e'.freeze
+  OS_SHA = 'd5269793f1'.freeze
 end

server/app/models/analysis.rb

@@ -71,7 +71,7 @@ class Analysis
 
   # Indexes
   index({ uuid: 1 }, unique: true)
-  index(id: 1)
+  #index(id: 1)
   index(name: 1)
   index(created_at: 1)
   index(updated_at: -1)
@@ -82,6 +82,7 @@ class Analysis
   validates_attachment_content_type :seed_zip, content_type: ['application/zip']
 
   # Callbacks
+  before_create :set_uuid_from_id
   after_create :verify_uuid
   before_destroy :queue_delete_files
 
@@ -510,6 +511,10 @@ def queue_delete_files
     end
   end
 
+  def set_uuid_from_id
+    self.uuid = id
+  end
+
   def verify_uuid
     self.uuid = id if uuid.nil?
     save!

server/app/models/data_point.rb

@@ -37,7 +37,7 @@ class DataPoint
 
   # Indexes
   index({ uuid: 1 }, unique: true)
-  index(id: 1)
+  #index(id: 1)
   index(name: 1)
   index(status: 1)
   index(analysis_id: 1, created_at: 1)
@@ -50,6 +50,7 @@ class DataPoint
   index(analysis_id: 1, status: 1, status_message: 1, created_at: 1)
 
   # Callbacks
+  before_create :set_uuid_from_id
   after_create :verify_uuid
   before_destroy :destroy_background_job
 
@@ -125,6 +126,10 @@ def set_queued_state
 
   protected
 
+  def set_uuid_from_id
+    self.uuid = id
+  end
+
   def verify_uuid
     self.uuid = id if uuid.nil?
     save!

server/app/models/job.rb

@@ -22,7 +22,7 @@ class Job
 
   belongs_to :analysis
 
-  index(id: 1)
+  #index(id: 1)
   index(created_at: 1)
   index(analysis_id: 1)
   index(analysis_id: 1, index: 1, analysis_type: 1)