diff --git a/.github/workflows/openstudio-server-tests.yml b/.github/workflows/openstudio-server-tests.yml index 9bea5d26e..d5dc610fc 100644 --- a/.github/workflows/openstudio-server-tests.yml +++ b/.github/workflows/openstudio-server-tests.yml @@ -10,8 +10,8 @@ on: [push, pull_request] env: USE_TESTING_TIMEOUTS: "true" - OPENSTUDIO_VERSION: 3.6.1 - OPENSTUDIO_VERSION_SHA: bb9481519e + OPENSTUDIO_VERSION: 3.7.0 + OPENSTUDIO_VERSION_SHA: d5269793f1 OPENSTUDIO_VERSION_EXT: "" DOCKER_COMPOSE_VERSION: 1.21.1 BUNDLE_WITHOUT: native_ext @@ -97,7 +97,7 @@ jobs: - name: docker shell: bash run: | - export OPENSTUDIO_TAG=3.6.1 + export OPENSTUDIO_TAG=3.7.0 sed -i -E "s/.git//g" .dockerignore docker volume create --name=osdata docker images --all diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 3180f0d08..6cd6c4437 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -10,8 +10,8 @@ on: [push, pull_request] env: USE_TESTING_TIMEOUTS: "true" - OPENSTUDIO_VERSION: 3.6.1 - OPENSTUDIO_VERSION_SHA: bb9481519e + OPENSTUDIO_VERSION: 3.7.0 + OPENSTUDIO_VERSION_SHA: d5269793f1 OPENSTUDIO_VERSION_EXT: "" DOCKER_COMPOSE_VERSION: 1.21.1 BUNDLE_WITHOUT: native_ext @@ -31,7 +31,7 @@ jobs: - name: docker shell: bash run: | - export OPENSTUDIO_TAG=3.6.1 + export OPENSTUDIO_TAG=3.7.0 sed -i -E "s/.git//g" .dockerignore docker volume create --name=osdata docker images --all diff --git a/CHANGELOG.md b/CHANGELOG.md index 72ca4e4b7..e98c5e400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ OpenStudio Server ================= +Version 3.7.0 +------------- +* update Analysis-gem to 1.3.5 in https://github.com/NREL/OpenStudio-server/pull/722 +* update Mongo to 6.0.9 in https://github.com/NREL/OpenStudio-server/pull/709 +* update Passenger to 6.0.18 in https://github.com/NREL/OpenStudio-server/pull/714 +* Fix indicies in Mongo Database in https://github.com/NREL/OpenStudio-server/pull/713 +* add XML and MAT files as valid downloadable Report Types https://github.com/NREL/OpenStudio-server/pull/720 + + Version 3.5.1 ------------- * update urbanopt to 0.9.0 diff --git a/Dockerfile b/Dockerfile index 13bc702ed..1eb1133d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,8 @@ # NOTES: Currently this is one big dockerfile and non-optimal. #may include suffix -ARG OPENSTUDIO_VERSION=3.6.1 -FROM nrel/openstudio:3.6.1 as base +ARG OPENSTUDIO_VERSION=3.7.0 +FROM nrel/openstudio:3.7.0 as base MAINTAINER Nicholas Long nicholas.long@nrel.gov ENV DEBIAN_FRONTEND=noninteractive @@ -59,7 +59,7 @@ RUN apt-get update && apt-get install -y wget gnupg \ && rm -rf /var/lib/apt/lists/* # Install passenger (this also installs nginx) -ENV PASSENGER_VERSION 6.0.2 +ENV PASSENGER_VERSION 6.0.18 RUN gem install passenger -v $PASSENGER_VERSION RUN passenger-install-nginx-module @@ -104,7 +104,7 @@ RUN bundle exec rake assets:precompile # Bundle app source ADD /server /opt/openstudio/server # Add in /spec for testing -#ADD /spec /opt/openstudio/spec +#ADD /server/spec /opt/openstudio/server/spec ADD .rubocop.yml /opt/openstudio/.rubocop.yml # Run bundle again, because if the user has a local Gemfile.lock it will have been overriden RUN rm Gemfile.lock diff --git a/README.md b/README.md index 6ccfebe7f..87713250c 100644 --- a/README.md +++ b/README.md @@ -3,20 +3,26 @@ [![Build Status][gh-img]][gh-url] [![Coverage Status][coveralls-img]][coveralls-url] -Windows Build (Under Development): [![Windows Build Status][appveyor-img]][appveyor-url] +Please refer to the [wiki](https://github.com/NREL/OpenStudio-server/wiki) for additional documentation. -The below documentation has not been recently reviewed. Please refer to the [wiki](https://github.com/NREL/OpenStudio-server/wiki) for current documentation. + + + +## About + +OpenStudio Server is a web application and distributed computing tool, which is the backbone of the OpenStudio Analysis Framework (OSAF). +It is intended to make parametric analysis of building energy models accessible to architects, engineers, and designers via the [OpenStudio PAT](http://nrel.github.io/OpenStudio-user-documentation/reference/parametric_studies/) GUI or the [OpenStudio Analysis Gem](https://github.com/NREL/OpenStudio-analysis-gem). +OpenStudio Server analyses are defined by PAT projects or OSA's. Each analysis may include many OpenStudio simulations, as determined by project configuration. + +Journal of Building Performance Simulation article: [An open source analysis framework for large-scale building energy modeling](https://www.tandfonline.com/doi/full/10.1080/19401493.2020.1778788) ## Application Development and Deployment -There are primarily three ways to utilize and deploy this codebase. +There are primarily two ways to utilize and deploy this codebase. -* [openstudio_meta](./bin/openstudio_meta) CLI: Allows for the server to be deployed on a local -desktop without docker through a pre-compilation process of all required gem dependencies. Additionally, it allows for -cloud instances to be created and analyses run on them. -* [Docker Compose](https://docs.docker.com/compose/): This is the preferred environment for application development, as -it is allows for rapid iteration and does not encumber developers with deployment configuration details. -* [Docker Swarm](https://docs.docker.com/engine/swarm/): This is the recommended deployment pathway. Swarm is an +* [openstudio-server-helm](https://github.com/NREL/openstudio-server-helm) This helm chart installs a OpenStudio-server instance deployment on a AWS, Azure, or Google Kubernetes cluster using the Helm package manager. You can interface with the OpenStudio-server cluster using the Parametric Analysis Tool or the [openstudio_meta](./bin/openstudio_meta) CLI. + +* [Docker Swarm](https://docs.docker.com/engine/swarm/): This is the recommended local deployment pathway. Swarm is an orchestration engine which allows for multi-node clusters and provides significant benefits in the forms of customization and hardening of network and storage fundamentals. @@ -25,9 +31,8 @@ fundamentals. The [openstudio_meta](./bin/openstudio_meta) file is a ruby script which provides access to packaging and execution commands which allow for this codebase to be embedded in applications deployed to computers without docker. Deployment -requires that [MongoDB v3.2](https://www.mongodb.com/download-center#previous) and [Ruby v2.2](https://www.ruby-lang.org/en/news/2014/12/25/ruby-2-2-0-released/) -are additionally packaged. For an example of cross-platform deployment please see the OpenStudio build guide for the -[2.X releases](https://github.com/NREL/OpenStudio/wiki/Configuring-OpenStudio-Build-Environments) and the [CMake lists](https://github.com/NREL/OpenStudio/blob/develop/openstudiocore/CMakeLists.txt). +requires that [MongoDB 6.0.7](https://www.mongodb.com/download-center/community/releases/archive) and [Ruby v2.7](https://www.ruby-lang.org/en/downloads/) +are additionally packaged. The openstudio_meta deployment relies on the `install_gems` command, which uses local system libraries to build all required gem dependencies of the server. Additionally, the export flag allows for the resulting package to be @@ -35,32 +40,24 @@ automatically assembled and zipped for deployment. It is important to note that it is critical to not specify the export path with home (`~`) substitution. Instead, pass a fully specified path to the desired output directory. -Once compiled or unpacked, the openstudio_meta file can be used for starting and stopping local and remote server, and -submitting analyses to both. Assembling the required files for the analysis is left to either the OpenStudio Analysis -Spreadsheet (the Spreadsheet) or the Parametric Assessment Tool (PAT). The Spreadsheet has a similar interface for -submitting analyses to servers, and PAT makes complete use of the openstudio_meta features. For more details, please -refer to the [wiki](https://github.com/NREL/OpenStudio-server/wiki/CLI). +Once compiled or unpacked, the openstudio_meta file can be used for starting and stopping the local server for the [Parametric Analysis Tool (PAT)](https://github.com/NREL/OpenStudio-PAT) and +submitting analyses to it. Assembling the required files for the analysis is done with the [Analysis-gem](https://github.com/NREL/OpenStudio-analysis-gem) or the export OSA function in PAT. For more details, please +refer to the [wiki](https://github.com/NREL/OpenStudio-server/wiki/CLI). For examples, please refer to [OSAF notebooks](https://github.com/NREL/docker-openstudio-jupyter/tree/master). ### Local Docker Development To develop locally the following dependency stack is recommended. -* Install Docker (Version 17.09.0 or greater is required) +* Install Docker (Version 20.10.5 or greater is required) * OSX Users: [install Docker CE for Mac](https://docs.docker.com/docker-for-mac/install/). Please refer to [this guide](https://docs.docker.com/docker-for-mac/install/) - * Windows 10 Users: [Docker CE for Windows](https://docs.docker.com/docker-for-windows/install/). More information - can be found in [this guide](https://docs.docker.com/docker-for-windows/). - * Pre Windows 10 Users: Use Docker Toolbox. You will need to install and configure dependencies, including [VirtualBox](https://docs.docker.com/toolbox/toolbox_install_windows/#next-steps). + * Windows 10 Users: [Docker Desktop](https://www.docker.com/products/docker-desktop/). * Linux Users: Follow the instructions in the [appropriate guide](https://www.docker.com/community-edition) *Note: Although generally newer versions of docker will behave as expected, certain CLI interactions change between releases, leading to scripts breaking and default behaviours, particularly regarding persistence, changing. The docker version installed and running can be found by typing `docker info` on the command line.* -* Install Docker Compose (Version 1.17.0 or greater is required) - * Docker compose will be installed on Mac and Windows by default - * Linux Users: See instructions [here](https://docs.docker.com/compose/install/) - -#### Run Docker Compose +#### Docker Compose ```bash docker-compose build @@ -105,23 +102,13 @@ git checkout -- .dockerignore && git checkout -- Dockerfile docker-compose rm -f ``` - - -### Docker Deployment +### Local Docker Swarm Deployment To deploy the OpenStudio Server in a docker-based production environment one or more machines need to be running Docker -Server version 17.9.01. If using docker on a Linux machine it is recommended that significant storage be available to -the `/var` folder. This is where Docker reads and writes all data to by default. In addition, advanced users may wish -to consider using specialized [storage drivers](https://docs.docker.com/engine/userguide/storagedriver/). Please refer -to the [wiki](https://github.com/NREL/OpenStudio-server/wiki) page for additional details and -a [configuration and reset guide](). Deploying a production docker swarm system outside of AWS (where complications -are managed and support by NREL) can be a non-trivial problem that may require significant systems administration -experience. Those embarking on this process are encouraged to refer to the scripts used by Packer to configure -[Ubuntu](https://github.com/NREL/OpenStudio-server/blob/develop/docker/deployment/scripts/aws_system_init.sh) and -[docker](https://github.com/NREL/OpenStudio-server/blob/develop/docker/deployment/scripts/aws_osserver_init.sh) in the -base AMI images, as well as the scripts used to provision the [server](https://github.com/NREL/OpenStudio-server/blob/develop/docker/deployment/scripts/server_provision.sh) -and [worker](https://github.com/NREL/OpenStudio-server/blob/develop/docker/deployment/scripts/worker_provision.sh) -nodes upon instantiation in a cluster. +Server version 20.10.05. If using docker on a Linux machine it is recommended that significant storage be available to +the `/var` folder. This is where Docker reads and writes all data to by default unless changed in the docker-compose.yml file. +There are scripts to help with docker swarm deployment [here](https://github.com/NREL/OpenStudio-server/tree/develop/local_setup_scripts). +Make sure to change the defaults to be applicable to your hardware requirements. ## Testing procedure diff --git a/appveyor.yml b/appveyor.yml index 17aaa03af..125f27e4d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,8 +4,8 @@ image: Visual Studio 2019 environment: USE_TESTING_TIMEOUTS: "true" - OPENSTUDIO_VERSION: 3.6.1 - OPENSTUDIO_VERSION_SHA: bb9481519e + OPENSTUDIO_VERSION: 3.7.0 + OPENSTUDIO_VERSION_SHA: d5269793f1 OPENSTUDIO_VERSION_EXT: "" OPENSTUDIO_TEST_EXE: C:\projects\openstudio\bin\openstudio.exe BUILD_TYPE: "test" diff --git a/ci/appveyor/setup.cmd b/ci/appveyor/setup.cmd index 89b84f537..9bd33da6a 100644 --- a/ci/appveyor/setup.cmd +++ b/ci/appveyor/setup.cmd @@ -10,7 +10,7 @@ set OS_INSTALL_NAME=OpenStudio-%OPENSTUDIO_VERSION%%OPENSTUDIO_VERSION_EXT%%%2B% echo Install name is %OS_INSTALL_NAME% REM curl -SLO --insecure https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/develop/%OS_INSTALL_NAME% -curl -SLO --insecure https://github.com/NREL/OpenStudio/releases/download/v3.6.1/%OS_INSTALL_NAME% +curl -SLO --insecure https://github.com/NREL/OpenStudio/releases/download/v3.7.0/%OS_INSTALL_NAME% dir . REM Install OpenStudio %OS_INSTALL_NAME% --script ci/appveyor/install-windows.qs diff --git a/ci/github-actions/install_openstudio.sh b/ci/github-actions/install_openstudio.sh index 91fc70003..bf4a217e0 100755 --- a/ci/github-actions/install_openstudio.sh +++ b/ci/github-actions/install_openstudio.sh @@ -11,7 +11,7 @@ if [ ! -z ${OPENSTUDIO_VERSION} ] && [ ! -z ${OPENSTUDIO_SHA} ]; then OPENSTUDIO_DOWNLOAD_FILENAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}%2B${OPENSTUDIO_SHA}-Ubuntu-20.04-x86_64.deb echo "Installing OpenStudio ${OPENSTUDIO_DOWNLOAD_FILENAME}" #OPENSTUDIO_DOWNLOAD_BASE_URL=https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/develop - OPENSTUDIO_DOWNLOAD_BASE_URL=https://github.com/NREL/OpenStudio/releases/download/v3.6.1 + OPENSTUDIO_DOWNLOAD_BASE_URL=https://github.com/NREL/OpenStudio/releases/download/v3.7.0 OPENSTUDIO_DOWNLOAD_URL=$OPENSTUDIO_DOWNLOAD_BASE_URL/$OPENSTUDIO_DOWNLOAD_FILENAME # copying this from the docker-openstudio dockerfile diff --git a/ci/github-actions/setup.sh b/ci/github-actions/setup.sh index 114c5e3c2..91f732e86 100755 --- a/ci/github-actions/setup.sh +++ b/ci/github-actions/setup.sh @@ -37,7 +37,7 @@ else export OS_NAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}%2B${OPENSTUDIO_VERSION_SHA}-Darwin-x86_64 export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-x86_64 #curl -SL --insecure https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/develop/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz - curl -SL --insecure https://github.com/NREL/OpenStudio/releases/download/v3.6.1/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz + curl -SL --insecure https://github.com/NREL/OpenStudio/releases/download/v3.7.0/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz # OSX downloads with %2B but installs with + sign. These are the encoded chars in url strings. #hdiutil attach ${OS_NAME}.dmg #sed -i -e "s|REPLACEME|$HOME/openstudio|" ci/github-actions/install-mac.qs diff --git a/docker/server/nginx.conf b/docker/server/nginx.conf index c9d6bbd33..bbaca21df 100644 --- a/docker/server/nginx.conf +++ b/docker/server/nginx.conf @@ -1,6 +1,6 @@ daemon off; user nobody nogroup; -worker_processes 1; +worker_processes auto; error_log /dev/stderr; events { @@ -8,7 +8,7 @@ events { } http { - passenger_root /usr/local/lib/ruby/gems/2.7.0/gems/passenger-6.0.2; + passenger_root /usr/local/lib/ruby/gems/2.7.0/gems/passenger-6.0.18; passenger_ruby /usr/local/bin/ruby; passenger_friendly_error_pages on; diff --git a/server/Gemfile b/server/Gemfile index d39d634c6..ebd3ed397 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -20,13 +20,8 @@ gem 'tzinfo-data', '~>1.2021.1' gem 'loofah', '2.20.0' # database modules -gem 'mongoid', '7.4.3' -#gem 'mongoid-paperclip' -# forked gem is neccessary as mongoid-paperclip relies on mimemagic which now requires freedesktop.org.xml at run time" -gem 'mongoid-paperclip', :github => 'tijcolem/mongoid-paperclip', :ref => 'master' -# Once this PR is merged (https://github.com/kreeti/kt-paperclip/pull/54) and released change to gem version -gem 'kt-paperclip', :github => 'tijcolem/kt-paperclip', :ref => 'use-marcel' -#gem 'paperclip' +gem 'mongoid', '7.5.4' +gem 'mongoid-paperclip' # R gem 'rserve-simpler', '~> 0.0.6' @@ -69,7 +64,7 @@ gem 'sassc', '~> 2.4.0' gem 'openstudio-workflow', '= 2.3.1' #gem 'openstudio-analysis', :github => 'NREL/OpenStudio-analysis-gem', :ref => 'develop' -gem 'openstudio-analysis', '= 1.3.4' +gem 'openstudio-analysis', '= 1.3.5' # Remove urbanopt cli gems for minor for releases as they use different versions of ext gems gem 'urbanopt-cli', '= 0.9.2' diff --git a/server/app/jobs/dj_jobs/run_simulate_data_point.rb b/server/app/jobs/dj_jobs/run_simulate_data_point.rb index df6f041b9..68b99c46d 100644 --- a/server/app/jobs/dj_jobs/run_simulate_data_point.rb +++ b/server/app/jobs/dj_jobs/run_simulate_data_point.rb @@ -180,9 +180,9 @@ def perform else #OS CLI workflow @sim_logger.info "analysis is configured with #{@data_point.analysis.to_json}" if @data_point.analysis.gemfile - cmd = "#{Utility::Oss.oscli_cmd_bundle_args(@sim_logger, analysis_dir)} #{@data_point.analysis.cli_verbose} run --workflow #{osw_path} #{@data_point.analysis.cli_debug}" + cmd = "#{Utility::Oss.oscli_cmd_bundle_args(@sim_logger, analysis_dir)} classic #{@data_point.analysis.cli_verbose} run --workflow #{osw_path} #{@data_point.analysis.cli_debug}" else - cmd = "#{Utility::Oss.oscli_cmd_no_bundle_args(@sim_logger)} #{@data_point.analysis.cli_verbose} run --workflow #{osw_path} #{@data_point.analysis.cli_debug}" + cmd = "#{Utility::Oss.oscli_cmd_no_bundle_args(@sim_logger)} classic #{@data_point.analysis.cli_verbose} run --workflow #{osw_path} #{@data_point.analysis.cli_debug}" end process_log = File.join(simulation_dir, 'oscli_simulation.log') @sim_logger.info "Running workflow using cmd #{cmd} and writing log to #{process_log}" @@ -289,10 +289,10 @@ def perform # Post the reports back to the server uploads_successful = [] if @data_point.analysis.download_reports - @sim_logger.info 'downloading reports/*.{html,json,csv}' - Dir["#{simulation_dir}/reports/*.{html,json,csv}"].each { |rep| uploads_successful << upload_file(rep, 'Report') } + @sim_logger.info 'downloading reports/*.{html,json,csv,xml,mat}' + Dir["#{simulation_dir}/reports/*.{html,json,csv,xml,mat}"].each { |rep| uploads_successful << upload_file(rep, 'Report') } else - @sim_logger.info "NOT downloading /reports/*.{html,json,csv} since download_reports value is: #{@data_point.analysis.download_reports}" + @sim_logger.info "NOT downloading /reports/*.{html,json,csv,xml,mat} since download_reports value is: #{@data_point.analysis.download_reports}" end report_file = "#{run_dir}/objectives.json" uploads_successful << upload_file(report_file, 'Report', 'objectives', 'application/json') if File.exist?(report_file) diff --git a/server/app/lib/openstudio_server/version.rb b/server/app/lib/openstudio_server/version.rb index b292497d6..8f58a6242 100644 --- a/server/app/lib/openstudio_server/version.rb +++ b/server/app/lib/openstudio_server/version.rb @@ -4,9 +4,9 @@ # ******************************************************************************* module OpenstudioServer - VERSION = '3.6.1'.freeze + VERSION = '3.7.0'.freeze # format should be ^.*\-{1}[a-z]+[0-9]+ # for example: -rc1, -beta6, -customusecase0 VERSION_EXT = ''.freeze # with preceding - or + - OS_SHA = 'bb9481519e'.freeze + OS_SHA = 'd5269793f1'.freeze end diff --git a/server/app/models/analysis.rb b/server/app/models/analysis.rb index 34a58a7f0..188f38386 100644 --- a/server/app/models/analysis.rb +++ b/server/app/models/analysis.rb @@ -71,7 +71,7 @@ class Analysis # Indexes index({ uuid: 1 }, unique: true) - index(id: 1) + #index(id: 1) index(name: 1) index(created_at: 1) index(updated_at: -1) @@ -82,6 +82,7 @@ class Analysis validates_attachment_content_type :seed_zip, content_type: ['application/zip'] # Callbacks + before_create :set_uuid_from_id after_create :verify_uuid before_destroy :queue_delete_files @@ -510,6 +511,10 @@ def queue_delete_files end end + def set_uuid_from_id + self.uuid = id + end + def verify_uuid self.uuid = id if uuid.nil? save! diff --git a/server/app/models/data_point.rb b/server/app/models/data_point.rb index b62996193..f3009f781 100644 --- a/server/app/models/data_point.rb +++ b/server/app/models/data_point.rb @@ -37,7 +37,7 @@ class DataPoint # Indexes index({ uuid: 1 }, unique: true) - index(id: 1) + #index(id: 1) index(name: 1) index(status: 1) index(analysis_id: 1, created_at: 1) @@ -50,6 +50,7 @@ class DataPoint index(analysis_id: 1, status: 1, status_message: 1, created_at: 1) # Callbacks + before_create :set_uuid_from_id after_create :verify_uuid before_destroy :destroy_background_job @@ -125,6 +126,10 @@ def set_queued_state protected + def set_uuid_from_id + self.uuid = id + end + def verify_uuid self.uuid = id if uuid.nil? save! diff --git a/server/app/models/job.rb b/server/app/models/job.rb index f3c7255bb..9ba37ca18 100644 --- a/server/app/models/job.rb +++ b/server/app/models/job.rb @@ -22,7 +22,7 @@ class Job belongs_to :analysis - index(id: 1) + #index(id: 1) index(created_at: 1) index(analysis_id: 1) index(analysis_id: 1, index: 1, analysis_type: 1) diff --git a/server/app/models/measure.rb b/server/app/models/measure.rb index 9c6ffc833..72a8b875a 100644 --- a/server/app/models/measure.rb +++ b/server/app/models/measure.rb @@ -24,13 +24,14 @@ class Measure # Indexes index({ uuid: 1 }, unique: true) - index(id: 1) + #index(id: 1) index(name: 1) index(analysis_id: 1) index(analysis_id: 1, uuid: 1) index({ analysis_id: 1, name: 1 }, unique: true) # Callbacks + before_create :set_uuid_from_id after_create :verify_uuid # Parse Analysis JSON to pull out the measures and variables @@ -145,6 +146,10 @@ def self.create_from_os_json(analysis_id, os_json, pat_json) protected + def set_uuid_from_id + self.uuid = id + end + def verify_uuid self.uuid = id if uuid.nil? save! diff --git a/server/app/models/pareto.rb b/server/app/models/pareto.rb index 3c7ff776f..08d336347 100644 --- a/server/app/models/pareto.rb +++ b/server/app/models/pareto.rb @@ -20,7 +20,7 @@ class Pareto # Indexes index({ uuid: 1 }, unique: true) - index(id: 1) + #index(id: 1) index(analysis_id: 1) # Validation diff --git a/server/app/models/project.rb b/server/app/models/project.rb index fafcb2ea6..772dacbd0 100644 --- a/server/app/models/project.rb +++ b/server/app/models/project.rb @@ -17,10 +17,11 @@ class Project # Indexes index({ uuid: 1 }, unique: true) - index(id: 1) + #index(id: 1) index(name: 1) # Callbacks + before_create :set_uuid_from_id after_create :verify_uuid def create_single_analysis(analysis_uuid, analysis_name, problem_uuid, problem_name) @@ -36,6 +37,10 @@ def create_single_analysis(analysis_uuid, analysis_name, problem_uuid, problem_n protected + def set_uuid_from_id + self.uuid = id + end + def verify_uuid self.uuid = id if uuid.nil? save! diff --git a/server/app/models/variable.rb b/server/app/models/variable.rb index 56e0028ec..034d5af13 100644 --- a/server/app/models/variable.rb +++ b/server/app/models/variable.rb @@ -63,7 +63,7 @@ class Variable # Indexes index({ uuid: 1 }, unique: true) - index(id: 1) + #index(id: 1) index(name: 1) index(r_index: 1) index(analysis_id: 1) @@ -76,6 +76,7 @@ class Variable # validates_attachment :seed_zip, content_type: { content_type: "application/zip" } # Callbacks + before_create :set_uuid_from_id after_create :verify_uuid before_destroy :destroy_preflight_images @@ -382,6 +383,10 @@ def map_discrete_hash_to_array protected + def set_uuid_from_id + self.uuid = id + end + def verify_uuid self.uuid = id if uuid.nil? save! diff --git a/server/spec/features/docker_stack_algo_spec.rb b/server/spec/features/docker_stack_algo_spec.rb index e24032cf7..1362c02d7 100644 --- a/server/spec/features/docker_stack_algo_spec.rb +++ b/server/spec/features/docker_stack_algo_spec.rb @@ -7,10 +7,10 @@ # To Run this test manually: # # start a server stack with /spec added and ssh into the Web container +# you may need to ADD the spec folder in the Dockerfile # >ruby /opt/openstudio/bin/openstudio_meta install_gems -# >cd /opt/openstudio/spec/ -# >gem install rspec -# >rspec openstudio_algo_spec.rb +# >bundle install --with development test +# >rspec spec/features/docker_stack_algo_spec.rb # ################################################################################# @@ -213,6 +213,14 @@ sleep Random.new.rand(1.0..10.0) retry if get_count <= get_count_max end + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # cli_test it 'run spea_nrel analysis', :spea_nrel, js: true do @@ -346,6 +354,14 @@ sleep Random.new.rand(1.0..10.0) retry if get_count <= get_count_max end + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # spea_nrel it 'run pso analysis', :pso, js: true do @@ -479,6 +495,14 @@ sleep Random.new.rand(1.0..10.0) retry if get_count <= get_count_max end + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # pso it 'run rgenoud analysis', :rgenoud, js: true do @@ -613,6 +637,14 @@ sleep Random.new.rand(1.0..10.0) retry if get_count <= get_count_max end + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # rgenoud it 'run sobol analysis', :sobol, js: true do @@ -770,6 +802,14 @@ expect(a.headers[:content_type]).to eq("application/zip") expect(a.size).to be >(30000) expect(a.size).to be <(40000) + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # sobol it 'run lhs analysis', :lhs, js: true do @@ -908,6 +948,14 @@ sleep Random.new.rand(1.0..10.0) retry if get_count <= get_count_max end + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # lhs it 'run lhs_discrete analysis', :lhs_discrete, js: true do @@ -1051,6 +1099,14 @@ sleep Random.new.rand(1.0..10.0) retry if get_count <= get_count_max end + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # lhs_discrete it 'run morris analysis', :morris, js: true do @@ -1196,6 +1252,14 @@ expect(a.headers[:content_type]).to eq("application/zip") expect(a.size).to be >(170000) expect(a.size).to be <(200000) + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # morris it 'run single_run analysis', :single_run, js: true do @@ -1319,11 +1383,33 @@ compare = single_run_bad.include?(sim) expect(compare).to be false + + a = RestClient.get "http://#{@host}/data_points/#{data_point[:_id]}/download_result_file?filename=calibration_reports_enhanced_20_report_xml_file.xml" + expect(a).not_to be_empty + #expect(a.size).to be >(1000) + #expect(a.size).to be <(2000) + expect(a.headers[:status]).to eq("200 OK") + expect(a.headers[:content_type]).to eq("application/xml") + expect(a.headers[:content_disposition]).to include("calibration_reports_enhanced_20_report_xml_file.xml") end rescue RestClient::ExceptionWithResponse => e - puts "rescue: #{e} get_count: #{get_count}" - sleep Random.new.rand(1.0..10.0) - retry if get_count <= get_count_max + if e.http_code == 422 + # Handle the 422 Unprocessable Entity error here if .xml file not found + fail("Received a 422 error: calibration_reports_enhanced_20_report_xml_file.xml not avail for download") + else + puts "rescue: #{e} get_count: #{get_count}" + sleep Random.new.rand(1.0..10.0) + get_count = get_count + 1 + retry if get_count <= get_count_max + end end + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # single_run end diff --git a/server/spec/features/docker_stack_test_apis_spec.rb b/server/spec/features/docker_stack_test_apis_spec.rb index 321010642..fe776d719 100644 --- a/server/spec/features/docker_stack_test_apis_spec.rb +++ b/server/spec/features/docker_stack_test_apis_spec.rb @@ -7,10 +7,10 @@ # To Run this test manually: # # start a server stack with /spec added and ssh into the Web container +# you may need to ADD the spec folder in the Dockerfile # >ruby /opt/openstudio/bin/openstudio_meta install_gems -# >cd /opt/openstudio/spec/ -# >gem install rspec -# >rspec openstudio_algo_spec.rb +# >bundle install --with development test +# >rspec spec/features/docker_stack_test_apis_spec.rb # ################################################################################# diff --git a/server/spec/features/docker_stack_urbanopt_algo_spec.rb b/server/spec/features/docker_stack_urbanopt_algo_spec.rb index ba1c48466..cfcd7482c 100644 --- a/server/spec/features/docker_stack_urbanopt_algo_spec.rb +++ b/server/spec/features/docker_stack_urbanopt_algo_spec.rb @@ -7,9 +7,10 @@ # To Run this test manually: # # start a server stack with /spec added and ssh into the Web container -# >cd /opt/openstudio/server/spec/ -# >gem install rest-client rails_helper json rspec rspec-retry -# >rspec openstudio_algo_spec.rb +# you may need to ADD the spec folder in the Dockerfile +# >ruby /opt/openstudio/bin/openstudio_meta install_gems +# >bundle install --with development test +# >rspec spec/features/docker_stack_urbanopt_algo_spec.rb # ################################################################################# @@ -267,5 +268,13 @@ sleep Random.new.rand(1.0..10.0) retry if get_count <= get_count_max end + + puts 'check logs for mongo index errors' + a = RestClient.get "http://#{@host}/analyses/#{analysis_id}/debug_log" + expect(a.headers[:status]).to eq("200 OK") + expect(a.body).not_to include "OperationFailure" + expect(a.body).not_to include "FATAL" + expect(a.body).to include "Created indexes" + end # urbanopt_single_run end diff --git a/server/spec/files/SEB_calibration_NSGA_2013.zip b/server/spec/files/SEB_calibration_NSGA_2013.zip index 9395c09c4..6c442530d 100644 Binary files a/server/spec/files/SEB_calibration_NSGA_2013.zip and b/server/spec/files/SEB_calibration_NSGA_2013.zip differ