diff --git a/.dockerignore b/.dockerignore index 184623a..78fad34 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,4 @@ build/ .vscode/.git/ .vscode/ transfer/ +*.code-workspace diff --git a/.gitignore b/.gitignore index 38afbb2..597a3d1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ node_modules/ build/ .vscode/ transfer/ +*.code-workspace diff --git a/Dockerfile b/Dockerfile index 535e38e..3a95e18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="ODK Build maintainers" LABEL description="ODK Build, a drag-and-drop ODK Xforms designer" # Install system dependencies -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - +RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - RUN DEBIAN_FRONTEND=noninteractive apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --yes \ -o Acquire::Retries=10 --no-install-recommends \ @@ -14,7 +14,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \ # Install Ruby gems and ODK Build ENV RACK_ENV production -# The build2xlsform hostname is "odkbuild2xlsform" when run with docker-compose, +# The build2xlsform hostname is "odkbuild2xlsform" when run with docker-compose, # not "localhost" (the default) as in a source install. ENV B2X_HOST odkbuild2xlsform WORKDIR /srv/odkbuild/current diff --git a/Gemfile b/Gemfile index 8981ad8..a9d0520 100644 --- a/Gemfile +++ b/Gemfile @@ -1,12 +1,12 @@ source 'https://rubygems.org' -gem 'rake' +gem 'rake', '13.0.6' gem 'ffi', '1.13.1' gem 'sequel', '>= 4.42' gem 'pg', '>= 0.18' -gem 'sinatra', '2.1.0' -gem "rack", ">= 2.1.4" +gem 'sinatra', '2.2.0' +gem "rack", ">= 2.2.3.1" gem 'pony', '1.3' gem 'warden', '1.0.6' gem 'deep_merge', '1.1.1' diff --git a/Gemfile.lock b/Gemfile.lock index ee00a56..96eb9fb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,31 +10,31 @@ GEM mini_mime (>= 0.1.1) mini_mime (1.0.2) mocha (0.9.12) - mustermann (1.1.1) + mustermann (1.1.2) ruby2_keywords (~> 0.0.1) net-http-digest_auth (1.4) pg (1.2.3) pony (1.3) mail (> 2.0) - rack (2.2.3) - rack-protection (2.1.0) + rack (2.2.7) + rack-protection (2.2.0) rack rack-test (0.5.7) rack (>= 1.0) raindrops (0.19.1) - rake (13.0.1) + rake (13.0.6) riot (0.12.1) rr term-ansicolor rr (1.2.1) - ruby2_keywords (0.0.4) + ruby2_keywords (0.0.5) sequel (5.39.0) shotgun (0.9.2) rack (>= 1.0) - sinatra (2.1.0) + sinatra (2.2.0) mustermann (~> 1.0) rack (~> 2.2) - rack-protection (= 2.1.0) + rack-protection (= 2.2.0) tilt (~> 2.0) sync (0.5.0) term-ansicolor (1.7.1) @@ -43,7 +43,7 @@ GEM daemons (~> 1.0, >= 1.0.9) eventmachine (~> 1.0, >= 1.0.4) rack (>= 1, < 3) - tilt (2.0.10) + tilt (2.1.0) tins (1.26.0) sync unicorn (5.7.0) @@ -63,13 +63,13 @@ DEPENDENCIES net-http-digest_auth (= 1.4) pg (>= 0.18) pony (= 1.3) - rack (>= 2.1.4) + rack (>= 2.2.3.1) rack-test (= 0.5.7) - rake + rake (= 13.0.6) riot (= 0.12.1) sequel (>= 4.42) shotgun - sinatra (= 2.1.0) + sinatra (= 2.2.0) thin unicorn warden (= 1.0.6) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ef62c2e..8fad7a0 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,9 +1,9 @@ -# This docker-compose file will deploy the latest tagged image for build2xlsform from ghcr.io, +# This docker compose file will deploy the latest tagged image for build2xlsform from ghcr.io, # and build the local version of ODK Build from the Dockerfile. # Use this file to test ODK Build during development. -# Run: docker-compose -f docker-compose.dev.yml up -d --build +# Run: docker compose -f docker-compose.dev.yml up -d --build # The bind mount "transfer" serves to transfer database snapshots in and out of the container. -# This is useful to migrate deployment from source to docker-compose. +# This is useful to migrate deployment from source to docker compose. version: "3" services: postgres: diff --git a/docker-compose.yml b/docker-compose.yml index 6d38695..4aa7396 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ -# This docker-compose file will deploy the latest tagged images from ghcr.io, +# This docker compose file will deploy the latest tagged images from ghcr.io, # which are built and pushed whenever a new Git tag is pushed to GitHub. # Use this file to deploy ODK Build to a staging or production server. -# Run: docker-compose up -d +# Run: docker compose up -d version: "3" services: postgres: diff --git a/docs/architecture.md b/docs/architecture.md index 01ce0a0..47d28b0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,9 +1,9 @@ # Architecture This document provides a high-level overview of how Build works. -Build is a combination between a Ruby Rack-based application built on Sinatra and a large Javascript frontend. -Everything it needs is kicked off by the config.ru Rackup file. -We use [shotgun](https://github.com/rtomayko/shotgun) for local development and +Build is a combination between a Ruby Rack-based application built on Sinatra and a large Javascript frontend. +Everything it needs is kicked off by the config.ru Rackup file. +We use [shotgun](https://github.com/rtomayko/shotgun) for local development and [docker-compose](https://docs.docker.com/compose/) for staging and production deployment. ## Secrets @@ -13,7 +13,7 @@ The file `config.yml` created from template `config.yml.sample` holds database c Ruby Rack, Sinatra The export to XLSForm depends on `build2xlsform`. Follow its README to install and run `build2xlsform` locally on its default port 8686. -## Frontend +## Frontend The frontend is written in Vanilla JS, it uses no frameworks and no modern syntax. ## Database @@ -23,11 +23,11 @@ A source install requires an existing Postgres database to be created with corre ## Dependencies -All Rubygem dependencies are managed by Ruby Bundler. There are config files present for `rbenv`/`rvm`. -Run bundle install in the application root to resolve and install the appropriate dependencies. +All Rubygem dependencies are managed by Ruby Bundler. There are config files present for `rbenv`/`rvm`. +Run bundle install in the application root to resolve and install the appropriate dependencies. You can do `--without test` to skip some gems if you're short on bandwidth. -We depend on one native binding, to connect to a PostgreSQL database. +We depend on one native binding, to connect to a PostgreSQL database. To satisfy the binding, you can install `libpq-dev` on apt, or `postgresql` on homebrew. ## Navigating the code @@ -59,16 +59,16 @@ Otherwise, files of note include: Some changes, like adding support for a new field type, might require an addition to `build2xlsform`. Make sure to test that the export to XLSForm yields a valid XLSForm. ## Deployment architecture - -Build is most recently deployed via `docker-compose`, which brings its own database and `build2xlsform` containers. + +Build is most recently deployed via `docker compose`, which brings its own database and `build2xlsform` containers. A bind mount provides a pathway for database dumps to be transferred in and out of the container. -In the distant past, Build up to version 0.3.5 was deployed via +In the distant past, Build up to version 0.3.5 was deployed via [ansible](https://github.com/getodk/build/tree/ansible-deployment) as automated source install behind an nginx server. After the Great Spring Clean of 2021™, Build 0.3.6 through 0.4.1 was deployed as a manual source install. ## Offline version - -The offline version consists of the Build code run through `docker-compose` on a local machine. + +The offline version consists of the Build code run through `docker compose` on a local machine. diff --git a/docs/deploy.md b/docs/deploy.md index 2bf4ac2..52994fd 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -1,21 +1,21 @@ # Deployment This section contains instructions to deploy Build and the related service build2xlsform to a test or production server. -The two alternatives are a deployment via `docker-compose` and a source install. +The two alternatives are a deployment via `docker compose` and a source install. -## docker-compose -This section explains how to run ODK Build with `docker-compose`. +## docker compose +This section explains how to run ODK Build with `docker compose`. This is useful for several audiences: * End users can run this as an offline or self-hosted version of ODK Build. Note, the installation process requires internet connectivity. * Developers can verify that the Docker builds still work after code changes. -* Maintainers can deploy Build with the same toolset (`docker-compose`) as Central. +* Maintainers can deploy Build with the same toolset (`docker compose`) as Central. ### First time start-up -With Docker and docker-compose installed, run +With Docker and docker compose installed, run ``` # via HTTPS -git clone https://github.com/getodk/build.git +git clone https://github.com/getodk/build.git # or with SSH keys git clone git@github.com:getodk/build.git @@ -25,15 +25,15 @@ git clone git@github.com:getodk/build.git The file `docker-compose.dev.yml` will deploy the latest tagged image for build2xlsform from ghcr.io, and build the current master as ODK Build image. Developers will use this option to test ODK Build during development. -A copy of this command is also mentioned in `docker-compose.dev.yml`. +A copy of this command is also mentioned in `docker-compose.dev.yml`. ``` cd build git pull -docker-compose -f docker-compose.dev.yml up -d --build +docker compose -f docker-compose.dev.yml up -d --build ``` -This will run ODK Build on `http://0.0.0.0:9393/`, together with its Postgres database +This will run ODK Build on `http://0.0.0.0:9393/`, together with its Postgres database and the related service `build2xlsform` providing an export to XLSForm. #### Latest official images @@ -42,14 +42,14 @@ which are built and pushed whenever a commit is pushed to GitHub. Maintainers will use this option to deploy ODK Build to a staging or production server. ``` -docker-compose up -d +docker compose up -d ``` ### Stop Build -To stop the images, run +To stop the images, run ``` -docker-compose stop +docker compose stop ``` The named database volume will survive even a destructive `docker-copmose down`, which removes @@ -61,12 +61,12 @@ Pull the latest changes and rebuild/restart the images. ``` cd build git pull -docker-compose up -d --build +docker compose up -d --build ``` ### Transfer database snapshots -Both `docker-compose` files mount a transfer directory. -On the host, `docker-compose` is run from the cloned Build repository. +Both `docker-compose` files mount a transfer directory. +On the host, `docker-compose` is run from the cloned Build repository. A folder `transfer` is created inside the cloned repository, which is mounted inside the `postgres` container as `/var/transfer`. This bind mount can be used to either transfer a database dump from the host (or another machine) into the Docker container, @@ -80,12 +80,12 @@ To restore the database dump into the container: ``` # Find the container name: build_odkbuild_1 -> docker-compose ps - Name Command State Ports +> docker compose ps + Name Command State Ports --------------------------------------------------------------------------------------------------------- build_build2xlsform_1 docker-entrypoint.sh node ... Up 0.0.0.0:8686->8686/tcp,:::8686->8686/tcp build_odkbuild_1 ./contrib/wait-for-it.sh p ... Up 0.0.0.0:9393->9393/tcp,:::9393->9393/tcp -build_postgres_1 docker-entrypoint.sh postgres Up 5432/tcp +build_postgres_1 docker-entrypoint.sh postgres Up 5432/tcp # Attach to the running container > docker exec -it build_postgres_1 /bin/bash -c "export TERM=xterm; exec bash" @@ -107,7 +107,7 @@ root@build-staging:/srv/odkbuild/current# pg_restore -h localhost -U odkbuild -d ## Production deployment from source This section documents how to deploy Build to a production server via a source install. -This deployment method has been superseded by the docker-compose deployment. +This deployment method has been superseded by the docker compose deployment. ### Deploy build2xlsform The symlink `/srv/xls_service/current` points to the unpacked and built production release. @@ -191,4 +191,4 @@ irb irb(main):001:0> require 'digest/sha1' irb(main):002:0> Digest::SHA1.hexdigest "--[your_new_password]==[pepper]--" Put that hash in the password column in the DB. -``` \ No newline at end of file +``` diff --git a/docs/develop.md b/docs/develop.md index 1424784..82c2133 100644 --- a/docs/develop.md +++ b/docs/develop.md @@ -1,49 +1,93 @@ # Development + This chapter provides a detailed guide to install Build locally for development. This setup is best if you want to contribute to the codebase. ## Ruby, NodeJS, and dependencies -As root user, upgrade the system, install [NodeJS](https://nodejs.org/) and -[Ruby](https://www.ruby-lang.org/en/documentation/installation/). -``` +As root user, upgrade the system, install a Java Runtime, +[NodeJS](https://nodejs.org/), and Ruby. + +While there are many options to install Ruby, we show here the +installation via rbenv and its plugin ruby-build. + + + +This project depends on an older version of Ruby (2.7.2) and needs +some code changes to work with Ruby >3.0. + +```{bash} apt update apt upgrade +# Install a Java runtime and review version +sudo apt install default-jre +java --version + # See: https://github.com/nodesource/distributions/blob/master/README.md#debinstall cd ~ -curl -sL https://deb.nodesource.com/setup_14.x -o nodesource_setup.sh -sh nodesource_setup.sh -apt install nodejs -# Verify version, here 14.x: -nodejs -v +curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - +sudo apt-get install -y nodejs +# Verify version, here 20.x: +node -v + +# Install rbenv, then Ruby +sudo apt-get install autoconf patch build-essential rustc libssl-dev libyaml-dev \ +libreadline6-dev zlib1g-dev libgmp-dev libncurses5-dev libffi-dev libgdbm6 \ +libgdbm-dev libdb-dev uuid-dev rbenv + +# Initialize rbenv +rbenv init + +# First time: Install ruby-build +git clone https://github.com/rbenv/ruby-build.git "$(rbenv root)"/plugins/ruby-build + +# Returning: Update ruby-build +git -C "$(rbenv root)"/plugins/ruby-build pull + +# Let the rbenv plugin ruby-build install Ruby at the version given by .ruby-version +rbenv install # Review Ruby version ruby -v -# Build 0.3.5 to 0.4.0 requires a Ruby upgrade + +# Historic documentation: Build 0.3.5 to 0.4.0 requires a Ruby upgrade gem update --system ``` -If you run into trouble while resolving dependencies, try updating Bundler: `gem update --system && gem install bundler`. +If you run into trouble while resolving dependencies, +try updating Bundler: `gem update --system && gem install bundler`. Note: If you run into this error while executing `bundle install` -``` + +```{text} An error occurred while installing pg (), and Bundler cannot continue. Make sure that `gem install pg -v ''` succeeds before bundling. ``` -Then installing `pg` with `gem install pg -v '0.19.0'` would solve the issue. -Sometimes it might complain that the `libpq` library is not found. This might happen for several reasons: +Then installing `pg` with `gem install pg -v '0.19.0'` would solve the issue. +Sometimes it might complain that the `libpq` library is not found. +This might happen for several reasons: * Lib paths are mis-configured. -* `libpq` library is not installed at all. -* On some platforms setting the `ARCHFLAGS` env to `-arch x86_64` and then installing pg would work. - So run this command: `sudo ARCHFLAGS="-arch x86_64" gem install pg -v ''` +* `libpq` library is not installed at all. +* On some platforms setting the `ARCHFLAGS` env to `-arch x86_64` and then + installing pg would work. So run this command: + `sudo ARCHFLAGS="-arch x86_64" gem install pg -v ''` ## Code -Clone the Build code to a local folder of your choice. -``` +Clone the Build code to a local folder of your choice. + +```{bash} cd ~/projects git clone git@github.com:getodk/build.git cd build @@ -52,43 +96,51 @@ cd build The subsequent commands are run inside the cloned Build repository. ## Database + Follow the official [Postgres docs](https://www.postgresql.org/) to setup a cluster. You can find the currently used Postgres version in `docker-compose.yml`. Configure access for local users, then create a role and database `odkbuild`. -``` +```{psql} create role odkbuild with login password 'odkbuild'; create database odkbuild with owner='odkbuild' encoding='utf8'; ``` Create `config.yml` as a copy of the template `config.yml.sample`. -If you have used different credentials to the code example above, update them in `config.yml`. -As `config.yml` contains a number of secret keys and tokens, it is excluded from both source control (via `.gitignore`) -and the Docker build (via `.dockerignore`). +If you have used different credentials to the code example above, update them in +`config.yml`. As `config.yml` contains a number of secret keys and tokens, it is +excluded from both source control (via `.gitignore`) and the Docker build (via `.dockerignore`). Finally, run migrations against the database. -``` -rake db:migrate +```{bash} +RACK_ENV=development rake db:migrate ``` ## Install + Install Build's Ruby gems and compile the Javascript assets. -``` +```{bash} bundle config set --local deployment 'true' bundle install bundle exec rake deploy:build ``` ## build2xlsform -The export to XLSForm depends on [build2xlsform](https://github.com/getodk/build2xlsform). -Follow its README to install `build2xlsform` locally as a separate project and run it on its default port 8686. + +The export to XLSForm depends on [build2xlsform](https://github.com/getodk/build2xlsform). +Follow its README to install `build2xlsform` locally as a separate project and +run it on its default port 8686. ## Run -To finally run Build, run either `bundle exec rackup config.ru` to start the server, or `bundle exec shotgun config.ru` if you want the application to automatically detect your changes to source code and load them up when you refresh the app in your web browser. -``` +To finally run Build, run either `bundle exec rackup config.ru` to start the +server, or `bundle exec shotgun config.ru` if you want the application to +automatically detect your changes to source code and load them up when you +refresh the app in your web browser. + +```{bash} # Static bundle exec rackup config.ru @@ -96,28 +148,41 @@ bundle exec rackup config.ru bundle exec shotgun config.ru ``` -In production, assets are bundled by a pre-processor. It is worth running this to verify that the assets build correctly. -In some cases, Javascript code can run fine through `bundle exec shotgun config.ru`, but fails to bundle correctly and therefore crashes the production deployment. +In production, assets are bundled by a pre-processor. +It is worth running this to verify that the assets build correctly. +In some cases, Javascript code can run fine through `bundle exec shotgun config.ru`, +but fails to bundle correctly and therefore crashes the production deployment. -``` -rake deploy:build +```{bash} +bundle exec rake deploy:build ``` -If JS files are not compiled cleanly by the JS compressor (Yui), the asset compilation tool may not throw any error. We just get a silent failure and blank line in the compiled assets. These errors can come e.g. from XForms terms which are reserved keywords, or at least trip over the compiler. One such example is the key "short". Be also aware that the JS code does not support ES6 syntax such as parameter defaults. +If JS files are not compiled cleanly by the JS compressor (Yui), the asset +compilation tool may not throw any error. We just get a silent failure and blank +line in the compiled assets. These errors can come e.g. from XForms terms which +are reserved keywords, or at least trip over the compiler. +One such example is the key "short". Be also aware that the JS code does not +support ES6 syntax such as parameter defaults. To view compilation errors, run the Yui jar file directly: -* Determine the path of the Java JAR `yuicompressor.jar` with `bundle info yui-compressor`. - Note that the Ruby Gem `yui-compressor` (in the example v0.9.3) contains the Java JAR `yuicompressor-x.x.x.jar` (in the example v2.4.2). -* Use the path to `yui-compressor.jar` to compile each asset you changed with option `-v` to see any resulting errors. +* Determine the path of the Java JAR `yuicompressor.jar` with `bundle info yui-compressor`. + Note that the Ruby Gem `yui-compressor` (in the example v0.9.3) contains the + Java JAR `yuicompressor-x.x.x.jar` (in the example v2.4.2). +* Use the path to `yui-compressor.jar` to compile each asset you changed with + option `-v` to see any resulting errors. -``` +```{bash} bundle info yui-compressor java -jar /srv/odkbuild/releases/0.3.6/vendor/bundle/ruby/2.7.0/gems/yui-compressor-0.9.3/lib/yuicompressor-2.4.2.jar -v --type js public/javascripts/data.js ``` ## Contribute -Once a contribution is ready, consult the [contributing guide](contribute.md) on preparing a Pull Request. + +Once a contribution is ready, consult the [contributing guide](contribute.md) +on preparing a Pull Request. ## Release -Once Build is ready for a new release, a team member with "maintainer" privileges will create a "release issue" and follow instructions therein. + +Once Build is ready for a new release, a team member with "maintainer" privileges +will create a "release issue" and follow instructions therein. diff --git a/docs/offline.md b/docs/offline.md index 0a6b5e2..a06f70b 100644 --- a/docs/offline.md +++ b/docs/offline.md @@ -1,17 +1,17 @@ # Build Offline -To prepare build to run offline, follow the `docker-compose` deployment steps in the [deployment guide](deploy.md). +To prepare build to run offline, follow the `docker compose` deployment steps in the [deployment guide](deploy.md). Note, you need to be online for these steps. Once the Docker images are built and the local storage volumes are created, you can run Build offline through ``` cd build -docker-compose up -d +docker compose up -d ``` and stop Build through ``` cd build -docker-compose stop +docker compose stop ```