New light Docker image for serverless deployment

ansible/group_vars/all.yml

@@ -30,7 +30,7 @@ kaui_system_properties: ''
 kb_system_properties: ''
 
 # Tomcat and JVM properties -- recommended defaults
-jvm_initial_memory: 4G
+jvm_initial_memory: 1G
 jvm_max_memory: 4G
 jvm_jdwp_port: 12345
 jvm_jdwp_server: y

ansible/templates/tomcat/conf/setenv.sh.j2

@@ -7,8 +7,16 @@ CATALINA_OPTS="-server
                -XX:+HeapDumpOnOutOfMemoryError
                -XX:HeapDumpPath={{ catalina_base }}/logs/
                -XX:+UseCodeCacheFlushing
+{% if lookup('env', 'TOMCAT_JAVA_XMS') %}
+               -Xms{{ lookup('env', 'TOMCAT_JAVA_XMS') }}
+{% else %}
                -Xms{{ jvm_initial_memory }}
+{% endif %}
+{% if lookup('env', 'TOMCAT_JAVA_XMX') %}
+               -Xmx{{ lookup('env', 'TOMCAT_JAVA_XMX') }}
+{% else %}
                -Xmx{{ jvm_max_memory }}
+{% endif %}
                -Xrunjdwp:transport=dt_socket,server={{ jvm_jdwp_server }},suspend=n,address={{ jvm_jdwp_port }}
                -XX:+CMSClassUnloadingEnabled
                -XX:-OmitStackTraceInFastThrow
@@ -20,10 +28,19 @@ CATALINA_OPTS="-server
                -XX:+CMSParallelRemarkEnabled
                -XX:+UseCMSInitiatingOccupancyOnly
                -XX:CMSInitiatingOccupancyFraction={{ jvm_cms_initiating_fraction_threshold }}
+{% if lookup('env', 'TOMCAT_JAVA_NEW_SIZE') %}
+               -XX:NewSize={{ lookup('env', 'TOMCAT_JAVA_NEW_SIZE') }}
+{% else %}
                -XX:NewSize={{ jvm_new_size }}
+{% endif %}
+{% if lookup('env', 'TOMCAT_JAVA_MAX_NEW_SIZE') %}
+               -XX:MaxNewSize={{ lookup('env', 'TOMCAT_JAVA_MAX_NEW_SIZE') }}
+{% else %}
                -XX:MaxNewSize={{ jvm_max_new_size }}
+{% endif %}
                -XX:SurvivorRatio={{ jvm_survivor_ratio }}
                -XX:+DisableExplicitGC
+{% if not lookup('env', 'TOMCAT_DISABLE_GC_LOGGING') %}
                -Xloggc:{{ catalina_base }}/logs/gc.log
                -XX:+PrintGCApplicationConcurrentTime
                -XX:+PrintGCApplicationStoppedTime
@@ -32,7 +49,9 @@ CATALINA_OPTS="-server
                -XX:+PrintTenuringDistribution
                -XX:+UseGCLogFileRotation
                -XX:NumberOfGCLogFiles=14
-               -XX:GCLogFileSize=100M"
+               -XX:GCLogFileSize=100M
+{% endif %}
+               "
 
 # Java Properties
 export CATALINA_OPTS="$CATALINA_OPTS

docker/README.adoc

@@ -84,7 +84,7 @@ The following environment variables will populate the default `killbill.properti
 * `KILLBILL_DAO_MAX_CONNECTION_AGE` (default `0m`)
 * `KILLBILL_DAO_MIN_IDLE` (default `5`)
 * `KILLBILL_DAO_MYSQL_SERVER_VERSION` (default `4.0`)
-* `KILLBILL_DAO_PASSWORD` (default `killkill`)
+* `KILLBILL_DAO_PASSWORD` (default `killbill`)
 * `KILLBILL_DAO_POOLING_TYPE` (default `HIKARICP`)
 * `KILLBILL_DAO_PREP_STMT_CACHE_SIZE` (default `500`)
 * `KILLBILL_DAO_PREP_STMT_CACHE_SQL_LIMIT` (default `2048`)

docker/templates/killbill/light/Dockerfile

@@ -0,0 +1,71 @@
+FROM killbill/killbill:0.20.11
+MAINTAINER Kill Bill core team <[email protected]>
+
+# Install envsubst (see killbill.sh)
+USER root
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && \
+    apt-get install -y \
+      gettext-base && \
+    rm -rf /var/lib/apt/lists/*
+ENV DEBIAN_FRONTEND teletype
+USER tomcat
+
+# Disable Ansible (requires /dev/shm)
+ENV KPM_INSTALL_CMD /bin/true
+
+# No JRuby plugin and no OSGI bundle, to keep memory usage low
+RUN kpm uninstall kpm --destination=$KILLBILL_INSTALL_DIR/bundles
+RUN rm -f $KILLBILL_INSTALL_DIR/bundles/platform/*
+
+# Pre-expand the Kill Bill WAR
+RUN cd $TOMCAT_HOME/webapps && \
+    mkdir ROOT && \
+    cd ROOT && \
+    jar -xvf ../ROOT.war && \
+    touch -r ../ROOT.war META-INF/war-tracker && \
+    cd - && \
+    rm -f ROOT.war && \
+    cd -
+
+# TODO
+# * The SocketFactory jdbc driver doesn't seem to be working on Cloud Run
+# * The Logback Stackdriver LoggingAppender isn't reliable
+#
+# Custom libraries
+#COPY lib/*.jar $TOMCAT_HOME/webapps/ROOT/WEB-INF/lib/
+# Hack for now...
+#RUN cd $TOMCAT_HOME/webapps/ROOT/WEB-INF/lib/ && \
+#    rm -f animal-sniffer-annotations-1.14.jar annotations-3.0.1u2.jar asm-5.0.3.jar commons-codec-1.9.jar commons-lang3-3.2.1.jar error_prone_annotations-2.1.3.jar google-api-services-sqladmin-v1beta4-rev20190510-1.28.0.jar j2objc-annotations-1.1.jar jackson-core-2.9.6.jar && \
+#    cd -
+
+# Add Kaui
+COPY webapps/kaui.war $TOMCAT_HOME/webapps/kaui.war
+
+# Pre-expand the Kaui WAR
+RUN cd $TOMCAT_HOME/webapps && \
+    mkdir kaui && \
+    cd kaui && \
+    jar -xvf ../kaui.war && \
+    touch -r ../kaui.war META-INF/war-tracker && \
+    cd - && \
+    rm -f kaui.war && \
+    cd -
+
+# TODO See above
+# Custom libraries
+#COPY lib/*.jar $TOMCAT_HOME/webapps/kaui/WEB-INF/lib/
+# Hack for now...
+#RUN cd $TOMCAT_HOME/webapps/kaui/WEB-INF/lib/ && \
+#    rm -f animal-sniffer-annotations-1.14.jar annotations-3.0.1u2.jar asm-5.0.3.jar commons-codec-1.9.jar commons-lang3-3.2.1.jar error_prone_annotations-2.1.3.jar google-api-services-sqladmin-v1beta4-rev20190510-1.28.0.jar j2objc-annotations-1.1.jar jackson-core-2.9.6.jar && \
+#    cd -
+
+# Note that classic configuration via environment variables won't work since Ansible isn't invoked
+COPY webapp-context.xml $TOMCAT_HOME/webapps/ROOT/META-INF/context.xml
+COPY webapp-context.xml $TOMCAT_HOME/webapps/kaui/META-INF/context.xml
+COPY context.xml $TOMCAT_HOME/conf/context.xml
+COPY server.xml $TOMCAT_HOME/conf/server.xml
+COPY setenv.sh $TOMCAT_HOME/bin/setenv.sh
+COPY logback.xml $KILLBILL_INSTALL_DIR/logback.xml
+COPY killbill.properties.template $KILLBILL_INSTALL_DIR/killbill.properties.template
+COPY killbill.sh $KILLBILL_INSTALL_DIR

docker/templates/killbill/light/README.md

@@ -0,0 +1,6 @@
+# Kill Bill light
+
+Kill Bill light is suitable for serverless deployments such as Google Cloud Run:
+
+* Memory usage low
+* No `/dev/shm` requirement

docker/templates/killbill/light/context.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Context unloadDelay="31000">
+</Context>

docker/templates/killbill/light/killbill.properties.template

@@ -0,0 +1,14 @@
+org.killbill.security.shiroNbHashIterations=1
+org.killbill.jaxrs.threads.pool.nb=5
+org.killbill.payment.plugin.threads.nb=1
+org.killbill.billing.osgi.bundles.jruby.conf.dir=/var/lib/killbill/config
+org.killbill.osgi.bundle.install.dir=/var/lib/killbill/bundles
+org.killbill.server.baseUrl=http://localhost:8080
+org.killbill.billing.osgi.dao.user=$KILLBILL_DAO_USER
+org.killbill.billing.osgi.dao.password=$KILLBILL_DAO_PASSWORD
+org.killbill.billing.osgi.dao.url=$KILLBILL_DAO_URL
+org.killbill.dao.user=$KILLBILL_DAO_USER
+org.killbill.dao.password=$KILLBILL_DAO_PASSWORD
+org.killbill.dao.url=$KILLBILL_DAO_URL
+org.killbill.dao.maxActive=10
+org.killbill.billing.osgi.dao.maxActive=10

docker/templates/killbill/light/killbill.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export KILLBILL_DAO_USER=${KILLBILL_DAO_USER:-killbill}
+export KILLBILL_DAO_PASSWORD=${KILLBILL_DAO_PASSWORD:-killbill}
+export KILLBILL_DAO_URL=${KILLBILL_DAO_URL:-jdbc:h2:mem:killbill}
+
+envsubst < $KILLBILL_INSTALL_DIR/killbill.properties.template > $KILLBILL_INSTALL_DIR/killbill.properties
+
+exec /usr/share/tomcat/bin/catalina.sh run

docker/templates/killbill/light/lib/download.sh

@@ -0,0 +1 @@
+mvn dependency:copy-dependencies -DoutputDirectory=.

docker/templates/killbill/light/lib/pom.xml

@@ -0,0 +1,19 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.kill-bill.docker</groupId>
+    <artifactId>killbill-light</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <dependencies>
+       <dependency>
+         <groupId>com.google.cloud.sql</groupId>
+         <artifactId>mysql-socket-factory-connector-j-8</artifactId>
+         <version>1.0.14</version>
+       </dependency>
+       <dependency>
+         <groupId>com.google.cloud</groupId>
+         <artifactId>google-cloud-logging-logback</artifactId>
+         <version>0.106.0-alpha</version>
+       </dependency>
+    </dependencies>
+</project>

docker/templates/killbill/light/logback.xml

@@ -0,0 +1,23 @@
+<configuration scan="false">
+  <!-- <appender name="CLOUD" class="com.google.cloud.logging.logback.LoggingAppender">
+    <log>killbill.log</log>
+    <flushLevel>WARN</flushLevel>
+  </appender> -->
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>%-4relative [%thread] %-5level %logger{35} - %msg %n</pattern>
+    </encoder>
+  </appender>
+
+  <!-- Silence verbose loggers -->
+  <logger name="com.dmurph" level="OFF"/>
+  <logger name="jdbc" level="OFF"/>
+  <logger name="org.jooq.Constants" level="OFF"/>
+  <logger name="org.eclipse" level="WARN"/>
+  <logger name="org.killbill.commons.jdbi.guice.DBIProvider" level="OFF"/>
+
+  <root level="INFO">
+    <!-- <appender-ref ref="CLOUD" /> -->
+    <appender-ref ref="STDOUT" />
+  </root>
+</configuration>

docker/templates/killbill/light/server.xml

@@ -0,0 +1,36 @@
+<?xml version='1.0' encoding='utf-8'?>
+<Server port="8005" shutdown="SHUTDOWN">
+  <Listener className="org.apache.catalina.startup.VersionLoggerListener" />
+  <!-- APR library loader. Documentation at /docs/apr.html -->
+  <Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
+  <!-- Prevent memory leaks due to use of particular java/javax APIs-->
+  <Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
+  <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
+  <Listener className="org.apache.catalina.core.ThreadLocalLeakPreventionListener" />
+
+  <Service name="Catalina">
+    <Executor name="tomcatThreadPool"
+              namePrefix="catalina-exec-"
+              maxThreads="20"
+              maxIdleTime="30000"
+              minSpareThreads="4"
+              prestartminSpareThreads="true" />
+
+    <Connector executor="tomcatThreadPool"
+               port="8080"
+               protocol="HTTP/1.1"
+               connectionTimeout="20000" />
+
+    <Engine name="Catalina" defaultHost="localhost">
+      <Host name="localhost"
+            appBase="webapps"
+            unpackWARs="true"
+            autoDeploy="false">
+
+        <Valve className="org.apache.catalina.valves.RemoteIpValve"
+               protocolHeader="x-forwarded-proto"
+               portHeader="x-forwarded-port" />
+      </Host>
+    </Engine>
+  </Service>
+</Server>

docker/templates/killbill/light/setenv.sh

@@ -0,0 +1,37 @@
+export JAVA_HOME=${JAVA_HOME-"/usr/lib/jvm/default-java"}
+
+# JVM Options
+CATALINA_OPTS="-server
+               -showversion
+               -XX:+PrintCommandLineFlags
+               -XX:+UseCodeCacheFlushing
+               -Xms512m
+               -Xmx1024m
+               -XX:+CMSClassUnloadingEnabled
+               -XX:-OmitStackTraceInFastThrow
+               -XX:+UseParNewGC
+               -XX:+UseConcMarkSweepGC
+               -XX:+CMSConcurrentMTEnabled
+               -XX:+ScavengeBeforeFullGC
+               -XX:+CMSScavengeBeforeRemark
+               -XX:+CMSParallelRemarkEnabled
+               -XX:+UseCMSInitiatingOccupancyOnly
+               -XX:CMSInitiatingOccupancyFraction=50
+               -XX:NewSize=100m
+               -XX:MaxNewSize=256m
+               -XX:SurvivorRatio=10
+               -XX:+DisableExplicitGC"
+
+# Java Properties
+export CATALINA_OPTS="$CATALINA_OPTS
+                      -Dkaui.url=http://127.0.0.1:8080
+                      -Dkaui.db.url=$KILLBILL_DAO_URL
+                      -Dkaui.db.password=$KILLBILL_DAO_PASSWORD
+                      -Dkaui.db.username=$KILLBILL_DAO_USER
+                      -Dlogback.configurationFile=/var/lib/killbill/logback.xml
+                      -Dorg.killbill.server.properties=file:///var/lib/killbill/killbill.properties
+                      -Dcom.sun.xml.bind.v2.bytecode.ClassTailor.noOptimize=true
+                      -Djava.rmi.server.hostname=$ENV_HOST_IP
+                      -Djava.security.egd=file:/dev/./urandom
+                      -Djruby.compile.invokedynamic=false
+                      -Dlog4jdbc.sqltiming.error.threshold=1000"

docker/templates/killbill/light/webapp-context.xml

@@ -0,0 +1,5 @@
+<Context>
+  <JarScanner>
+    <JarScanFilter pluggabilitySkip="*" tldSkip="*" defaultTldScan="false" />
+  </JarScanner>
+</Context>