feat(deploy): Mount monitoring secrets for kubectl (#238)

halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/metricStores/datadog/DatadogStore.java

@@ -32,4 +32,7 @@ public String getNodeName() {
 
   @JsonProperty("api_key")
   private String apiKey;
+
+  @JsonProperty("app_key")
+  private String appKey;
 }

halyard-core/src/main/java/com/netflix/spinnaker/halyard/core/registry/v1/BillOfMaterials.java

@@ -40,8 +40,10 @@ public static class Artifacts {
     Artifact igor;
     Artifact orca;
     Artifact rosco;
-    @JsonProperty("spinnaker-monitoring")
-    Artifact spinnakerMonitoring;
+    @JsonProperty("spinnaker-monitoring-third-party")
+    Artifact spinnakerMonitoringThirdParty;
+    @JsonProperty("spinnaker-monitoring-daemon")
+    Artifact spinnakerMonitoringDaemon;
 
     public String getArtifactVersion(String artifactName) {
       Optional<Field> field = Arrays.stream(Artifacts.class.getDeclaredFields())

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/deployment/v1/FlotillaDeployment.java

@@ -60,7 +60,8 @@ private void waitForServiceUp(SpinnakerService service) {
 
   @Override
   public DeployResult deploy(String spinnakerOutputPath) {
-    SpinnakerEndpoints.Services services = getEndpoints().getServices();
+    SpinnakerEndpoints endpoints = getEndpoints();
+    SpinnakerEndpoints.Services services = endpoints.getServices();
     DaemonTaskHandler.newStage("Bootstrapping a minimal Spinnaker installation");
     providerInterface.bootstrapSpinnaker(deploymentDetails, services);
 
@@ -71,17 +72,15 @@ public DeployResult deploy(String spinnakerOutputPath) {
     DaemonTaskHandler.newStage("Deploying remainder of Spinnaker services");
     OrcaService.Orca orca = providerInterface.connectTo(deploymentDetails, services.getOrcaBootstrap());
     providerInterface.ensureServiceIsRunning(deploymentDetails, services.getRedis());
-    providerInterface.deployService(deploymentDetails, orca, services.getClouddriver());
-    providerInterface.deployService(deploymentDetails, orca, services.getDeck());
-
-    providerInterface.deployService(deploymentDetails, orca, services.getEcho());
-    providerInterface.deployService(deploymentDetails, orca, services.getFront50());
-    providerInterface.deployService(deploymentDetails, orca, services.getGate());
-    providerInterface.deployService(deploymentDetails, orca, services.getIgor());
-    providerInterface.deployService(deploymentDetails, orca, services.getOrca());
-    providerInterface.deployService(deploymentDetails, orca, services.getRosco());
-
-    providerInterface.deployService(deploymentDetails, orca, services.getFiat());
+    providerInterface.deployService(deploymentDetails, orca, endpoints, "clouddriver");
+    providerInterface.deployService(deploymentDetails, orca, endpoints, "deck");
+    providerInterface.deployService(deploymentDetails, orca, endpoints, "echo");
+    providerInterface.deployService(deploymentDetails, orca, endpoints, "front50");
+    providerInterface.deployService(deploymentDetails, orca, endpoints, "gate");
+    providerInterface.deployService(deploymentDetails, orca, endpoints, "igor");
+    providerInterface.deployService(deploymentDetails, orca, endpoints, "orca");
+    providerInterface.deployService(deploymentDetails, orca, endpoints, "rosco");
+    providerInterface.deployService(deploymentDetails, orca, endpoints, "fiat");
 
     DeployResult deployResult = new DeployResult();
     deployResult.setScriptDescription("Use the provided script to connect to your Spinnaker installation.");

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/deployment/v1/LocalhostDebianDeployment.java

@@ -125,6 +125,7 @@ public DeployResult deploy(String spinnakerOutputPath) {
     bindings.put("pin-files", pinFiles);
     bindings.put("spinnaker-artifacts", artifacts);
     bindings.put("install-redis", "true");
+    bindings.put("install-java", "true");
     bindings.put("install-spinnaker", "true");
     bindings.put("etc-init", etcInit);
     bindings.put("config-dir", spinnakerOutputPath);

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/monitor/v1/DashboardFactory.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2017 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.netflix.spinnaker.halyard.deploy.monitor.v1;
+
+public class DashboardFactory {
+}

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/provider/v1/OperationFactory.java

@@ -19,6 +19,7 @@
 
 import com.netflix.spinnaker.clouddriver.orchestration.AtomicOperations;
 import com.netflix.spinnaker.halyard.config.model.v1.node.Provider;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.SpinnakerMonitoringDaemonService;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.SpinnakerService;
 import lombok.Data;
 
@@ -27,7 +28,18 @@
 import java.util.Map;
 
 abstract public class OperationFactory {
-  abstract public Map<String, Object> createDeployPipeline(String accountName, SpinnakerService service, String artifact, List<ConfigSource> configSources, boolean update);
+  abstract public Map<String, Object> createDeployPipeline(String accountName,
+      SpinnakerService service,
+      String artifact,
+      SpinnakerMonitoringDaemonService monitoringService,
+      String monitoringArtifact,
+      List<ConfigSource> configSources,
+      boolean update);
+  abstract public Map<String, Object> createDeployPipeline(String accountName,
+      SpinnakerService service,
+      String artifact,
+      List<ConfigSource> configSources,
+      boolean update);
   abstract public Map<String, Object> createUpsertPipeline(String accountName, SpinnakerService service);
 
   abstract protected Provider.ProviderType getProviderType();

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/provider/v1/ProviderInterface.java

@@ -29,6 +29,7 @@
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.SpinnakerEndpoints;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.OrcaService.Orca;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.ServiceInterfaceFactory;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.SpinnakerMonitoringDaemonService;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.SpinnakerService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
@@ -70,10 +71,13 @@ public abstract class ProviderInterface<T extends Account> {
   /**
    * Deploy a service using Orca's orchestration engine.
    * @param details are the deployment details for the current deployment.
-   * @param orca is the orca service being used for deployment.
-   * @param service is the service to deploy.
+   * @param orca is the instance of orca used to orchestrate the deployment.
+   * @param endpoints are the endpoints spinnaker is conforming to.
+   * @param name is the service being deployed.
    */
-  public void deployService(AccountDeploymentDetails<T> details, Orca orca, SpinnakerService service) {
+  public void deployService(AccountDeploymentDetails<T> details, Orca orca, SpinnakerEndpoints endpoints, String name) {
+    SpinnakerService service = endpoints.getService(name);
+    SpinnakerMonitoringDaemonService monitoringService = endpoints.getServices().getSpinnakerMonitoringDaemon();
     String artifactName = service.getArtifact().getName();
     DaemonTaskHandler.newStage("Deploying " + service.getArtifact().getName());
     boolean update = serviceExists(details, service);
@@ -85,15 +89,15 @@ public void deployService(AccountDeploymentDetails<T> details, Orca orca, Spinna
       monitorOrcaTask(idSupplier, orca);
     }
 
-    Map<String, Object> pipeline = deployServerGroupPipeline(details, service, update);
+    Map<String, Object> pipeline = deployServerGroupPipeline(details, service, monitoringService, update);
     idSupplier = () -> orca.orchestrate(pipeline).get("ref");
     DaemonTaskHandler.log("Orchestrating " + artifactName + " deployment");
     monitorOrcaTask(idSupplier, orca);
   }
 
   abstract protected Map<String, Object> upsertLoadBalancerTask(AccountDeploymentDetails<T> details, SpinnakerService service);
 
-  abstract protected Map<String, Object> deployServerGroupPipeline(AccountDeploymentDetails<T> details, SpinnakerService service, boolean update);
+  abstract protected Map<String, Object> deployServerGroupPipeline(AccountDeploymentDetails<T> details, SpinnakerService service, SpinnakerMonitoringDaemonService monitoringService, boolean update);
 
   /**
    * Creates a service only if it isn't running yet. This is useful for dealing with dependent services that can't

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/provider/v1/kubernetes/KubernetesOperationFactory.java

@@ -26,6 +26,7 @@
 import com.netflix.spinnaker.clouddriver.kubernetes.deploy.description.servergroup.*;
 import com.netflix.spinnaker.halyard.config.model.v1.node.Provider;
 import com.netflix.spinnaker.halyard.deploy.provider.v1.OperationFactory;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.SpinnakerMonitoringDaemonService;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.SpinnakerService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
@@ -73,12 +74,10 @@ private KubernetesLoadBalancerDescription baseLoadBalancerDescription(String acc
 
   private DeployKubernetesAtomicOperationDescription baseDeployDescription(String accountName, SpinnakerService service, String artifact, List<ConfigSource> configSources) {
     String address = service.getAddress();
-    int port = service.getPort();
     DeployKubernetesAtomicOperationDescription description = new DeployKubernetesAtomicOperationDescription();
 
     String namespace = KubernetesProviderInterface.getNamespaceFromAddress(address);
     String name = KubernetesProviderInterface.getServiceFromAddress(address);
-    String artifactName = service.getArtifact().getName();
     Names parsedName = Names.parseName(name);
 
     description.setNamespace(namespace);
@@ -106,9 +105,19 @@ private DeployKubernetesAtomicOperationDescription baseDeployDescription(String
     loadBalancers.add(name);
     description.setLoadBalancers(loadBalancers);
 
+    KubernetesContainerDescription container = buildContainer(service, artifact, configSources);
+    List<KubernetesContainerDescription> containers = new ArrayList<>();
+    containers.add(container);
+    description.setContainers(containers);
+
+    return description;
+  }
+
+  private KubernetesContainerDescription buildContainer(SpinnakerService service, String artifactVersion, List<ConfigSource> configSources) {
     KubernetesContainerDescription container = new KubernetesContainerDescription();
     KubernetesProbe readinessProbe = new KubernetesProbe();
     KubernetesHandler handler = new KubernetesHandler();
+    int port = service.getPort();
 
     String httpHealth = service.getHttpHealth();
     if (httpHealth != null) {
@@ -127,9 +136,9 @@ private DeployKubernetesAtomicOperationDescription baseDeployDescription(String
     readinessProbe.setHandler(handler);
     container.setReadinessProbe(readinessProbe);
 
-    KubernetesImageDescription imageDescription = KubernetesUtil.buildImageDescription(artifact);
+    KubernetesImageDescription imageDescription = KubernetesUtil.buildImageDescription(artifactVersion);
     container.setImageDescription(imageDescription);
-    container.setName(artifactName);
+    container.setName(service.getName());
 
     List<KubernetesContainerPort> ports = new ArrayList<>();
     KubernetesContainerPort containerPort = new KubernetesContainerPort();
@@ -150,7 +159,7 @@ private DeployKubernetesAtomicOperationDescription baseDeployDescription(String
     List<KubernetesEnvVar> envVars = new ArrayList<>();
     if (!service.getProfiles().isEmpty()) {
       KubernetesEnvVar envVar = new KubernetesEnvVar();
-      envVar.setName(artifactName.toUpperCase() + "_OPTS");
+      envVar.setName(service.getArtifact().getName().toUpperCase() + "_OPTS");
       envVar.setValue("-Dspring.profiles.active=" + service.getProfiles().stream().reduce((a, b) -> a + "," + b).get());
 
       envVars.add(envVar);
@@ -166,11 +175,7 @@ private DeployKubernetesAtomicOperationDescription baseDeployDescription(String
 
     container.setEnvVars(envVars);
 
-    List<KubernetesContainerDescription> containers = new ArrayList<>();
-    containers.add(container);
-    description.setContainers(containers);
-
-    return description;
+    return container;
   }
 
   private List<String> healthProviders() {
@@ -198,9 +203,13 @@ public Map<String, Object> createUpsertPipeline(String accountName, SpinnakerSer
   }
 
   @Override
-  public Map<String, Object> createDeployPipeline(String accountName, SpinnakerService service, String artifact, List<ConfigSource> configSources, boolean update) {
+  public Map<String, Object> createDeployPipeline(String accountName, SpinnakerService service, String artifact, SpinnakerMonitoringDaemonService monitoringService, String monitoringArtifact, List<ConfigSource> configSources, boolean update) {
     List<Map<String, Object>> stages = new ArrayList<>();
-    Map<String, Object> deploy = objectMapper.convertValue(baseDeployDescription(accountName, service, artifact, configSources), Map.class);
+    DeployKubernetesAtomicOperationDescription description = baseDeployDescription(accountName, service, artifact, configSources);
+    if (monitoringArtifact != null) {
+      description.getContainers().add(buildContainer(monitoringService, monitoringArtifact, configSources));
+    }
+    Map<String, Object> deploy = objectMapper.convertValue(description, Map.class);
     String namespace = KubernetesProviderInterface.getNamespaceFromAddress(service.getAddress());
     if (update) {
       deploy = redBlackStage(deploy, healthProviders(), namespace);
@@ -211,6 +220,11 @@ public Map<String, Object> createDeployPipeline(String accountName, SpinnakerSer
     return buildPipeline("Deploy server group for " + service.getArtifact().getName(), stages);
   }
 
+  @Override
+  public Map<String, Object> createDeployPipeline(String accountName, SpinnakerService service, String artifact, List<ConfigSource> configSources, boolean update) {
+    return createDeployPipeline(accountName, service, artifact, null, null, configSources, update);
+  }
+
 
   @Override
   protected Provider.ProviderType getProviderType() {