Merge pull request #107 from FraunhoferISST/dev

Dev
FraunhoferISST · Nov 18, 2021 · 5f96f1d · 5f96f1d
2 parents f75ded2 + 01539cc
commit 5f96f1d
Show file tree

Hide file tree

Showing 175 changed files with 4,821 additions and 3,144 deletions.
diff --git a/.github/workflows/publish-service-management-image.yml b/.github/workflows/publish-service-management-image.yml
@@ -0,0 +1,63 @@
+name: Build and publish service-management image
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ["master", "dev"]
+    paths:
+      - "core/services/service-management/**"
+      - "!*.md"
+  pull_request:
+    branches: ["master", "dev"]
+    paths:
+      - "core/services/service-management/**"
+      - "!*.md"
+
+env:
+  REGISTRY: ghcr.io/fraunhoferisst/diva
+  IMAGE_NAME: service-management
+  DIVA_DOCKER_CONTEXT: core
+  DIVA_DOCKERFILE_PATH: core/services/Dockerfile
+  SERVICE_PATH: core/services/service-management
+  SERVICE_RELATIVE_PATH: service-management
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: get-npm-version
+        id: package-version
+        uses: martinbeentjes/npm-get-version-action@master
+        with:
+          path: ${{ env.SERVICE_PATH }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@548e2346a9987b56d8a4104fe776321ff8e23440
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: ${{ steps.package-version.outputs.current-version}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
+        with:
+          context: ${{ env.DIVA_DOCKER_CONTEXT }}
+          file: ${{ env.DIVA_DOCKERFILE_PATH }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: SERVICE_NAME=${{ env.SERVICE_RELATIVE_PATH }}
diff --git a/.github/workflows/templates/publish-generic-image.yml b/.github/workflows/templates/publish-generic-image.yml
@@ -42,7 +42,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@3a3bb3a81753dc99f090d24ee7e5343838b73a96
+        uses: docker/metadata-action@548e2346a9987b56d8a4104fe776321ff8e23440
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: ${{ env.IMAGE_VERSION }}

diff --git a/.github/workflows/templates/publish-node-image.yml b/.github/workflows/templates/publish-node-image.yml
@@ -47,7 +47,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@3a3bb3a81753dc99f090d24ee7e5343838b73a96
+        uses: docker/metadata-action@548e2346a9987b56d8a4104fe776321ff8e23440
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: ${{ steps.package-version.outputs.current-version}}

diff --git a/.github/workflows/templates/publish-python-image.yml b/.github/workflows/templates/publish-python-image.yml
@@ -46,7 +46,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@3a3bb3a81753dc99f090d24ee7e5343838b73a96
+        uses: docker/metadata-action@548e2346a9987b56d8a4104fe776321ff8e23440
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: ${{ steps.package-version.outputs.VERSION }}

diff --git a/README.md b/README.md
@@ -12,7 +12,7 @@
 
 <br />
 
-[![version](https://img.shields.io/badge/diva-v3.1.0-green)](https://gitlab.cc-asp.fraunhofer.de/diva/drm)
+[![version](https://img.shields.io/badge/diva-v3.2.0-green)](https://gitlab.cc-asp.fraunhofer.de/diva/drm)
 
 <br />
 

diff --git a/build-dist.sh b/build-dist.sh
@@ -4,7 +4,6 @@ CORE_FOLDER="core"
 KONG_CONFIG_FOLDER="$CORE_FOLDER/kong-gateway"
 ELASTICSEARCH_CONFIG_FOLDER="$CORE_FOLDER/elasticsearch"
 AIRFLOW_BASE_FOLDER="$CORE_FOLDER/airflow"
-SCHEMAS_FOLDER="$CORE_FOLDER/schemata"
 KEYCLOAK_FOLDER="$CORE_FOLDER/keycloak"
 
 echo "🏭 Building Distribution..."
@@ -19,17 +18,18 @@ DOCKER_FOLDER="docker"
 DOCKER_DIST_FOLDER="dist/docker-deployment"
 
 cp -r "$DOCKER_FOLDER"* "$DOCKER_DIST_FOLDER/docker" # copy docker-compose files
-rm -f "$DOCKER_DIST_FOLDER/docker/docker-compose.airflow.build.yml"
 rm -f "$DOCKER_DIST_FOLDER/docker/docker-compose.monitoring.yml"
 rm -f "$DOCKER_DIST_FOLDER/docker/build_all.sh"
 rm -f "$DOCKER_DIST_FOLDER"/docker/*faas*
 rm -f "$DOCKER_DIST_FOLDER/docker/docker-compose.override.yml" # remove override as it's only for local dev
+rm -fr "$DOCKER_DIST_FOLDER/docker/docker-compose.airflow.build.yml" # remove override as it's only for local dev
+rm -fr "$DOCKER_DIST_FOLDER/docker/docker-compose.migration.build.yml" # remove override as it's only for local dev
 rm -rf "$DOCKER_DIST_FOLDER/docker/base-images"
 rm -rf "$DOCKER_DIST_FOLDER"/docker/proxy/certs/* # remove keys if present
 rm -f "$DOCKER_DIST_FOLDER/docker/proxy/.env" # remove personal proxy .env if present
 rm -f "$DOCKER_DIST_FOLDER/docker/proxy/.gitignore"
 rm -f "$DOCKER_DIST_FOLDER/docker/proxy/nginx.conf"
-rm -f "$DOCKER_DIST_FOLDER/docker/.env" # remove personal .env if present
+rm -fr "$DOCKER_DIST_FOLDER/docker/.env" # remove personal .env if present
 
 mkdir "$DOCKER_DIST_FOLDER/$CORE_FOLDER/airflow"
 mkdir "$DOCKER_DIST_FOLDER/$CORE_FOLDER/kong-gateway"
@@ -39,13 +39,12 @@ cp -r "$KONG_CONFIG_FOLDER/plugins" "$DOCKER_DIST_FOLDER/$CORE_FOLDER/kong-gatew
 cp -r "$KONG_CONFIG_FOLDER/kong.conf" "$DOCKER_DIST_FOLDER/$CORE_FOLDER/kong-gateway/"
 cp -r "$KONG_CONFIG_FOLDER/kong.production.yml" "$DOCKER_DIST_FOLDER/$CORE_FOLDER/kong-gateway/kong.yml"
 cp -r "$ELASTICSEARCH_CONFIG_FOLDER" "$DOCKER_DIST_FOLDER/$CORE_FOLDER"
-cp -r "$SCHEMAS_FOLDER" "$DOCKER_DIST_FOLDER/$CORE_FOLDER"
 cp -r "$KEYCLOAK_FOLDER" "$DOCKER_DIST_FOLDER/$KEYCLOAK_FOLDER"
 
 echo "
 # DIVA Docker distribution
 
-[Repository](https://github.com/FraunhoferISST/diva) | [Documentation](https://fraunhoferisst.github.io/diva-docs/)
+[Repository](https://github.com/FraunhoferISST/diva) | [Documentation](https://fraunhoferisst.github.io/diva-docs/) | [Migration](https://github.com/FraunhoferISST/diva/blob/dev/migration/README.md)
 
 Quick start:
 \`\`\`

diff --git a/changelog-template.hbs b/changelog-template.hbs
@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+Have a look check our [Migration guide](https://github.com/FraunhoferISST/diva/blob/dev/migration/README.md)
+to see if there is anything to keep in mind when updating to the latest version.
+
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
@@ -53,4 +56,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
     ---
 
-{{/each}}
+{{/each}}
+
+[comment]: # (Generated with npx auto-changelog -o CHANGELOG.md -t changelog-template.hbs)
diff --git a/core/kong-gateway/kong.production.yml b/core/kong-gateway/kong.production.yml
@@ -92,6 +92,52 @@ services:
         config:
           allowed_iss: [ "http://172.17.0.1:7000/auth/realms/diva-kc-realm" ]
 
+  - name: "Service-Management"
+    url: "http://service-management:3004/"
+    routes:
+      - name: services-route
+        paths:
+          - "/services/"
+        protocols:
+          - "http"
+          - "https"
+        strip_path: false
+        plugins:
+          - name: jwt-keycloak
+            config:
+              allowed_iss: [ "http://172.17.0.1:7000/auth/realms/diva-kc-realm" ]
+
+      - name: services-route-images
+        methods:
+          - POST
+          - PUT
+          - GET
+          - DELETE
+          - OPTIONS
+        paths:
+          - "/serviceImages"
+        protocols:
+          - "http"
+          - "https"
+        strip_path: false
+        plugins:
+          - name: jwt-keycloak
+            config:
+              allowed_iss: [ "http://172.17.0.1:7000/auth/realms/diva-kc-realm" ]
+
+      - name: services-route-get-images
+        methods:
+          - GET
+        paths:
+          - "/serviceImages"
+        protocols:
+          - "http"
+          - "https"
+        strip_path: false
+
+    plugins:
+      - name: cors
+
   - name: "Search-Assistant"
     url: "http://search-assistant:3005/search"
     routes:

diff --git a/core/kong-gateway/kong.yml b/core/kong-gateway/kong.yml
@@ -104,6 +104,55 @@ services:
         config:
           allowed_iss: [ "http://172.17.0.1:7000/auth/realms/diva-kc-realm" ]
 
+  - name: "Service-Management"
+    protocol: http
+    host: "172.17.0.1"
+    port: 3004
+    path: "/"
+    routes:
+      - name: services-route
+        paths:
+          - "/services/"
+        protocols:
+          - "http"
+          - "https"
+        strip_path: false
+        plugins:
+          - name: jwt-keycloak
+            config:
+              allowed_iss: [ "http://172.17.0.1:7000/auth/realms/diva-kc-realm" ]
+
+      - name: services-route-images
+        methods:
+          - POST
+          - PUT
+          - GET
+          - DELETE
+          - OPTIONS
+        paths:
+          - "/serviceImages"
+        protocols:
+          - "http"
+          - "https"
+        strip_path: false
+        plugins:
+          - name: jwt-keycloak
+            config:
+              allowed_iss: [ "http://172.17.0.1:7000/auth/realms/diva-kc-realm" ]
+
+      - name: services-route-get-images
+        methods:
+          - GET
+        paths:
+          - "/serviceImages"
+        protocols:
+          - "http"
+          - "https"
+        strip_path: false
+
+    plugins:
+      - name: cors
+
   - name: "Search-Assistant"
     protocol: http
     host: "172.17.0.1"

diff --git a/core/services/adapter-services/diva-lake-adapter/package.json b/core/services/adapter-services/diva-lake-adapter/package.json
@@ -1,6 +1,6 @@
 {
   "name": "diva-lake-adapter",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Adapter for our DIVA Lake",
   "keywords": [
     "DIVA Lake",
@@ -24,7 +24,7 @@
     "url": "https://gitlab.cc-asp.fraunhofer.de/diva/drm"
   },
   "dependencies": {
-    "@diva/common": "^1.1.1",
+    "@diva/common": "^1.1.2",
     "axios": "^0.21.1",
     "file-type": "^16.3.0",
     "hasha": "^5.2.2",
@@ -37,6 +37,9 @@
     "pkg": "^5.3.1"
   },
   "pkg": {
-    "assets": ["package.json", "apiDoc"]
+    "assets": [
+      "package.json",
+      "apiDoc"
+    ]
   }
 }
diff --git a/core/services/adapter-services/dsc-adapter/package.json b/core/services/adapter-services/dsc-adapter/package.json
@@ -1,6 +1,6 @@
 {
   "name": "dsc-adapter",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -13,7 +13,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@diva/common": "^1.1.1",
+    "@diva/common": "^1.1.2",
     "axios": "^0.21.1",
     "url-join": "^4.0.1",
     "uuid": "^8.3.2"

diff --git a/core/services/adapter-services/urban-pulse-adapter/apiDoc/openapi.yml b/core/services/adapter-services/urban-pulse-adapter/apiDoc/openapi.yml
@@ -28,6 +28,15 @@ paths:
             type: boolean
             default: false
             example: true
+        - name: streamResponse
+          in: query
+          description: Indicates whether response should be a stream
+          required: false
+          explode: true
+          schema:
+            type: boolean
+            default: false
+            example: true
         - name: assetId
           in: query
           description: The id of the existing asset under wich the sensors should be grouped. If provided, `createAsset`

diff --git a/core/services/adapter-services/urban-pulse-adapter/package.json b/core/services/adapter-services/urban-pulse-adapter/package.json
@@ -1,6 +1,6 @@
 {
   "name": "urban-pulse-adapter",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "",
   "keywords": [
     "Urban Pulse",
@@ -24,7 +24,7 @@
     "url": "https://gitlab.cc-asp.fraunhofer.de/diva/drm"
   },
   "dependencies": {
-    "@diva/common": "^1.1.1",
+    "@diva/common": "^1.1.2",
     "axios": "^0.21.1",
     "url-join": "^4.0.1"
   },
@@ -33,6 +33,9 @@
     "pkg": "^5.3.1"
   },
   "pkg": {
-    "assets": ["package.json", "apiDoc"]
+    "assets": [
+      "package.json",
+      "apiDoc"
+    ]
   }
 }
diff --git a/core/services/adapter-services/urban-pulse-adapter/routes/adapter.js b/core/services/adapter-services/urban-pulse-adapter/routes/adapter.js
@@ -5,14 +5,19 @@ const router = express.Router();
 
 router.post("/import", async (req, res, next) => {
   try {
-    const { createAsset, assetId } = req.query;
+    const { createAsset, assetId, streamResponse } = req.query;
     const result = await UrbanPulseService.import(
       req.body,
       req.headers["x-actorid"],
+      streamResponse ? { req, res } : {},
       createAsset,
       assetId
     );
-    res.status(207).send(result);
+    if (streamResponse) {
+      res.end();
+    } else {
+      res.status(207).send(result);
+    }
   } catch (e) {
     return next(e?.response?.data || e);
   }