Improve prepare application and PDS solution

- Added storage variables as non-spring parameter
- Added additional tags to skopeo wrapper
- Make git-clone into repository name

.gitignore

@@ -18,6 +18,7 @@ src-gen/
 .idea/
 *.ipr
 *.iws
+out/
 
 # vscode parts
 .vscode/

sechub-commons-archive/src/main/java/com/mercedesbenz/sechub/commons/archive/DirectoryAndFileSupport.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.commons.archive;
 
 import java.io.File;

sechub-commons-archive/src/test/java/com/mercedesbenz/sechub/commons/archive/DirectoryAndFileSupportTest.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.commons.archive;
 
 import static org.junit.jupiter.api.Assertions.*;

sechub-commons-core/src/main/java/com/mercedesbenz/sechub/commons/core/environment/SecureEnvironmentVariableKeyValueRegistry.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.commons.core.environment;
 
 import java.util.ArrayList;

sechub-commons-pds/src/test/java/com/mercedesbenz/sechub/commons/pds/DefaultProcessBuilderFactoryTest.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.commons.pds;
 
 import static org.junit.jupiter.api.Assertions.*;

sechub-commons-pds/src/test/resources/process-adapter-test.sh

@@ -1,4 +1,5 @@
 #!/bin/bash 
+# SPDX-License-Identifier: MIT
 
 #
 # Usage: inside a junit test we call this to verify the process adapter implementation works

sechub-pds-commons-core/src/main/java/com/mercedesbenz/sechub/pds/commons/core/PDSLogSanitizer.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.pds.commons.core;
 
 import java.util.regex.Pattern;

sechub-pds-commons-core/src/main/java/com/mercedesbenz/sechub/pds/commons/core/config/PDSStorageConstants.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.pds.commons.core.config;
 
 public class PDSStorageConstants {

sechub-pds-solutions/pds-base/docker/copy/README.adoc

@@ -1,4 +1,13 @@
 // SPDX-License-Identifier: MIT
-. Place a single PDS Jar into this folder. 
-. Name it `sechub-pds-0.0.0.jar`
-. Run `10-create-image.sh` to build the image
+. Build PDS jar local with gradle command
+. Copy the pds jar into the copy folder sechub-pds-0.0.0.jar
++
+----
+cp sechub-pds/build/libs/sechub-pds-0.0.0.jar .
+----
+. Run `10-create-image.sh` with the copy parameter to build the image
++
+----
+./10-create-image.sh pds-base-pds latest 0.0.0 debian:12-slim copy
+----
+

sechub-pds-solutions/pds-base/docker/run.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/bin/sh
 # SPDX-License-Identifier: MIT
 
 DEFAULT_PDS_MAX_FILE_UPLOAD_BYTES=52428800  # 50 MB
@@ -89,7 +89,7 @@ start_server() {
     -Dfile.encoding=UTF-8 \
     -Dserver.port=8444 \
     -Dserver.address=0.0.0.0 \
-    -jar "$PDS_FOLDER/sechub-pds-*.jar" &
+    -jar "$PDS_FOLDER/"sechub-pds-*.jar &
 
   # Get process pid and wait until it ends
   #   The pid will be needed by function trigger_shutdown() in case we receive a termination signal.

sechub-pds-solutions/prepare/README.adoc

@@ -37,7 +37,7 @@ The folder contains a start script which does the manual steps for you:
 
 The container will be started and attached to the `sechub` network.
 
-WARNING: Make sure the SecHub container is running and executor for Prepare is set up.
+WARNING: Make sure the SecHub container is running with compose prepare and executor for Prepare is set up. (./sechub-solution/01-start-single-docker-compose-prepare.sh)
 
 . Start container in local sechub network:
 +

sechub-pds-solutions/prepare/docker-compose_pds_prepare_external-network.yaml

@@ -17,6 +17,7 @@ services:
             - "sechub"
         volumes:
           - ./docker/scripts:/pds/scripts
+          - /tmp/sechub-shared-volume:/shared_volumes/uploads
 
 networks:
     sechub:

sechub-pds-solutions/prepare/docker/pds-config.json

@@ -5,7 +5,7 @@
     {
       "id": "PDS_PREPARE",
       "path": "/pds/scripts/prepare.sh",
-      "envWhitelist" : [ "TOOL_FOLDER" ],
+      "envWhitelist" : [ "TOOL_FOLDER","PDS_STORAGE_*","HTTPS_PROXY","NO_PROXY"],
       "scanType": "prepare",
       "description": "Runs the prepare wrapper application to prepare remote data for SecHub scans.",
 

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/PDSStartupAssertEnvironmentVariablesUsed.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.pds;
 
 import org.slf4j.Logger;

sechub-pds/src/test/java/com/mercedesbenz/sechub/pds/PDSStartupAssertEnvironmentVariablesUsedSpringBootTest.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.pds;
 
 import static org.mockito.Mockito.*;

sechub-pds/src/test/java/com/mercedesbenz/sechub/pds/PDSStartupAssertEnvironmentVariablesUsedTest.java

@@ -1,4 +1,4 @@
-//SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.pds;
 
 import static org.junit.jupiter.api.Assertions.*;

sechub-solution/01-start-single-docker-compose-prepare.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+
+ENVIRONMENT_FILE=".env-single"
+
+resource_limits_enabled="$1"
+compose_file="docker-compose_sechub-debian-prepare"
+
+mkdir /tmp/sechub-shared-volume
+chmod -R 777 /tmp/sechub-shared-volume
+
+cd $(dirname "$0")
+source ../sechub-solutions-shared/scripts/9999-env-file-helper.sh
+
+# Only variables from .env can be used in the Docker-Compose file
+# all other variables are only available in the container
+setup_environment_file ".env" "env"
+setup_environment_file "$ENVIRONMENT_FILE" "env-sechub"
+
+echo "Copying install-java scripts into the docker directory"
+cp --recursive --force ../sechub-solutions-shared/install-java/ docker/
+
+# Use Docker BuildKit
+export BUILDKIT_PROGRESS=plain
+export DOCKER_BUILDKIT=1
+
+if [[ "$resource_limits_enabled" == "yes" ]]
+then
+    compose_file="docker-compose_sechub_resource_limits-debian"
+fi
+
+echo "Compose file: $compose_file"
+
+docker compose --file "$compose_file.yaml" up --build --remove-orphans

sechub-solution/README.adoc

@@ -14,6 +14,14 @@ To start a single container run the script:
 ./01-start-single-docker-compose.sh
 ----
 
+=== Single Container with Prepare (remote Data)
+
+. Start the container with the script:
+
+----
+./01-start-single-docker-compose-prepare.sh
+----
+
 === SecHub with PDS Solution
 
 . First start a SecHub container.

sechub-solution/docker-compose_sechub-debian-prepare.yaml

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: MIT
+
+version: "3"
+services:
+  sechub:
+    build:
+      args:
+        - BASE_IMAGE=debian:12-slim
+        - BUILD_TYPE=${BUILD_TYPE}
+        - JAVA_DISTRIBUTION=${JAVA_DISTRIBUTION}
+        - JAVA_VERSION=${JAVA_VERSION}
+        - SECHUB_VERSION=${SECHUB_VERSION}
+        - GO=${GO}
+        - TAG=${TAG}
+        - BRANCH=${BRANCH}
+      context: docker/
+      dockerfile: SecHub-Debian.dockerfile
+    container_name: sechub
+    hostname: sechub
+    env_file:
+      - .env
+      - .env-single
+    ports:
+      - "127.0.0.1:${PORT}:8443"
+      - "127.0.0.1:${JAVA_DEBUG_PORT}:15023"
+    networks:
+      - sechub
+    volumes:
+      - /tmp/sechub-shared-volume:/shared_volumes/uploads
+networks:
+  sechub:
+    name: sechub

sechub-solution/setup-pds/8900-helper.sh

@@ -64,3 +64,12 @@ function setup_project_user_executor_profile() {
     ./8801-create-executor-and-profile.sh "$executor_file_name" "$profile"
     ./8802-assign-profile-to-project.sh "$project" "$profile"
 }
+
+function setup_second_executor_profile_to_existing_project() {
+      local project="$1"
+      local executor_file_name="$2"
+      local profile="$3"
+
+      ./8801-create-executor-and-profile.sh "$executor_file_name" "$profile"
+      ./8802-assign-profile-to-project.sh "$project" "$profile"
+}

sechub-solution/setup-pds/setup-prepare.sh

@@ -1,6 +1,15 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: MIT
 
+# Print helpFunction in case parameters are empty
+if [ -z "$1" ]
+then
+   echo "Error: no second executor was provided! Please provide the seconds executor name as a parameter, e.g. gosec, xray or other.";
+   echo "Example usage: ./setup-prepare.sh gosec";
+   exit 1
+fi
+
+
 declare -r SCRIPT_PARAMETERS="<project-id> <user>"
 
 cd $(dirname "$0")
@@ -17,9 +26,17 @@ user="prepare-user"
 project="test-prepare"
 executor_file_name="prepare"
 profile="pds-prepare"
+echo ""
+# defining a second profile with the tool to scan
+second_profile="pds-$1"
+second_executor_filename="$1"
+echo "Second profile: $second_profile with executor $second_executor_filename"
+echo""
 
 # main setup execution
 setup_project_user_executor_profile "$project" "$user" "$executor_file_name" "$profile"
 
+setup_second_executor_profile_to_existing_project "$project" "$second_executor_filename" "$second_profile"
+
 # print sechub scan usage message
 setup_complete_message_for_tool "Prepare" "$user" "$project"

sechub-wrapper-prepare/build.gradle

@@ -21,6 +21,7 @@ dependencies {
 
     implementation spring_boot_dependency.logback_classic
 
+    implementation library.logstashLogbackEncoder
     implementation library.jgit_core
     implementation library.apache_commons_io
 

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/LogSanitizerProvider.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.wrapper.prepare;
 
 import com.mercedesbenz.sechub.pds.commons.core.PDSLogSanitizer;

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/PrepareAcceptFilter.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.wrapper.prepare;
 
 public interface PrepareAcceptFilter {

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/PrepareInputValidationSupport.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.wrapper.prepare;
 
 import static com.mercedesbenz.sechub.wrapper.prepare.InputValidatorExitcode.*;

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/PrepareWrapperUsageException.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.wrapper.prepare;
 
 import com.mercedesbenz.sechub.wrapper.prepare.modules.UsageExceptionExitCode;

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/cli/PrepareWrapperCLI.java

@@ -54,7 +54,7 @@ public void run(String... args) {
         }
 
         storeResultOrFail(result);
-
+        LOG.info("Prepare wrapper has finished successfully.");
     }
 
     private static AdapterExecutionResult getAdapterExecutionResultFailed(String message) {

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/AbstractPrepareWrapperModule.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.wrapper.prepare.modules;
 
 import java.io.IOException;

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareToolContext.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.wrapper.prepare.modules;
 
 import java.nio.file.Path;

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/UsageExceptionExitCode.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.wrapper.prepare.modules;
 
 public enum UsageExceptionExitCode {

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/git/AutoCleanupAdditionalGitFilesFilter.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.wrapper.prepare.modules.git;
 
 import java.io.File;

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/git/AutoCleanupGitFoldersFilter.java

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.wrapper.prepare.modules.git;
 
 import java.io.File;

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/git/GitContext.java

@@ -8,8 +8,9 @@
 public class GitContext extends AbstractPrepareToolContext {
 
     static final String DOWNLOAD_DIRECTORY_NAME = "git-download";
+    static final String DEFAULT_REPOSITORY_NAME = "git-repository";
     private boolean cloneWithoutHistory;
-    private String repositoryName = "git-repository";
+    private String repositoryName = DEFAULT_REPOSITORY_NAME;
     private Path toolDownloadDirectory;
 
     public void setCloneWithoutHistory(boolean cloneWithoutHistory) {
@@ -27,7 +28,9 @@ public String getRepositoryName() {
     }
 
     public void setRepositoryName(String repositoryName) {
-        this.repositoryName = repositoryName;
+        if (!(repositoryName == null || repositoryName.isBlank())) {
+            this.repositoryName = repositoryName;
+        }
     }
 
     public boolean isCloneWithoutHistory() {

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/git/GitLocationConverter.java

@@ -30,6 +30,13 @@ public class GitLocationConverter {
     @Autowired
     PDSLogSanitizer pdsLogSanitizer;
 
+    public String convertLocationToRepositoryName(String location) {
+        String[] parts = location.split("/");
+        String repository = parts[parts.length - 1];
+        repository = repository.replace(".git", "");
+        return repository;
+    }
+
     public URL convertLocationToHttpsBasedURL(String originLocation) {
         if (originLocation == null) {
             throw new IllegalArgumentException("Location may not be null!");

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/git/GitPrepareWrapperModule.java

@@ -60,6 +60,9 @@ public class GitPrepareWrapperModule extends AbstractPrepareWrapperModule {
     @Autowired
     PDSLogSanitizer pdsLogSanitizer;
 
+    @Autowired
+    GitLocationConverter gitLocationConverter;
+
     @Override
     public boolean isEnabled() {
         return enabled;
@@ -118,10 +121,12 @@ private void beforeUpload(GitContext gitContext) throws IOException {
 
     private GitContext initializeGitContext(PrepareWrapperContext context, SecHubRemoteDataConfiguration secHubRemoteDataConfiguration) {
         Path workingDirectory = Paths.get(context.getEnvironment().getPdsJobWorkspaceLocation());
+        String location = secHubRemoteDataConfiguration.getLocation();
 
         GitContext gitContext = new GitContext();
         gitContext.setCloneWithoutHistory(cloneWithoutGitHistory);
-        gitContext.setLocation(secHubRemoteDataConfiguration.getLocation());
+        gitContext.setLocation(location);
+        gitContext.setRepositoryName(gitLocationConverter.convertLocationToRepositoryName(location));
         gitContext.init(workingDirectory);
 
         return gitContext;