update to version v1.0.1. support report download

Makefile

@@ -5,6 +5,7 @@ else
 endif
 IMAGE_BASE		:= reversinglabs/rl-scanner-cloud
 IMAGE_NAME		:= $(IMAGE_BASE):$(BUILD_VERSION)
+REPORT_DIR		:= reports
 
 LINE_LENGTH	:=	120
 DIST		:=	dist
@@ -37,7 +38,7 @@ build:
 	docker image inspect $(IMAGE_NAME) --format '{{ .Config.Labels }}'
 	docker image inspect $(IMAGE_NAME) --format '{{ .RepoTags }}'
 
-test: testFail test_ok test_err
+test: testFail test_ok test_err test_ok_with_report test_err_with_report
 
 testFail:
 	# we know that specifying no arguments alt all should print "Valid commands are" and fail
@@ -56,6 +57,17 @@ test_ok:
 			$(TEST_PARAMS_SCAN) --file-path=/input/$(ARTIFACT_OK)
 	ls -laR input >./tmp/list_in_out_ok.txt
 
+test_ok_with_report:
+	rm -rf output input
+	mkdir -m 777 -p input output
+	cp /bin/$(ARTIFACT_OK) ./input/$(ARTIFACT_OK)
+	rm -rf $(REPORT_DIR) && mkdir $(REPORT_DIR)
+	docker run $(COMMON_DOCKER) $(VOLUMES) -v ./$(REPORT_DIR):/$(REPORT_DIR) $(IMAGE_NAME) \
+		rl-scan \
+			$(TEST_PARAMS_SCAN) --file-path=/input/$(ARTIFACT_OK) --report-path /$(REPORT_DIR) --report-format all
+	ls -laR input >./tmp/list_in_out_ok.txt
+	ls -laR $(REPORT_DIR)
+
 test_err:
 	rm -rf output input
 	mkdir -m 777 -p input output
@@ -67,6 +79,18 @@ test_err:
 			$(TEST_PARAMS_SCAN) --file-path=/input/$(ARTIFACT_ERR)
 	ls -laR input >./tmp/list_in_out_err.txt
 
+test_err_with_report:
+	rm -rf output input
+	mkdir -m 777 -p input output
+	curl -o $(ARTIFACT_ERR) -sS https://secure.eicar.org/$(ARTIFACT_ERR)
+	cp $(ARTIFACT_ERR) ./input/$(ARTIFACT_ERR)
+	rm -rf $(REPORT_DIR) && mkdir $(REPORT_DIR)
+	# as we are now scanning a item that makes the scan fail (non zero exit code) we have to ignore the error in the makefile
+	-docker run $(COMMON_DOCKER) $(VOLUMES) -v ./$(REPORT_DIR):/$(REPORT_DIR) $(IMAGE_NAME) \
+		rl-scan \
+			$(TEST_PARAMS_SCAN) --file-path=/input/$(ARTIFACT_ERR) --report-path /$(REPORT_DIR) --report-format all
+	ls -laR input >./tmp/list_in_out_err.txt
+
 clean:
 	rm -rf dist
 	-docker rmi $(IMAGE_NAME)
@@ -78,8 +102,7 @@ format:
 	black --line-length $(LINE_LENGTH) scripts/*
 
 pycheck:
-	# pylama -l "eradicate,mccabe,mypy,pycodestyle,pyflakes,pylint" -i E501,C0114,C0115,C0116,C0301,R1705,R0903,W0603,W1510 scripts/
-	pylama -l "eradicate,mccabe,pycodestyle,pyflakes" scripts/
+	pylama --max-line-length $(LINE_LENGTH) -l "eradicate,mccabe,pycodestyle,pyflakes" scripts/
 
 dist: format pycheck
 	rm -rf $(DIST) && mkdir -p $(DIST) && mkdir -p $(DIST)/scripts

README.md

@@ -2,12 +2,12 @@
 
 ![Header image](https://github.com/reversinglabs/rl-scanner-cloud/raw/main/armando-docker-cloud.png)
 
-`reversinglabs/rl-scanner-cloud` is the official Docker image created by ReversingLabs 
+`reversinglabs/rl-scanner-cloud` is the official Docker image created by ReversingLabs
 for users who want to automate their workflows on the secure.software Portal and integrate it with CI/CD tools.
 
 The image provides access to the main Portal Projects workflows - uploading package versions to a project,
 scanning them, and generating analysis reports.
-You can also compare different package versions in a project, and analyze reproducible build artifacts for a package version. 
+You can also compare different package versions in a project, analyze reproducible build artifacts for a package version, and save analysis reports to local storage.
 All successfully analyzed files are visible in the Portal interface, and accessible by you and any other Portal users who can access your projects.
 
 This Docker image is based on Rocky Linux 9.
@@ -41,7 +41,7 @@ The following table lists more detailed differences between these two Docker ima
 |             | **rl-scanner** | **rl-scanner-cloud** |
 |:----        |:----           | :----                |
 | Scanning | Software packages are scanned inside the Docker container, on the local system where the container is running. | Software packages are scanned in the cloud, on the Portal instance to which they are uploaded. |
-| Reports  | Users can choose the report format(s) they want to generate, and automatically save the reports to local storage. | All report formats supported by the Portal are always generated (not configurable). Users can manually download the reports from the Portal web interface or via the [Portal Public APIs](https://docs.secure.software/api/). The rl-json and rl-checks reports can only be downloaded via the APIs. The HTML report (rl-html format) cannot be downloaded. |
+| Reports  | Users can choose the report format(s) they want to generate, and automatically save the reports to local storage. | Users can choose the report format(s) they want to generate and save them to local storage. The HTML report (rl-html format) is always generated and displayed in the Portal web interface. |
 | Accounts and licensing | A valid `rl-secure` license with site key is required to use the Docker image. The size of analyzed files is deducted from the quota allocated to the user's `rl-secure` account. | An active secure.software Portal account with a Personal Access Token is required to use the Docker image. The size of analyzed files is deducted from the analysis capacity that is allocated to the user's group and reserved for projects. |
 
 
@@ -75,13 +75,13 @@ This makes it easier for cautious customers to use versioned tag images and migr
 The most common workflow for this Docker image is to upload a file for analysis to a secure.software Portal instance, where it is added as a package version to a new or an existing project and package.
 Portal users can then view the analysis report and [manage the analyzed file](https://docs.secure.software/portal/projects#work-with-package-versions-releases) like any other package version.
 
-Access to input data (files you want to scan) is provided by using [Docker volume mounts](https://docs.docker.com/storage/volumes/).
+Access to input data (files you want to scan) and the reports destination directory (to optionally save analysis reports) is provided by using [Docker volume mounts](https://docs.docker.com/storage/volumes/).
 To prevent issues with file ownership and access, the `-u` option is used to provide current user identification to the container.
 
 The image wraps the functionality of several Portal Public API endpoints into a single command with [configurable parameters](#configuration-parameters).
 As a result, users don't have to send multiple API requests, because the whole workflow can be completed in a single run.
 
-To use the provided Portal functionality, an active account on a Portal instance is required, together with a Personal Access Token for API authentication. 
+To use the provided Portal functionality, an active account on a Portal instance is required, together with a Personal Access Token for API authentication.
 Before you start using the image, make sure all [prerequisites](#prerequisites) are satisfied.
 
 In some cases, a proxy server may be required to access the internet and connect to ReversingLabs servers.
@@ -96,7 +96,7 @@ To successfully use this Docker image, you need:
 
 2. **An active secure.software Portal account and a Personal Access Token generated for it**. If you don't already have a Portal account, you may need to contact the administrator of your Portal organization to [invite you](https://docs.secure.software/portal/members#invite-a-new-member). Alternatively, if you're not a secure.software customer yet, you can [contact ReversingLabs](https://docs.secure.software/portal/#get-access-to-securesoftware-portal) to sign up for a Portal account. When you have an account set up, follow the instructions to [generate a Personal Access Token](https://docs.secure.software/api/generate-api-token).
 
-3. **One or more software packages to analyze**. Your packages must be stored in a location that Docker will be able to access. 
+3. **One or more software packages to analyze**. Your packages must be stored in a location that Docker will be able to access.
 
 
 ## Environment variables
@@ -130,6 +130,8 @@ The `rl-scanner-cloud` image supports the following parameters.
 | `--submit-only`      | No | By default, the Docker container runs until the uploaded file is analyzed on the Portal and returns the result in the output. This optional parameter lets you skip waiting for the analysis result. |
 | `--timeout`          | No | This optional parameter lets you specify how long the container should wait for analysis to complete before exiting (in minutes). The parameter accepts any integer from 10 to 1440. The default timeout is 20 minutes. |
 | `--message-reporter` | No  | Optional parameter that changes the format of output messages (STDOUT) for easier integration with CI tools. Supported values: `text`, `teamcity` |
+| `--report-path`      | No  | Path to the location where you want to store analysis reports. The specified path must exist in the reports destination directory mounted to the container. |
+| `--report-format`    | No  | A comma-separated list of report formats to generate. Supported values: cyclonedx, sarif, spdx, rl-json, rl-checks, all |
 
 
 ## Return codes
@@ -157,21 +159,53 @@ The file is added to the specified organization and group, and assigned as a ver
 After the file is uploaded to the Portal, it's visible in the web interface while the analysis is pending.
 
 
-```
     docker run \
       -u $(id -u):$(id -g) \
       -v "$(pwd)/packages:/packages:ro" \
       -e RLPORTAL_ACCESS_TOKEN=exampletoken \
       reversinglabs/rl-scanner-cloud \
-        --rl-portal-server=demo \
-        --rl-portal-org=ExampleOrg \
-        --rl-portal-group=demo-group \
-        --purl=my-project/[email protected] \
-        --file-path=/packages/demo-packages/MyPackage_1.exe
-```
+        --rl-portal-server demo \
+        --rl-portal-org ExampleOrg \
+        --rl-portal-group demo-group \
+        --purl my-project/[email protected] \
+        --file-path /packages/demo-packages/MyPackage_1.exe
 
 
-4. The container exits automatically when the analysis is complete. You can then access the analysis report in the Portal web interface and continue to work with the package version you just uploaded. 
+
+4. The container exits automatically when the analysis is complete. You can then access the analysis report in the Portal web interface and continue to work with the package version you just uploaded.
+
+
+### Scan a package version and download analysis reports
+
+To download analysis reports, you can use the `--report-path` and `--report-format` parameters when scanning a file.
+These parameters are optional, but they must be used together.
+
+To store the reports to a specific location, you must use an additional volume and make sure Docker can write to it.
+In this example, we're adding the volume with `-v "$(pwd)/reports:/reports"`, so the destination directory is going to be called `reports`. 
+This destination directory must be created empty before starting the container.
+You will then specify it in the Docker command with the `--report-path` parameter.
+
+The `--report-format` parameter accepts any of the [supported report formats](https://docs.secure.software/cli/commands/report).
+To request multiple formats at once, specify them as a comma-separated list.
+The special value `all` will download all supported report formats.
+
+The following command will scan a package version (1.0) and save all supported report formats into the `/reports` directory on the mounted volume.
+Other configuration parameters are the same in this example as in the other examples in this text.
+
+
+    docker run \
+      -u $(id -u):$(id -g) \
+      -v "$(pwd)/packages:/packages:ro" \
+      -v "$(pwd)/reports:/reports" \
+      -e RLPORTAL_ACCESS_TOKEN=exampletoken \
+      reversinglabs/rl-scanner-cloud \
+        --rl-portal-server demo \
+        --rl-portal-org ExampleOrg \
+        --rl-portal-group demo-group \
+        --purl my-project/[email protected] \
+        --file-path /packages/demo-packages/MyPackage_1.exe \
+        --report-path /reports \
+        --report-format all 
 
 
 ### Compare package versions in a Portal project
@@ -183,19 +217,17 @@ The following command will scan a new package version (1.1) and generate a repor
 Other configuration parameters are the same in this example as in the other examples in this text.
 
 
-```
     docker run \
       -u $(id -u):$(id -g) \
       -v "$(pwd)/packages:/packages:ro" \
       -e RLPORTAL_ACCESS_TOKEN=exampletoken \
       reversinglabs/rl-scanner-cloud \
-        --rl-portal-server=demo \
-        --rl-portal-org=ExampleOrg \
-        --rl-portal-group=demo-group \
-        --purl=my-project/[email protected] \
-        --file-path=/packages/demo-packages/MyPackage_1-1.exe \
+        --rl-portal-server demo \
+        --rl-portal-org ExampleOrg \
+        --rl-portal-group demo-group \
+        --purl my-project/[email protected] \
+        --file-path /packages/demo-packages/MyPackage_1-1.exe \
         --diff-with=1.0
-```
 
 
 The analysis report of the new version will contain the **Diff** tab with all the differences between the two versions.
@@ -217,18 +249,16 @@ The following command will scan the new artifact and perform a build reproducibi
 Other configuration parameters are the same in this example as in the other examples in this text.
 
 
-```
     docker run \
       -u $(id -u):$(id -g) \
       -v "$(pwd)/packages:/packages:ro" \
       -e RLPORTAL_ACCESS_TOKEN=exampletoken \
       reversinglabs/rl-scanner-cloud \
-        --rl-portal-server=demo \
-        --rl-portal-org=ExampleOrg \
-        --rl-portal-group=demo-group \
-        --purl=my-project/[email protected]?build=repro \
-        --file-path=/packages/demo-packages/MyPackage_1-build1.exe \
-```
+        --rl-portal-server demo \
+        --rl-portal-org ExampleOrg \
+        --rl-portal-group demo-group \
+        --purl my-project/[email protected]?build=repro \
+        --file-path /packages/demo-packages/MyPackage_1-build1.exe 
 
 
 The analysis report will contain the **Reproducibility** tab with the reproducibility check status and a summary of differences between the reproducible build artifact and the main artifact ("Reference Version" in the report).

scripts/analysis_report_exporter.py

@@ -0,0 +1,54 @@
+from params import Params
+from request_invoker import RequestInvoker
+from cimessages import Messages
+
+from typing import List
+
+
+class AnalysisReportExporter:
+    def __init__(self, params: Params, reporter: Messages):
+        self.params = params
+        self.request_invoker = RequestInvoker(reporter)
+        self.reporter = reporter
+
+    def export_analysis_report(self):
+        report_formats = AnalysisReportExporter.parse_report_formats(self.params.report_format)
+
+        for report_format in report_formats:
+            response = self.request_invoker.export_analysis_report(self.params, report_format)
+            self.reporter.info(f"Started {report_format} export")
+
+            report_filename = AnalysisReportExporter._get_report_name(report_format)
+
+            with open(f"{self.params.report_path}/{report_filename}", "wb") as f:
+                for chunk in response.iter_content(chunk_size=1024):
+                    f.write(chunk)
+                self.reporter.info(f"Finished {report_format} export")
+
+    @staticmethod
+    def _get_report_name(report_format: str):
+        # https://docs.secure.software/cli/commands/report?_highlight=report&_highlight=format#usage
+        if report_format == "sarif":
+            return "report.sarif.json"
+        if report_format == "cyclonedx":
+            return "report.cyclonedx.json"
+        if report_format == "spdx":
+            return "report.spdx.json"
+        if report_format == "rl-json":
+            return "report.rl.json"
+        if report_format == "rl-checks":
+            return "report.checks.json"
+
+    @staticmethod
+    def parse_report_formats(report_formats: str) -> List[str]:
+        valid_report_formats = ["sarif", "cyclonedx", "spdx", "rl-json", "rl-checks"]
+        parsed_report_formats = [report_format for report_format in report_formats.split(",")]
+
+        for report_format in parsed_report_formats:
+            if report_format == "all":
+                return valid_report_formats
+
+            if report_format not in valid_report_formats:
+                raise RuntimeError("Invalid report format provided")
+
+        return parsed_report_formats

scripts/checks_fetcher.py

@@ -0,0 +1,53 @@
+from time import sleep
+from params import Params
+from cimessages import Messages
+from request_invoker import RequestInvoker
+
+
+ATTEMPT_TIMEOUT_SEC = 30
+
+DEFAULT_ATTEMPT_TIMEOUT_MIN = 20
+LOWER_ATTEMPT_TIMEOUT_MIN = 10
+UPPER_ATTEMPT_TIMEOUT_MIN = 1440  # 24h
+
+
+class ChecksFetcher:
+    def __init__(self, params: Params, reporter: Messages):
+        timeout = params.timeout
+        if timeout not in range(LOWER_ATTEMPT_TIMEOUT_MIN, UPPER_ATTEMPT_TIMEOUT_MIN):
+            timeout = DEFAULT_ATTEMPT_TIMEOUT_MIN
+            reporter.info(
+                f"""
+                Value of timeout parameter is out of bounds ({LOWER_ATTEMPT_TIMEOUT_MIN} - {UPPER_ATTEMPT_TIMEOUT_MIN}).
+                Will set it to default {DEFAULT_ATTEMPT_TIMEOUT_MIN} minutes
+            """
+            )
+
+        self.number_of_attempts = (timeout * 60) // ATTEMPT_TIMEOUT_SEC
+        self.request_invoker = RequestInvoker(reporter)
+        self.reporter = reporter
+        self.params = params
+
+    def get_scan_status(self) -> str:
+        while True:
+            if self.number_of_attempts == 0:
+                self.reporter.info("Preset timeout time expired")
+                return "fail"
+
+            self.reporter.info("Attempting to fetch analysis status")
+            sleep(ATTEMPT_TIMEOUT_SEC)
+
+            response = self.request_invoker.get_performed_checks(self.params)
+
+            if response.status_code == 202:
+                self.number_of_attempts -= 1
+                continue
+
+            return (
+                response.json()
+                .get("analysis", {})
+                .get("report", {})
+                .get("info", {})
+                .get("summary", {})
+                .get("scan_status", "fail")
+            )

scripts/cimessages.py

@@ -17,7 +17,7 @@ class Messages:
     @classmethod
     def create(cls, name: str) -> Any:
         if name == "teamcity":
-            return TeamCityMesages()
+            return TeamCityMessages()
         else:
             return TextMessages()
 
@@ -46,7 +46,7 @@ def progress_block(self, msg: str) -> Any:
         self.block_end(msg)
 
 
-class TeamCityMesages(Messages):
+class TeamCityMessages(Messages):
     @classmethod
     def service_message(cls, name: str, msg: Any) -> str:
         def escape(m: str) -> str:
@@ -67,24 +67,24 @@ def escape(m: str) -> str:
             return f"##teamcity[{name} '{escape(msg)}']"
 
     def block_start(self, msg: str) -> None:
-        print(TeamCityMesages.service_message("progressStart", msg), flush=True)
-        print(TeamCityMesages.service_message("blockOpened", {"name": msg}), flush=True)
+        print(TeamCityMessages.service_message("progressStart", msg), flush=True)
+        print(TeamCityMessages.service_message("blockOpened", {"name": msg}), flush=True)
 
     def block_end(self, msg: str) -> None:
-        print(TeamCityMesages.service_message("blockClosed", {"name": msg}), flush=True)
-        print(TeamCityMesages.service_message("progressFinish", msg), flush=True)
+        print(TeamCityMessages.service_message("blockClosed", {"name": msg}), flush=True)
+        print(TeamCityMessages.service_message("progressFinish", msg), flush=True)
 
     def __build_problem(self, msg: str) -> None:
         print(
-            TeamCityMesages.service_message("buildProblem", {"description": msg}),
+            TeamCityMessages.service_message("buildProblem", {"description": msg}),
             flush=True,
         )
 
     def __build_status(self, msg: str) -> None:
-        print(TeamCityMesages.service_message("buildStatus", {"text": msg}), flush=True)
+        print(TeamCityMessages.service_message("buildStatus", {"text": msg}), flush=True)
 
     def info(self, msg: str) -> None:
-        print(TeamCityMesages.service_message("message", {"text": msg}), flush=True)
+        print(TeamCityMessages.service_message("message", {"text": msg}), flush=True)
 
     def scan_result(self, passed: bool) -> bool:
         if passed: