cpm-cmake · gerhardol · May 3, 2024
diff --git a/README.md b/README.md
@@ -193,6 +193,12 @@ You can use `CPM_SOURCE_CACHE` on GitHub Actions workflows [cache](https://githu
 The directory where the version for a project is stored is by default the hash of the arguments to `CPMAddPackage()`.
 If for instance the patch command uses external files, the directory name can be set with the argument `CUSTOM_CACHE_KEY`.
 
+It is possible to check the integrity of the downloaded content with a checksum by adding a [checksum command](test/unit/checksum_directory.sh) to `CPMAddPackage()`.
+Checksum validation can be done in two ways:
+
+- Setting the option `CPM_CHECK_CACHE_CHECKSUM` to validate to the checksum calculated when downloading the project.
+- Providing the checksum in the call to `CPMAddPackage()`.
+
 ### CPM_DOWNLOAD_ALL
 
 If set, CPM will forward all calls to `CPMFindPackage` as `CPMAddPackage`.
@@ -216,6 +222,14 @@ Note that this does not apply to dependencies that have been defined with a trut
 If set, CPM use additional directory level in cache to improve readability of packages names in IDEs like CLion. It changes cache structure, so all dependencies are downloaded again. There is no problem to mix both structures in one cache directory but then there may be 2 copies of some dependencies.
 This can also be set as an environmental variable.
 
+### CPM_CHECK_CACHE_CHECKSUM
+
+Enable validation of the checksum for a cache directory if a command to checksum the directory is provided. The validation is performed to a supplied checksum if provided, otherwise the checksum detected when downloading the dependency.
+
+If `GIT_TAG` is set, `git-status` will check the status, checksum command is not required.
+
+If the check fails, an existing directory will be deleted and downloaded again.
+
 ## Local package override
 
 Library developers are often in the situation where they work on a locally checked out dependency at the same time as on a consumer project.

diff --git a/cmake/CPM.cmake b/cmake/CPM.cmake
@@ -129,6 +129,10 @@ option(CPM_USE_NAMED_CACHE_DIRECTORIES
        "Use additional directory of package name in cache on the most nested level."
        $ENV{CPM_USE_NAMED_CACHE_DIRECTORIES}
 )
+option(CPM_CHECK_CACHE_CHECKSUM
+       "If a package is stored in cache and there is a command to provide checksum, check the checksum when the cache dir exists."
+       $ENV{CPM_CHECK_CACHE_CHECKSUM}
+)
 
 set(CPM_VERSION
     ${CURRENT_CPM_VERSION}
@@ -535,9 +539,10 @@ function(CPMAddPackage)
       EXCLUDE_FROM_ALL
       SOURCE_SUBDIR
       CUSTOM_CACHE_KEY
+      CUSTOM_CACHE_CHECKSUM_VALUE
   )
 
-  set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND)
+  set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND CUSTOM_CACHE_CHECKSUM_COMMAND)
 
   cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
 
@@ -720,31 +725,73 @@ function(CPMAddPackage)
     get_filename_component(download_directory ${download_directory} ABSOLUTE)
     list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${download_directory})
 
-    if(CPM_SOURCE_CACHE)
-      file(LOCK ${download_directory}/../cmake.lock)
+    file(LOCK ${download_directory}/../cmake.lock)
+
+    if(EXISTS ${download_directory} AND NOT EXISTS ${download_directory}.download)
+      message(WARNING "Cache for ${CPM_ARGS_NAME} is missing .download, downloading. (${download_directory}.download)")
+      file(REMOVE_RECURSE ${download_directory})
     endif()
 
-    if(EXISTS ${download_directory})
-      if(CPM_SOURCE_CACHE)
-        file(LOCK ${download_directory}/../cmake.lock RELEASE)
+    if(EXISTS ${download_directory}
+      AND CPM_ARGS_CUSTOM_CACHE_CHECKSUM_COMMAND
+      AND (CPM_CHECK_CACHE_CHECKSUM
+        OR DEFINED CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE))
+      if (CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE)
+        # Explicit checksum provided, ignore value in .downloaded
+        set(expected_checksum ${CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE})
+      else()
+        file(READ ${download_directory}.download expected_checksum)
+        string(STRIP "${expected_checksum}" expected_checksum)
+      endif()
+
+      if(expected_checksum)
+        execute_process(
+          COMMAND ${CPM_ARGS_CUSTOM_CACHE_CHECKSUM_COMMAND}
+          WORKING_DIRECTORY ${download_directory}
+          OUTPUT_VARIABLE checksum
+          OUTPUT_STRIP_TRAILING_WHITESPACE
+          COMMAND_ERROR_IS_FATAL ANY
+        )
+        if(NOT expected_checksum STREQUAL checksum)
+          message(
+            WARNING
+              "Checksum mismatch for ${CPM_ARGS_NAME}, removing (${expected_checksum} != ${checksum})"
+          )
+          file(REMOVE_RECURSE ${download_directory})
+        endif()
+      else()
+        message(
+          WARNING
+            "Checksum cannot be verified for ${CPM_ARGS_NAME}, no existing value (${expected_checksum})"
+        )
+      endif()
+    endif()
+
+    if(EXISTS ${download_directory}
+      AND DEFINED CPM_ARGS_GIT_TAG
+       AND NOT (PATCH_COMMAND IN_LIST CPM_ARGS_UNPARSED_ARGUMENTS))
+      # warn if cache has been changed since checkout
+      cpm_check_git_working_dir_is_clean(${download_directory} ${CPM_ARGS_GIT_TAG} IS_CLEAN)
+      if(NOT ${IS_CLEAN})
+        message(
+          WARNING "${CPM_INDENT} Cache for ${CPM_ARGS_NAME} (${download_directory}) is dirty"
+        )
+        if(CPM_CHECK_CACHE_CHECKSUM OR DEFINED CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE)
+          file(REMOVE_RECURSE ${download_directory})
+        endif()
       endif()
+    endif()
+
+    if(EXISTS ${download_directory})
+      # Directory content is considered OK
+      file(LOCK ${download_directory}/../cmake.lock RELEASE)
 
       cpm_store_fetch_properties(
         ${CPM_ARGS_NAME} "${download_directory}"
         "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-build"
       )
       cpm_get_fetch_properties("${CPM_ARGS_NAME}")
 
-      if(DEFINED CPM_ARGS_GIT_TAG AND NOT (PATCH_COMMAND IN_LIST CPM_ARGS_UNPARSED_ARGUMENTS))
-        # warn if cache has been changed since checkout
-        cpm_check_git_working_dir_is_clean(${download_directory} ${CPM_ARGS_GIT_TAG} IS_CLEAN)
-        if(NOT ${IS_CLEAN})
-          message(
-            WARNING "${CPM_INDENT} Cache for ${CPM_ARGS_NAME} (${download_directory}) is dirty"
-          )
-        endif()
-      endif()
-
       cpm_add_subdirectory(
         "${CPM_ARGS_NAME}"
         "${DOWNLOAD_ONLY}"
@@ -801,8 +848,30 @@ function(CPMAddPackage)
     )
     cpm_fetch_package("${CPM_ARGS_NAME}" populated)
     if(CPM_SOURCE_CACHE AND download_directory)
+      if(${populated})
+        if(CPM_ARGS_CUSTOM_CACHE_CHECKSUM_COMMAND)
+          execute_process(
+            COMMAND ${CPM_ARGS_CUSTOM_CACHE_CHECKSUM_COMMAND}
+            WORKING_DIRECTORY ${download_directory}
+            OUTPUT_VARIABLE checksum
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            COMMAND_ERROR_IS_FATAL ANY
+          )
+          if(CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE AND NOT CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE STREQUAL checksum)
+            message(
+              FATAL_ERROR
+                "Checksum mismatch for ${CPM_ARGS_NAME} (${CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE} != ${checksum})"
+            )
+          endif()
+        else()
+          set(checksum "")
+        endif()
+        file(WRITE ${download_directory}.download ${checksum})
+      endif()
+
       file(LOCK ${download_directory}/../cmake.lock RELEASE)
     endif()
+
     if(${populated})
       cpm_add_subdirectory(
         "${CPM_ARGS_NAME}"

diff --git a/test/unit/cache.cmake b/test/unit/cache.cmake
@@ -153,3 +153,127 @@ execute_process(
 
 assert_equal(${ret} "0")
 assert_exists("${CPM_SOURCE_CACHE_DIR}/fibonacci/my_custom_unique_dir")
+
+# Cache checksum
+
+reset_test()
+set(FIBONACCI_VERSION 1.1)
+set(FIBONACCI_GIT_TAG "GIT_TAG e9ebf168ca0fffaa4ef8c6fefc6346aaa22f6ed5")
+set(TEST_CHECKSUM_DIR "${CPM_SOURCE_CACHE_DIR}/fibonacci/my_checksummed_dir")
+set(TEST_CHECKSUM_VALUE d2e7c040116d3f4f153eee84cb884fd5008a31b480b739bf86d3872d978cfaa74d82e07581a1861fdb07e717be5b658eb520e2f4e29fbf0ce248bfef478c1971)
+
+# OK download
+
+set(FIBONACCI_PACKAGE_ARGS
+    "${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh"
+)
+update_cmake_lists()
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
+          "-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
+)
+
+assert_equal(${ret} "0")
+assert_exists("${TEST_CHECKSUM_DIR}.download")
+file(READ "${TEST_CHECKSUM_DIR}.download" chksum)
+assert_equal("${chksum}" "${TEST_CHECKSUM_VALUE}")
+
+# Test download again if .download file is missing
+
+file(REMOVE "${TEST_CHECKSUM_DIR}.download")
+file(REMOVE "${TEST_CHECKSUM_DIR}/include/fibonacci.h")
+
+set(FIBONACCI_PACKAGE_ARGS
+    "${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh"
+)
+update_cmake_lists()
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
+          "-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
+)
+
+assert_equal(${ret} "0")
+assert_exists("${TEST_CHECKSUM_DIR}.download")
+assert_exists("${TEST_CHECKSUM_DIR}/include/fibonacci.h")
+
+# check checksum for download
+
+set(FIBONACCI_PACKAGE_ARGS
+    "${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh"
+)
+update_cmake_lists()
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
+          "-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
+)
+
+assert_equal(${ret} "0")
+
+# check checksum for download, provided
+
+set(FIBONACCI_PACKAGE_ARGS
+    "${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh CUSTOM_CACHE_CHECKSUM_VALUE ${TEST_CHECKSUM_VALUE}"
+)
+update_cmake_lists()
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
+          "-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
+)
+
+assert_equal(${ret} "0")
+
+# check checksum for download, provided incorrect
+# This will print a fatal_error (red) error to the console
+
+set(FIBONACCI_PACKAGE_ARGS
+    "${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh CUSTOM_CACHE_CHECKSUM_VALUE invalid_checksum_value"
+)
+update_cmake_lists()
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
+          "-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
+)
+
+assert_equal(${ret} "1")
+
+# redownload when checksum is changed
+
+set(FIBONACCI_PACKAGE_ARGS
+    "${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh CUSTOM_CACHE_CHECKSUM_VALUE ${TEST_CHECKSUM_VALUE}"
+)
+update_cmake_lists()
+
+# dummy change, to trigger checksum mismatch
+file(WRITE "${TEST_CHECKSUM_DIR}/fail_checksum.txt" "dummy")
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
+          "-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
+)
+
+assert_equal(${ret} "0")
+assert_not_exists("${TEST_CHECKSUM_DIR}/fail_checksum.txt")
+
+# redownload when checksum is changed
+
+set(FIBONACCI_PACKAGE_ARGS
+    "${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_VALUE ${TEST_CHECKSUM_VALUE}"
+)
+update_cmake_lists()
+
+# dummy change, to trigger checksum mismatch
+file(WRITE "${TEST_CHECKSUM_DIR}/fail_checksum.txt" "dummy")
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
+          "-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
+)
+
+assert_equal(${ret} "0")
+assert_not_exists("${TEST_CHECKSUM_DIR}/fail_checksum.txt")
+
diff --git a/test/unit/checksum_directory.sh b/test/unit/checksum_directory.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Script to checksum contents recursively in a directory
+
+set -o errexit
+set -o nounset
+
+function usage {
+    echo
+    echo "Checksum the contents of a directory"
+    echo "Usage: $0 [-d <directory>]"
+    echo ""
+    echo "  -d directory Default '.'"
+    echo "  -h           Help, this message"
+    echo "  -t           Use alternative tar method (requires zstd binary)"
+    echo "  -v           Verbose output"
+}
+
+dir=.
+use_tar=
+# sha512 is faster than sha256 for large files, sha1 is even faster
+SHA_ALGORITHM=sha512sum
+
+while getopts "d:htv" o; do
+    case "${o}" in
+        d)
+            dir=${OPTARG}
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        t)
+            use_tar=1
+            ;;
+        v)
+            set -x
+            ;;
+        *)
+            echo "Incorrect argument switch"
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+cd $dir
+if [ ! -z $use_tar ]; then
+  # This is faster for single threads but requires more memory and requires the separate zstd binary
+  # For a 3 GB data this is 3s vs 'find' below: 5s (one thread) below, 2.5s with 28 threads, 0.7s with 100 files on each line
+  # Without --fast, just ZSTD_CLEVEL=1 ZSTD_NBTHREADS=0 is about 6s
+  tar -I "zstd --fast -1 -T0" -cf - . | $SHA_ALGORITHM | cut -f1 -d ' '
+else
+  # In general, there is no point in checksumming Git repos, filter .git here as this is used in tests
+  find . \( -name .git -prune \) -o -type f -print0  | xargs -n 100 --max-procs=$(nproc) -0 $SHA_ALGORITHM | sort -k 2 | $SHA_ALGORITHM | cut -f1 -d ' '
+fi