GitLab CI: enable runtime tests for gcc and clang cpu jobs

alpaka-group · Sep 5, 2023 · ef103d6 · ef103d6
1 parent 8bcee69
commit ef103d6
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 27 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -26,15 +26,18 @@ generate:
   script:
     - apk update && apk add python3~=3.11 py3-pip
     - pip3 install -r script/job_generator/requirements.txt
-    # it's enough to verify one time, because --compile-only und --runtime-only generates the same job matrix and filter it
-    - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --verify --compile-only -o compile_only.yml
-    - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --runtime-only -o runtime.yml
+    # it is sufficient to verify once, as the same job matrix is generated, verified and then filtered each time
+    - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --verify --wave compile_only_job -o compile_only.yml
+    - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave runtime_job_cpu -o runtime_cpu.yml
+    - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave runtime_job_gpu -o runtime_gpu.yml
     - cat compile_only.yml
-    - cat runtime.yml
+    - cat runtime_cpu.yml
+    - cat runtime_gpu.yml
   artifacts:
     paths:
       - compile_only.yml
-      - runtime.yml
+      - runtime_cpu.yml
+      - runtime_gpu.yml
     expire_in: 1 week
   interruptible: true
 
@@ -46,10 +49,18 @@ run-compile-only-test:
         job: generate
     strategy: depend
 
-run-runtime-test:
+run-runtime-cpu-test:
   stage: run-test-jobs
   trigger:
     include:
-      - artifact: runtime.yml
+      - artifact: runtime_cpu.yml
+        job: generate
+    strategy: depend
+
+run-runtime-gpu-test:
+  stage: run-test-jobs
+  trigger:
+    include:
+      - artifact: runtime_gpu.yml
         job: generate
     strategy: depend
diff --git a/script/job_generator/generate_job_yaml.py b/script/job_generator/generate_job_yaml.py
@@ -14,7 +14,8 @@
 from util import print_warn, exit_error
 
 JOB_COMPILE_ONLY = "compile_only_job"
-JOB_RUNTIME = "runtime_job"
+JOB_RUNTIME = "runtime_job_gpu"
+JOB_CPU_RUNTIME = "runtime_job_cpu"
 JOB_ROCM_RUNTIME = "rocm_runtime_job"
 JOB_NVCC_GCC_RUNTIME = "nvcc_gcc_runtime_job"
 JOB_NVCC_CLANG_RUNTIME = "nvcc_clng_runtime_job"
@@ -25,8 +26,8 @@
 WAVE_GROUP_NAMES = [
     JOB_COMPILE_ONLY,
     JOB_RUNTIME,
+    JOB_CPU_RUNTIME,
     # can be enabled again, if fine granular scheduling is required
-    # JOB_CPU_RUNTIME,
     # JOB_ROCM_RUNTIME,
     # JOB_NVCC_GCC_RUNTIME,
     # JOB_NVCC_CLANG_RUNTIME,
@@ -562,12 +563,13 @@ def distribute_to_waves(
         job_name: str = next(iter(job))
         if "_compile_only_" in job_name:
             sorted_groups[JOB_COMPILE_ONLY].append(job)
+        elif job_name.startswith("linux_gcc"):
+            sorted_groups[JOB_CPU_RUNTIME].append(job)
         # Clang as C++ compiler without CUDA backend
         elif job_name.startswith("linux_clang") and not job_name.startswith(
             "linux_clang-cuda"
         ):
-            # sorted_groups[JOB_CPU_RUNTIME].append(job)
-            sorted_groups[JOB_COMPILE_ONLY].append(job)
+            sorted_groups[JOB_CPU_RUNTIME].append(job)
         elif job_name.startswith("linux_hipcc"):
             # sorted_groups[JOB_ROCM_RUNTIME].append(job)
             sorted_groups[JOB_RUNTIME].append(job)

diff --git a/script/job_generator/job_generator.py b/script/job_generator/job_generator.py
@@ -88,11 +88,12 @@ def get_args() -> argparse.Namespace:
     )
 
     parser.add_argument(
-        "--compile-only", action="store_true", help="Generate only compile only jobs."
-    )
-
-    parser.add_argument(
-        "--runtime-only", action="store_true", help="Generate only runtime jobs."
+        "--wave",
+        type=str,
+        # add `all` and remove `JOB_UNKNOWN` from the choices
+        choices=["all"] + list(set(WAVE_GROUP_NAMES) - set([JOB_UNKNOWN])),
+        default="all",
+        help="Generates only jobs for a specific wave group.",
     )
 
     parser.add_argument(
@@ -201,14 +202,9 @@ def get_args() -> argparse.Namespace:
             for job in wave:
                 print(job)
 
-    if args.compile_only:
-        wave_job_matrix = {JOB_COMPILE_ONLY: wave_job_matrix[JOB_COMPILE_ONLY]}
-        for wave_name in WAVE_GROUP_NAMES:
-            if not wave_name in wave_job_matrix:
-                wave_job_matrix[wave_name] = []
-
-    if args.runtime_only:
-        wave_job_matrix = {JOB_RUNTIME: wave_job_matrix[JOB_RUNTIME]}
+    if args.wave != "all":
+        filter_wave_name = args.wave
+        wave_job_matrix = {filter_wave_name: wave_job_matrix[filter_wave_name]}
         for wave_name in WAVE_GROUP_NAMES:
             if not wave_name in wave_job_matrix:
                 wave_job_matrix[wave_name] = []

diff --git a/script/job_generator/job_modifier.py b/script/job_generator/job_modifier.py
@@ -39,7 +39,7 @@ def add_job_parameters(job_matrix: List[Dict[str, Tuple[str, str]]]):
     #        ):
     #            job[JOB_EXECUTION_TYPE] = (JOB_EXECUTION_TYPE, JOB_EXECUTION_RUNTIME)
     #            break
-            
+
     ##############################################
     ## Defining runtime test for the CUDA backend
     ##############################################
@@ -66,7 +66,7 @@ def add_job_parameters(job_matrix: List[Dict[str, Tuple[str, str]]]):
     for job in job_matrix:
         if (
             ALPAKA_ACC_GPU_CUDA_ENABLE in job
-            and job[ALPAKA_ACC_GPU_CUDA_ENABLE][VERSION] !=OFF_VER
+            and job[ALPAKA_ACC_GPU_CUDA_ENABLE][VERSION] != OFF_VER
         ):
             v = version.parse(job[ALPAKA_ACC_GPU_CUDA_ENABLE][VERSION])
             if not v.major in latest_CUDA_SDK_minor_versions[job[HOST_COMPILER][NAME]]:
@@ -142,8 +142,13 @@ def add_job_parameters(job_matrix: List[Dict[str, Tuple[str, str]]]):
             missing_nvcc_versions.remove(job[DEVICE_COMPILER][VERSION])
         elif (
             ALPAKA_ACC_GPU_CUDA_ENABLE in job
-            and ALPAKA_ACC_GPU_CUDA_ENABLE[VERSION] !=OFF_VER
+            and ALPAKA_ACC_GPU_CUDA_ENABLE[VERSION] != OFF_VER
         ):
             job[SM_LEVEL] = (SM_LEVEL, STANDARD_SM_LEVEL)
         else:
             job[SM_LEVEL] = (SM_LEVEL, "")
+
+    # run tests each time if a CPU backend is used
+    for job in job_matrix:
+        if job[DEVICE_COMPILER][NAME] in [GCC, CLANG]:
+            job[JOB_EXECUTION_TYPE] = (JOB_EXECUTION_TYPE, JOB_EXECUTION_RUNTIME)