Merge branch 'master' into feat/related-studies-update

Jenkinsfile

@@ -134,8 +134,8 @@ spec:
       readOnly: true
       mountPath: "/usr/local/share/ca-certificates/cdis/cdis-ca.crt"
       subPath: "ca.pem"
-    - name: dockersock
-      mountPath: "/var/run/docker.sock"
+    - name: containerdsock
+      mountPath: "/var/run/containerd/containerd.sock"
   serviceAccount: jenkins-service
   serviceAccountName: jenkins-service
   volumes:
@@ -145,9 +145,9 @@ spec:
   - name: ca-volume
     secret:
       secretName: "service-ca"
-  - name: dockersock
+  - name: containerdsock
     hostPath:
-      path: /var/run/docker.sock
+      path: /var/run/containerd/containerd.sock
 '''
         defaultContainer 'shell'
         }
@@ -293,8 +293,8 @@ spec:
                 script {
                     try {
                         if(!skipUnitTests) {
-                            sh '/usr/bin/pip3 install boto3 --upgrade --user'
-                            sh '/usr/bin/pip3 install kubernetes --upgrade --user'
+                            sh '/usr/local/bin/pip3 install boto3 --upgrade --user'
+                            sh '/usr/local/bin/pip3 install kubernetes --upgrade --user'
                             sh 'python3 -m pytest cloud-automation/apis_configs/'
                             sh 'python3 -m pytest cloud-automation/gen3/lib/dcf/'
                             sh 'cd cloud-automation/tf_files/aws/modules/common-logging && python3 -m pytest testLambda.py'

gen3/bin/awsrole.sh

@@ -25,16 +25,16 @@ gen3_awsrole_help() {
 function gen3_awsrole_ar_policy() {
   local serviceAccount="$1"
   shift || return 1
-  if [[ ! -z $1 ]]; then
-    local namespace=$1
+  if [[ -z $1 ]] || [[ $1 == -* ]]; then
+    namespace=$(gen3 db namespace)
   else
-    local namespace=$(gen3 db namespace)
+    namespace=$1
+    shift
   fi
   local issuer_url
   local account_id
   local vpc_name
-  shift || return 1
-  local flag=$1
+  local flag=$flag
 
   vpc_name="$(gen3 api environment)" || return 1
   issuer_url="$(aws eks describe-cluster \
@@ -46,7 +46,7 @@ function gen3_awsrole_ar_policy() {
 
   local provider_arn="arn:aws:iam::${account_id}:oidc-provider/${issuer_url}"
 
-  if [[ "$flag" == "all_namespaces" ]]; then
+  if [[ "$flag" == "-all_namespaces" ]]; then
     # Use a trust policy that allows role to be used by multiple namespaces.
     cat - <<EOF
 {
@@ -110,7 +110,6 @@ EOF
   fi
 }
 
-
 #
 # Annotate the given service account with the given IAM role
 #
@@ -162,22 +161,19 @@ _get_entity_type() {
 #
 # @param rolename
 # @param saName for assume-role policy document
+# @param flag (optional) - specify a flag to use a different trust policy
 #
 _tfplan_role() {
   local rolename="$1"
   shift || return 1
   local saName="$1"
   shift || return 1
   local namespace="$1"
-  shift || return 1
-  local flag=""
-  # Check if the "all_namespaces" flag is provided
-  if [[ "$1" == "-f" || "$1" == "--flag" ]]; then
-    flag="$2"
-    shift 2
-  fi
   local arDoc
+  local flag=$flag
+
   arDoc="$(gen3_awsrole_ar_policy "$saName" "$namespace" "$flag")" || return 1
+
   gen3 workon default "${rolename}_role"
   gen3 cd
   cat << EOF > config.tfvars
@@ -230,10 +226,14 @@ gen3_awsrole_create() {
     gen3_log_err "use: gen3 awsrole create roleName saName"
     return 1
   fi
-  if [[ ! -z $1 ]]; then
-    local namespace=$1
+  if [[ -z $1 ]] || [[ $1 == -* ]]; then
+    namespace=$(gen3 db namespace)
   else
-    local namespace=$(gen3 db namespace)
+    namespace=$1
+    shift
+  fi
+  if [[ ! -z $1 ]]; then
+    flag=$1
   fi
   # do simple validation of name
   local regexp="^[a-z][a-z0-9\-]*$"
@@ -247,13 +247,7 @@ EOF
     gen3_log_err $errMsg
     return 1
   fi
-  shift || return 1
-  local flag=""
-  # Check if the "all_namespaces" flag is provided
-  if [[ "$1" == "-f" || "$1" == "--flag" ]]; then
-    flag="$2"
-    shift 2
-  fi
+
 
   # check if the name is already used by another entity
   local entity_type
@@ -271,9 +265,11 @@ EOF
   fi
 
   TF_IN_AUTOMATION="true"
-  if ! _tfplan_role $rolename $saName $namespace -f $flag; then
+
+  if ! _tfplan_role $rolename $saName $namespace $flag; then
     return 1
   fi
+
   if ! _tfapply_role $rolename; then
     return 1
   fi
@@ -422,4 +418,4 @@ gen3_awsrole() {
 # Let testsuite source file
 if [[ -z "$GEN3_SOURCE_ONLY" ]]; then
   gen3_awsrole "$@"
-fi
+fi

gen3/bin/healthcheck.sh

@@ -137,6 +137,10 @@ gen3_healthcheck() {
     internetAccessExplicitProxy=false
   fi
 
+  gen3_log_info "Clearing Evicted pods"
+  sleep 5
+  clear_evicted_pods
+
   local healthJson=$(cat - <<EOM
   {
     "pendingTimeoutPods": $pendingTimeoutPods,
@@ -210,5 +214,3 @@ clear_evicted_pods() {
 }
 
 gen3_healthcheck "$@"
-
-clear_evicted_pods

gen3/bin/kube-setup-argo.sh

@@ -156,7 +156,7 @@ EOF
       g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} --overwrite -n $argo_namespace
       g3kubectl annotate serviceaccount argo eks.amazonaws.com/role-arn=${roleArn} --overwrite -n $nameSpace
   else
-        gen3 awsrole create $roleName argo $nameSpace -f all_namespaces
+        gen3 awsrole create $roleName argo $nameSpace -all_namespaces
         roleArn=$(aws iam get-role --role-name "${roleName}" --query 'Role.Arn' --output text)
         g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n $argo_namespace
   fi

gen3/lib/logs/snapshot.sh

@@ -39,7 +39,7 @@ gen3_logs_snapshot_all() {
   # For each pod for which we can list the containers, get the pod name and get its list of containers
   # (container names + initContainers names). Diplay them as lines of "<pod name>  <container name>".
   g3kubectl get pods -o json | \
-    jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | map( {pod: .metadata.name, containers: [(.spec.containers | select(.!=null) | map(.name)), (.spec.initContainers | select(.!=null) | map(.name)) | add ] } ) | map( .pod as $pod | .containers | map( { pod: $pod, cont: .})[]) | map(select(.cont != "pause" and .cont != "jupyterhub"))[] | .pod + "  " + .cont' | \
+    jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | .[] | .metadata.name as $pod | (.spec.containers + .spec.initContainers) | map(select(.name != "pause" and .name != "jupyterhub")) | .[] | {pod: $pod, cont: .name} | "\(.pod)  \(.cont)"' | \
     while read -r line; do
       gen3_logs_snapshot_container $line
     done

gen3/test/ec2Test.sh

@@ -1,6 +1,6 @@
 
 
-if ! EC2_TEST_IP="$(g3kubectl get nodes -o json | jq -r -e '.items[0].status.addresses[] | select(.type == "InternalIP") | .address')" || [[ -z "$EC2_TEST_IP" ]]; then
+if ! EC2_TEST_IP="$(g3kubectl get nodes -o json | jq -r -e '.items[3].status.addresses[] | select(.type == "InternalIP") | .address')" || [[ -z "$EC2_TEST_IP" ]]; then
   gen3_log_err "ec2Test failed to acquire IP address of a k8s node to test against"
 fi
 

gen3/test/ecrTest.sh

@@ -10,8 +10,8 @@ test_ecr_login() {
 
 test_ecr_setup() {
   if [[ -n "$JENKINS_HOME" ]]; then
-    # give ourselves read/write permissions on /var/run/docker.sock
-    sudo chmod a+rw /var/run/docker.sock; because $? "ecr_setup modified docker.sock"
+    # give ourselves permissions on /run/containerd/containerd.sock
+    sudo chown root:sudo /run/containerd/containerd.sock; because $? "ecr_setup modified containerd.sock"
   fi
 }
 

gen3/test/jobTest.sh

@@ -6,7 +6,7 @@
 excludeJob() {
   local jobKey="$1"
   local excludeList=(
-    /aws-bucket- /bucket- /covid19- /data-ingestion- /google- /nb-etl- /remove-objects-from- /replicate- /s3sync- /fence-cleanup
+    /aws-bucket- /bucket- /covid19- /data-ingestion- /google- /nb-etl- /remove-objects-from- /replicate- /s3sync- /fence-cleanup /etl- /indexd- /metadata-
   )
   for exclude in "${excludeList[@]}"; do
     if [[ "$it" =~ $exclude ]]; then return 0; fi

gen3/test/jupyterTest.sh

@@ -30,7 +30,7 @@ test_jupyter_metrics() {
 }
 
 shunit_runtest "test_jupyter_idle" "jupyter"
-shunit_runtest "test_jupyter_metrics" "jupyter"
+# shunit_runtest "test_jupyter_metrics" "jupyter"
 shunit_runtest "test_jupyter_prepuller" "local,jupyter"
 shunit_runtest "test_jupyter_namespace" "local,jupyter"
 shunit_runtest "test_jupyter_setup" "jupyter"