Jenkinsfile.okeext

// Copyright (c) 2022, 2023, Oracle and/or its affiliates.
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
//

pipeline {
    agent { label 'VM.Standard2.8' }
    options {
        timeout(time: 1800, unit: 'MINUTES')
        disableConcurrentBuilds()
    }

    tools {
        maven 'maven-3.8.7'
        jdk 'jdk21'
    }

    environment {
        github_url = "${env.GIT_URL}"
        github_creds = 'ecnj_github'
        ocr_username_creds = 'OCR username'
        ocr_password_creds = 'OCR Password'
        ocir_registry_creds = 'ocir-server'
        ocir_email_creds = 'ocir-email'
        ocir_username_creds = 'ocir-username'
        ocir_password_creds = 'ocir-token'
        image_pull_secret_weblogic_creds = 'image-pull-secret-weblogic'
        terraform_property_file = 'tvarsFile'
        ocipk_file = 'oci_pk_file'
        wkobuild_oci_config = 'wkobuild_profile_oci_config_file'
        wkobuild_cert_fingerprint = 'wkobuildcert_fingerprint'
        weblogick8s_tenancy_ocid = 'WK8_TENANCY_OCID'
        weblogick8s_qa_compartment_ocid = 'WK8_QA_COMP_OCID'
        wkobuild_user_ocid = 'WKOBUILD_USER_OCID'
        wko_oke_test_sshpubcert = 'OKE_PUB_CERT'
        oke_test_ssh_pk = 'oke_test_ssh_pk'

        OKE_CLUSTER='true'

        outdir = "${WORKSPACE}/staging"
        result_root = "${outdir}/wl_k8s_test_results"
        pv_root = "${outdir}/k8s-pvroot"
        oci_config_file = credentials("${wkobuild_oci_config}")
        OCI_PROP_FILE="${WORKSPACE}/terraform/oci.prop"

        start_time = sh(script: 'date +"%Y-%m-%d %H:%M:%S"', returnStdout: true).trim()
        wle_download_url="https://github.com/oracle/weblogic-logging-exporter/releases/latest"

        CLUSTER_NAME = "${env.JOB_NAME == 'wko-oke-nightly-parallel' ? 'parcluster' : 'seqcluster'}"
        kubeconfig_file = "${WORKSPACE}/terraform/${CLUSTER_NAME}_kubeconfig"
        MAVEN_PROFILE_NAME = "${env.JOB_NAME == 'wko-oke-nightly-parallel' ? 'oke-parallel' : 'oke-sequential'}"
        availability_domain = "${env.JOB_NAME == 'wko-oke-nightly-parallel' ? 'VPGL:PHX-AD-3' : 'VPGL:PHX-AD-2'}"
        PARALLEL_RUN = "${env.JOB_NAME == 'wko-oke-nightly-parallel' ? true : false}"
    }

    parameters {

        string(name: 'IT_TEST',
               description: 'Comma separated list of individual It test classes to be run e.g., ItParameterizedDomain, ItMiiUpdateDomainConfig, ItMiiDynamicUpdate*, ItMiiMultiMode',
               defaultValue: ''
        )

        string(name: 'OPERATOR_LOG_LEVEL',
               description: 'The default log level is not set',
               defaultValue: ''
        )	

        string(name: 'KUBE_VERSION',
               description: 'kube version for oke cluster',
               defaultValue: '1.28.2'
        )
        string(name: 'IMAGE_ID',
                       description: 'oci image id for node pool, find image OCID for your region from https://docs.oracle.com/iaas/images/',
                       //defaultValue7.6: 'ocid1.image.oc1.phx.aaaaaaaancucg25pze6m52agxwxmmfszvsdnhp3kvim53pts2tw6mik4xtha'
                       //oke1.27.2 defaultValue: 'ocid1.image.oc1.phx.aaaaaaaaypr5r5drojwytghw6e6mvpjsscrnkuwtmqlmvmix7kjb2zcnc7wa'
                       defaultValue: 'ocid1.image.oc1.phx.aaaaaaaajqv2odwgofwohn2chxfytl4uiupz2filik5e2rrepe3izq26pyrq'
                )
        string(name: 'KUBECTL_VERSION',
                       description: 'kubectl version',
                       defaultValue: '1.26.2'
                )
        string(name: 'HELM_VERSION',
               description: 'Helm version',
               defaultValue: '3.11.2'
        )
        choice(name: 'ISTIO_VERSION',
               description: 'Istio version',
               choices: [
	           '1.23.0',
                   '1.17.2',
                   '1.16.1',
                   '1.13.2',
                   '1.12.6',
                   '1.11.1',
                   '1.10.4',
                   '1.9.9'
               ]
        )

        string(name: 'NUMBER_OF_THREADS',
               description: 'Number of threads to run the classes in parallel, default is 3.',
               defaultValue: "3"
        )
        string(name: 'WDT_DOWNLOAD_URL',
               description: 'URL to download WDT.',
               defaultValue: 'https://objectstorage.us-phoenix-1.oraclecloud.com/n/weblogick8s/b/wko-system-test-files/o/weblogic-deploy-main.zip'
        )
        string(name: 'WIT_DOWNLOAD_URL',
               description: 'URL to download WIT.',
               defaultValue: 'https://objectstorage.us-phoenix-1.oraclecloud.com/n/weblogick8s/b/wko-system-test-files/o/imagetool-main.zip'
        )
        string(name: 'REPO_REGISTRY',
               description: '',
               defaultValue: 'phx.ocir.io'
        )
        choice(name: 'BASE_IMAGES_REPO',
               choices: ['phx.ocir.io', 'container-registry.oracle.com'],
               description: 'Repository to pull the base images. Make sure to modify the image names if you are modifying this parameter value.'
        )
        string(name: 'TEST_IMAGES_REPO',
               description: '',
               defaultValue: 'phx.ocir.io'
        )
        string(name: 'NODE_SHAPE',
                       description: '',
                       defaultValue: 'VM.Standard.E3.Flex'
                )
        string(name: 'BRANCH',
                       description: '',
                       defaultValue: 'main'
                )
        string(name: 'WEBLOGIC_IMAGE_NAME',
               description: 'WebLogic base image name. Default is the image name in OCIR. Use middleware/weblogic for OCR.',
               defaultValue: 'test-images/weblogic'
        )
        string(name: 'WEBLOGIC_IMAGE_TAG',
               description: '12.2.1.4,  12.2.1.4-dev(12.2.1.4-dev-ol7) , 12.2.1.4-slim(12.2.1.4-slim-ol7), 12.2.1.4-ol8, 12.2.1.4-dev-ol8, 12.2.1.4-slim-ol8, 14.1.1.0-11-ol7, 14.1.1.0-dev-11-ol7, 14.1.1.0-slim-11-ol7, 14.1.1.0-8-ol7, 14.1.1.0-dev-8-ol7, 14.1.1.0-slim-8-ol7, 14.1.1.0-11-ol8, 14.1.1.0-dev-11-ol8, 14.1.1.0-slim-11-ol8, 14.1.1.0-8-ol8, 14.1.1.0-dev-8-ol8, 14.1.1.0-slim-8-ol8',
               defaultValue: '12.2.1.4'
        )
        string(name: 'FMWINFRA_IMAGE_NAME',
               description: 'FWM Infra image name. Default is the image name in OCIR. Use middleware/fmw-infrastructure for OCR.',
               defaultValue: 'test-images/fmw-infrastructure'
        )
        string(name: 'FMWINFRA_IMAGE_TAG',
               description: 'FWM Infra image tag',
               defaultValue: '12.2.1.4'
        )
        string(name: 'DB_IMAGE_NAME',
               description: 'Oracle DB image name. Default is the image name in OCIR, use database/enterprise for OCR.',
               defaultValue: 'test-images/database/enterprise'
        )
        string(name: 'DB_IMAGE_TAG',
               description: 'Oracle DB image tag',
               defaultValue: '12.2.0.1-slim'
        )
        string(name: 'MONITORING_EXPORTER_BRANCH',
               description: '',
               defaultValue: 'main'
        )
        string(name: 'MONITORING_EXPORTER_WEBAPP_VERSION',
               description: '',
               defaultValue: '2.1.3'
        )
        string(name: 'PROMETHEUS_CHART_VERSION',
               description: '',
               defaultValue: '17.0.0'
        )
        string(name: 'GRAFANA_CHART_VERSION',
               description: '',
               defaultValue: '6.44.11'
        )
        booleanParam(name: 'COLLECT_LOGS_ON_SUCCESS',
                     description: 'Collect logs for successful runs. Default is false.',
                     defaultValue: false
        )
        string(name: 'REMOTECONSOLE_VERSION',
               description: 'RemoteConsole version.',
               defaultValue: '2.4.7'
        )
    }


            stages {
                stage('Workaround JENKINS-41929 Parameters bug') {
                    steps {
                        echo 'Initialize parameters as environment variables due to https://issues.jenkins-ci.org/browse/JENKINS-41929'
                        evaluate """${def script = ""; params.each { k, v -> script += "env.${k} = '''${v}'''\n" }; return script}"""
                    }
                }
                stage ('Echo environment') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                    }
                    steps {
                        sh '''
                            env|sort
                            java -version
                            mvn --version
                            python --version
                            docker version
                            ulimit -a
                            ulimit -aH
                        '''
                    }
                }

                // Use explicit checkout so that we can clean up the workspace.
                // Cannot use skipDefaultCheckout option since we want to use
                // the GIT_COMMIT environment variable to make sure that we are
                // checking out the exactly commit that triggered the build.
                //
                stage('GitHub Checkout') {
                    steps {
                        sh "sudo rm -rf ${WORKSPACE}/*"
                        sh "sudo rm -rf ${WORKSPACE}/.mvn"
                        checkout([$class: 'GitSCM', branches: [[name: "${GIT_COMMIT}"]],
                                  doGenerateSubmoduleConfigurations: false,
                                  extensions: [], submoduleCfg: [],
                                  userRemoteConfigs: [[credentialsId: "${github_creds}", url: "${github_url}"]]])
                    }
                }

                // Creating terraform property file.

                stage('Setup terraform property file  ') {
                    environment {
                        OKE_CLUSTER_NAME="${CLUSTER_NAME}"
                        NODE_SHAPE="${NODE_SHAPE}"
                        IMAGE_ID="${IMAGE_ID}"
                        OCI_PROP_FILE="${WORKSPACE}/terraform/oci.prop"
                        TVARS_FILE = credentials("${terraform_property_file}")
                        OCIPK_FILE = credentials("${ocipk_file}")
                        FINGERPRINT = credentials("${wkobuild_cert_fingerprint}")
                        TENANCY_OCID = credentials("${weblogick8s_tenancy_ocid}")
                        COMPARTMENT_OCID = credentials("${weblogick8s_qa_compartment_ocid}")
                        USER_OCID = credentials("${wkobuild_user_ocid}")
                        WKOBUILD_PUBLIC_CERT = credentials("${wko_oke_test_sshpubcert}")
                    }
                    steps {
                        withCredentials([file(credentialsId: 'oci_pk_file', variable: 'oci_private_key')]) {
                                                sh "mkdir -p ${WORKSPACE}/terraform"
                                                sh "cp -rf \$oci_private_key ${WORKSPACE}/terraform/oci_api_key.pem"
                                                sh " ls -al ${WORKSPACE}/terraform/*"
                                                sh "chmod 666 ${WORKSPACE}/terraform/oci_api_key.pem"

                        }

                        sh '''

                        export OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING=True

                        echo 'Set env vars needed for integration tests...'


                        export OKE_CLUSTER="true"
                        if [ ${MAVEN_PROFILE_NAME} = 'oke-parallel' ]; then
                           VCN_CIDR_PREFIX="10.4"
                        else
                           VCN_CIDR_PREFIX="10.3"
                        fi
                        export VCN_CIDR_PREFIX=${VCN_CIDR_PREFIX}

                        export OKE_CLUSTER_NAME=${OKE_CLUSTER_NAME}
                        export NODE_SHAPE=${NODE_SHAPE}
                        export IMAGE_ID=${IMAGE_ID}

                        export OCI_PROP_FILE=${OCI_PROP_FILE}

                        #export IMAGE_TAG_WEBLOGIC="12.2.1.4"
                        #export IMAGE_TAG_FMWINFRA="12.2.1.4"
                        ssh_pubkey=`cat ${WKOBUILD_PUBLIC_CERT}`

                        #################
                        echo "Generating property file oci.prop for terraform scripts"

                        cat > $OCI_PROP_FILE <<EOF

                        user.ocid=${USER_OCID}
                        okeclustername=${OKE_CLUSTER_NAME}
                        tfvars.filename=${OKE_CLUSTER_NAME}
                        region=us-phoenix-1
                        tenancy.ocid=${TENANCY_OCID}
                        compartment.ocid=${COMPARTMENT_OCID}
                        compartment.name=QualityAssurance
                        ociapi.pubkey.fingerprint=${FINGERPRINT}
                        ocipk.path=${WORKSPACE}/terraform/oci_api_key.pem
                        vcn.cidr.prefix=${VCN_CIDR_PREFIX}
                        vcn.cidr=${VCN_CIDR_PREFIX}.0.0/16
                        nodepool.shape=${NODE_SHAPE}
                        nodepool.imagename=${IMAGE_ID}
                        k8s.version=v${KUBE_VERSION}
                        nodepool.ssh.pubkey=${ssh_pubkey}
                        terraform.installdir=${WORKSPACE}/terraform/terraforminstall
EOF

                        ##################

                        cp -rf ${WORKSPACE}/kubernetes/samples/scripts/terraform/template.tfvars ${WORKSPACE}/terraform/.
                        mkdir -p ${WORKSPACE}/terraform/terraforminstall
                        '''

                    }
                }

                stage('Build WebLogic Kubernetes Operator') {
                    steps {
                        sh "mvn -DtrimStackTrace=false clean install"
                    }
                }

                stage('Make Workspace bin directory') {
                    steps {
                        sh "mkdir -m777 -p ${WORKSPACE}/bin"
                    }
                }
                stage ('Install oci cli') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                    }
                    steps {
                        sh '''
                            export PATH=${runtime_path}
                            mkdir -p ${WORKSPACE}/oci
                            cd ${WORKSPACE}/oci
                            curl -LO https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh
                            chmod +x ${WORKSPACE}/oci/install.sh
                            ${WORKSPACE}/oci/install.sh --install-dir ${PWD} --exec-dir  ${WORKSPACE}/bin --script-dir ${PWD} --optional-features n --accept-all-defaults
                            oci --version
                        '''
                    }
                }
                stage('Install Helm') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                    }
                    steps {
                        sh '''
                            export PATH=${runtime_path}
                            curl -Lo "helm.tar.gz" "https://objectstorage.us-phoenix-1.oraclecloud.com/n/weblogick8s/b/wko-system-test-files/o/helm%2Fhelm-v${HELM_VERSION}.tar.gz"
                            tar zxf helm.tar.gz
                            mv linux-amd64/helm ${WORKSPACE}/bin/helm
                            rm -rf linux-amd64
                            helm version
                        '''
                    }
                }

                stage ('Install kubectl') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                    }
                    steps {
                        sh '''
                            export PATH=${runtime_path}
                            curl -Lo "${WORKSPACE}/bin/kubectl" "https://objectstorage.us-phoenix-1.oraclecloud.com/n/weblogick8s/b/wko-system-test-files/o/kubectl%2Fkubectl-v${KUBECTL_VERSION}"
                            chmod +x ${WORKSPACE}/bin/kubectl
                            kubectl version --client=true
                        '''
                    }
                }

                stage('Preparing Integration Test Environment') {
                    steps {
                        sh 'mkdir -m777 -p ${result_root}'
                        echo "Results will be in ${result_root}"
                        sh 'mkdir -m777 -p ${pv_root}'
                        echo "Persistent volume files, if any, will be in ${pv_root}"
                    }
                }

                stage('Create OKE cluster') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                        oci_config_file = credentials("${wkobuild_oci_config}")
                        OCI_PROP_FILE="${WORKSPACE}/terraform/oci.prop"
                        CLUSTER_NAME="${CLUSTER_NAME}"
                        KUBECONFIG ="${WORKSPACE}/terraform/${CLUSTER_NAME}_kubeconfig"
                        terraform_script_dir_name = "${WORKSPACE}/integration-tests/src/test/resources/oke/terraform/"

                    }
                    steps {
                        sh '''
                            export PATH=${runtime_path}
                            export KUBECONFIG=${KUBECONFIG}
                            mkdir -p ${WORKSPACE}/oci
                            cp -rf ${oci_config_file} ${WORKSPACE}/oci/config
                            chmod 777 ${WORKSPACE}/oci/config
                            sed -i -e "s:@WORKSPACE@:${WORKSPACE}:g" ${WORKSPACE}/oci/config
                            export OCI_CLI_CONFIG_FILE=${WORKSPACE}/oci/config
                            export OCI_CLI_PROFILE=WKOBUILD
                            echo 'Create a OKE cluster ${CLUSTER_NAME}'

                            cp -rf ${terraform_script_dir_name}/*.tf ${WORKSPACE}/terraform/.
                            if [ "${MAVEN_PROFILE_NAME}" = 'oke-parallel' ]; then
                                cp -rf ${terraform_script_dir_name}/twoclusters/mount_target.tf ${WORKSPACE}/terraform/.
                                cp -rf ${terraform_script_dir_name}/twoclusters/file_system.tf ${WORKSPACE}/terraform/.
                            fi
                            cp -rf ${WORKSPACE}/kubernetes/samples/scripts/terraform/template.tfvars ${WORKSPACE}/terraform/.
                            cp -rf ${WORKSPACE}/kubernetes/samples/scripts/terraform/*.sh ${WORKSPACE}/terraform/.
                            chmod 777 ${WORKSPACE}/terraform/*.sh
                            mkdir -p ${WORKSPACE}/terraform/terraforminstall
                            sh ${WORKSPACE}/terraform/oke.create.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform
			    export SERVICE_ACCOUNT_NAME=${CLUSTER_NAME}-sa
    			    ${KUBERNETES_CLI:-kubectl} -n kube-system create serviceaccount ${SERVICE_ACCOUNT_NAME}
                            ${KUBERNETES_CLI:-kubectl} create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:${SERVICE_ACCOUNT_NAME}
			    sed -i 's@%SERVICE_ACCOUNT_NAME%@'"${SERVICE_ACCOUNT_NAME}"'@' ${WORKSPACE}/integration-tests/src/test/resources/oke/secret.yaml
			    ${KUBERNETES_CLI:-kubectl} apply -f ${WORKSPACE}/integration-tests/src/test/resources/oke/secret.yaml

    			    TOKEN=`${KUBERNETES_CLI:-kubectl} -n kube-system get secret secret-${SERVICE_ACCOUNT_NAME} -o jsonpath='{.data.token}'| base64 --decode`
                            ${KUBERNETES_CLI:-kubectl} config set-credentials ${SERVICE_ACCOUNT_NAME} --token=$TOKEN
                            ${KUBERNETES_CLI:-kubectl} config set-context --current --user=${SERVICE_ACCOUNT_NAME}
                        '''
                    }
                }

                 stage('Setup OFSS') {
                                    environment {
                                        runtime_path = "${WORKSPACE}/bin:${PATH}:${WORKSPACE}/terraform/terraforminstall"
                                        OCI_PROP_FILE = "${WORKSPACE}/terraform/oci.prop"
                                        compartment_ocid = credentials("${weblogick8s_qa_compartment_ocid}")
                                        clusterName = "${CLUSTER_NAME}"
                                        terraform_script_dir_name = "${WORKSPACE}/integration-tests/src/test/resources/oke/terraform/"


                                    }
                                    steps {
                                        sh '''
                                        export KUBECONFIG=${kubeconfig_file}
                                        export OCI_CLI_CONFIG_FILE=${WORKSPACE}/oci/config
                                        export OCI_CLI_PROFILE=WKOBUILD
                                        export PATH=${runtime_path}

                                        echo "creating storage class to setup OFSS ..."
                                        echo "getting MountTarget ID"
                                        mount_target_id=`oci fs mount-target  list --compartment-id=${compartment_ocid}  --display-name=${clusterName}-mt --availability-domain=${availability_domain} | jq -r '.data[] | .id'`

                                        echo "Creating Storage Class to mount OFSS"
					export MOUNT_TARGET_ID=${mount_target_id}
					export AVAILABILITY_DOMAIN=${availability_domain}
					sed -i 's@%MOUNT_TARGET_ID%@'"$MOUNT_TARGET_ID"'@' ${WORKSPACE}/integration-tests/src/test/resources/oke/ofssnew.yaml
					sed -i 's@%AVAILABILITY_DOMAIN%@'"$AVAILABILITY_DOMAIN"'@' ${WORKSPACE}/integration-tests/src/test/resources/oke/ofssnew.yaml
					kubectl apply -f ${WORKSPACE}/integration-tests/src/test/resources/oke/ofssnew.yaml
					kubectl delete csiDriver fss.csi.oraclecloud.com
					kubectl create -f ${WORKSPACE}/integration-tests/src/test/resources/oke/fss_csi_driver.yaml

                                        '''
                                    }
                                }

                stage('Run integration tests') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                        IMAGE_PULL_SECRET_WEBLOGIC = credentials("${image_pull_secret_weblogic_creds}")
                        OCR_USERNAME = credentials("${ocr_username_creds}")
                        OCR_PASSWORD = credentials("${ocr_password_creds}")
                        OCR_EMAIL = credentials("${ocr_username_creds}")
                        OCIR_REGISTRY = credentials("${ocir_registry_creds}")
                        OCIR_USERNAME = credentials("${ocir_username_creds}")
                        OCIR_PASSWORD = credentials("${ocir_password_creds}")
                        OCIR_EMAIL = credentials("${ocir_email_creds}")
                        OKE_SSH_PRIVATE_KEY = credentials("${oke_test_ssh_pk}")
                        BASE_IMAGES_REPO_USERNAME = credentials("${ocir_username_creds}")
                        BASE_IMAGES_REPO_PASSWORD = credentials("${ocir_password_creds}")
                        BASE_IMAGES_REPO_EMAIL = credentials("${ocir_email_creds}")
                        TEST_IMAGES_REPO_USERNAME = credentials("${ocir_username_creds}")
                        TEST_IMAGES_REPO_PASSWORD = credentials("${ocir_password_creds}")
                        TEST_IMAGES_REPO_EMAIL = credentials("${ocir_email_creds}")
                        compartment_ocid = credentials("${weblogick8s_qa_compartment_ocid}")
                        DOMAIN_IMAGES_REPO = "phx.ocir.io"
                        BASE_IMAGES_REPO = "phx.ocir.io"
                        REPO_REGISTRY = "phx.ocir.io"
                        clusterName = "${CLUSTER_NAME}"
                        FSS_DIR = "/oketest1,/oketest2,/oketest3,/oketest4,/oketest5,/oketest6,/oketest7,/oketest8,/oketest9,/oketest10,/oketest11,/oketest12,/oketest13,/oketest14,/oketest15"

                    }
                    steps {
                        sh '''
                            export PATH=${runtime_path}
                            export OCI_CLI_CONFIG_FILE=${WORKSPACE}/oci/config
                            export OCI_CLI_PROFILE=WKOBUILD
                            mkdir -m777 -p "${WORKSPACE}/.mvn"
                            cp ${OKE_SSH_PRIVATE_KEY} ${WORKSPACE}/rsaokepk
                            chmod 600 ${WORKSPACE}/rsaokepk
                            touch ${WORKSPACE}/.mvn/maven.config
                            export KUBECONFIG=${kubeconfig_file}
                            NODE_IP=`kubectl get nodes -o wide| awk '{print $7}'| tail -n+3`
                            if [ -z "${NODE_IP}" ]; then
                                    echo "retry get node ip ";
                              sleep 15;
                              NODE_IP=`kubectl get nodes -o wide| awk '{print $7}'| tail -n+3`
                            fi

                            export K8S_NODEPORT_HOST=${NODE_IP}
                            NODE_IP=`kubectl get nodes -o wide| awk '{print $7}'| head -n2 | tail -n1`
                            echo "second node external IP ${NODE_IP}"
                            export NODE_IP=${NODE_IP}
                            echo "expand disk volume for nodes"
                            ssh -o StrictHostKeyChecking=no -i ${WORKSPACE}/rsaokepk opc@${K8S_NODEPORT_HOST} 'sudo /usr/libexec/oci-growfs -y'
                            ssh -o StrictHostKeyChecking=no -i ${WORKSPACE}/rsaokepk opc@${NODE_IP} 'sudo /usr/libexec/oci-growfs -y'
                            mt_privateip_id=`oci fs mount-target  list --compartment-id=${compartment_ocid}  --display-name=${clusterName}-mt --availability-domain=${availability_domain} | jq -r '.data[] | ."private-ip-ids"[]'`
                           
                            mt_private_ip=`oci network private-ip get --private-ip-id "${mt_privateip_id}" | jq -r '.data | ."ip-address"'`

                            if [ -z "${mt_private_ip}" ]; then
                               echo "Mount Target was not setup properly , clean up Kubernetes cluster"
                               sh ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform
                               exit -1
                            fi
                            export NFS_SERVER=$mt_private_ip
                            export FSS_DIR=${FSS_DIR}
                            echo "Using NFS Server ${NFS_SERVER}"
                            export NO_PROXY="${K8S_NODEPORT_HOST}"

                            if [ "${IT_TEST}" = '**/It*' ] && [ "${MAVEN_PROFILE_NAME}" = "integration-tests" ]; then
                                echo "-Dit.test=\"${IT_TEST}, !ItExternalRmiTunneling, !ItMiiSampleWlsMain, !ItSamples, !ItLBTwoDomainsApache, !ItLBTwoDomainsNginx, !ItLBTwoDomainsTraefik, !ItTwoDomainsManagedByTwoOperators, !ItMiiSample, !ItMonitoringExporter*, !ItAuxV8DomainImplicitUpgrade, !ItOperatorWlsUpgrade, !ItFmwDomainInPVUsingWDT, !ItFmwDynamicDomainInPV, !ItDedicatedMode, !ItT3Channel, !ItOperatorFmwUpgrade, !ItMiiSampleFmwMain\"" >> ${WORKSPACE}/.mvn/maven.config
                            elif [ ! -z "${IT_TEST}" ]; then
                                echo "-Dit.test=\"${IT_TEST}\"" >> ${WORKSPACE}/.mvn/maven.config
                            fi
                            echo "-Dwko.it.wle.download.url=\"${wle_download_url}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.result.root=\"${result_root}\""                                               >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.pv.root=\"${pv_root}\""                                                       >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.k8s.nodeport.host=\"${K8S_NODEPORT_HOST}\""                                   >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.istio.version=\"${ISTIO_VERSION}\""                                           >> ${WORKSPACE}/.mvn/maven.config
                            echo "-DPARALLEL_CLASSES=\"${PARALLEL_RUN}\""                                                >> ${WORKSPACE}/.mvn/maven.config
                            echo "-DNUMBER_OF_THREADS=\"${NUMBER_OF_THREADS}\""                                          >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.wdt.download.url=\"${WDT_DOWNLOAD_URL}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.wit.download.url=\"${WIT_DOWNLOAD_URL}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.repo.registry=\"${REPO_REGISTRY}\""                                           >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.base.images.repo=\"${BASE_IMAGES_REPO}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.weblogic.image.name=\"${WEBLOGIC_IMAGE_NAME}\""                               >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.weblogic.image.tag=\"${WEBLOGIC_IMAGE_TAG}\""                                 >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.fmwinfra.image.name=\"${FMWINFRA_IMAGE_NAME}\""                               >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.fmwinfra.image.tag=\"${FMWINFRA_IMAGE_TAG}\""                                 >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.db.image.name=\"${DB_IMAGE_NAME}\""                                           >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.db.image.tag=\"${DB_IMAGE_TAG}\""                                             >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.test.images.repo=\"${TEST_IMAGES_REPO}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.monitoring.exporter.branch=\"${MONITORING_EXPORTER_BRANCH}\""                 >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.monitoring.exporter.webapp.version=\"${MONITORING_EXPORTER_WEBAPP_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.prometheus.chart.version=\"${PROMETHEUS_CHART_VERSION}\""                     >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.grafana.chart.version=\"${GRAFANA_CHART_VERSION}\""                           >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.collect.logs.on.success=\"${COLLECT_LOGS_ON_SUCCESS}\""                       >> ${WORKSPACE}/.mvn/maven.config
                            echo "-Dwko.it.remoteconsole.version=\"${REMOTECONSOLE_VERSION}\""                           >> ${WORKSPACE}/.mvn/maven.config
			    echo "-DOPERATOR_LOG_LEVEL=\"${OPERATOR_LOG_LEVEL}\""                                        >> ${WORKSPACE}/.mvn/maven.config

                            echo "${WORKSPACE}/.mvn/maven.config contents:"
                            cat "${WORKSPACE}/.mvn/maven.config"
                            cp "${WORKSPACE}/.mvn/maven.config" "${result_root}"

                            export OCR_USERNAME=${OCR_USERNAME}
                            export OCR_PASSWORD=${OCR_PASSWORD}
                            export OCR_EMAIL=${OCR_EMAIL}
                            export OCIR_USERNAME=${OCIR_USERNAME}
                            export OCIR_PASSWORD=${OCIR_PASSWORD}
                            export OCIR_EMAIL=${OCIR_EMAIL}
                            export DOMAIN_IMAGES_REPO=${DOMAIN_IMAGES_REPO}
                            export BASE_IMAGES_REPO=${BASE_IMAGES_REPO}
                            export OKE_CLUSTER="true"
                            export OKD="false"
                            export BASE_IMAGES_REPO_USERNAME=${BASE_IMAGES_REPO_USERNAME}
                            export BASE_IMAGES_REPO_PASSWORD=${BASE_IMAGES_REPO_PASSWORD}
                            export BASE_IMAGES_REPO_EMAIL=${BASE_IMAGES_REPO_EMAIL}


                            if [ ! -z "${http_proxy}" ]; then
                                export http_proxy
                            elif [ ! -z "${HTTP_PROXY}" ]; then
                                export HTTP_PROXY
                            fi

                            if [ ! -z "${https_proxy}" ]; then
                                export https_proxy
                            elif [ ! -z "${HTTPS_PROXY}" ]; then
                                export HTTPS_PROXY
                            fi

                            if [ ! -z "${no_proxy}" ]; then
                                export no_proxy
                            elif [ ! -z "${NO_PROXY}" ]; then
                                export NO_PROXY
                            fi

                            if ! mvn -pl integration-tests -P ${MAVEN_PROFILE_NAME} verify 2>&1 | tee "${result_root}/oketest.log"; then
                                echo "integration-tests failed"
                            fi
                        '''
                    }
                    post {
                        always {
                            sh '''
                                export PATH="${WORKSPACE}/bin:${PATH}"
                                export KUBECONFIG=${kubeconfig_file}
                                export OCI_CLI_CONFIG_FILE=${WORKSPACE}/oci/config
                                export OCI_CLI_PROFILE=WKOBUILD
                                mkdir -m777 -p ${result_root}/kubelogs

                                mkdir -m777 -p "${WORKSPACE}/logdir/${BUILD_TAG}/wl_k8s_test_results"
                                sudo mv -f ${result_root}/* "${WORKSPACE}/logdir/${BUILD_TAG}/wl_k8s_test_results"
                                ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform
                            '''
                            archiveArtifacts(artifacts:
                            "logdir/${BUILD_TAG}/wl_k8s_test_results/diagnostics/**/*,logdir/${BUILD_TAG}/wl_k8s_test_results/workdir/liftandshiftworkdir/**/*")

                            junit(testResults: 'integration-tests/target/failsafe-reports/*.xml', allowEmptyResults: true)
                        }
                    }
                }
            }
            post {
                always {
                    sh '''
                        export PATH="${WORKSPACE}/bin:${PATH}"
                        export KUBECONFIG=${kubeconfig_file}
                        export OCI_CLI_CONFIG_FILE=${WORKSPACE}/oci/config
                        export OCI_CLI_PROFILE=WKOBUILD
                        echo 'Remove old OKE cluster (if any)...'

                        if [ -f "$OCI_PROP_FILE" ] && [ -f "${WORKSPACE}/terraform/oke.delete.sh" ]; then
                            ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform
                        fi

                    '''
                }
            }
}