salt/.ci/lint

// Define the maximum time, in hours, that a test run should run for
def global_timeout = 3
def salt_target_branch = '2019.2'

properties([
    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
])

def shell_header

node('lint') {
    timeout(time: global_timeout, unit: 'HOURS') {
        ansiColor('xterm') {
            timestamps {
                try {
                    // Set the GH status even before cloning the repo
                    if (env.NODE_NAME.startsWith('jenkins-pr-')) {
                        stage('github-pending') {
                            githubNotify credentialsId: 'test-jenkins-credentials',
                                description: 'Python lint begins...',
                                status: 'PENDING',
                                context: "jenkins/pr/lint"
                        }
                        shell_header = 'export PYENV_ROOT="/usr/local/pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"'
                    } else {
                        shell_header = ''
                    }

                    withEnv(["SALT_TARGET_BRANCH=${salt_target_branch}"]) {
                        // Checkout the repo
                        stage('checkout-scm') {
                            cleanWs notFailBuild: true
                            checkout scm
                            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
                        }

                        // Setup the kitchen required bundle
                        stage('Setup') {
                            sh shell_header + '''
                            # Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M
                            # -M is on by default in git 2.9+
                            git diff --name-status -l99999 -C "origin/${SALT_TARGET_BRANCH}" > file-list-status.log
                            # the -l increase the search limit, lets use awk so we do not need to repeat the search above.
                            gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log
                            gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log
                            (git diff --name-status -l99999 -C "origin/${SALT_TARGET_BRANCH}" "origin/$BRANCH_NAME";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log
                            eval "$(pyenv init -)"
                            pyenv --version
                            pyenv install --skip-existing 2.7.15
                            pyenv shell 2.7.15
                            python --version
                            pip install tox
                            tox --version
                            '''
                        }
                        archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log'
                    }

                    stage('Lint Changes') {
                        try {
                            parallel(
                                lintSalt: {
                                    stage('Lint Salt Changes') {
                                        if (readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/) {
                                            sh shell_header + '''
                                            eval "$(pyenv init - --no-rehash)"
                                            pyenv shell 2.7.15
                                            # tee makes the exit/return code always 0
                                            grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | (xargs -r '--delimiter=\\n' tox -e pylint-salt ; echo "$?" >  pylint-salt-chg.exit) | tee pylint-report-salt-chg.log
                                            # remove color escape coding
                                            sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt-chg.log
                                            read rc_exit < pylint-salt-chg.exit
                                            exit "$rc_exit"
                                            '''
                                        }
                                    }
                                },
                                lintTests: {
                                    stage('Lint Test Changes') {
                                        if (readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/) {
                                            sh shell_header + '''
                                            eval "$(pyenv init - --no-rehash)"
                                            pyenv shell 2.7.15
                                            # tee makes the exit/return code always 0
                                            grep -Ei '^tests/.*\\.py$' file-list-changed.log | (xargs -r '--delimiter=\\n' tox -e pylint-tests ; echo "$?" > pylint-tests-chg.exit) | tee pylint-report-tests-chg.log
                                            # remove color escape coding
                                            sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests-chg.log
                                            read rc_exit < pylint-tests-chg.exit
                                            exit "$rc_exit"
                                            '''
                                        }
                                    }
                                }
                            )
                        } finally {
                            def changed_logs_pattern = 'pylint-report-*-chg.log'
                            archiveArtifacts artifacts: changed_logs_pattern, allowEmptyArchive: true
                            if (env.NODE_NAME.startsWith('jenkins-pr-')) {
                                step([$class: 'WarningsPublisher',
                                    parserConfigurations: [[
                                        parserName: 'PyLint',
                                        pattern: changed_logs_pattern
                                    ]],
                                    failedTotalAll: '0',
                                    useDeltaValues: false,
                                    canRunOnFailed: true,
                                    usePreviousBuildAsReference: true
                                ])
                            } else {
                                recordIssues(enabledForFailure: true, tool: pyLint(pattern: changed_logs_pattern, reportEncoding: 'UTF-8'))
                            }
                        }
                    }
                    stage('Lint Full') {
                        if (env.CHANGE_BRANCH =~ /(?i)^merge[._-]/) {
                            // perform a full linit if this is a merge forward and the change only lint passed.
                            try {
                                if (env.NODE_NAME.startsWith('jenkins-pr-')) {
                                    githubNotify credentialsId: 'test-jenkins-credentials',
                                        description: 'Python lint on everything begins...',
                                        status: 'PENDING',
                                        context: "jenkins/pr/lint"
                                }
                                parallel(
                                    lintSaltFull: {
                                        stage('Lint Salt Full') {
                                            sh shell_header + '''
                                            eval "$(pyenv init - --no-rehash)"
                                            pyenv shell 2.7.15
                                            # tee makes the exit/return code always 0
                                            (tox -e pylint-salt ; echo "$?" > pylint-salt-full.exit) | tee pylint-report-salt-full.log
                                            # remove color escape coding
                                            sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt-full.log
                                            read rc_exit <  pylint-salt-full.exit
                                            '''
                                        }
                                    },
                                    lintTestsFull: {
                                        stage('Lint Tests Full') {
                                            sh shell_header + '''
                                            eval "$(pyenv init - --no-rehash)"
                                            pyenv shell 2.7.15
                                            # tee makes the exit/return code always 0
                                            (tox -e pylint-tests ; echo "$?" > pylint-tests-full.exit) | tee pylint-report-tests-full.log
                                            # remove color escape coding
                                            sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests-full.log
                                            read rc_exit <  pylint-tests-full.exit
                                            exit "$rc_exit"
                                            '''
                                        }
                                    }
                                )
                            } finally {
                                def full_logs_pattern = 'pylint-report-*-full.log'
                                archiveArtifacts artifacts: full_logs_pattern, allowEmptyArchive: true
                                if (env.NODE_NAME.startsWith('jenkins-pr-')) {
                                    step([$class: 'WarningsPublisher',
                                        parserConfigurations: [[
                                            parserName: 'PyLint',
                                            pattern: full_logs_pattern
                                        ]],
                                        failedTotalAll: '0',
                                        useDeltaValues: false,
                                        canRunOnFailed: true,
                                        usePreviousBuildAsReference: true
                                    ])
                                } else {
                                    recordIssues(enabledForFailure: true, tool: pyLint(pattern: full_logs_pattern, reportEncoding: 'UTF-8'))
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    currentBuild.result = 'FAILURE'
                } finally {
                    cleanWs notFailBuild: true
                    if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
                        if (env.NODE_NAME.startsWith('jenkins-pr-')) {
                            githubNotify credentialsId: 'test-jenkins-credentials',
                                description: 'The lint test passed',
                                status: 'SUCCESS',
                                context: "jenkins/pr/lint"
                        }
                    } else {
                        if (env.NODE_NAME.startsWith('jenkins-pr-')) {
                            githubNotify credentialsId: 'test-jenkins-credentials',
                                description: 'The lint test failed',
                                status: 'FAILURE',
                                context: "jenkins/pr/lint"
                        }
                        try {
                            slackSend channel: "#jenkins-prod-pr",
                                color: '#FF0000',
                                message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})"
                        } catch (Exception e) {
                            sh 'echo Failed to send the Slack notification'
                        }
                    }
                }
            }
        }
    }
}

// vim: ft=groovy