From 9fe69841eb729fe7e8357bd39a12949e0f02bb5b Mon Sep 17 00:00:00 2001 From: Andris Raugulis Date: Wed, 5 Apr 2017 00:56:17 +0300 Subject: [PATCH] Integrate SonarQube analysis. --- .gitignore | 8 +-- .travis.yml | 2 + test/tools/ci-linux.sh | 154 ++++++++++++++++++++++++++++++++++++++++- tox.ini | 10 +-- 4 files changed, 164 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 0edf4f4..9dc68e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ *~ *.pyc -html/ -venv/ +venv*/ .cache/ .tox -.coverage -coverage.xml +.coverage* +reports/ +.scannerwork/ diff --git a/.travis.yml b/.travis.yml index b0cb261..08daa94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ matrix: python: 2.6 - os: linux python: 2.7 + env: SQ=1 - os: linux python: 3.3 - os: linux @@ -60,6 +61,7 @@ cache: - pip - directories: - $HOME/.pyenv.cache + - $HOME/.bin before_install: - source test/tools/ci-linux.sh diff --git a/test/tools/ci-linux.sh b/test/tools/ci-linux.sh index 9453adc..ba423e5 100755 --- a/test/tools/ci-linux.sh +++ b/test/tools/ci-linux.sh @@ -2,7 +2,8 @@ CI_VERBOSE=1 -ci_err() { [ $1 -ne 0 ] && echo "err: $2" >&2 && exit 1; } +ci_err_msg() { echo "[ci] error: $1" >&2; } +ci_err() { [ $1 -ne 0 ] && ci_err_msg "$2" && exit 1; } ci_is_osx() { [ X"$(uname -s)" == X"Darwin" ]; } ci_get_pypy_ver() { @@ -58,7 +59,7 @@ ci_get_py_env() { pypy|pypy2|pypy-*|pypy2-*) echo "pypy" ;; pypy3|pypy3*) echo "pypy3" ;; *) - local _v=$(echo "$1" | head -1 | sed -e 's/[^0-9]//g' | cut -c1-2;) + local _v=$(echo "$1" | head -1 | sed -e 's/[^0-9]//g' | cut -c1-2) echo "py${_v}" esac return 0 @@ -166,6 +167,152 @@ ci_venv_use() { return 0 } +ci_get_filedir() { + local _sdir=$(cd -- "$(dirname "$0")" && pwd) + local _pdir=$(pwd) + if [ -z "${_pdir##${_sdir}*}" ]; then + _sdir="${_pdir}" + fi + local _first=1 + while [ X"${_sdir}" != X"/" ]; do + if [ ${_first} -eq 1 ]; then + _first=0 + local _f=$(find "${_sdir}" -name "$1" | head -1) + if [ -n "${_f}" ]; then + echo $(dirname -- "${_f}") + return 0 + fi + else + _f=$(find "${_sdir}" -mindepth 1 -maxdepth 1 -name "$1" | head -1) + fi + [ -n "${_f}" ] && echo "${_sdir}" && return 0 + _sdir=$(cd -- "${_sdir}/.." && pwd) + done + return 1 +} + +ci_sq_ensure_java() { + type java >/dev/null 2>&1 + if [ $? -ne 0 ]; then + ci_err_msg "java not found" + return 1 + fi + local _java_ver=$(java -version 2>&1 | head -1 | sed -e 's/[^0-9\._]//g') + if [ -z "${_java_ver##1.8*}" ]; then + return 0 + fi + ci_err_msg "unsupported java version: ${_java_ver}" + return 1 +} + +ci_sq_ensure_scanner() { + local _cli_version="3.0.0.702" + local _cli_basedir="$HOME/.bin" + local _cli_postfix="" + case "$(uname -s)" in + Linux) + [ X"$(uname -m)" = X"x86_64" ] && _cli_postfix="-linux" + [ X"$(uname -m)" = X"amd64" ] && _cli_postfix="-linux" + ;; + Darwin) _cli_postfix="-macosx" ;; + esac + if [ X"${_cli_postfix}" = X"" ]; then + ci_sq_ensure_java || return 1 + fi + if [ X"${SONAR_SCANNER_PATH}" != X"" ]; then + if [ -e "${SONAR_SCANNER_PATH}" ]; then + return 0 + fi + fi + local _cli_fname="sonar-scanner-cli-${_cli_version}${_cli_postfix}" + [ ${CI_VERBOSE} -gt 0 ] && echo "[ci] ensure scanner ${_cli_fname}" + local _cli_dname="sonar-scanner-${_cli_version}${_cli_postfix}" + local _cli_archive="${_cli_basedir}/${_cli_fname}.zip" + local _cli_dir="${_cli_basedir}/${_cli_dname}" + local _cli_url="https://sonarsource.bintray.com/Distribution/sonar-scanner-cli/${_cli_fname}.zip" + if [ ! -e "${_cli_archive}" ]; then + mkdir -p -- "${_cli_basedir}" > /dev/null 2>&1 + if [ $? -ne 0 ]; then + ci_err_msg "could not create ${_cli_basedir}" + return 1 + fi + [ ${CI_VERBOSE} -gt 0 ] && echo "[ci] downloading ${_cli_fname}" + curl -kL -o "${_cli_archive}" "${_cli_url}" + [ $? -ne 0 ] && ci_err_msg "download failed" && return 1 + [ ! -e "${_cli_archive}" ] && ci_err_msg "download verify" && return 1 + fi + if [ ! -d "${_cli_dir}" ]; then + [ ${CI_VERBOSE} -gt 0 ] && echo "[ci] extracting ${_cli_fname}" + unzip -od "${_cli_basedir}" "${_cli_archive}" + [ $? -ne 0 ] && ci_err_msg "extract failed" && return 1 + [ ! -d "${_cli_dir}" ] && ci_err_msg "extract verify" && return 1 + fi + if [ ! -e "${_cli_dir}/bin/sonar-scanner" ]; then + ci_err_msg "sonar-scanner binary not found." + return 1 + fi + SONAR_SCANNER_PATH="${_cli_dir}/bin/sonar-scanner" + return 0 +} + +ci_sq_run() { + if [ X"${SONAR_SCANNER_PATH}" = X"" ]; then + ci_err_msg "environment variable SONAR_SCANNER_PATH not set" + return 1 + fi + if [ X"${SONAR_HOST_URL}" = X"" ]; then + ci_err_msg "environment variable SONAR_HOST_URL not set" + return 1 + fi + if [ X"${SONAR_AUTH_TOKEN}" = X"" ]; then + ci_err_msg "environment variable SONAR_AUTH_TOKEN not set" + return 1 + fi + local _pdir=$(ci_get_filedir "ssh-audit.py") + if [ -z "${_pdir}" ]; then + ci_err_msg "failed to find project directory" + return 1 + fi + local _odir=$(pwd) + cd -- "${_pdir}" + local _branch=$(git name-rev --name-only HEAD | cut -d '~' -f 1) + case "${_branch}" in + master) ;; + develop) ;; + *) ci_err_msg "unknown branch: ${_branch}"; return 1 ;; + esac + local _junit=$(cd -- "${_pdir}" && ls -1 reports/junit.*.xml | sort -r | head -1) + if [ X"${_junit}" = X"" ]; then + ci_err_msg "no junit.xml found" + return 1 + fi + local _project_ver=$(grep VERSION ssh-audit.py | head -1 | cut -d "'" -f 2) + if [ -z "${_project_ver}" ]; then + ci_err_msg "failed to get project version" + return 1 + fi + if [ -z "${_project_ver##*dev}" ]; then + local _git_rc=$(git rev-list --count `git rev-parse HEAD`) + _project_ver="${_project_ver}.${_git_rc}" + fi + [ ${CI_VERBOSE} -gt 0 ] && echo "[ci] run sonar-scanner for ${_project_ver}" + "${SONAR_SCANNER_PATH}" -X \ + -Dsonar.projectKey=arthepsy-github:ssh-audit \ + -Dsonar.sources=ssh-audit.py \ + -Dsonar.tests=test \ + -Dsonar.test.inclusions=test/*.py \ + -Dsonar.host.url="${SONAR_HOST_URL}" \ + -Dsonar.projectName=ssh-audit \ + -Dsonar.projectVersion="${_project_ver}" \ + -Dsonar.branch="${_branch}" \ + -Dsonar.python.coverage.overallReportPath=reports/coverage.xml \ + -Dsonar.python.xunit.reportPath="${_junit}" \ + -Dsonar.organization=arthepsy-github \ + -Dsonar.login="${SONAR_AUTH_TOKEN}" + cd -- "${_odir}" + return 0 +} + ci_run_wrapped() { local _versions=$(echo "${PY_VER}" | sed -e 's/,/ /g') [ -z "${_versions}" ] && eval "$1" @@ -247,6 +394,9 @@ ci_step_script_wrapped() { ci_step_success_wrapped() { local _py_ver="$1" local _py_ori="$2" + if [ X"${SQ}" = X"1" ]; then + ci_sq_ensure_scanner && ci_sq_run + fi ci_venv_use "${_py_ver}" || return 1 coveralls codecov diff --git a/tox.ini b/tox.ini index 9e5e3be..4bb7ef5 100644 --- a/tox.ini +++ b/tox.ini @@ -20,11 +20,12 @@ setenv = SSHAUDIT = {toxinidir}/ssh-audit.py test: COVERAGE_FILE = {toxinidir}/.coverage.{envname} type,mypy: MYPYPATH = {toxinidir}/test/stubs - type,mypy: MYPYHTML = {toxinidir}/html/mypy + type,mypy: MYPYHTML = {toxinidir}/reports/html/mypy commands = - test: coverage run --source ssh-audit -m -- pytest -v {posargs:test} + test: coverage run --source ssh-audit -m -- \ + test: pytest -v --junitxml={toxinidir}/reports/junit.{envname}.xml {posargs:test} test: coverage report --show-missing - test: - coverage html -d {toxinidir}/html/coverage.{envname} + test: coverage html -d {toxinidir}/reports/html/coverage.{envname} py{33,34,35,36,37}-{type,mypy}: {[testenv:mypy]commands} py{27,py,py3,33,34,35,36,37}-{lint,pylint}: {[testenv:pylint]commands} py{27,py,py3,33,34,35,36,37}-{lint,flake8}: {[testenv:flake8]commands} @@ -42,7 +43,8 @@ commands = coverage erase coverage combine coverage report --show-missing - - coverage html -d {toxinidir}/html/coverage + coverage xml -i -o {toxinidir}/reports/coverage.xml + coverage html -d {toxinidir}/reports/html/coverage [testenv:mypy] deps =