From 328f076d2639d8664436e2b620cf9eed4f7dee53 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 4 May 2017 15:22:14 +0200 Subject: [PATCH] Add performance benchmarking scripts And run them by Travis-CI --- .travis.yml | 4 +- tests/performance/compare_perfs.py | 120 ++++++++++++++++++++ tests/performance/perf_test.py | 137 +++++++++++++++++++++++ tests/performance/perf_test_filelist.csv | 7 ++ tools/travis-ci/run.sh | 45 +++++++- 5 files changed, 310 insertions(+), 3 deletions(-) create mode 100755 tests/performance/compare_perfs.py create mode 100755 tests/performance/perf_test.py create mode 100644 tests/performance/perf_test_filelist.csv diff --git a/.travis.yml b/.travis.yml index f0ac2d7c..e4c9e89f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ matrix: env: OPJ_CI_ARCH=x86_64 OPJ_CI_BUILD_CONFIGURATION=Release OPJ_CI_INCLUDE_IF_DEPLOY=1 - os: linux compiler: gcc - env: OPJ_CI_ARCH=x86_64 OPJ_CI_BUILD_CONFIGURATION=Release OPJ_CI_INCLUDE_IF_DEPLOY=1 + env: OPJ_CI_ARCH=x86_64 OPJ_CI_BUILD_CONFIGURATION=Release OPJ_CI_INCLUDE_IF_DEPLOY=1 OPJ_CI_PERF_TESTS=1 - os: linux compiler: gcc env: OPJ_CI_ARCH=x86_64 OPJ_CI_BUILD_CONFIGURATION=Release OPJ_NUM_THREADS=2 @@ -26,7 +26,7 @@ matrix: env: OPJ_CI_ARCH=x86_64 OPJ_CI_BUILD_CONFIGURATION=Debug OPJ_CI_ASAN=1 - os: linux compiler: clang-3.8 - env: OPJ_CI_ARCH=x86_64 OPJ_CI_BUILD_CONFIGURATION=Release + env: OPJ_CI_ARCH=x86_64 OPJ_CI_BUILD_CONFIGURATION=Release OPJ_CI_PERF_TESTS=1 addons: apt: sources: diff --git a/tests/performance/compare_perfs.py b/tests/performance/compare_perfs.py new file mode 100755 index 00000000..db142bbd --- /dev/null +++ b/tests/performance/compare_perfs.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017, IntoPIX SA +# Contact: support@intopix.com +# Author: Even Rouault +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS `AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +import sys + + +def Usage(): + print('Usage: compare_perfs.py [-noise_threshold val_in_pct]') + print(' [-warning_threshold val_in_pct]') + print(' [-error_threshold val_in_pct]') + print(' [-global_error_threshold val_in_pct]') + print(' ref.csv new.csv') + sys.exit(1) + +ref_filename = None +new_filename = None +noise_threshold = 2 +warning_threshold = 4 +error_threshold = 6 +global_error_threshold = 2 +i = 1 +while i < len(sys.argv): + if sys.argv[i] == '-noise_threshold' and i + 1 < len(sys.argv): + i += 1 + noise_threshold = int(sys.argv[i]) + elif sys.argv[i] == '-warning_threshold' and i + 1 < len(sys.argv): + i += 1 + warning_threshold = int(sys.argv[i]) + elif sys.argv[i] == '-error_threshold' and i + 1 < len(sys.argv): + i += 1 + error_threshold = int(sys.argv[i]) + elif sys.argv[i] == '-global_error_threshold' and i + 1 < len(sys.argv): + i += 1 + global_error_threshold = int(sys.argv[i]) + elif sys.argv[i][0] == '-': + Usage() + elif ref_filename is None: + ref_filename = sys.argv[i] + elif new_filename is None: + new_filename = sys.argv[i] + else: + Usage() + i += 1 +if ref_filename is None or new_filename is None: + Usage() + +assert noise_threshold < warning_threshold +assert warning_threshold < error_threshold +assert global_error_threshold >= noise_threshold +assert global_error_threshold <= error_threshold + +ref_lines = open(ref_filename, 'rt').readlines()[1:] +new_lines = open(new_filename, 'rt').readlines()[1:] +if len(ref_lines) != len(new_lines): + raise Exception('files are not comparable') + +ret_code = 0 +for i in range(len(ref_lines)): + line = ref_lines[i].replace('\n', '') + filename_ref, num_iterations_ref, num_threads_ref, command_ref, \ + _, time_ms_ref = line.split(',') + line = new_lines[i].replace('\n', '') + filename_new, num_iterations_new, num_threads_new, command_new, \ + _, time_ms_new = line.split(',') + assert filename_ref == filename_new + assert num_iterations_ref == num_iterations_new + assert num_threads_ref == num_threads_new + assert command_ref == command_new + time_ms_ref = int(time_ms_ref) + time_ms_new = int(time_ms_new) + if filename_ref == 'TOTAL': + display = 'TOTAL' + else: + display = '%s, %s iterations, %s threads, %s' % \ + (filename_ref, num_iterations_ref, num_threads_ref, command_ref) + display += ': ref_time %d ms, new_time %d ms' % (time_ms_ref, time_ms_new) + var_pct = 100.0 * (time_ms_new - time_ms_ref) / time_ms_ref + if abs(var_pct) <= noise_threshold: + display += ', (stable) %0.1f %%' % var_pct + elif var_pct < 0: + display += ', (improvement) %0.1f %%' % var_pct + else: + display += ', (regression) %0.1f %%' % var_pct + if filename_ref == 'TOTAL' and var_pct > global_error_threshold: + display += ', ERROR_THRESHOLD' + ret_code = 1 + elif var_pct > error_threshold: + display += ', ERROR_THRESHOLD' + ret_code = 1 + elif var_pct > warning_threshold: + display += ', WARNING_THRESHOLD' + print(display) + +sys.exit(ret_code) diff --git a/tests/performance/perf_test.py b/tests/performance/perf_test.py new file mode 100755 index 00000000..e4c91167 --- /dev/null +++ b/tests/performance/perf_test.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017, IntoPIX SA +# Contact: support@intopix.com +# Author: Even Rouault +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS `AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +import os +import subprocess +import sys +import time + + +def Usage(): + print('Usage: perf_test.py [-kakadu] [-i filelist.csv] [-o out.csv] [-q]') + sys.exit(1) + +opj_decompress_path = 'opj_decompress' +opj_compress_path = 'opj_compress' +kdu_expand_path = 'kdu_expand' +kdu_compress_path = 'kdu_compress' + +in_filename = 'perf_test_filelist.csv' +out_filename = None +i = 1 +quiet = False +kakadu = False +while i < len(sys.argv): + if sys.argv[i] == '-o' and i + 1 < len(sys.argv): + i += 1 + out_filename = sys.argv[i] + elif sys.argv[i] == '-q': + quiet = True + elif sys.argv[i] == '-kakadu': + kakadu = True + else: + Usage() + i += 1 + +i = 0 +while i < 10 * 1024 * 1024: + i += 1 + +out_file = None +if out_filename is not None: + out_file = open(out_filename, 'wt') + out_file.write('filename,iterations,threads,command,comment,time_ms\n') + +total_time = 0 +for line in open(in_filename, 'rt').readlines()[1:]: + line = line.replace('\n', '') + filename, num_iterations, num_threads, command, comment = line.split(',') + num_threads = int(num_threads) + num_iterations = int(num_iterations) + start = time.time() + for i in range(num_iterations): + env = None + if kakadu: + if command == 'DECOMPRESS': + args = [kdu_expand_path, + '-i', filename, + '-num_threads', str(num_threads), + '-o', 'out_perf_test.pgm'] + elif command == 'COMPRESS': + args = [kdu_compress_path, + '-i', filename, + '-num_threads', str(num_threads), + 'Creversible=yes', + '-o', 'out_perf_test.jp2'] + else: + assert False, command + else: + env = os.environ + if num_threads > 1: + env['OPJ_NUM_THREADS'] = str(num_threads) + else: + env['OPJ_NUM_THREADS'] = '0' + if command == 'DECOMPRESS': + args = [opj_decompress_path, + '-i', filename, '-o', 'out_perf_test.pgm'] + elif command == 'COMPRESS': + args = [opj_compress_path, + '-i', filename, '-o', 'out_perf_test.jp2'] + else: + assert False, command + p = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + env=env) + p.wait() + stop = time.time() + if os.path.exists('out_perf_test.pgm'): + os.unlink('out_perf_test.pgm') + if os.path.exists('out_perf_test.jp2'): + os.unlink('out_perf_test.jp2') + spent_time = stop - start + total_time += spent_time + if not quiet: + if len(comment) != 0: + print('%s (%s), %d iterations, %d threads, %s: %.02f s' % + (filename, comment, num_iterations, num_threads, + command, spent_time)) + else: + print('%s, %d iterations, %d threads, %s: %.02f s' % + (filename, num_iterations, num_threads, command, spent_time)) + if out_file is not None: + out_file.write('%s,%d,%d,%s,%s,%d\n' % + (filename, num_iterations, num_threads, command, + comment, spent_time * 1000)) + +if not quiet: + print('Total time: %.02f s' % total_time) +if out_file is not None: + out_file.write('%s,,,,,%d\n' % ('TOTAL', total_time * 1000)) diff --git a/tests/performance/perf_test_filelist.csv b/tests/performance/perf_test_filelist.csv new file mode 100644 index 00000000..7bcf4c84 --- /dev/null +++ b/tests/performance/perf_test_filelist.csv @@ -0,0 +1,7 @@ +filename,iterations,threads,command,comment +../../data/input/nonregression/kodak_2layers_lrcp.j2c,3,1,DECOMPRESS, +../../data/input/nonregression/kodak_2layers_lrcp.j2c,5,2,DECOMPRESS, +../../data/input/nonregression/kodak_2layers_lrcp.j2c,10,4,DECOMPRESS, +../../data/input/conformance/p0_07.j2k,3,1,DECOMPRESS,128x128 tiles RLCP 3decomp 8layers no MCT reversible +../../data/input/conformance/p0_04.j2k,10,1,DECOMPRESS,precincts 128x128 irreversible +../../data/input/nonregression/X_4_2K_24_185_CBR_WB_000.tif,3,1,COMPRESS, diff --git a/tools/travis-ci/run.sh b/tools/travis-ci/run.sh index 782622b5..e59bdc06 100755 --- a/tools/travis-ci/run.sh +++ b/tools/travis-ci/run.sh @@ -184,7 +184,9 @@ export OPJ_BINARY_DIR=$(opjpath -m ${PWD}/build) export OPJ_BUILD_CONFIGURATION=${OPJ_CI_BUILD_CONFIGURATION} export OPJ_DO_SUBMIT=${OPJ_DO_SUBMIT} -ctest -S ${OPJ_SOURCE_DIR}/tools/ctest_scripts/travis-ci.cmake -V || true +if [ "${OPJ_SKIP_REBUILD:-}" != "1" ]; then + ctest -S ${OPJ_SOURCE_DIR}/tools/ctest_scripts/travis-ci.cmake -V || true +fi # ctest will exit with various error codes depending on version. # ignore ctest exit code & parse this ourselves set +x @@ -303,4 +305,45 @@ New/unknown test failure found!!! fi fi +if [ "${OPJ_CI_PERF_TESTS:-}" == "1" ]; then + cd tests/performance + echo "Running performance tests on current version (dry-run)" + PATH=../../build/bin:$PATH python ./perf_test.py + echo "Running performance tests on current version" + PATH=../../build/bin:$PATH python ./perf_test.py -o /tmp/new.csv + if [ "${OPJ_NONCOMMERCIAL:-}" == "1" ] && [ -d ../../kdu ]; then + echo "Running performances tests with Kakadu" + LD_LIBRARY_PATH=../../kdu PATH=../../kdu::$PATH python ./perf_test.py -kakadu -o /tmp/kakadu.csv + echo "Comparing current version with Kakadu" + python compare_perfs.py /tmp/kakadu.csv /tmp/new.csv || true + fi + cd ../.. + + REF_VERSION=master + if [ "${TRAVIS_PULL_REQUEST:-false}" == "false" ]; then + REF_VERSION=v2.1.2 + fi + if [ ! -d ref_opj ]; then + git clone https://github.com/uclouvain/openjpeg ref_opj + fi + echo "Building reference version (${REF_VERSION})" + cd ref_opj + git checkout ${REF_VERSION} + mkdir -p build + cd build + cmake .. -DCMAKE_BUILD_TYPE=${OPJ_BUILD_CONFIGURATION} + make -j3 + cd ../.. + cd tests/performance + echo "Running performance tests on ${REF_VERSION} version (dry-run)" + PATH=../../ref_opj/build/bin:$PATH python ./perf_test.py + echo "Running performance tests on ${REF_VERSION} version" + PATH=../../ref_opj/build/bin:$PATH python ./perf_test.py -o /tmp/ref.csv + echo "Comparing current version with ${REF_VERSION} version" + # we should normally set OPJ_CI_RESULT=1 in case of failure, but + # this is too unreliable + python compare_perfs.py /tmp/ref.csv /tmp/new.csv || true + cd ../.. +fi + exit ${OPJ_CI_RESULT}