Merge pull request #8665 from jtattermusch/protobuf_fix_benchmark_script
Fix OSS benchmarking script
diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh
index e563e79..f078280 100755
--- a/kokoro/linux/benchmark/build.sh
+++ b/kokoro/linux/benchmark/build.sh
@@ -2,25 +2,4 @@
cd $(dirname $0)/../../..
-# prepare php environments
-sudo apt-get update && sudo apt-get install -y --force-yes php5
-sudo ln -sf /usr/include/x86_64-linux-gnu/gmp.h /usr/include/gmp.h
-mkdir php_temp
-cd php_temp
-curl -sS https://getcomposer.org/installer | php
-sudo mv composer.phar /usr/local/bin/composer
-git clone https://github.com/php/php-src
-cd php-src && git checkout PHP-7.2.13 && ./buildconf --force
-./configure \
- --enable-bcmatch \
- --with-gmp --with-openssl \
- --with-zlib \
- --prefix=/usr/local/php-7.2 && \
-make -j8 && sudo make install && make clean
-wget -O phpunit https://phar.phpunit.de/phpunit-7.phar && \
- chmod +x phpunit && \
- sudo cp phpunit /usr/local/php-7.2/bin
-sudo apt-get install -y --force-yes valgrind
-cd ../..
-
./tests.sh benchmark
diff --git a/kokoro/linux/benchmark/run.sh b/kokoro/linux/benchmark/run.sh
index ee80db3..da22d8a 100755
--- a/kokoro/linux/benchmark/run.sh
+++ b/kokoro/linux/benchmark/run.sh
@@ -3,36 +3,40 @@
# Change to repo root
cd $(dirname $0)/../../..
+set -ex
+
export OUTPUT_DIR=testoutput
-oldpwd=`pwd`
+repo_root="$(pwd)"
# tcmalloc
if [ ! -f gperftools/.libs/libtcmalloc.so ]; then
git clone https://github.com/gperftools/gperftools.git
- cd gperftools
+ pushd gperftools
./autogen.sh
./configure
make -j8
- cd ..
+ popd
fi
# download datasets for benchmark
-cd benchmarks
+pushd benchmarks
datasets=$(for file in $(find . -type f -name "dataset.*.pb" -not -path "./tmp/*"); do echo "$(pwd)/$file"; done | xargs)
echo $datasets
-cd $oldpwd
+popd
# build Python protobuf
./autogen.sh
./configure CXXFLAGS="-fPIC -O2"
make -j8
-cd python
+pushd python
python setup.py build --cpp_implementation
pip install . --user
-
+popd
# build and run Python benchmark
-cd ../benchmarks
+# We do this before building protobuf C++ since C++ build
+# will rewrite some libraries used by protobuf python.
+pushd benchmarks
make python-pure-python-benchmark
make python-cpp-reflection-benchmark
make -j8 python-cpp-generated-code-benchmark
@@ -41,64 +45,59 @@
./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json
echo "," >> "tmp/python_result.json"
echo "benchmarking python cpp reflection..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json
+env LD_PRELOAD="${repo_root}/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="${repo_root}/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json
echo "," >> "tmp/python_result.json"
echo "benchmarking python cpp generated code..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
+env LD_PRELOAD="${repo_root}/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="${repo_root}/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
echo "]" >> "tmp/python_result.json"
-cd $oldpwd
+popd
# build CPP protobuf
./configure
make clean && make -j8
# build Java protobuf
-cd java
-mvn package
-cd ..
+pushd java
+mvn package -B -Dmaven.test.skip=true
+popd
-# build CPP benchmark
-cd benchmarks
+pushd benchmarks
+
+# build and run C++ benchmark
+# "make clean" deletes the contents of the tmp/ directory, so we move it elsewhere and then restore it once build is done.
+# TODO(jtattermusch): find a less clumsy way of protecting python_result.json contents
mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp
echo "benchmarking cpp..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets
-cd $oldpwd
+env LD_PRELOAD="${repo_root}/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets
-# build go protobuf
-export PATH="`pwd`/src:$PATH"
-export GOPATH="$HOME/gocode"
-mkdir -p "$GOPATH/src/github.com/google"
-rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf"
-ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf"
-export PATH="$GOPATH/bin:$PATH"
-go get github.com/golang/protobuf/protoc-gen-go
+# TODO(jtattermusch): add benchmarks for https://github.com/protocolbuffers/protobuf-go.
+# The original benchmarks for https://github.com/golang/protobuf were removed
+# because:
+# * they were broken and haven't been producing results for a long time
+# * the https://github.com/golang/protobuf implementation has been superseded by
+# https://github.com/protocolbuffers/protobuf-go
-# build go benchmark
-cd benchmarks
-make go-benchmark
-echo "benchmarking go..."
-./go-benchmark $datasets > tmp/go_result.txt
-
-# build java benchmark
+# build and run java benchmark
make java-benchmark
echo "benchmarking java..."
./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets
+# build and run js benchmark
make js-benchmark
echo "benchmarking js..."
./js-benchmark $datasets --json_output=$(pwd)/tmp/node_result.json
-make -j8 generate_proto3_data
-proto3_datasets=$(for file in $datasets; do echo $(pwd)/tmp/proto3_data/${file#$(pwd)}; done | xargs)
-echo $proto3_datasets
+# TODO(jtattermusch): add php-c-benchmark. Currently its build is broken.
-# build php benchmark
-make -j8 php-c-benchmark
-echo "benchmarking php_c..."
-./php-c-benchmark $proto3_datasets --json --behavior_prefix="php_c" > tmp/php_c_result.json
-
-# upload result to bq
+# upload results to bq
+# TODO(jtattermusch): the upload to bq is currently broken.
make python_add_init
-env LD_LIBRARY_PATH="$oldpwd/src/.libs" python -m util.result_uploader -php_c="../tmp/php_c_result.json" \
- -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" -go="../tmp/go_result.txt" -python="../tmp/python_result.json" -node="../tmp/node_result.json"
-cd $oldpwd
+env LD_LIBRARY_PATH="${repo_root}/src/.libs" python -m util.result_uploader \
+ -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" -python="../tmp/python_result.json" -node="../tmp/node_result.json"
+popd
+
+# also persist the results in the build job log (for better debuggability)
+cat benchmarks/tmp/cpp_result.json
+cat benchmarks/tmp/java_result.json
+cat benchmarks/tmp/python_result.json
+cat benchmarks/tmp/node_result.json
