Merge pull request #4014 from BSBandme/JavaCaliper
Add caliper supported to java benchmark
diff --git a/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java b/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java
index 2c272a7..9c0997c 100644
--- a/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java
+++ b/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java
@@ -37,22 +37,22 @@
/**
* Test field deprecation
- *
+ *
* @author birdo@google.com (Roberto Scaramuzzi)
*/
public class DeprecatedFieldTest extends TestCase {
private String[] deprecatedGetterNames = {
"hasDeprecatedInt32",
"getDeprecatedInt32"};
-
+
private String[] deprecatedBuilderGetterNames = {
"hasDeprecatedInt32",
"getDeprecatedInt32",
"clearDeprecatedInt32"};
-
+
private String[] deprecatedBuilderSetterNames = {
- "setDeprecatedInt32"};
-
+ "setDeprecatedInt32"};
+
public void testDeprecatedField() throws Exception {
Class<?> deprecatedFields = TestDeprecatedFields.class;
Class<?> deprecatedFieldsBuilder = TestDeprecatedFields.Builder.class;
@@ -72,7 +72,15 @@
isDeprecated(method));
}
}
-
+
+ public void testDeprecatedFieldInOneof() throws Exception {
+ Class<?> oneofCase = TestDeprecatedFields.OneofFieldsCase.class;
+ String name = "DEPRECATED_INT32_IN_ONEOF";
+ java.lang.reflect.Field enumValue = oneofCase.getField(name);
+ assertTrue("Enum value " + name + " should be deprecated.",
+ isDeprecated(enumValue));
+ }
+
private boolean isDeprecated(AnnotatedElement annotated) {
return annotated.isAnnotationPresent(Deprecated.class);
}
diff --git a/kokoro/linux/32-bit/Dockerfile b/kokoro/linux/32-bit/Dockerfile
new file mode 100644
index 0000000..1278889
--- /dev/null
+++ b/kokoro/linux/32-bit/Dockerfile
@@ -0,0 +1,143 @@
+# This Dockerfile specifies the recipe for creating an image for the tests
+# to run in.
+#
+# We install as many test dependencies here as we can, because these setup
+# steps can be cached. They do *not* run every time we run the build.
+# The Docker image is only rebuilt when the Dockerfile (ie. this file)
+# changes.
+
+# Base Dockerfile for gRPC dev images
+FROM 32bit/debian:latest
+
+# Apt source for php
+RUN echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu trusty main" | tee /etc/apt/sources.list.d/various-php.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F4FCBB07
+
+# Install dependencies. We start with the basic ones require to build protoc
+# and the C++ build
+RUN apt-get clean && apt-get update && apt-get install -y --force-yes \
+ autoconf \
+ autotools-dev \
+ build-essential \
+ bzip2 \
+ ccache \
+ curl \
+ gcc \
+ git \
+ libc6 \
+ libc6-dbg \
+ libc6-dev \
+ libgtest-dev \
+ libtool \
+ make \
+ parallel \
+ time \
+ wget \
+ unzip \
+ # -- For python --
+ python-setuptools \
+ python-pip \
+ python-dev \
+ # -- For C++ benchmarks --
+ cmake \
+ # -- For PHP --
+ php5.5 \
+ php5.5-dev \
+ php5.5-xml \
+ php5.6 \
+ php5.6-dev \
+ php5.6-xml \
+ php7.0 \
+ php7.0-dev \
+ php7.0-xml \
+ phpunit \
+ valgrind \
+ libxml2-dev \
+ && apt-get clean
+
+##################
+# PHP dependencies.
+RUN wget http://am1.php.net/get/php-5.5.38.tar.bz2/from/this/mirror
+RUN mv mirror php-5.5.38.tar.bz2
+RUN tar -xvf php-5.5.38.tar.bz2
+RUN cd php-5.5.38 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.5-zts && \
+ make && make install && make clean && cd ..
+RUN cd php-5.5.38 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.5 && \
+ make && make install && make clean && cd ..
+
+RUN wget http://am1.php.net/get/php-5.6.30.tar.bz2/from/this/mirror
+RUN mv mirror php-5.6.30.tar.bz2
+RUN tar -xvf php-5.6.30.tar.bz2
+RUN cd php-5.6.30 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.6-zts && \
+ make && make install && cd ..
+RUN cd php-5.6.30 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.6 && \
+ make && make install && cd ..
+
+RUN wget http://am1.php.net/get/php-7.0.18.tar.bz2/from/this/mirror
+RUN mv mirror php-7.0.18.tar.bz2
+RUN tar -xvf php-7.0.18.tar.bz2
+RUN cd php-7.0.18 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.0-zts && \
+ make && make install && cd ..
+RUN cd php-7.0.18 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.0 && \
+ make && make install && cd ..
+
+RUN wget http://am1.php.net/get/php-7.1.4.tar.bz2/from/this/mirror
+RUN mv mirror php-7.1.4.tar.bz2
+RUN tar -xvf php-7.1.4.tar.bz2
+RUN cd php-7.1.4 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.1-zts && \
+ make && make install && cd ..
+RUN cd php-7.1.4 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.1 && \
+ make && make install && cd ..
+
+RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+RUN php composer-setup.php
+RUN mv composer.phar /usr/bin/composer
+RUN php -r "unlink('composer-setup.php');"
+RUN composer config -g -- disable-tls true
+RUN composer config -g -- secure-http false
+RUN cd /tmp && \
+ git clone https://github.com/google/protobuf.git && \
+ cd protobuf/php && \
+ git reset --hard 49b44bff2b6257a119f9c6a342d6151c736586b8 && \
+ ln -sfn /usr/local/php-5.5/bin/php /usr/bin/php && \
+ ln -sfn /usr/local/php-5.5/bin/php-config /usr/bin/php-config && \
+ ln -sfn /usr/local/php-5.5/bin/phpize /usr/bin/phpize && \
+ composer install && \
+ mv vendor /usr/local/vendor-5.5 && \
+ ln -sfn /usr/local/php-5.6/bin/php /usr/bin/php && \
+ ln -sfn /usr/local/php-5.6/bin/php-config /usr/bin/php-config && \
+ ln -sfn /usr/local/php-5.6/bin/phpize /usr/bin/phpize && \
+ composer install && \
+ mv vendor /usr/local/vendor-5.6 && \
+ ln -sfn /usr/local/php-7.0/bin/php /usr/bin/php && \
+ ln -sfn /usr/local/php-7.0/bin/php-config /usr/bin/php-config && \
+ ln -sfn /usr/local/php-7.0/bin/phpize /usr/bin/phpize && \
+ composer install && \
+ mv vendor /usr/local/vendor-7.0 && \
+ ln -sfn /usr/local/php-7.1/bin/php /usr/bin/php && \
+ ln -sfn /usr/local/php-7.1/bin/php-config /usr/bin/php-config && \
+ ln -sfn /usr/local/php-7.1/bin/phpize /usr/bin/phpize && \
+ composer install && \
+ mv vendor /usr/local/vendor-7.1
+
+##################
+# Python dependencies
+
+# These packages exist in apt-get, but their versions are too old, so we have
+# to get updates from pip.
+
+RUN pip install pip --upgrade
+RUN pip install virtualenv tox yattag
+
+##################
+# Prepare ccache
+
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+# Define the default command.
+CMD ["bash"]
diff --git a/kokoro/linux/32-bit/build.sh b/kokoro/linux/32-bit/build.sh
new file mode 100755
index 0000000..04383a57
--- /dev/null
+++ b/kokoro/linux/32-bit/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request 32" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/32-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="php_all_32"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/32-bit/continuous.cfg b/kokoro/linux/32-bit/continuous.cfg
new file mode 100644
index 0000000..28b66c0
--- /dev/null
+++ b/kokoro/linux/32-bit/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/32-bit/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/32-bit/presubmit.cfg b/kokoro/linux/32-bit/presubmit.cfg
new file mode 100644
index 0000000..28b66c0
--- /dev/null
+++ b/kokoro/linux/32-bit/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/32-bit/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/64-bit/Dockerfile b/kokoro/linux/64-bit/Dockerfile
new file mode 100644
index 0000000..3a279e6
--- /dev/null
+++ b/kokoro/linux/64-bit/Dockerfile
@@ -0,0 +1,244 @@
+# This Dockerfile specifies the recipe for creating an image for the tests
+# to run in.
+#
+# We install as many test dependencies here as we can, because these setup
+# steps can be cached. They do *not* run every time we run the build.
+# The Docker image is only rebuilt when the Dockerfile (ie. this file)
+# changes.
+
+# Base Dockerfile for gRPC dev images
+FROM debian:latest
+
+# Apt source for old Python versions.
+RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C
+
+# Apt source for Oracle Java.
+RUN echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \
+ echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
+
+# Apt source for Mono
+RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \
+ echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+
+# Apt source for php
+RUN echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu trusty main" | tee /etc/apt/sources.list.d/various-php.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F4FCBB07
+
+# Install dotnet SDK based on https://www.microsoft.com/net/core#debian
+# (Ubuntu instructions need apt to support https)
+RUN apt-get update && apt-get install -y --force-yes curl libunwind8 gettext && \
+ curl -sSL -o dotnet.tar.gz https://go.microsoft.com/fwlink/?LinkID=847105 && \
+ mkdir -p /opt/dotnet && tar zxf dotnet.tar.gz -C /opt/dotnet && \
+ ln -s /opt/dotnet/dotnet /usr/local/bin
+
+# Install dependencies. We start with the basic ones require to build protoc
+# and the C++ build
+RUN apt-get clean && apt-get update && apt-get install -y --force-yes \
+ autoconf \
+ autotools-dev \
+ build-essential \
+ bzip2 \
+ ccache \
+ curl \
+ gcc \
+ git \
+ libc6 \
+ libc6-dbg \
+ libc6-dev \
+ libgtest-dev \
+ libtool \
+ make \
+ parallel \
+ time \
+ wget \
+ # -- For csharp --
+ mono-devel \
+ referenceassemblies-pcl \
+ nunit \
+ # -- For all Java builds -- \
+ maven \
+ # -- For java_jdk6 -- \
+ # oops! not in jessie. too old? openjdk-6-jdk \
+ # -- For java_jdk7 -- \
+ openjdk-7-jdk \
+ # -- For java_oracle7 -- \
+ oracle-java7-installer \
+ # -- For python / python_cpp -- \
+ python-setuptools \
+ python-pip \
+ python-dev \
+ python2.6-dev \
+ python3.3-dev \
+ python3.4-dev \
+ # -- For Ruby --
+ ruby \
+ # -- For C++ benchmarks --
+ cmake \
+ # -- For PHP --
+ php5.6 \
+ php5.6-dev \
+ php5.6-xml \
+ php7.0 \
+ php7.0-dev \
+ php7.0-xml \
+ phpunit \
+ valgrind \
+ libxml2-dev \
+ && apt-get clean
+
+##################
+# C# dependencies
+
+RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe
+
+##################
+# Python dependencies
+
+# These packages exist in apt-get, but their versions are too old, so we have
+# to get updates from pip.
+
+RUN pip install pip --upgrade
+RUN pip install virtualenv tox yattag
+
+##################
+# Ruby dependencies
+
+# Install rvm
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+RUN \curl -sSL https://get.rvm.io | bash -s stable
+
+# Install Ruby 2.1, Ruby 2.2 and JRuby 1.7
+RUN /bin/bash -l -c "rvm install ruby-2.1"
+RUN /bin/bash -l -c "rvm install ruby-2.2"
+RUN /bin/bash -l -c "rvm install jruby-1.7"
+RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
+RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
+RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
+
+##################
+# Java dependencies
+
+# This step requires compiling protoc. :(
+
+ENV MAVEN_REPO /var/maven_local_repository
+ENV MVN mvn --batch-mode
+
+RUN cd /tmp && \
+ git clone https://github.com/google/protobuf.git && \
+ cd protobuf && \
+ git reset --hard 129a6e2aca95dcfb6c3e717d7b9cca1f104fde39 && \
+ ./autogen.sh && \
+ ./configure && \
+ make -j4 && \
+ cd java && \
+ $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \
+ cd ../javanano && \
+ $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO
+
+##################
+# PHP dependencies.
+RUN wget http://am1.php.net/get/php-5.5.38.tar.bz2/from/this/mirror
+RUN mv mirror php-5.5.38.tar.bz2
+RUN tar -xvf php-5.5.38.tar.bz2
+RUN cd php-5.5.38 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.5-zts && \
+ make && make install && cd ..
+RUN cd php-5.5.38 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.5 && \
+ make && make install && cd ..
+
+RUN wget http://am1.php.net/get/php-5.6.30.tar.bz2/from/this/mirror
+RUN mv mirror php-5.6.30.tar.bz2
+RUN tar -xvf php-5.6.30.tar.bz2
+RUN cd php-5.6.30 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.6-zts && \
+ make && make install && cd ..
+RUN cd php-5.6.30 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.6 && \
+ make && make install && cd ..
+
+RUN wget http://am1.php.net/get/php-7.0.18.tar.bz2/from/this/mirror
+RUN mv mirror php-7.0.18.tar.bz2
+RUN tar -xvf php-7.0.18.tar.bz2
+RUN cd php-7.0.18 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.0-zts && \
+ make && make install && cd ..
+RUN cd php-7.0.18 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.0 && \
+ make && make install && cd ..
+
+RUN wget http://am1.php.net/get/php-7.1.4.tar.bz2/from/this/mirror
+RUN mv mirror php-7.1.4.tar.bz2
+RUN tar -xvf php-7.1.4.tar.bz2
+RUN cd php-7.1.4 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.1-zts && \
+ make && make install && cd ..
+RUN cd php-7.1.4 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.1 && \
+ make && make install && cd ..
+
+RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+RUN php composer-setup.php
+RUN mv composer.phar /usr/bin/composer
+RUN php -r "unlink('composer-setup.php');"
+RUN composer config -g -- disable-tls true
+RUN composer config -g -- secure-http false
+RUN cd /tmp && \
+ rm -rf protobuf && \
+ git clone https://github.com/google/protobuf.git && \
+ cd protobuf && \
+ git reset --hard 49b44bff2b6257a119f9c6a342d6151c736586b8 && \
+ cd php && \
+ ln -sfn /usr/local/php-5.5/bin/php /usr/bin/php && \
+ ln -sfn /usr/local/php-5.5/bin/php-config /usr/bin/php-config && \
+ ln -sfn /usr/local/php-5.5/bin/phpize /usr/bin/phpize && \
+ composer install && \
+ mv vendor /usr/local/vendor-5.5 && \
+ ln -sfn /usr/local/php-5.6/bin/php /usr/bin/php && \
+ ln -sfn /usr/local/php-5.6/bin/php-config /usr/bin/php-config && \
+ ln -sfn /usr/local/php-5.6/bin/phpize /usr/bin/phpize && \
+ composer install && \
+ mv vendor /usr/local/vendor-5.6 && \
+ ln -sfn /usr/local/php-7.0/bin/php /usr/bin/php && \
+ ln -sfn /usr/local/php-7.0/bin/php-config /usr/bin/php-config && \
+ ln -sfn /usr/local/php-7.0/bin/phpize /usr/bin/phpize && \
+ composer install && \
+ mv vendor /usr/local/vendor-7.0 && \
+ ln -sfn /usr/local/php-7.1/bin/php /usr/bin/php && \
+ ln -sfn /usr/local/php-7.1/bin/php-config /usr/bin/php-config && \
+ ln -sfn /usr/local/php-7.1/bin/phpize /usr/bin/phpize && \
+ composer install && \
+ mv vendor /usr/local/vendor-7.1
+
+##################
+# Go dependencies.
+RUN apt-get install -y \
+ # -- For go -- \
+ golang
+
+##################
+# Javascript dependencies.
+RUN apt-get install -y \
+ # -- For javascript -- \
+ npm
+
+##################
+# Python 3.5 3.6 dependencies.
+RUN apt-get clean && apt-get update && apt-get install -y --force-yes \
+ python3.5-dev \
+ python3.6-dev \
+ && apt-get clean
+
+# On Debian/Ubuntu, nodejs binary is named 'nodejs' because the name 'node'
+# is taken by another legacy binary. We don't have that legacy binary and
+# npm expects the binary to be named 'node', so we just create a symbol
+# link here.
+RUN ln -s `which nodejs` /usr/bin/node
+
+##################
+# Prepare ccache
+
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+# Define the default command.
+CMD ["bash"]
diff --git a/kokoro/linux/64-bit/build.sh b/kokoro/linux/64-bit/build.sh
new file mode 100755
index 0000000..48ddbce
--- /dev/null
+++ b/kokoro/linux/64-bit/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="csharp java_jdk7 javanano_jdk7 java_oracle7 javanano_oracle7 python python_cpp ruby_all javascript golang php_all"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/64-bit/continuous.cfg b/kokoro/linux/64-bit/continuous.cfg
new file mode 100644
index 0000000..3a4faac
--- /dev/null
+++ b/kokoro/linux/64-bit/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/64-bit/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/64-bit/presubmit.cfg b/kokoro/linux/64-bit/presubmit.cfg
new file mode 100644
index 0000000..3a4faac
--- /dev/null
+++ b/kokoro/linux/64-bit/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/64-bit/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/build_and_run_docker.sh b/kokoro/linux/build_and_run_docker.sh
new file mode 100755
index 0000000..6a1f327
--- /dev/null
+++ b/kokoro/linux/build_and_run_docker.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Builds docker image and runs a command under it.
+# This is a generic script that is configured with the following variables:
+#
+# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
+# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
+# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
+# $@ - Extra args to pass to docker run
+
+
+set -ex
+
+cd $(dirname $0)/../..
+git_root=$(pwd)
+cd -
+
+# Use image name based on Dockerfile sha1
+DOCKERHUB_ORGANIZATION=grpctesting/protobuf
+DOCKER_IMAGE_NAME=${DOCKERHUB_ORGANIZATION}_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
+
+# Pull dockerimage from Dockerhub
+docker pull $DOCKER_IMAGE_NAME
+
+# Ensure existence of ccache directory
+CCACHE_DIR=/tmp/protobuf-ccache
+mkdir -p $CCACHE_DIR
+
+# Choose random name for docker container
+CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
+
+echo $git_root
+
+# Run command inside docker
+docker run \
+ "$@" \
+ -e CCACHE_DIR=$CCACHE_DIR \
+ -e EXTERNAL_GIT_ROOT="/var/local/kokoro/protobuf" \
+ -e TEST_SET="$TEST_SET" \
+ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
+ -v "$git_root:/var/local/kokoro/protobuf:ro" \
+ -v $CCACHE_DIR:$CCACHE_DIR \
+ -w /var/local/git/protobuf \
+ --name=$CONTAINER_NAME \
+ $DOCKER_IMAGE_NAME \
+ bash -l "/var/local/kokoro/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
+
+# Copy output artifacts
+if [ "$OUTPUT_DIR" != "" ]
+then
+ docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "${git_root}/kokoro" || FAILED="true"
+fi
+
+# remove the container, possibly killing it first
+docker rm -f $CONTAINER_NAME || true
+
+if [ "$FAILED" != "" ]
+then
+ exit 1
+fi
diff --git a/kokoro/linux/golang/build.sh b/kokoro/linux/golang/build.sh
new file mode 100755
index 0000000..79cddff
--- /dev/null
+++ b/kokoro/linux/golang/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="golang"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/golang/continuous.cfg b/kokoro/linux/golang/continuous.cfg
new file mode 100644
index 0000000..2282f78
--- /dev/null
+++ b/kokoro/linux/golang/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/golang/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/golang/presubmit.cfg b/kokoro/linux/golang/presubmit.cfg
new file mode 100644
index 0000000..2282f78
--- /dev/null
+++ b/kokoro/linux/golang/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/golang/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/java_jdk7/build.sh b/kokoro/linux/java_jdk7/build.sh
new file mode 100755
index 0000000..0a5604b
--- /dev/null
+++ b/kokoro/linux/java_jdk7/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="java_jdk7"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/java_jdk7/continuous.cfg b/kokoro/linux/java_jdk7/continuous.cfg
new file mode 100644
index 0000000..07a9df5
--- /dev/null
+++ b/kokoro/linux/java_jdk7/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/java_jdk7/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/java_jdk7/presubmit.cfg b/kokoro/linux/java_jdk7/presubmit.cfg
new file mode 100644
index 0000000..07a9df5
--- /dev/null
+++ b/kokoro/linux/java_jdk7/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/java_jdk7/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/java_oracle7/build.sh b/kokoro/linux/java_oracle7/build.sh
new file mode 100755
index 0000000..9ab90da
--- /dev/null
+++ b/kokoro/linux/java_oracle7/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="java_oracle7"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/java_oracle7/continuous.cfg b/kokoro/linux/java_oracle7/continuous.cfg
new file mode 100644
index 0000000..610469b
--- /dev/null
+++ b/kokoro/linux/java_oracle7/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/java_oracle7/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/java_oracle7/presubmit.cfg b/kokoro/linux/java_oracle7/presubmit.cfg
new file mode 100644
index 0000000..610469b
--- /dev/null
+++ b/kokoro/linux/java_oracle7/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/java_oracle7/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/javanano_jdk7/build.sh b/kokoro/linux/javanano_jdk7/build.sh
new file mode 100755
index 0000000..2fc0647
--- /dev/null
+++ b/kokoro/linux/javanano_jdk7/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="javanano_jdk7"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/javanano_jdk7/continuous.cfg b/kokoro/linux/javanano_jdk7/continuous.cfg
new file mode 100644
index 0000000..7293648
--- /dev/null
+++ b/kokoro/linux/javanano_jdk7/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/javanano_jdk7/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/javanano_jdk7/presubmit.cfg b/kokoro/linux/javanano_jdk7/presubmit.cfg
new file mode 100644
index 0000000..7293648
--- /dev/null
+++ b/kokoro/linux/javanano_jdk7/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/javanano_jdk7/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/javanano_oracle7/build.sh b/kokoro/linux/javanano_oracle7/build.sh
new file mode 100755
index 0000000..651d17e
--- /dev/null
+++ b/kokoro/linux/javanano_oracle7/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="javanano_oracle7"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/javanano_oracle7/continuous.cfg b/kokoro/linux/javanano_oracle7/continuous.cfg
new file mode 100644
index 0000000..9cac897
--- /dev/null
+++ b/kokoro/linux/javanano_oracle7/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/javanano_oracle7/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/javanano_oracle7/presubmit.cfg b/kokoro/linux/javanano_oracle7/presubmit.cfg
new file mode 100644
index 0000000..9cac897
--- /dev/null
+++ b/kokoro/linux/javanano_oracle7/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/javanano_oracle7/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/javascript/build.sh b/kokoro/linux/javascript/build.sh
new file mode 100755
index 0000000..c2585fb
--- /dev/null
+++ b/kokoro/linux/javascript/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="javascript"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/javascript/continuous.cfg b/kokoro/linux/javascript/continuous.cfg
new file mode 100644
index 0000000..b42f522
--- /dev/null
+++ b/kokoro/linux/javascript/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/javascript/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/javascript/presubmit.cfg b/kokoro/linux/javascript/presubmit.cfg
new file mode 100644
index 0000000..b42f522
--- /dev/null
+++ b/kokoro/linux/javascript/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/javascript/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/make_test_output.py b/kokoro/linux/make_test_output.py
new file mode 100644
index 0000000..9853685
--- /dev/null
+++ b/kokoro/linux/make_test_output.py
@@ -0,0 +1,94 @@
+"""Gathers output from test runs and create an XML file in JUnit format.
+
+The output files from the individual tests have been written in a directory
+structure like:
+
+ $DIR/joblog (output from "parallel --joblog joblog")
+ $DIR/logs/1/cpp/stdout
+ $DIR/logs/1/cpp/stderr
+ $DIR/logs/1/csharp/stdout
+ $DIR/logs/1/csharp/stderr
+ $DIR/logs/1/java_jdk7/stdout
+ $DIR/logs/1/java_jdk7/stderr
+ etc.
+
+This script bundles them into a single output XML file so Jenkins can show
+detailed test results. It runs as the last step before the Jenkins build
+finishes.
+"""
+
+import os
+import sys
+from yattag import Doc
+from collections import defaultdict
+
+
+def readtests(basedir):
+ tests = defaultdict(dict)
+
+ # Sample input (note: separators are tabs).
+ #
+ # Seq Host Starttime Runtime Send Receive Exitval Signal Command
+ # 1 : 1456263838.313 0.005 0 0 0 0 echo A
+ with open(basedir + "/joblog") as jobs:
+ firstline = next(jobs)
+ for line in jobs:
+ values = line.split("\t")
+
+ name = values[8].split()[-1]
+ test = tests[name]
+ test["name"] = name
+ test["time"] = values[3]
+
+ exitval = values[6]
+ if int(exitval):
+ # We don't have a more specific message. User should look at stderr.
+ test["failure"] = "TEST FAILURE"
+ else:
+ test["failure"] = False
+
+ for testname in os.listdir(basedir + "/logs/1"):
+ test = tests[testname]
+
+ with open(basedir + "/logs/1/" + testname + "/stdout") as f:
+ test["stdout"] = f.read()
+
+ with open(basedir + "/logs/1/" + testname + "/stderr") as f:
+ test["stderr"] = f.read()
+
+ # The cpp test is special since it doesn't run under parallel so doesn't show
+ # up in the job log.
+ tests["cpp"]["name"] = "cpp"
+
+ with open(basedir + '/logs/1/cpp/build_time', 'r') as f:
+ tests["cpp"]["time"] = f.read().strip()
+ tests["cpp"]["failure"] = False
+
+ ret = tests.values()
+ ret.sort(key=lambda x: x["name"])
+
+ return ret
+
+
+def genxml(tests):
+ doc, tag, text = Doc().tagtext()
+
+ with tag("testsuites"):
+ with tag("testsuite", name="Protobuf Tests"):
+ for test in tests:
+ with tag("testcase", name=test["name"], classname=test["name"],
+ time=test["time"]):
+ with tag("system-out"):
+ text(test["stdout"])
+ with tag("system-err"):
+ text(test["stderr"])
+ if test["failure"]:
+ with tag("failure"):
+ text(test["failure"])
+
+ return doc.getvalue()
+
+
+sys.stderr.write("make_test_output.py: writing XML from directory: " +
+ sys.argv[1] + "\n")
+print(genxml(readtests(sys.argv[1])))
diff --git a/kokoro/linux/php_all/build.sh b/kokoro/linux/php_all/build.sh
new file mode 100755
index 0000000..e9ab84c
--- /dev/null
+++ b/kokoro/linux/php_all/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="php_all"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/php_all/continuous.cfg b/kokoro/linux/php_all/continuous.cfg
new file mode 100644
index 0000000..dfa8037
--- /dev/null
+++ b/kokoro/linux/php_all/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/php_all/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/php_all/presubmit.cfg b/kokoro/linux/php_all/presubmit.cfg
new file mode 100644
index 0000000..dfa8037
--- /dev/null
+++ b/kokoro/linux/php_all/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/php_all/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/pull_request_in_docker.sh b/kokoro/linux/pull_request_in_docker.sh
new file mode 100755
index 0000000..23e6396
--- /dev/null
+++ b/kokoro/linux/pull_request_in_docker.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# This is the script that runs inside Docker, once the image has been built,
+# to execute all tests for the "pull request" project.
+
+WORKSPACE_BASE=`pwd`
+MY_DIR="$(dirname "$0")"
+TEST_SCRIPT=$MY_DIR/../../tests.sh
+BUILD_DIR=/tmp/protobuf
+
+set -e # exit immediately on error
+set -x # display all commands
+
+# The protobuf repository is mounted into our Docker image, but read-only.
+# We clone into a directory inside Docker (this is faster than cp).
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+cd $BUILD_DIR
+git clone /var/local/kokoro/protobuf
+cd protobuf
+
+# Set up the directory where our test output is going to go.
+OUTPUT_DIR=`mktemp -d`
+LOG_OUTPUT_DIR=$OUTPUT_DIR/logs
+mkdir -p $LOG_OUTPUT_DIR/1/cpp
+
+################################################################################
+# cpp build needs to run first, non-parallelized, so that protoc is available
+# for other builds.
+
+# Output filenames to follow the overall scheme used by parallel, ie:
+# $DIR/logs/1/cpp/stdout
+# $DIR/logs/1/cpp/stderr
+# $DIR/logs/1/csharp/stdout
+# $DIR/logs/1/csharp/stderr
+# $DIR/logs/1/java_jdk7/stdout
+# $DIR/logs/1/java_jdk7/stderr
+CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout
+CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr
+
+# Time the C++ build, so we can put this info in the test output.
+# It's important that we get /usr/bin/time (which supports -f and -o) and not
+# the bash builtin "time" which doesn't.
+TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time"
+
+$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
+
+# Other tests are run in parallel. TEST_SET is defined in
+# buildcmds/pull_request{_32}.sh
+
+parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
+ $TEST_SET \
+ || true # Process test results even if tests fail.
+
+cat $OUTPUT_DIR/joblog
+
+# The directory that is copied from Docker back into the Kokoro workspace.
+COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput
+mkdir -p $COPY_FROM_DOCKER
+TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/sponge_log.xml
+
+# Process all the output files from "parallel" and package them into a single
+# .xml file with detailed, broken-down test output.
+python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
+
+ls -l $TESTOUTPUT_XML_FILE
diff --git a/kokoro/linux/python/build.sh b/kokoro/linux/python/build.sh
new file mode 100755
index 0000000..86964f5
--- /dev/null
+++ b/kokoro/linux/python/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/python/continuous.cfg b/kokoro/linux/python/continuous.cfg
new file mode 100644
index 0000000..e2fc413
--- /dev/null
+++ b/kokoro/linux/python/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/python/presubmit.cfg b/kokoro/linux/python/presubmit.cfg
new file mode 100644
index 0000000..e2fc413
--- /dev/null
+++ b/kokoro/linux/python/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/python_cpp/build.sh b/kokoro/linux/python_cpp/build.sh
new file mode 100755
index 0000000..add6465
--- /dev/null
+++ b/kokoro/linux/python_cpp/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python_cpp"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/python_cpp/continuous.cfg b/kokoro/linux/python_cpp/continuous.cfg
new file mode 100644
index 0000000..b1b0e55
--- /dev/null
+++ b/kokoro/linux/python_cpp/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/python_cpp/presubmit.cfg b/kokoro/linux/python_cpp/presubmit.cfg
new file mode 100644
index 0000000..b1b0e55
--- /dev/null
+++ b/kokoro/linux/python_cpp/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/ruby_all/build.sh b/kokoro/linux/ruby_all/build.sh
new file mode 100755
index 0000000..ea75895
--- /dev/null
+++ b/kokoro/linux/ruby_all/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="ruby_all"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/ruby_all/continuous.cfg b/kokoro/linux/ruby_all/continuous.cfg
new file mode 100644
index 0000000..02d7643
--- /dev/null
+++ b/kokoro/linux/ruby_all/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/ruby_all/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/ruby_all/presubmit.cfg b/kokoro/linux/ruby_all/presubmit.cfg
new file mode 100644
index 0000000..02d7643
--- /dev/null
+++ b/kokoro/linux/ruby_all/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/ruby_all/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh
index 7e1e9a0..b34e28f 100755
--- a/objectivec/DevTools/full_mac_build.sh
+++ b/objectivec/DevTools/full_mac_build.sh
@@ -261,11 +261,11 @@
-destination "platform=iOS Simulator,name=iPhone 7,OS=latest" # 64bit
)
;;
- 9.[0-1]* )
+ 9.[0-2]* )
XCODEBUILD_TEST_BASE_IOS+=(
-destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
-destination "platform=iOS Simulator,name=iPhone 7,OS=latest" # 64bit
- # 9.0/9.1 both seem to often fail running destinations in parallel
+ # 9.0-9.2 all seem to often fail running destinations in parallel
-disable-concurrent-testing
)
;;
diff --git a/objectivec/DevTools/pddm_tests.py b/objectivec/DevTools/pddm_tests.py
index 8a73b84..9ac6a85 100755
--- a/objectivec/DevTools/pddm_tests.py
+++ b/objectivec/DevTools/pddm_tests.py
@@ -311,7 +311,7 @@
mc = pddm.MacroCollection(f)
try:
result = mc.Expand('foo(A,B)')
- self.fail('Should throw exception, entry %d' % idx)
+ self.fail('Should throw exception! Test failed to catch recursion.')
except pddm.PDDMError as e:
self.assertEqual(e.message,
'Found macro recusion, invoking "foo(1, A)":\n...while expanding "bar(1, A)".\n...while expanding "foo(A,B)".')
@@ -483,7 +483,7 @@
sf = pddm.SourceFile(f)
try:
sf.ProcessContent()
- self.fail('Should throw exception, entry %d' % idx)
+ self.fail('Should throw exception! Test failed to catch macro parsing error.')
except pddm.PDDMError as e:
self.assertEqual(e.message,
'Attempt to redefine macro: "PDDM-DEFINE mumble(x_)"\n'
@@ -503,7 +503,7 @@
sf = pddm.SourceFile(f)
try:
sf.ProcessContent()
- self.fail('Should throw exception, entry %d' % idx)
+ self.fail('Should throw exception! Test failed to catch expand error.')
except pddm.PDDMError as e:
self.assertEqual(e.message,
'No macro named "foobar".\n'
diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m
index 90485bd..3be20b6 100644
--- a/objectivec/GPBMessage.m
+++ b/objectivec/GPBMessage.m
@@ -969,7 +969,8 @@
newValue = [value copyWithZone:zone];
}
} else {
- if (field.mapKeyDataType == GPBDataTypeString) {
+ if ((field.mapKeyDataType == GPBDataTypeString) &&
+ GPBFieldDataTypeIsObject(field)) {
// NSDictionary
newValue = [value mutableCopyWithZone:zone];
} else {
diff --git a/objectivec/Tests/GPBMessageTests.m b/objectivec/Tests/GPBMessageTests.m
index 4d75f1e..a314909 100644
--- a/objectivec/Tests/GPBMessageTests.m
+++ b/objectivec/Tests/GPBMessageTests.m
@@ -2052,4 +2052,55 @@
XCTAssertEqual([msg1 hash], [msg1Prime hash]);
}
+- (void)testCopyingMapFileds {
+ TestMessageOfMaps *msg = [TestMessageOfMaps message];
+
+ msg.strToStr[@"foo"] = @"bar";
+
+ [msg.strToInt setInt32:1 forKey:@"mumble"];
+ [msg.intToStr setObject:@"wee" forKey:42];
+ [msg.intToInt setInt32:123 forKey:321];
+
+ [msg.strToBool setBool:YES forKey:@"one"];
+ [msg.boolToStr setObject:@"something" forKey:YES];
+ [msg.boolToBool setBool:YES forKey:NO];
+
+ [msg.intToBool setBool:YES forKey:13];
+ [msg.boolToInt setInt32:111 forKey:NO];
+
+ TestAllTypes *subMsg1 = [TestAllTypes message];
+ subMsg1.optionalInt32 = 1;
+ TestAllTypes *subMsg2 = [TestAllTypes message];
+ subMsg1.optionalInt32 = 2;
+ TestAllTypes *subMsg3 = [TestAllTypes message];
+ subMsg1.optionalInt32 = 3;
+
+ msg.strToMsg[@"baz"] = subMsg1;
+ [msg.intToMsg setObject:subMsg2 forKey:222];
+ [msg.boolToMsg setObject:subMsg3 forKey:YES];
+
+ TestMessageOfMaps *msg2 = [[msg copy] autorelease];
+ XCTAssertNotNil(msg2);
+ XCTAssertEqualObjects(msg2, msg);
+ XCTAssertTrue(msg2 != msg); // ptr compare
+ XCTAssertTrue(msg.strToStr != msg2.strToStr); // ptr compare
+ XCTAssertTrue(msg.intToStr != msg2.intToStr); // ptr compare
+ XCTAssertTrue(msg.intToInt != msg2.intToInt); // ptr compare
+ XCTAssertTrue(msg.strToBool != msg2.strToBool); // ptr compare
+ XCTAssertTrue(msg.boolToStr != msg2.boolToStr); // ptr compare
+ XCTAssertTrue(msg.boolToBool != msg2.boolToBool); // ptr compare
+ XCTAssertTrue(msg.intToBool != msg2.intToBool); // ptr compare
+ XCTAssertTrue(msg.boolToInt != msg2.boolToInt); // ptr compare
+ XCTAssertTrue(msg.strToMsg != msg2.strToMsg); // ptr compare
+ XCTAssertTrue(msg.intToMsg != msg2.intToMsg); // ptr compare
+ XCTAssertTrue(msg.boolToMsg != msg2.boolToMsg); // ptr compare
+
+ XCTAssertTrue(msg.strToMsg[@"baz"] != msg2.strToMsg[@"baz"]); // ptr compare
+ XCTAssertEqualObjects(msg.strToMsg[@"baz"], msg2.strToMsg[@"baz"]);
+ XCTAssertTrue([msg.intToMsg objectForKey:222] != [msg2.intToMsg objectForKey:222]); // ptr compare
+ XCTAssertEqualObjects([msg.intToMsg objectForKey:222], [msg2.intToMsg objectForKey:222]);
+ XCTAssertTrue([msg.boolToMsg objectForKey:YES] != [msg2.boolToMsg objectForKey:YES]); // ptr compare
+ XCTAssertEqualObjects([msg.boolToMsg objectForKey:YES], [msg2.boolToMsg objectForKey:YES]);
+}
+
@end
diff --git a/objectivec/Tests/unittest_objc.proto b/objectivec/Tests/unittest_objc.proto
index e5577fa..b0eb472 100644
--- a/objectivec/Tests/unittest_objc.proto
+++ b/objectivec/Tests/unittest_objc.proto
@@ -58,6 +58,27 @@
map<string, string> str_to_str = 5;
}
+// Message with a few types of maps to cover the different custom flows
+// in the runtime.
+message TestMessageOfMaps {
+ map<string, string> str_to_str = 1;
+
+ map<string, int32> str_to_int = 2;
+ map<int32, string> int_to_str = 3;
+ map<int32, int32> int_to_int = 4;
+
+ map<string, bool> str_to_bool = 5;
+ map<bool, string> bool_to_str = 6;
+ map<bool, bool> bool_to_bool = 7;
+
+ map<int32, bool> int_to_bool = 8;
+ map<bool, int32> bool_to_int = 9;
+
+ map<string, TestAllTypes> str_to_msg = 10;
+ map<int32, TestAllTypes> int_to_msg = 11;
+ map<bool, TestAllTypes> bool_to_msg = 12;
+}
+
// Recursive message and extension to for testing autocreators at different
// depths.
message TestRecursiveExtension {
diff --git a/python/google/protobuf/internal/containers.py b/python/google/protobuf/internal/containers.py
index 68be9e5..c6a3692 100755
--- a/python/google/protobuf/internal/containers.py
+++ b/python/google/protobuf/internal/containers.py
@@ -549,10 +549,10 @@
self._values = {}
def __getitem__(self, key):
+ key = self._key_checker.CheckValue(key)
try:
return self._values[key]
except KeyError:
- key = self._key_checker.CheckValue(key)
new_element = self._message_descriptor._concrete_class()
new_element._SetListener(self._message_listener)
self._values[key] = new_element
@@ -584,12 +584,14 @@
return default
def __contains__(self, item):
+ item = self._key_checker.CheckValue(item)
return item in self._values
def __setitem__(self, key, value):
raise ValueError('May not set values directly, call my_map[key].foo = 5')
def __delitem__(self, key):
+ key = self._key_checker.CheckValue(key)
del self._values[key]
self._message_listener.Modified()
diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py
index f1d5934..02a43d1 100755
--- a/python/google/protobuf/internal/descriptor_test.py
+++ b/python/google/protobuf/internal/descriptor_test.py
@@ -742,6 +742,19 @@
deprecated: true
>
>
+ field {
+ name: "deprecated_int32_in_oneof"
+ number: 2
+ label: LABEL_OPTIONAL
+ type: TYPE_INT32
+ options {
+ deprecated: true
+ }
+ oneof_index: 0
+ }
+ oneof_decl {
+ name: "oneof_fields"
+ }
"""
self._InternalTestCopyToProto(
diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py
index a303b1a..8dae637 100755
--- a/python/google/protobuf/internal/message_test.py
+++ b/python/google/protobuf/internal/message_test.py
@@ -1480,12 +1480,8 @@
submsg = msg.map_int32_foreign_message[5]
self.assertIs(submsg, msg.map_int32_foreign_message.get(5))
- # TODO(jieluo): Fix python and cpp extension diff.
- if api_implementation.Type() == 'cpp':
- with self.assertRaises(TypeError):
- msg.map_int32_foreign_message.get('')
- else:
- self.assertEqual(None, msg.map_int32_foreign_message.get(''))
+ with self.assertRaises(TypeError):
+ msg.map_int32_foreign_message.get('')
def testScalarMap(self):
msg = map_unittest_pb2.TestMap()
@@ -1695,12 +1691,8 @@
del msg2.map_int32_foreign_message[222]
self.assertFalse(222 in msg2.map_int32_foreign_message)
- if api_implementation.Type() == 'cpp':
- with self.assertRaises(TypeError):
- del msg2.map_int32_foreign_message['']
- else:
- with self.assertRaises(KeyError):
- del msg2.map_int32_foreign_message['']
+ with self.assertRaises(TypeError):
+ del msg2.map_int32_foreign_message['']
def testMergeFromBadType(self):
msg = map_unittest_pb2.TestMap()
diff --git a/python/release.sh b/python/release.sh
index b0bf3b3..01fc5e2 100755
--- a/python/release.sh
+++ b/python/release.sh
@@ -47,6 +47,7 @@
# Make sure all files are world-readable.
find python -type d -exec chmod a+r,a+x {} +
find python -type f -exec chmod a+r {} +
+umask 0022
# Check that the supplied version number matches what's inside the source code.
SOURCE_VERSION=`get_source_version`
diff --git a/src/google/protobuf/compiler/java/java_enum_field.cc b/src/google/protobuf/compiler/java/java_enum_field.cc
index 9f7bb34..642cdd3 100644
--- a/src/google/protobuf/compiler/java/java_enum_field.cc
+++ b/src/google/protobuf/compiler/java/java_enum_field.cc
@@ -80,7 +80,7 @@
// Use deprecated valueOf() method to be compatible with old generated code
// for v2.5.0/v2.6.1.
// TODO(xiaofeng): Use "forNumber" when we no longer support compatibility
- // with v2.5.0/v2.6.1.
+ // with v2.5.0/v2.6.1, and remove the @SuppressWarnings annotations.
(*variables)["for_number"] = "valueOf";
if (SupportFieldPresence(descriptor->file())) {
@@ -199,6 +199,7 @@
WriteFieldDocComment(printer, descriptor_);
printer->Print(variables_,
"$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n"
+ " @SuppressWarnings(\"deprecation\")\n"
" $type$ result = $type$.$for_number$($name$_);\n"
" return result == null ? $unknown$ : result;\n"
"}\n");
@@ -237,6 +238,7 @@
WriteFieldDocComment(printer, descriptor_);
printer->Print(variables_,
"$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n"
+ " @SuppressWarnings(\"deprecation\")\n"
" $type$ result = $type$.$for_number$($name$_);\n"
" return result == null ? $unknown$ : result;\n"
"}\n");
@@ -320,6 +322,7 @@
} else {
printer->Print(variables_,
"int rawValue = input.readEnum();\n"
+ " @SuppressWarnings(\"deprecation\")\n"
"$type$ value = $type$.$for_number$(rawValue);\n"
"if (value == null) {\n"
" unknownFields.mergeVarintField($number$, rawValue);\n"
@@ -412,6 +415,7 @@
printer->Print(variables_,
"$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n"
" if ($has_oneof_case_message$) {\n"
+ " @SuppressWarnings(\"deprecation\")\n"
" $type$ result = $type$.$for_number$(\n"
" (java.lang.Integer) $oneof_name$_);\n"
" return result == null ? $unknown$ : result;\n"
@@ -456,6 +460,7 @@
printer->Print(variables_,
"$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n"
" if ($has_oneof_case_message$) {\n"
+ " @SuppressWarnings(\"deprecation\")\n"
" $type$ result = $type$.$for_number$(\n"
" (java.lang.Integer) $oneof_name$_);\n"
" return result == null ? $unknown$ : result;\n"
@@ -517,6 +522,7 @@
} else {
printer->Print(variables_,
"int rawValue = input.readEnum();\n"
+ "@SuppressWarnings(\"deprecation\")\n"
"$type$ value = $type$.$for_number$(rawValue);\n"
"if (value == null) {\n"
" unknownFields.mergeVarintField($number$, rawValue);\n"
@@ -626,6 +632,7 @@
" new com.google.protobuf.Internal.ListAdapter.Converter<\n"
" java.lang.Integer, $type$>() {\n"
" public $type$ convert(java.lang.Integer from) {\n"
+ " @SuppressWarnings(\"deprecation\")\n"
" $type$ result = $type$.$for_number$(from);\n"
" return result == null ? $unknown$ : result;\n"
" }\n"
@@ -879,6 +886,7 @@
} else {
printer->Print(variables_,
"int rawValue = input.readEnum();\n"
+ "@SuppressWarnings(\"deprecation\")\n"
"$type$ value = $type$.$for_number$(rawValue);\n"
"if (value == null) {\n"
" unknownFields.mergeVarintField($number$, rawValue);\n"
diff --git a/src/google/protobuf/compiler/java/java_message.cc b/src/google/protobuf/compiler/java/java_message.cc
index f2ff280..eca7cf9 100644
--- a/src/google/protobuf/compiler/java/java_message.cc
+++ b/src/google/protobuf/compiler/java/java_message.cc
@@ -437,11 +437,10 @@
for (int j = 0; j < descriptor_->oneof_decl(i)->field_count(); j++) {
const FieldDescriptor* field = descriptor_->oneof_decl(i)->field(j);
printer->Print(
- "$field_name$($field_number$),\n",
- "field_name",
- ToUpper(field->name()),
- "field_number",
- SimpleItoa(field->number()));
+ "$deprecation$$field_name$($field_number$),\n",
+ "deprecation", field->options().deprecated() ? "@java.lang.Deprecated " : "",
+ "field_name", ToUpper(field->name()),
+ "field_number", SimpleItoa(field->number()));
}
printer->Print(
"$cap_oneof_name$_NOT_SET(0);\n",
diff --git a/src/google/protobuf/stubs/platform_macros.h b/src/google/protobuf/stubs/platform_macros.h
index 7a54060..c3a64dd 100644
--- a/src/google/protobuf/stubs/platform_macros.h
+++ b/src/google/protobuf/stubs/platform_macros.h
@@ -50,6 +50,9 @@
#elif defined(_M_ARM) || defined(__ARMEL__)
#define GOOGLE_PROTOBUF_ARCH_ARM 1
#define GOOGLE_PROTOBUF_ARCH_32_BIT 1
+#elif defined(_M_ARM64)
+#define GOOGLE_PROTOBUF_ARCH_ARM 1
+#define GOOGLE_PROTOBUF_ARCH_64_BIT 1
#elif defined(__aarch64__)
#define GOOGLE_PROTOBUF_ARCH_AARCH64 1
#define GOOGLE_PROTOBUF_ARCH_64_BIT 1
diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto
index 45a0eda..2a288da 100644
--- a/src/google/protobuf/unittest.proto
+++ b/src/google/protobuf/unittest.proto
@@ -189,6 +189,9 @@
message TestDeprecatedFields {
optional int32 deprecated_int32 = 1 [deprecated=true];
+ oneof oneof_fields {
+ int32 deprecated_int32_in_oneof = 2 [deprecated=true];
+ }
}
message TestDeprecatedMessage {
diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc
index ce3569c..25e78a6 100644
--- a/src/google/protobuf/util/json_util.cc
+++ b/src/google/protobuf/util/json_util.cc
@@ -50,7 +50,9 @@
namespace internal {
ZeroCopyStreamByteSink::~ZeroCopyStreamByteSink() {
- stream_->BackUp(buffer_size_);
+ if (buffer_size_ > 0) {
+ stream_->BackUp(buffer_size_);
+ }
}
void ZeroCopyStreamByteSink::Append(const char* bytes, size_t len) {
diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc
index 25c7e96..5b714bb 100644
--- a/src/google/protobuf/util/json_util_test.cc
+++ b/src/google/protobuf/util/json_util_test.cc
@@ -40,6 +40,7 @@
#include <google/protobuf/util/json_format_proto3.pb.h>
#include <google/protobuf/util/type_resolver.h>
#include <google/protobuf/util/type_resolver_util.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <gtest/gtest.h>
namespace google {
@@ -457,6 +458,29 @@
}
}
+TEST_F(JsonUtilTest, TestWrongJsonInput) {
+ using namespace google::protobuf;
+ const char json[] = "{\"unknown_field\":\"some_value\"}";
+ io::ArrayInputStream input_stream(json, strlen(json));
+ char proto_buffer[10000];
+ io::ArrayOutputStream output_stream(proto_buffer, sizeof(proto_buffer));
+ std::string message_type = "type.googleapis.com/proto3.TestMessage";
+ TypeResolver* resolver = NewTypeResolverForDescriptorPool(
+ "type.googleapis.com",
+ DescriptorPool::generated_pool());
+
+ util::Status result_status = util::JsonToBinaryStream(resolver,
+ message_type,
+ &input_stream,
+ &output_stream);
+
+ delete resolver;
+
+ EXPECT_FALSE(result_status.ok());
+ EXPECT_EQ(result_status.error_code(),
+ google::protobuf::util::error::INVALID_ARGUMENT);
+}
+
} // namespace
} // namespace util
} // namespace protobuf
