From d6e90be651ccc153840fd6a118ae6f50fb0fd9f5 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Fri, 12 Jul 2019 10:12:30 -0700 Subject: Refactor Dockerfiles (#6) --- ci/kokoro/Dockerfile.fedora | 38 +++++++++++++++++ ci/kokoro/Dockerfile.fedora-install | 38 ----------------- ci/kokoro/Dockerfile.ubuntu | 81 ------------------------------------- ci/kokoro/Dockerfile.ubuntu-16.04 | 53 ++++++++++++++++++++++++ ci/kokoro/Dockerfile.ubuntu-18.04 | 45 +++++++++++++++++++++ ci/kokoro/create-docker-image.sh | 13 ++++-- ci/kokoro/docker/build.sh | 2 +- 7 files changed, 147 insertions(+), 123 deletions(-) create mode 100644 ci/kokoro/Dockerfile.fedora delete mode 100644 ci/kokoro/Dockerfile.fedora-install delete mode 100644 ci/kokoro/Dockerfile.ubuntu create mode 100644 ci/kokoro/Dockerfile.ubuntu-16.04 create mode 100644 ci/kokoro/Dockerfile.ubuntu-18.04 diff --git a/ci/kokoro/Dockerfile.fedora b/ci/kokoro/Dockerfile.fedora new file mode 100644 index 000000000000..af3aa38037fc --- /dev/null +++ b/ci/kokoro/Dockerfile.fedora @@ -0,0 +1,38 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG DISTRO_VERSION=30 +FROM fedora:${DISTRO_VERSION} + +# Fedora includes packages for gRPC, libcurl, and OpenSSL that are recent enough +# for `google-cloud-cpp`. Install these packages and additional development +# tools to compile the dependencies: +RUN dnf makecache && \ + dnf install -y clang clang-tools-extra cmake doxygen findutils gcc-c++ git \ + grpc-devel grpc-plugins libcxx-devel libcxxabi-devel libcurl-devel \ + make openssl-devel pkgconfig protobuf-compiler python-pip ShellCheck \ + tar wget zlib-devel + +# Install the the buildifier tool, which does not compile with the default +# golang compiler for Ubuntu 16.04 and Ubuntu 18.04. +RUN wget -q -O /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/0.17.2/buildifier +RUN chmod 755 /usr/bin/buildifier + +# Install cmake_format to automatically format the CMake list files. +# https://github.com/cheshirekow/cmake_format +# Pin this to an specific version because the formatting changes when the +# "latest" version is updated, and we do not want the builds to break just +# because some third party changed something. +RUN pip install --upgrade pip +RUN pip install numpy cmake_format==0.5.2 diff --git a/ci/kokoro/Dockerfile.fedora-install b/ci/kokoro/Dockerfile.fedora-install deleted file mode 100644 index af3aa38037fc..000000000000 --- a/ci/kokoro/Dockerfile.fedora-install +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG DISTRO_VERSION=30 -FROM fedora:${DISTRO_VERSION} - -# Fedora includes packages for gRPC, libcurl, and OpenSSL that are recent enough -# for `google-cloud-cpp`. Install these packages and additional development -# tools to compile the dependencies: -RUN dnf makecache && \ - dnf install -y clang clang-tools-extra cmake doxygen findutils gcc-c++ git \ - grpc-devel grpc-plugins libcxx-devel libcxxabi-devel libcurl-devel \ - make openssl-devel pkgconfig protobuf-compiler python-pip ShellCheck \ - tar wget zlib-devel - -# Install the the buildifier tool, which does not compile with the default -# golang compiler for Ubuntu 16.04 and Ubuntu 18.04. -RUN wget -q -O /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/0.17.2/buildifier -RUN chmod 755 /usr/bin/buildifier - -# Install cmake_format to automatically format the CMake list files. -# https://github.com/cheshirekow/cmake_format -# Pin this to an specific version because the formatting changes when the -# "latest" version is updated, and we do not want the builds to break just -# because some third party changed something. -RUN pip install --upgrade pip -RUN pip install numpy cmake_format==0.5.2 diff --git a/ci/kokoro/Dockerfile.ubuntu b/ci/kokoro/Dockerfile.ubuntu deleted file mode 100644 index 32af422dec4f..000000000000 --- a/ci/kokoro/Dockerfile.ubuntu +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG DISTRO_VERSION=18.04 -FROM ubuntu:${DISTRO_VERSION} - -RUN apt update && \ - apt install -y \ - build-essential \ - clang \ - cmake \ - curl \ - doxygen \ - gawk \ - git \ - gcc \ - golang \ - g++ \ - libc-ares-dev \ - libc-ares2 \ - libssl-dev \ - make \ - ninja-build \ - pkg-config \ - python-pip \ - shellcheck \ - tar \ - unzip \ - wget \ - zlib1g-dev - -# Install newer c-ares on Ubuntu 16.04. -RUN if grep -q 16.04 /etc/lsb-release; then \ - apt remove libc-ares-dev libc-ares2; \ - apt install -y automake libtool; \ - mkdir -p /var/tmp/Downloads; \ - cd /var/tmp/Downloads; \ - wget -q https://github.com/c-ares/c-ares/archive/cares-1_15_0.tar.gz; \ - tar -xf cares-1_15_0.tar.gz; \ - cd /var/tmp/Downloads/c-ares-cares-1_15_0; \ - ./buildconf && ./configure && make -j $(nproc); \ - make install; \ - ldconfig; \ - fi - -# By default, Ubuntu 18.04 does not install the alternatives for clang-format -# and clang-tidy, so we need to manually install those. -RUN if grep -q 18.04 /etc/lsb-release; then \ - apt update && apt install -y clang-tidy clang-format-7; \ - update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-6.0 100; \ - update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-7 100; \ - fi - -# Install the the buildifier tool, which does not compile with the default -# golang compiler for Ubuntu 16.04 and Ubuntu 18.04. -RUN wget -q -O /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/0.17.2/buildifier -RUN chmod 755 /usr/bin/buildifier - -# Install cmake_format to automatically format the CMake list files. -# https://github.com/cheshirekow/cmake_format -# Pin this to an specific version because the formatting changes when the -# "latest" version is updated, and we do not want the builds to break just -# because some third party changed something. -RUN pip install --upgrade pip -RUN pip install numpy cmake_format==0.5.2 - -# Install grpc from source -WORKDIR /var/tmp/ci -COPY install-grpc.sh /var/tmp/ci -RUN /var/tmp/ci/install-grpc.sh diff --git a/ci/kokoro/Dockerfile.ubuntu-16.04 b/ci/kokoro/Dockerfile.ubuntu-16.04 new file mode 100644 index 000000000000..8ef65dd9b2e6 --- /dev/null +++ b/ci/kokoro/Dockerfile.ubuntu-16.04 @@ -0,0 +1,53 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:16.04 + +RUN apt update && \ + apt install -y \ + automake \ + build-essential \ + clang \ + cmake \ + curl \ + doxygen \ + gawk \ + git \ + gcc \ + golang \ + g++ \ + libssl-dev \ + libtool \ + make \ + ninja-build \ + pkg-config \ + python-pip \ + shellcheck \ + tar \ + unzip \ + wget \ + zlib1g-dev + +WORKDIR /var/tmp/Downloads +RUN wget -q https://github.com/c-ares/c-ares/archive/cares-1_15_0.tar.gz && \ + tar -xf cares-1_15_0.tar.gz && \ + cd /var/tmp/Downloads/c-ares-cares-1_15_0 && \ + ./buildconf && ./configure && make -j $(nproc) && \ + make install && \ + ldconfig + +# Install grpc from source +WORKDIR /var/tmp/ci +COPY install-grpc.sh /var/tmp/ci +RUN /var/tmp/ci/install-grpc.sh diff --git a/ci/kokoro/Dockerfile.ubuntu-18.04 b/ci/kokoro/Dockerfile.ubuntu-18.04 new file mode 100644 index 000000000000..d3660dafa25a --- /dev/null +++ b/ci/kokoro/Dockerfile.ubuntu-18.04 @@ -0,0 +1,45 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 + +RUN apt update && \ + apt install -y \ + build-essential \ + clang \ + cmake \ + curl \ + doxygen \ + gawk \ + git \ + gcc \ + golang \ + g++ \ + libc-ares-dev \ + libc-ares2 \ + libssl-dev \ + make \ + ninja-build \ + pkg-config \ + python-pip \ + shellcheck \ + tar \ + unzip \ + wget \ + zlib1g-dev + +# Install grpc from source +WORKDIR /var/tmp/ci +COPY install-grpc.sh /var/tmp/ci +RUN /var/tmp/ci/install-grpc.sh diff --git a/ci/kokoro/create-docker-image.sh b/ci/kokoro/create-docker-image.sh index df1d7af12692..82882072fed6 100755 --- a/ci/kokoro/create-docker-image.sh +++ b/ci/kokoro/create-docker-image.sh @@ -23,6 +23,13 @@ fi source "${PROJECT_ROOT}/ci/kokoro/define-docker-variables.sh" cd "${PROJECT_ROOT}" -sudo docker build -t "${IMAGE}:tip" \ - --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ - -f "ci/kokoro/Dockerfile.${DISTRO}" ci + +# If there's a version specific Dockerfile, we use it. +if [[ -f "ci/kokoro/Dockerfile.${DISTRO}-${DISTRO_VERSION}" ]]; then + sudo docker build -t "${IMAGE}:tip" \ + -f "ci/kokoro/Dockerfile.${DISTRO}-${DISTRO_VERSION}" ci +else + sudo docker build -t "${IMAGE}:tip" \ + --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ + -f "ci/kokoro/Dockerfile.${DISTRO}" ci +fi diff --git a/ci/kokoro/docker/build.sh b/ci/kokoro/docker/build.sh index a8dd2492fc1a..b42c49de29eb 100755 --- a/ci/kokoro/docker/build.sh +++ b/ci/kokoro/docker/build.sh @@ -47,7 +47,7 @@ fi if [[ "${BUILD_NAME}" = "clang-tidy" ]]; then # Compile with clang-tidy(1) turned on. The build treats clang-tidy warnings # as errors. - export DISTRO=fedora-install + export DISTRO=fedora export DISTRO_VERSION=30 export CC=clang export CXX=clang++ -- cgit 1.4.1