[docker] Add support for publishing UBI based runtime images
This patch adds a `--use-ubi-runtime` option to the docker build.
When that option is passed for Centos 7 and 8 builds, the Kudu
runtime images (`kudu` & `kudu-python`) will be built using a
UBI base instead of the CentOS base. If the `--use-ubi-runtime`
flag is passed for any unsupported OS, a clear error message will
be printed.
We do not build the development images using the UBI base
because the package repositories in the UBI do not contain
all the packages necessary to build Kudu and support
development. Additionally the benefits of UBI are primarily for
runtime/deployment usage and continuing to use CentOS for the
development can simplify the build.
This patch also required some small changes to how we generated
the published image labels given the UBI images use a slightly
different format than was previously handled.
More can be read about RedHat UBI here:
https://www.redhat.com/en/blog/introducing-red-hat-universal-base-image
https://developers.redhat.com/articles/ubi-faq
I tested building images with `--use-ubi-runtime` on both
CentOS 7 and 8.
Change-Id: I7be121c4ef8422d8aa7556aa163254b43c226b07
Reviewed-on: http://gerrit.cloudera.org:8080/17564
Tested-by: Kudu Jenkins
Reviewed-by: Grant Henke <granthenke@apache.org>
(cherry picked from commit 76cb0dd808aaef548ef80682e13a00711e7dd6a4)
Reviewed-on: http://gerrit.cloudera.org:8080/17573
Reviewed-by: Bankim Bhavsar <bankim@cloudera.com>
diff --git a/docker/Dockerfile b/docker/Dockerfile
index b9eecc4..ea8d085 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -26,12 +26,16 @@
# Note: This file uses the shared label namespace for common labels. See:
# http://label-schema.org/rc1/
+# Base OS Image Arguments.
+# ARGs defined before the first FROM, thus outside of any build-stage, are global.
+ARG RUNTIME_BASE_OS
+ARG DEV_BASE_OS
+
#
# ---- Runtime ----
# Builds a base image that has all the runtime libraries for Kudu pre-installed.
#
-ARG BASE_OS
-FROM $BASE_OS as runtime
+FROM $RUNTIME_BASE_OS as runtime
COPY ./docker/bootstrap-runtime-env.sh /
RUN ./bootstrap-runtime-env.sh && rm bootstrap-runtime-env.sh
@@ -62,8 +66,7 @@
# ---- Dev ----
# Builds a base image that has all the development libraries for Kudu pre-installed.
#
-ARG BASE_OS
-FROM $BASE_OS as dev
+FROM $DEV_BASE_OS as dev
COPY ./docker/bootstrap-dev-env.sh /
COPY ./docker/bootstrap-java-env.sh /
diff --git a/docker/bootstrap-runtime-env.sh b/docker/bootstrap-runtime-env.sh
index d384907..b4c5904 100755
--- a/docker/bootstrap-runtime-env.sh
+++ b/docker/bootstrap-runtime-env.sh
@@ -36,7 +36,6 @@
yum install -y \
cyrus-sasl-gssapi \
cyrus-sasl-plain \
- krb5-server \
krb5-workstation \
openssl
@@ -44,7 +43,7 @@
# --no-install-recommends keeps the install smaller
yum install -y \
cyrus-sasl-devel \
- lzo-devel \
+ lzo \
tzdata \
which
diff --git a/docker/docker-build.py b/docker/docker-build.py
index 8d452d0..dc7ee9e 100755
--- a/docker/docker-build.py
+++ b/docker/docker-build.py
@@ -69,6 +69,18 @@
DEFAULT_REPOSITORY = 'apache/kudu'
DEFAULT_ACTION = 'load'
+# A map of supported base images to their UBI counterpart.
+# If no mapping exists then --use-ubi-runtime will fail for that base image.
+UBI_MAP = {
+ "centos:7" : "registry.access.redhat.com/ubi7/ubi",
+ "centos:8" : "registry.access.redhat.com/ubi8/ubi"
+}
+
+# Targets that use the dev base image.
+DEV_TARGETS = [ 'dev', 'thirdparty', 'build', 'impala-build' ]
+# Targets that use the runtime base image.
+RUNTIME_TARGETS = [ 'runtime', 'kudu', 'kudu-python', 'impala' ]
+
REQUIRED_CPUS = 4
REQUIRED_MEMORY_GIB = 4
@@ -85,12 +97,14 @@
'ubuntu:bionic', 'ubuntu:focal',
'opensuse/leap:15'],
help='The base operating systems to build with')
+ parser.add_argument('--use-ubi-runtime', action='store_true',
+ help='If passed, uses the equivalent RedHat UBI base image for Kudu runtime images'
+ ' of CentOS builds.')
# These targets are defined in the Dockerfile. Dependent targets of a passed image will be built,
# but not tagged. Note that if a target is not tagged it is subject removal by Dockers system
# and image pruning.
- parser.add_argument('--targets', nargs='+', default=DEFAULT_TARGETS, choices=[
- 'runtime', 'dev', 'thirdparty', 'build',
- 'kudu', 'kudu-python', 'impala-build', 'impala'],
+ parser.add_argument('--targets', nargs='+', default=DEFAULT_TARGETS,
+ choices=DEV_TARGETS + RUNTIME_TARGETS,
help='The targets to build and tag')
parser.add_argument('--repository', default=DEFAULT_REPOSITORY,
help='The repository string to use when tagging the image')
@@ -163,11 +177,12 @@
"""
# If the base contains a "/" remove the prefix. ex: `opensuse/leap:15`
if "/" in base:
- short_base = base.split('/')[1]
+ short_base = base[base.find('/') + 1:]
else:
short_base = base
- os_name = short_base.split(':')[0]
- os_version = short_base.split(':')[1]
+ # The OS name and version is delimited by ':' or '/'.
+ os_name = re.split(':|/', short_base)[0]
+ os_version = re.split(':|/', short_base)[1]
os_tag = os_name
if os_version.isdigit():
os_tag += os_version
@@ -210,10 +225,12 @@
else:
return [opts.targets]
-def build_args(base, version, vcs_ref, cache_from):
+def build_args(runtime_base, dev_base, version, vcs_ref, cache_from):
""" Constructs the build the arguments to pass to the Docker build. """
args = ''
- args += ' --build-arg BASE_OS="%s"' % base
+ args += ' --build-arg RUNTIME_BASE_OS="%s"' % runtime_base
+ args += ' --build-arg DEV_BASE_OS="%s"' % dev_base
+ args += ' --build-arg BASE_OS="%s"' % dev_base
args += ' --build-arg DOCKERFILE="docker/Dockerfile"'
args += ' --build-arg MAINTAINER="Apache Kudu <dev@kudu.apache.org>"'
args += ' --build-arg URL="https://kudu.apache.org"'
@@ -225,7 +242,15 @@
args += ' --cache-from %s' % cache_from
return args
-def build_tags(opts, base, version, vcs_ref, target):
+def build_tags(opts, runtime_base, dev_base, version, vcs_ref, target):
+ if target in RUNTIME_TARGETS:
+ base = runtime_base
+ elif target in DEV_TARGETS:
+ base = dev_base
+ else:
+ print('ERROR: invalid target in build_tags: %s' % target)
+ sys.exit(1)
+
os_tag = get_os_tag(base)
version_tag = get_version_tag(version, vcs_ref)
@@ -314,11 +339,23 @@
% (version, vcs_ref))
sys.exit(1)
+ # Validate the bases have a correlated UBI.
+ if opts.use_ubi_runtime:
+ for base in bases:
+ if base not in UBI_MAP:
+ print('ERROR: --use-ubi-runtime is only valid for the following bases: %s' % ", ".join(UBI_MAP.keys()))
+ sys.exit(1)
+
for base in bases:
print('Building targets for %s...' % base)
+ dev_base = base
+ runtime_base = base
+ if opts.use_ubi_runtime:
+ runtime_base = UBI_MAP[base]
+ print('Runtime images will use UBI image %s...' % runtime_base)
for target in targets:
- target_tags = build_tags(opts, base, version, vcs_ref, target)
+ target_tags = build_tags(opts, runtime_base, dev_base, version, vcs_ref, target)
# Build the target.
print('Building %s target...' % target)
# Note: It isn't currently possible to load multi-arch images into docker,
@@ -329,7 +366,7 @@
docker_build_cmd = 'docker buildx build --%s' % opts.action
if opts.platforms:
docker_build_cmd += ' --platform %s' % platforms_csv(opts)
- docker_build_cmd += build_args(base, version, vcs_ref, opts.cache_from)
+ docker_build_cmd += build_args(runtime_base, dev_base, version, vcs_ref, opts.cache_from)
docker_build_cmd += ' --file %s' % os.path.join(ROOT, 'docker', 'Dockerfile')
docker_build_cmd += ' --target %s' % target
docker_build_cmd += ''.join(' --tag %s' % tag for tag in target_tags)
