BIGTOP-3558. Fix build failure of HBase on ppc64le. (#789)
* BIGTOP-3558. Fix build failure of HBase on ppc64le.
* bumped asciidoctorj.pdf.version to avoid OOM. improved the readability by avoiding duplicate architecture specific conditionals.
* added ppc64le patch for protoc.
diff --git a/bigtop-packages/src/common/hbase/do-component-build b/bigtop-packages/src/common/hbase/do-component-build
index 0619ff9..9de0ca4 100644
--- a/bigtop-packages/src/common/hbase/do-component-build
+++ b/bigtop-packages/src/common/hbase/do-component-build
@@ -25,10 +25,13 @@
fi
if [ $HOSTTYPE = "powerpc64le" ] ; then
- sed -i "s|<asciidoctor.plugin.version>.*</asciidoctor.plugin.version>|<asciidoctor.plugin.version>1.5.3</asciidoctor.plugin.version>|" pom.xml
+ sed -i "s|<asciidoctor.plugin.version>.*</asciidoctor.plugin.version>|<asciidoctor.plugin.version>1.5.3</asciidoctor.plugin.version>|" pom.xml
+ sed -i 's|<asciidoctorj.pdf.version>.*</asciidoctorj.pdf.version>|<asciidoctorj.pdf.version>1.5.0-alpha.11</asciidoctorj.pdf.version>|' pom.xml
sed -i 's|<jruby.version>.*</jruby.version>|<jruby.version>1.7.23</jruby.version>|' pom.xml
- sed -i "s|<version>1.5.0-alpha.6</version>|<version>1.5.0-alpha.11</version>|" pom.xml
- CLASSIFIER="linux-ppcle_64"
+ mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 \
+ -Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc
+ mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=3.5.1-1 \
+ -Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/local/protobuf-3.5.1.1/bin/protoc
fi
if [ $HOSTTYPE = "aarch64" ] ; then
@@ -38,7 +41,8 @@
'\ <artifactId>jruby-complete<\/artifactId>\n'\
'\ <version>9.1.8.0</version>\n'\
'\ <\/dependency>' pom.xml
- CLASSIFIER="linux-aarch_64"
+ mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 \
+ -Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc
fi
MVN_ARGS="-Phadoop-3.0 "
@@ -50,9 +54,6 @@
MVN_ARGS+="-DskipTests "
MVN_ARGS+="-Dcheckstyle.skip=true "
-mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 \
- -Dclassifier=${CLASSIFIER} -Dpackaging=exe -Dfile=/usr/local/bin/protoc
-
# HBASE-21513: separate site and assembly:single to avoid assembly issues.
mvn ${MVN_ARGS} ${MAVEN_ADDITIONAL} clean install "$@"
mvn ${MVN_ARGS} ${MAVEN_ADDITIONAL} site "$@"
diff --git a/bigtop_toolchain/files/protoc-3.5.1-add-support-for-ppc64le.patch b/bigtop_toolchain/files/protoc-3.5.1-add-support-for-ppc64le.patch
new file mode 100644
index 0000000..581ac4e
--- /dev/null
+++ b/bigtop_toolchain/files/protoc-3.5.1-add-support-for-ppc64le.patch
@@ -0,0 +1,65 @@
+diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh
+index fe1dec2..f31597a 100755
+--- a/protoc-artifacts/build-protoc.sh
++++ b/protoc-artifacts/build-protoc.sh
+@@ -81,6 +81,8 @@ checkArch ()
+ assertEq $format "elf64-x86-64" $LINENO
+ elif [[ "$ARCH" == aarch_64 ]]; then
+ assertEq $format "elf64-little" $LINENO
++ elif [[ "$ARCH" == ppcle_64 ]]; then
++ assertEq $format "elf64-powerpcle" $LINENO
+ else
+ fail "Unsupported arch: $ARCH"
+ fi
+@@ -127,6 +129,8 @@ checkDependencies ()
+ elif [[ "$ARCH" == aarch_64 ]]; then
+ dump_cmd='objdump -p '"$1"' | grep NEEDED'
+ white_list="libpthread\.so\.0\|libc\.so\.6\|ld-linux-aarch64\.so\.1"
++ elif [[ "$ARCH" == ppcle_64 ]]; then
++ white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libz\.so\.1\|ld64\.so\.2"
+ fi
+ elif [[ "$OS" == osx ]]; then
+ dump_cmd='otool -L '"$1"' | fgrep dylib'
+@@ -193,6 +197,8 @@ elif [[ "$(uname)" == Linux* ]]; then
+ CXXFLAGS="$CXXFLAGS -m32"
+ elif [[ "$ARCH" == aarch_64 ]]; then
+ CONFIGURE_ARGS="$CONFIGURE_ARGS --host=aarch64-linux-gnu"
++ elif [[ "$ARCH" == ppcle_64 ]]; then
++ CXXFLAGS="$CXXFLAGS -m64"
+ else
+ fail "Unsupported arch: $ARCH"
+ fi
+diff --git a/protoc-artifacts/build-zip.sh b/protoc-artifacts/build-zip.sh
+index f08e275..e87bb39 100755
+--- a/protoc-artifacts/build-zip.sh
++++ b/protoc-artifacts/build-zip.sh
+@@ -20,6 +20,7 @@ included. Each invocation will create 6 zip packages:
+ dist/<TARGET>-<VERSION_NUMBER>-linux-x86_32.zip
+ dist/<TARGET>-<VERSION_NUMBER>-linux-x86_64.zip
+ dist/<TARGET>-<VERSION_NUMBER>-linux-aarch_64.zip
++ dist/<TARGET>-<VERSION_NUMBER>-linux-ppcle_64.zip
+ EOF
+ exit 1
+ fi
+@@ -35,6 +36,8 @@ declare -a FILE_NAMES=( \
+ linux-x86_32.zip linux-x86_32.exe \
+ linux-x86_64.zip linux-x86_64.exe \
+ linux-aarch_64.zip linux-aarch_64.exe \
++ linux-ppcle_64.zip linux-ppcle_64.exe \
++
+ )
+
+ # List of all well-known types to be included.
+diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml
+index 0f9dd9f..4a0892e 100644
+--- a/protoc-artifacts/pom.xml
++++ b/protoc-artifacts/pom.xml
+@@ -37,7 +37,7 @@
+ <extension>
+ <groupId>kr.motd.maven</groupId>
+ <artifactId>os-maven-plugin</artifactId>
+- <version>1.2.3.Final</version>
++ <version>1.5.0.Final</version>
+ </extension>
+ </extensions>
+ <plugins>
diff --git a/bigtop_toolchain/manifests/protobuf.pp b/bigtop_toolchain/manifests/protobuf.pp
index 808b702..3e8879e 100644
--- a/bigtop_toolchain/manifests/protobuf.pp
+++ b/bigtop_toolchain/manifests/protobuf.pp
@@ -40,4 +40,26 @@
require => EXEC["download protobuf"],
timeout => 3000
}
+
+ if ($architecture == 'ppc64le') {
+ exec { "download protobuf 3.5.1.1":
+ cwd => "/usr/src",
+ command => "/usr/bin/wget https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.5.1.1.tar.gz && mkdir -p protobuf-3.5.1.1 && /bin/tar -xvzf v3.5.1.1.tar.gz -C protobuf-3.5.1.1 --strip-components=1",
+ creates => "/usr/src/protobuf-3.5.1.1",
+ }
+
+ file { "/usr/src/protobuf-3.5.1.1/protoc-3.5.1-add-support-for-ppc64le.patch":
+ source => "puppet:///modules/bigtop_toolchain/protoc-3.5.1-add-support-for-ppc64le.patch",
+ require => Exec["download protobuf 3.5.1.1"],
+ }
+
+ exec { "install protobuf 3.5.1.1":
+ cwd => "/usr/src/protobuf-3.5.1.1",
+ command => "/usr/bin/patch -p1 < protoc-3.5.1-add-support-for-ppc64le.patch && /usr/src/protobuf-3.5.1.1/autogen.sh && /usr/src/protobuf-3.5.1.1/configure --prefix=/usr/local/protobuf-3.5.1.1 --disable-shared --with-pic && /usr/bin/make install",
+ creates => "/usr/local/protobuf-3.5.1.1",
+ require => File["/usr/src/protobuf-3.5.1.1/protoc-3.5.1-add-support-for-ppc64le.patch"],
+ timeout => 3000
+ }
+ }
+
}
