Testcontainers: Fixes for JDBC test suite (#108)
In this PR following issues addressed:
* Docker image size reduced from 4.52G to 3.34G (github provides 10G cache)
* Runs MS SQL tests with FDW plugin
* Always specify username/password for PostgreSQL JDBC tests (required for hardened PXF versions)
* minor cleanups
diff --git a/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/CloudberryApplication.java b/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/CloudberryApplication.java
index 5150a6f..0929572 100644
--- a/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/CloudberryApplication.java
+++ b/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/CloudberryApplication.java
@@ -49,6 +49,7 @@
private final PXFCloudberryContainer container;
private final String jdbcUrl;
private final String userName;
+ private final String password;
private Connection connection;
private Statement statement;
@@ -56,12 +57,14 @@
this.container = container;
this.jdbcUrl = getCloudberryMappedJdbcUrl();
this.userName = container.getCloudberryUser();
+ this.password = container.getCloudberryPassword();
}
public CloudberryApplication(PXFCloudberryContainer container, String dbName) {
this.container = container;
this.jdbcUrl = getCloudberryMappedJdbcUrl(dbName);
this.userName = container.getCloudberryUser();
+ this.password = container.getCloudberryPassword();
}
public void connect() throws Exception {
@@ -71,6 +74,7 @@
Properties props = new Properties();
if (userName != null) {
props.setProperty("user", userName);
+ props.setProperty("password", password);
}
Exception lastException = null;
@@ -276,6 +280,10 @@
return userName;
}
+ public String getPassword() {
+ return password;
+ }
+
public PXFCloudberryContainer getContainer() {
return container;
}
diff --git a/automation/src/main/java/org/apache/cloudberry/pxf/automation/structures/tables/pxf/ForeignTable.java b/automation/src/main/java/org/apache/cloudberry/pxf/automation/structures/tables/pxf/ForeignTable.java
index c01874f..5448fe4 100644
--- a/automation/src/main/java/org/apache/cloudberry/pxf/automation/structures/tables/pxf/ForeignTable.java
+++ b/automation/src/main/java/org/apache/cloudberry/pxf/automation/structures/tables/pxf/ForeignTable.java
@@ -113,7 +113,7 @@
if (params != null) {
for (String param : params) {
// parse parameter, each one is KEY=VALUE
- String[] paramPair = param.split("=");
+ String[] paramPair = param.split("=", 2);
appendOption(joiner, paramPair[0], paramPair[1]);
}
}
diff --git a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java
index b5bbb4c..5382c79 100644
--- a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java
+++ b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java
@@ -64,6 +64,7 @@
public static final int CLOUDBERRY_PORT = 7000;
public static final int PXF_PORT = 5888;
public static final String CLOUDBERRY_USER = "gpadmin";
+ public static final String CLOUDBERRY_PASSWORD = "cbdb@123";
private static final String CONTAINER_GRADLE_RO_CACHE = "/home/gpadmin/.gradle-host-cache";
private static final String CONTAINER_REPO_DIR = "/home/gpadmin/workspace/cloudberry-pxf";
@@ -234,6 +235,10 @@
return CLOUDBERRY_USER;
}
+ public String getCloudberryPassword() {
+ return CLOUDBERRY_PASSWORD;
+ }
+
public String getPxfInternalHost() {
return "localhost";
}
diff --git a/automation/src/main/resources/testcontainers/pxf-cbdb/Dockerfile b/automation/src/main/resources/testcontainers/pxf-cbdb/Dockerfile
index d063332..19ef014 100644
--- a/automation/src/main/resources/testcontainers/pxf-cbdb/Dockerfile
+++ b/automation/src/main/resources/testcontainers/pxf-cbdb/Dockerfile
@@ -42,9 +42,9 @@
-o /tmp/gradle.zip && \
sudo unzip -q /tmp/gradle.zip -d /opt && \
sudo ln -s "/opt/gradle-${GRADLE_VERSION}/bin/gradle" /usr/local/bin/gradle && \
- rm /tmp/gradle.zip
-RUN cd /tmp && gradle init && gradle wrapper --gradle-version ${GRADLE_VERSION} && ./gradlew javaToolchains
-RUN rm -rf /tmp/gradle /tmp/gradlew /tmp/gradlew.bat /tmp/.gradle
+ rm /tmp/gradle.zip && \
+ cd /tmp && gradle init && gradle wrapper --gradle-version ${GRADLE_VERSION} && ./gradlew javaToolchains &&\
+ rm -rf /tmp/gradle /tmp/gradlew /tmp/gradlew.bat /tmp/.gradle
ENV GRADLE_HOME="/opt/gradle-${GRADLE_VERSION}"
# Go toolchain for building pxf_regress inside the container
diff --git a/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_cloudberry.sh b/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_cloudberry.sh
index 7ad3204..ef65c28 100755
--- a/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_cloudberry.sh
+++ b/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_cloudberry.sh
@@ -209,3 +209,6 @@
/usr/local/cloudberry-db/bin/postgres --gp-version
/usr/local/cloudberry-db/bin/postgres --version
ldd /usr/local/cloudberry-db/bin/postgres
+
+# cleanup build tree (reduce image size)
+make clean
\ No newline at end of file
diff --git a/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_pxf.sh b/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_pxf.sh
index bc7bbc2..350cc91 100755
--- a/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_pxf.sh
+++ b/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_pxf.sh
@@ -33,14 +33,6 @@
source /usr/local/cloudberry-db/cloudberry-env.sh
export PATH=$GPHOME/bin:$PATH
-# Install Java 11 JDK and Maven
-if command -v apt-get >/dev/null 2>&1; then
- sudo apt update
- sudo apt install -y openjdk-11-jdk-headless maven
-elif command -v dnf >/dev/null 2>&1; then
- sudo dnf install -y java-11-openjdk-devel maven
-fi
-
cd /home/gpadmin/workspace/cloudberry-pxf
# Ensure gpadmin owns the source directory
diff --git a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/jdbc/JdbcMssqlTest.java b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/jdbc/JdbcMssqlTest.java
index ff19e3a..673a201 100644
--- a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/jdbc/JdbcMssqlTest.java
+++ b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/jdbc/JdbcMssqlTest.java
@@ -19,6 +19,7 @@
* under the License.
*/
+import annotations.WorksWithFDW;
import org.apache.cloudberry.pxf.automation.AbstractTestcontainersTest;
import org.apache.cloudberry.pxf.automation.structures.tables.pxf.ExternalTable;
import org.apache.cloudberry.pxf.automation.structures.tables.utils.TableFactory;
@@ -41,6 +42,7 @@
import java.util.TimeZone;
import java.util.UUID;
+@WorksWithFDW
public class JdbcMssqlTest extends AbstractTestcontainersTest {
private static final String MSSQL_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
diff --git a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/jdbc/JdbcTest.java b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/jdbc/JdbcTest.java
index 2263deb..e2cfe04 100755
--- a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/jdbc/JdbcTest.java
+++ b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/jdbc/JdbcTest.java
@@ -223,6 +223,7 @@
gpdb.getUserName());
pxfJdbcSingleFragment.setHost(pxfHost);
pxfJdbcSingleFragment.setPort(pxfPort);
+ pxfJdbcSingleFragment.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcSingleFragment);
}
@@ -242,6 +243,7 @@
null);
pxfJdbcMultipleFragmentsByEnum.setHost(pxfHost);
pxfJdbcMultipleFragmentsByEnum.setPort(pxfPort);
+ pxfJdbcMultipleFragmentsByEnum.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcMultipleFragmentsByEnum);
}
@@ -261,6 +263,7 @@
null);
pxfJdbcMultipleFragmentsByInt.setHost(pxfHost);
pxfJdbcMultipleFragmentsByInt.setPort(pxfPort);
+ pxfJdbcMultipleFragmentsByInt.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcMultipleFragmentsByInt);
}
@@ -280,6 +283,7 @@
null);
pxfJdbcMultipleFragmentsByDate.setHost(pxfHost);
pxfJdbcMultipleFragmentsByDate.setPort(pxfPort);
+ pxfJdbcMultipleFragmentsByDate.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcMultipleFragmentsByDate);
}
@@ -299,6 +303,8 @@
"database");
pxfJdbcReadServerConfigAll.setHost(pxfHost);
pxfJdbcReadServerConfigAll.setPort(pxfPort);
+ pxfJdbcReadServerConfigAll.addUserParameter("USER=" + gpdb.getUserName());
+ pxfJdbcReadServerConfigAll.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcReadServerConfigAll);
}
@@ -310,6 +316,8 @@
"database");
pxfJdbcReadViewNoParams.setHost(pxfHost);
pxfJdbcReadViewNoParams.setPort(pxfPort);
+ pxfJdbcReadViewNoParams.addUserParameter("USER=" + gpdb.getUserName());
+ pxfJdbcReadViewNoParams.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcReadViewNoParams);
pxfJdbcReadViewSessionParams = TableFactory.getPxfJdbcReadableTable(
@@ -319,6 +327,8 @@
"db-session-params");
pxfJdbcReadViewSessionParams.setHost(pxfHost);
pxfJdbcReadViewSessionParams.setPort(pxfPort);
+ pxfJdbcReadViewSessionParams.addUserParameter("USER=" + gpdb.getUserName());
+ pxfJdbcReadViewSessionParams.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcReadViewSessionParams);
}
@@ -332,6 +342,7 @@
gpdb.getUserName(), null);
pxfJdbcWritable.setHost(pxfHost);
pxfJdbcWritable.setPort(pxfPort);
+ pxfJdbcWritable.addUserParameter("PASS=" + gpdb.getPassword());
pxfJdbcWritable.addUserParameter("date_wide_range=false");
gpdb.createTableAndVerify(pxfJdbcWritable);
@@ -344,6 +355,7 @@
gpdb.getUserName(), null);
pxfJdbcDateTimeWritableWithDateWideRangeOn.setHost(pxfHost);
pxfJdbcDateTimeWritableWithDateWideRangeOn.setPort(pxfPort);
+ pxfJdbcDateTimeWritableWithDateWideRangeOn.addUserParameter("PASS=" + gpdb.getPassword());
pxfJdbcDateTimeWritableWithDateWideRangeOn.addUserParameter("date_wide_range=true");
gpdb.createTableAndVerify(pxfJdbcDateTimeWritableWithDateWideRangeOn);
@@ -356,6 +368,7 @@
gpdb.getUserName(), null);
pxfJdbcDateTimeWritableWithDateWideRangeOff.setHost(pxfHost);
pxfJdbcDateTimeWritableWithDateWideRangeOff.setPort(pxfPort);
+ pxfJdbcDateTimeWritableWithDateWideRangeOff.addUserParameter("PASS=" + gpdb.getPassword());
pxfJdbcDateTimeWritableWithDateWideRangeOff.addUserParameter("date_wide_range=false");
gpdb.createTableAndVerify(pxfJdbcDateTimeWritableWithDateWideRangeOff);
@@ -368,6 +381,7 @@
gpdb.getUserName(), "BATCH_SIZE=1");
pxfJdbcWritableNoBatch.setHost(pxfHost);
pxfJdbcWritableNoBatch.setPort(pxfPort);
+ pxfJdbcWritableNoBatch.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcWritableNoBatch);
pxfJdbcWritablePool = TableFactory.getPxfJdbcWritableTable(
@@ -379,6 +393,7 @@
gpdb.getUserName(), "POOL_SIZE=2");
pxfJdbcWritablePool.setHost(pxfHost);
pxfJdbcWritablePool.setPort(pxfPort);
+ pxfJdbcWritablePool.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcWritablePool);
}
@@ -392,6 +407,7 @@
gpdb.getUserName());
pxfJdbcColumns.setHost(pxfHost);
pxfJdbcColumns.setPort(pxfPort);
+ pxfJdbcColumns.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcColumns);
}
@@ -405,6 +421,7 @@
gpdb.getUserName());
pxfJdbcColumnProjectionSubset.setHost(pxfHost);
pxfJdbcColumnProjectionSubset.setPort(pxfPort);
+ pxfJdbcColumnProjectionSubset.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcColumnProjectionSubset);
}
@@ -418,6 +435,7 @@
gpdb.getUserName());
pxfJdbcColumnProjectionSuperset.setHost(pxfHost);
pxfJdbcColumnProjectionSuperset.setPort(pxfPort);
+ pxfJdbcColumnProjectionSuperset.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcColumnProjectionSuperset);
}
@@ -431,6 +449,7 @@
gpdb.getUserName(), "FETCH_SIZE=0");
pxfJdbcSingleFragment.setHost(pxfHost);
pxfJdbcSingleFragment.setPort(pxfPort);
+ pxfJdbcSingleFragment.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcSingleFragment);
}
@@ -444,6 +463,7 @@
gpdb.getUserName());
pxfJdbcDateWideRangeOn.setHost(pxfHost);
pxfJdbcDateWideRangeOn.setPort(pxfPort);
+ pxfJdbcDateWideRangeOn.addUserParameter("PASS=" + gpdb.getPassword());
pxfJdbcDateWideRangeOn.addUserParameter("date_wide_range=true");
gpdb.createTableAndVerify(pxfJdbcDateWideRangeOn);
@@ -456,6 +476,7 @@
gpdb.getUserName());
pxfJdbcDateWideRangeOff.setHost(pxfHost);
pxfJdbcDateWideRangeOff.setPort(pxfPort);
+ pxfJdbcDateWideRangeOff.addUserParameter("PASS=" + gpdb.getPassword());
pxfJdbcDateWideRangeOff.addUserParameter("date_wide_range=false");
gpdb.createTableAndVerify(pxfJdbcDateWideRangeOff);
}
@@ -468,6 +489,8 @@
"database");
pxfJdbcNamedQuery.setHost(pxfHost);
pxfJdbcNamedQuery.setPort(pxfPort);
+ pxfJdbcNamedQuery.addUserParameter("USER=" + gpdb.getUserName());
+ pxfJdbcNamedQuery.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcNamedQuery);
pxfJdbcNamedQuery = TableFactory.getPxfJdbcReadablePartitionedTable(
@@ -484,6 +507,8 @@
"database");
pxfJdbcNamedQuery.setHost(pxfHost);
pxfJdbcNamedQuery.setPort(pxfPort);
+ pxfJdbcNamedQuery.addUserParameter("USER=" + gpdb.getUserName());
+ pxfJdbcNamedQuery.addUserParameter("PASS=" + gpdb.getPassword());
gpdb.createTableAndVerify(pxfJdbcNamedQuery);
}
diff --git a/server/Makefile b/server/Makefile
index e6d05e6..bfd0ee3 100644
--- a/server/Makefile
+++ b/server/Makefile
@@ -109,7 +109,6 @@
echo "ERROR: PXF_HOME is not set"; exit 2; \
fi
mkdir -p "$(PXF_HOME)"/lib
- rm -f "$(PXF_HOME)"/lib/clickhouse-jdbc-*.jar
cp -R build/stage/lib/. "$(PXF_HOME)"/lib/
clean: prepare-gradle-wrapper
