Replace pip3 download with curl for Python deps
Replace pip3 download commands with curl for downloading psutil,
PyYAML, PyGreSQL and mock packages in gpMgmt/bin/Makefile.
This change unifies the download mechanism to use curl consistently
for all Python dependencies, avoiding potential pip3 download
compatibility issues across different environments.
Changes:
- Remove pip3 download commands for psutil, PyYAML, PyGreSQL & mock
- Use curl to download packages directly from PyPI
- Maintain existing wheel and cython installation for PyYAML build
- Keep the same package versions and download locations
Also, add curl and pip3 dependency detection in configure.ac file for
better user experience. When `--with-pythonsrc-ext` option is enabled,
configure can check the required dependencies early.
diff --git a/configure b/configure
index 3124f02..4297c80 100755
--- a/configure
+++ b/configure
@@ -742,6 +742,9 @@
krb_srvtab
with_gssapi
with_pythonsrc_ext
+PIP3
+CURL
+PYTHON3
with_python
with_perl
with_tcl
@@ -9846,6 +9849,142 @@
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_pythonsrc_ext" >&5
$as_echo "$with_pythonsrc_ext" >&6; }
+# Check for required dependencies when pythonsrc-ext is enabled
+if test "$with_pythonsrc_ext" = yes; then
+ # Extract the first word of "python3", so it can be a program name with args.
+set dummy python3; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_PYTHON3+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $PYTHON3 in
+ [\\/]* | ?:[\\/]*)
+ ac_cv_path_PYTHON3="$PYTHON3" # Let the user override the test with a path.
+ ;;
+ *)
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_path_PYTHON3="$as_dir/$ac_word$ac_exec_ext"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ ;;
+esac
+fi
+PYTHON3=$ac_cv_path_PYTHON3
+if test -n "$PYTHON3"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON3" >&5
+$as_echo "$PYTHON3" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ if test -z "$PYTHON3"; then
+ as_fn_error $? "python3 is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
+ fi
+
+ # Extract the first word of "curl", so it can be a program name with args.
+set dummy curl; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_CURL+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $CURL in
+ [\\/]* | ?:[\\/]*)
+ ac_cv_path_CURL="$CURL" # Let the user override the test with a path.
+ ;;
+ *)
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_path_CURL="$as_dir/$ac_word$ac_exec_ext"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ ;;
+esac
+fi
+CURL=$ac_cv_path_CURL
+if test -n "$CURL"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CURL" >&5
+$as_echo "$CURL" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ if test -z "$CURL"; then
+ as_fn_error $? "curl is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
+ fi
+
+ # Extract the first word of "pip3", so it can be a program name with args.
+set dummy pip3; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_PIP3+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $PIP3 in
+ [\\/]* | ?:[\\/]*)
+ ac_cv_path_PIP3="$PIP3" # Let the user override the test with a path.
+ ;;
+ *)
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_path_PIP3="$as_dir/$ac_word$ac_exec_ext"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ ;;
+esac
+fi
+PIP3=$ac_cv_path_PIP3
+if test -n "$PIP3"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PIP3" >&5
+$as_echo "$PIP3" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ if test -z "$PIP3"; then
+ as_fn_error $? "pip3 is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
+ fi
+fi
+
+
#
@@ -18442,7 +18581,7 @@
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
+#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -18488,7 +18627,7 @@
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
+#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -18512,7 +18651,7 @@
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
+#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -18557,7 +18696,7 @@
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
+#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -18581,7 +18720,7 @@
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
+#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
diff --git a/configure.ac b/configure.ac
index 3b1000a..2732c72 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1061,6 +1061,25 @@
AC_MSG_CHECKING([whether to build Python modules for gpMgmt])
PGAC_ARG_BOOL(with, pythonsrc-ext, no, [build Python modules for gpMgmt])
AC_MSG_RESULT([$with_pythonsrc_ext])
+
+# Check for required dependencies when pythonsrc-ext is enabled
+if test "$with_pythonsrc_ext" = yes; then
+ AC_PATH_PROG([PYTHON3], [python3])
+ if test -z "$PYTHON3"; then
+ AC_MSG_ERROR([python3 is required for --with-pythonsrc-ext but was not found])
+ fi
+
+ AC_PATH_PROG([CURL], [curl])
+ if test -z "$CURL"; then
+ AC_MSG_ERROR([curl is required for --with-pythonsrc-ext but was not found])
+ fi
+
+ AC_PATH_PROG([PIP3], [pip3])
+ if test -z "$PIP3"; then
+ AC_MSG_ERROR([pip3 is required for --with-pythonsrc-ext but was not found])
+ fi
+fi
+
AC_SUBST(with_pythonsrc_ext)
diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile
index c863cb7..3c4b6d2 100644
--- a/gpMgmt/bin/Makefile
+++ b/gpMgmt/bin/Makefile
@@ -90,14 +90,14 @@
download-python-deps:
@echo "--- Downloading Python dependencies for gpMgmt modules"
@mkdir -p $(PYLIB_SRC_EXT)
- # Download psutil
- pip3 download --no-deps --no-binary :all: --no-build-isolation psutil==$(PSUTIL_VERSION) --dest $(PYLIB_SRC_EXT)/
- # Download PyYAML, wheel is for metadata vefiry;
- # cython is used for building, see https://github.com/apache/cloudberry/issues/1201
- pip3 install wheel "cython<3.0.0"
- pip3 download --no-deps --no-binary :all: --no-build-isolation PyYAML==$(PYYAML_VERSION) --dest $(PYLIB_SRC_EXT)/
- # Download PyGreSQL, the `PATH` is needed to avoid cannot find the pg_config
- PATH=$(DESTDIR)$(bindir):$$PATH pip3 download --no-deps --no-binary :all: --no-build-isolation PyGreSQL==$(PYGRESQL_VERSION) --dest $(PYLIB_SRC_EXT)/
+ # Download psutil using curl
+ curl -sSL https://files.pythonhosted.org/packages/source/p/psutil/psutil-$(PSUTIL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/psutil-$(PSUTIL_VERSION).tar.gz
+ # Download PyYAML using curl
+ curl -sSL https://files.pythonhosted.org/packages/source/P/PyYAML/PyYAML-$(PYYAML_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyYAML-$(PYYAML_VERSION).tar.gz
+ # Download PyGreSQL using curl
+ curl -sSL https://files.pythonhosted.org/packages/source/P/PyGreSQL/PyGreSQL-$(PYGRESQL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyGreSQL-$(PYGRESQL_VERSION).tar.gz
+ # Install wheel and cython for PyYAML building
+ pip3 install --user wheel "cython<3.0.0"
#
# PyGreSQL
@@ -148,14 +148,14 @@
@echo "--- mock for platform $(UBUNTU_PLATFORM)"
@if [ "$(UBUNTU_PLATFORM)" = "Ubuntu" ]; then\
pip3 install mock;\
- else\
- mkdir -p $(PYLIB_SRC_EXT) && \
- pip3 download --no-deps --no-binary :all: --no-build-isolation mock==$(MOCK_VERSION) --dest $(PYLIB_SRC_EXT)/ && \
- mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
- cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
- cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
- PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
- fi;
+ else\
+ mkdir -p $(PYLIB_SRC_EXT) && \
+ curl -sSL https://files.pythonhosted.org/packages/source/m/mock/mock-$(MOCK_VERSION).zip -o $(PYLIB_SRC_EXT)/mock-$(MOCK_VERSION).zip && \
+ mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
+ cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
+ cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
+ PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
+ fi;
# -----------------------------------------------------------------------------
#
