refactor: replace lsof port preflight with ss/netstat (#3105)
Fix a startup hang in kind/Kubernetes environments — the case that surfaced while bringing up the Helm chart. In a pod where `ulimit -n` is very large, `lsof -i :PORT` walks an enormous file descriptor table and `start-hugegraph.sh` stalls before the server ever binds.
The fix replaces that `lsof` call in the server's `check_port`, and removes the dead `check_port` copies from PD/Store.
- retain a bounded real-clock startup smoke test
---------
Co-authored-by: imbajin <jin@apache.org>
diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml
index 8d31ff2..eb1f0d9 100644
--- a/.github/workflows/docker-build-ci.yml
+++ b/.github/workflows/docker-build-ci.yml
@@ -26,6 +26,7 @@
paths:
- '**/Dockerfile*'
- '.dockerignore'
+ - 'hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh'
jobs:
docker-build:
@@ -47,6 +48,28 @@
run: |
IMAGE_ID=$(docker build -q -f ${{ matrix.dockerfile }} .)
echo "Built: $IMAGE_ID"
+ echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV"
HC=$(docker inspect --format='{{json .Config.Healthcheck}}' "$IMAGE_ID")
echo "Healthcheck: $HC"
[[ "$HC" != "null" ]] || { echo "ERROR: HEALTHCHECK missing in ${{ matrix.dockerfile }}"; exit 1; }
+
+ # The startup preflight needs a socket-table tool, and the base image
+ # ships none of its own. Without one every start reports "unknown" and
+ # a duplicate start is no longer refused, so assert the image can
+ # actually answer. Only the server images run check_port.
+ # TODO(docker-ci): this pins the probe dependency, not the behaviour it
+ # protects. A full duplicate-start/stop check needs a booted server with
+ # a backend, which belongs with the e2e job rather than the image build.
+ - name: Port preflight can answer inside ${{ matrix.dockerfile }}
+ if: ${{ startsWith(matrix.dockerfile, 'hugegraph-server/') }}
+ run: |
+ STATE=$(docker run --rm "$IMAGE_ID" bash -c \
+ 'source /hugegraph-server/bin/util.sh && port_listen_state 8080')
+ echo "port_listen_state 8080 -> $STATE"
+ # Assert a positive answer rather than "not unknown": an empty $STATE
+ # would otherwise satisfy the check. The step's default `bash -e`
+ # already aborts on a failed `docker run`, so this only states intent.
+ [[ "$STATE" == "free" || "$STATE" == "busy" ]] || {
+ echo "ERROR: no usable socket-table tool (ss/netstat) in ${{ matrix.dockerfile }}"
+ exit 1
+ }
diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml
index 6200679..6f670e1 100644
--- a/.github/workflows/pd-store-ci.yml
+++ b/.github/workflows/pd-store-ci.yml
@@ -110,7 +110,9 @@
- name: Check startup test prerequisites (PD)
id: pd-preflight
run: |
- for tool in lsof curl java; do
+ # These jobs run on Linux, where the suites clean up ports with fuser.
+ # lsof is no longer required: check_port now uses ss/netstat.
+ for tool in fuser curl java; do
if ! command -v "$tool" >/dev/null 2>&1; then
echo "can_run=false" >> "$GITHUB_OUTPUT"
echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT"
@@ -190,7 +192,9 @@
- name: Check startup test prerequisites (Store)
id: store-preflight
run: |
- for tool in lsof curl java; do
+ # These jobs run on Linux, where the suites clean up ports with fuser.
+ # lsof is no longer required: check_port now uses ss/netstat.
+ for tool in fuser curl java; do
if ! command -v "$tool" >/dev/null 2>&1; then
echo "can_run=false" >> "$GITHUB_OUTPUT"
echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml
index 72f28b6..7b98021 100644
--- a/.github/workflows/server-ci.yml
+++ b/.github/workflows/server-ci.yml
@@ -79,17 +79,33 @@
run: |
mvn clean compile -U -Dmaven.javadoc.skip=true -ntp
+ - name: Run check_port unit tests
+ if: ${{ env.BACKEND == 'rocksdb' }}
+ run: |
+ # Validates the ss/netstat port preflight that replaced lsof
+ $TRAVIS_DIR/test-check-port.sh \
+ hugegraph-server/hugegraph-dist/src/assembly/static \
+ hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh
+
- name: Check startup test prerequisites
id: server-preflight
if: ${{ env.BACKEND == 'rocksdb' }}
run: |
- for tool in lsof crontab curl java; do
+ # Note: lsof removed from prerequisites; check_port now uses ss, falling back
+ # to netstat, and warns without blocking when neither can answer.
+ # This job always runs on Linux (ubuntu-22.04), which uses fuser for port cleanup.
+ for tool in crontab curl java; do
if ! command -v "$tool" >/dev/null 2>&1; then
echo "can_run=false" >> "$GITHUB_OUTPUT"
echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT"
exit 0
fi
done
+ if ! command -v fuser >/dev/null 2>&1; then
+ echo "can_run=false" >> "$GITHUB_OUTPUT"
+ echo "skip_reason=missing tool: fuser (required for Linux port cleanup)" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
echo "can_run=true" >> "$GITHUB_OUTPUT"
- name: Run start-hugegraph.sh foreground mode tests
@@ -187,6 +203,13 @@
run: |
mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp
+ - name: Run check_port unit tests
+ # Validates the ss/netstat port preflight that replaced lsof
+ run: |
+ $TRAVIS_DIR/test-check-port.sh \
+ hugegraph-server/hugegraph-dist/src/assembly/static \
+ hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh
+
- name: Run RocksDB core test
run: |
$TRAVIS_DIR/run-core-test.sh $BACKEND
diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile
index d55ac18..68e6e25 100644
--- a/hugegraph-pd/Dockerfile
+++ b/hugegraph-pd/Dockerfile
@@ -45,12 +45,14 @@
WORKDIR /hugegraph-pd/
# 1. Install runtime dependencies
+# Note: lsof is gone because its only consumer was the dead check_port removed
+# from hg-pd-dist bin/util.sh. PD runs no port preflight, so unlike the server
+# images this needs no socket-table tool in its place.
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends --no-install-suggests \
dumb-init \
procps \
curl \
- lsof \
vim \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh
index 0b7ad0f..b476d79 100644
--- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh
+++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh
@@ -20,20 +20,21 @@
# TODO: consider reuse it with server-dist module (almost same as it)
function command_available() {
local cmd=$1
- if [ $(command -v $cmd >/dev/null 2>&1) ]; then
- return 1
- else
+ if command -v "$cmd" >/dev/null 2>&1; then
return 0
fi
+ return 1
}
# read a property from .properties file
function read_property() {
# file path
+ local file_name
+ local property_name
file_name=$1
# replace "." to "\."
- property_name=$(echo $2 | sed 's/\./\\\./g')
- cat $file_name | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1
+ property_name=$(echo "$2" | sed 's/\./\\\./g')
+ cat "$file_name" | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1
}
function write_property() {
@@ -56,7 +57,7 @@
local version=$2
local module=$3
- cat $file | tr -d '\n {}'| awk -F',+|:' '''{
+ cat "$file" | tr -d '\n {}'| awk -F',+|:' '''{
pre="";
for(i=1; i<=NF; ) {
if(match($i, /version/)) {
@@ -72,27 +73,13 @@
}
function process_num() {
- num=`ps -ef | grep $1 | grep -v grep | wc -l`
- return $num
+ num=$(ps -ef | grep "$1" | grep -v grep | wc -l)
+ return "$num"
}
function process_id() {
- pid=`ps -ef | grep $1 | grep -v grep | awk '{print $2}'`
- return $pid
-}
-
-# check the port of rest server is occupied
-function check_port() {
- local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||')
- if ! command_available "lsof"; then
- echo "Required lsof but it is unavailable"
- exit 1
- fi
- lsof -i :$port >/dev/null
- if [ $? -eq 0 ]; then
- echo "The port $port has already been used"
- exit 1
- fi
+ pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}')
+ return "$pid"
}
function crontab_append() {
@@ -130,13 +117,14 @@
local server_url="$3"
local timeout_s="$4"
- local now_s=`date '+%s'`
- local stop_s=$(( $now_s + $timeout_s ))
+ local now_s
+ now_s=$(date '+%s')
+ local stop_s=$(( now_s + timeout_s ))
local status
echo -n "Connecting to $server_name ($server_url)"
- while [ $now_s -le $stop_s ]; do
+ while [ "$now_s" -le "$stop_s" ]; do
echo -n .
process_status "$server_name" "$pid" >/dev/null
if [ $? -eq 1 ]; then
@@ -144,14 +132,14 @@
return 1
fi
- status=`curl -o /dev/null -s -k -w %{http_code} $server_url`
- if [[ $status -eq 200 || $status -eq 401 ]]; then
+ status=$(curl -o /dev/null -s -k -w '%{http_code}' "$server_url")
+ if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then
echo "OK"
echo "Started [pid $pid]"
return 0
fi
sleep 2
- now_s=`date '+%s'`
+ now_s=$(date '+%s')
done
echo "The operation timed out when attempting to connect to $server_url" >&2
@@ -160,22 +148,28 @@
function free_memory() {
local free=""
- local os=`uname`
+ local os=$(uname)
if [ "$os" == "Linux" ]; then
- local mem_free=`cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}'`
- local mem_buffer=`cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}'`
- local mem_cached=`cat /proc/meminfo | grep -w "Cached" | awk '{print $2}'`
+ local mem_free=$(cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}')
+ local mem_buffer=$(cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}')
+ local mem_cached=$(cat /proc/meminfo | grep -w "Cached" | awk '{print $2}')
if [[ "$mem_free" == "" || "$mem_buffer" == "" || "$mem_cached" == "" ]]; then
echo "Failed to get free memory"
exit 1
fi
- free=`expr $mem_free + $mem_buffer + $mem_cached`
- free=`expr $free / 1024`
+ free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached")
+ free=$(expr "$free" / 1024)
elif [ "$os" == "Darwin" ]; then
- local pages_free=`vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "`
- local pages_inactive=`vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "`
- local pages_available=`expr $pages_free + $pages_inactive`
- free=`expr $pages_available \* 4096 / 1024 / 1024`
+ local pages_free pages_inactive
+ pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ")
+ pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ")
+ if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then
+ echo "Failed to get free memory"
+ exit 1
+ fi
+ local pages_available
+ pages_available=$(expr "$pages_free" + "$pages_inactive")
+ free=$(expr "$pages_available" \* 4096 / 1024 / 1024)
else
echo "Unsupported operating system $os"
exit 1
@@ -187,8 +181,8 @@
local min_mem=$1
local max_mem=$2
# Get machine available memory
- local free=`free_memory`
- local half_free=$[free/2]
+ local free=$(free_memory)
+ local half_free=$((free/2))
local xmx=$min_mem
if [[ "$free" -lt "$min_mem" ]]; then
@@ -236,47 +230,78 @@
}
function get_ip() {
- local os=`uname`
+ local os=$(uname)
local loopback="127.0.0.1"
local ip=""
case $os in
Linux)
if command_available "ifconfig"; then
- ip=`ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}'`
+ ip=$(ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}')
elif command_available "ip"; then
- ip=`ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}'`
+ ip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}')
else
ip=$loopback
fi
;;
FreeBSD|OpenBSD|Darwin)
if command_available "ifconfig"; then
- ip=`ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}'`
+ ip=$(ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}')
else
ip=$loopback
fi
;;
SunOS)
if command_available "ifconfig"; then
- ip=`ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} '`
+ ip=$(ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} ')
else
ip=$loopback
fi
;;
*) ip=$loopback;;
esac
- echo $ip
+ [[ -z "$ip" ]] && ip=$loopback
+ echo "$ip"
}
function download() {
local path=$1
local link_url=$2
- if command_available "wget"; then
- wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt=""
- wget ${link_url} -P ${path} $progress_opt
- elif command_available "curl"; then
- curl ${link_url} -o ${path}/${link_url}
+ if [ ! -d "$path" ]; then
+ mkdir -p "$path" || {
+ echo "Failed to create directory: $path"
+ exit 1
+ }
+ fi
+
+ local filename tmp
+ filename=$(basename "${link_url%%[?#]*}")
+ local dest="${path}/${filename}"
+ # mktemp, not $$: a PID is shared by concurrent background subshells.
+ tmp=$(mktemp -- "${path}/.${filename}.XXXXXX") || {
+ echo "Failed to create a temporary file in $path"
+ return 1
+ }
+
+ if command_available "curl"; then
+ # -o must appear before -- so it is parsed as an option, not an extra URL.
+ if curl -fL -o "$tmp" -- "${link_url}"; then
+ mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; }
+ else
+ rm -f -- "$tmp"
+ return 1
+ fi
+ elif command_available "wget"; then
+ local -a progress_opt=()
+ if wget --help 2>&1 | grep -q -- '--show-progress'; then
+ progress_opt=(-q --show-progress)
+ fi
+ if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then
+ mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; }
+ else
+ rm -f -- "$tmp"
+ return 1
+ fi
else
echo "Required wget or curl but they are unavailable"
exit 1
@@ -289,19 +314,17 @@
local tar=$3
local link=$4
- if [ ! -d ${path}/${dir} ]; then
- if [ ! -f ${path}/${tar} ]; then
+ if [ ! -d "${path}/${dir}" ]; then
+ if [ ! -f "${path}/${tar}" ]; then
echo "Downloading the compressed package '${tar}'"
- download ${path} ${link}
- if [ $? -ne 0 ]; then
+ if ! download "${path}" "${link}"; then
echo "Failed to download, please ensure the network is available and link is valid"
exit 1
fi
echo "[OK] Finished download"
fi
echo "Unzip the compressed package '$tar'"
- tar -zxvf ${path}/${tar} -C ${path} >/dev/null 2>&1
- if [ $? -ne 0 ]; then
+ if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then
echo "Failed to unzip, please check the compressed package"
exit 1
fi
@@ -316,7 +339,7 @@
local pid="$2"
local timeout_s="$3"
- local now_s=`date '+%s'`
+ local now_s=$(date '+%s')
local stop_s=$(( $now_s + $timeout_s ))
echo -n "Killing $process_name(pid $pid)" >&2
@@ -328,7 +351,7 @@
return 0
fi
sleep 2
- now_s=`date '+%s'`
+ now_s=$(date '+%s')
done
echo "$process_name shutdown timeout(exceeded $timeout_s seconds)" >&2
return 1
@@ -357,7 +380,7 @@
return 0
fi
- case "`uname`" in
+ case "$(uname)" in
CYGWIN*) taskkill /F /PID "$pid" ;;
*) kill "$pid" ;;
esac
diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile
index 2ca946e..5caadd2 100644
--- a/hugegraph-server/Dockerfile
+++ b/hugegraph-server/Dockerfile
@@ -46,12 +46,16 @@
WORKDIR /hugegraph-server/
# 1. Install runtime dependencies and configure server
+# Note: iproute2 provides `ss`, which the bin/util.sh port preflight needs. The
+# jammy base image ships neither ss nor netstat, so without it the preflight is
+# permanently inconclusive and a duplicate start is no longer caught. It
+# replaces lsof, which no shipped script calls any more.
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends --no-install-suggests \
dumb-init \
procps \
curl \
- lsof \
+ iproute2 \
vim \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore
index 06d2d4b..7cd64e8 100644
--- a/hugegraph-server/Dockerfile-hstore
+++ b/hugegraph-server/Dockerfile-hstore
@@ -48,12 +48,16 @@
WORKDIR /hugegraph-server/
# 1. Install runtime dependencies and configure server
+# Note: iproute2 provides `ss`, which the bin/util.sh port preflight needs. The
+# jammy base image ships neither ss nor netstat, so without it the preflight is
+# permanently inconclusive and a duplicate start is no longer caught. It
+# replaces lsof, which no shipped script calls any more.
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends --no-install-suggests \
dumb-init \
procps \
curl \
- lsof \
+ iproute2 \
vim \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh
index 91c0c3e..8ae46e8 100755
--- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh
+++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh
@@ -18,11 +18,10 @@
function command_available() {
local cmd=$1
- if [[ -x "$(command -v "$cmd")" ]]; then
+ if command -v "$cmd" >/dev/null 2>&1; then
return 0
- else
- return 1
fi
+ return 1
}
function configure_riscv64_libatomic() {
@@ -67,6 +66,8 @@
# read a property from .properties file
function read_property() {
# file path
+ local file_name
+ local property_name
file_name=$1
# replace "." to "\."
property_name=$(echo "$2" | sed 's/\./\\\./g')
@@ -118,18 +119,197 @@
return "$pid"
}
-# check the port of rest server is occupied
+# Extract a validated TCP port from a configured server URL.
+# Echoes the port on success. Returns 1 when the value carries no usable port
+# or is ambiguous, in which case the caller skips the preflight.
+function parse_port_from_url() {
+ local url="$1"
+
+ # ServerOptions tolerates surrounding whitespace, so strip it first.
+ url="${url#"${url%%[![:space:]]*}"}"
+ url="${url%"${url##*[![:space:]]}"}"
+ [[ -z "$url" ]] && return 1
+
+ # The scheme is optional and case-insensitive.
+ local scheme="" rest="$url"
+ if [[ "$url" == *"://"* ]]; then
+ scheme=$(echo "${url%%://*}" | tr '[:upper:]' '[:lower:]')
+ rest="${url#*://}"
+ fi
+
+ # The authority ends at the first '/', '?' or '#'.
+ local authority="${rest%%[/?#]*}"
+ [[ -z "$authority" ]] && return 1
+
+ # Drop any userinfo prefix; its colon would otherwise look like an
+ # unbracketed IPv6 separator.
+ authority="${authority##*@}"
+ [[ -z "$authority" ]] && return 1
+
+ local port=""
+ if [[ "$authority" =~ ^\[[^]]*\](:([0-9]+))?$ ]]; then
+ # Bracketed IPv6, with or without a port: [::1] or [::1]:8080
+ port="${BASH_REMATCH[2]}"
+ elif [[ "$authority" == *:*:* ]]; then
+ # Unbracketed IPv6 is ambiguous: in "::1:8080" the trailing group may
+ # be a port or another hextet. Refuse to guess.
+ # TODO(check_port): no preflight runs at all for this form. If
+ # ServerOptions ever guarantees a normalized bracketed value here, this
+ # branch can resolve the port instead of skipping the check.
+ echo "WARN: ambiguous IPv6 authority '$authority' in server URL;" \
+ "use bracket notation such as [::1]:8080." >&2
+ return 1
+ elif [[ "$authority" == *:* ]]; then
+ port="${authority##*:}"
+ fi
+
+ # Fall back to the scheme's default port.
+ # TODO(check_port): a scheme-less value with no explicit port (e.g. plain
+ # "127.0.0.1") has no derivable port, so it is skipped rather than guessed.
+ # Reading the configured default from ServerOptions would close this gap.
+ if [[ -z "$port" ]]; then
+ case "$scheme" in
+ http) port="80" ;;
+ https) port="443" ;;
+ *) return 1 ;;
+ esac
+ fi
+
+ [[ "$port" =~ ^[0-9]+$ ]] || return 1
+ # Normalise leading-zero forms textually; Java reads 08080 as decimal 8080.
+ # Arithmetic conversion must not happen before the value is bounded: Bash
+ # evaluates in 64-bit and wraps silently, so 18446744073709559616 would
+ # otherwise pass the range check as port 8000.
+ port="${port#"${port%%[!0]*}"}"
+ [[ -z "$port" ]] && return 1
+ (( ${#port} <= 5 )) || return 1
+ (( port >= 1 && port <= 65535 )) || return 1
+
+ echo "$port"
+}
+
+# Echo "busy", "free" or "unknown" for the given TCP port.
+#
+# The port is compared as text against the listener table, so the argument must
+# already be normalised - parse_port_from_url() strips leading zeroes for this
+# reason, and "08080" passed directly here would not match a listener on 8080.
+#
+# Detection is deliberately port-only, matching the conservative behaviour of
+# the `lsof -i :PORT` call this replaces. Reproducing kernel socket semantics
+# in Bash - dual-stack IPV6_V6ONLY, wildcard versus specific binds, address
+# canonicalisation - produced more wrong answers than it prevented, so we only
+# ask "is anything already listening on this port?".
+#
+# Only LISTEN rows and only the local-address column are inspected, so an
+# unrelated outbound connection to the same port number is never mistaken for
+# a local listener. A tool that is missing, fails, or yields no recognisable
+# listener row reports "unknown" rather than "free", and the next probe is
+# tried; the sole exception is `ss -H` succeeding with no output at all, which
+# says positively that the host has no TCP listeners.
+#
+# TODO(check_port): port-only matching ignores the listener's address, so a
+# listener bound to one local address (127.0.0.1:8080) reports the port busy
+# even when the server would bind a different one (192.168.1.5:8080). This is
+# deliberate - it is what `lsof -i :PORT` did, and it fails safe - but it can
+# refuse a bind that would have succeeded. If that is reported in practice,
+# revisit by comparing the local-address column instead of only its port.
+#
+# TODO(check_port): only the `ss` branch can tell "no listeners at all" from
+# "table unreadable", because -H removes the header and leaves a zero exit with
+# empty output as a positive signal. Both netstat branches print headers that
+# the parser drops, so an empty result there stays "unknown" and warns.
+function port_listen_state() {
+ local port="$1"
+ local out
+ local state
+ local os
+ os=$(uname)
+
+ # $4 is the local address for both `ss -ltn` and BSD `netstat -an`.
+ # Splitting on the last separator keeps IPv6 hextets (for example
+ # [2001:db8::80]:443) from being misread as the port.
+ local parser='
+ NF >= 4 && (!want_listen || $NF == "LISTEN") {
+ addr = $4
+ cut = 0
+ for (k = length(addr); k > 0; k--) {
+ if (substr(addr, k, 1) == sep) { cut = k; break }
+ }
+ if (cut == 0) next
+ rows++
+ if (substr(addr, cut + 1) == port) { found = 1; exit }
+ }
+ END {
+ if (found) print "busy"
+ else if (rows > 0) print "free"
+ else print "unknown"
+ }'
+
+ # Each probe answers only when it produced a usable listener table; an
+ # inconclusive one falls through to the next rather than ending the search.
+ if [[ "$os" == "Darwin" || "$os" == *BSD* ]]; then
+ if command_available "netstat" && out=$(netstat -an -p tcp 2>/dev/null) \
+ && [[ -n "$out" ]]; then
+ state=$(echo "$out" | awk -v port="$port" -v sep="." -v want_listen=1 "$parser")
+ case "$state" in
+ busy|free) echo "$state"; return 0 ;;
+ esac
+ fi
+ else
+ # `ss -H -ltn` already restricts output to listening sockets, and -H
+ # drops the header, so a zero exit with no output means "nothing is
+ # listening" - the one case where empty is an answer, not a failure.
+ if command_available "ss" && out=$(ss -H -ltn 2>/dev/null); then
+ if [[ -z "$out" ]]; then
+ echo "free"
+ return 0
+ fi
+ state=$(echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=0 "$parser")
+ case "$state" in
+ busy|free) echo "$state"; return 0 ;;
+ esac
+ fi
+ if command_available "netstat" && out=$(netstat -ltn 2>/dev/null) \
+ && [[ -n "$out" ]]; then
+ state=$(echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=1 "$parser")
+ case "$state" in
+ busy|free) echo "$state"; return 0 ;;
+ esac
+ fi
+ fi
+
+ # TODO(check_port): with neither ss nor netstat present there is no probe
+ # left, so the preflight is permanently "unknown" and never detects a busy
+ # port. The published server images carry iproute2 for this reason, but a
+ # hand-built minimal image can still ship neither. A dependency-free
+ # fallback would need a bounded connect, which was deliberately removed
+ # here; adding one back means re-solving the hang this PR fixes.
+ echo "unknown"
+}
+
+# Best-effort startup preflight. Exits 1 when the configured port is already
+# in use. The server's own bind stays authoritative, so an inconclusive
+# result only warns and lets startup proceed.
function check_port() {
- local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||')
- if ! command_available "lsof"; then
- echo "Required lsof but it is unavailable"
- exit 1
- fi
- lsof -i :"$port" >/dev/null
- if [ $? -eq 0 ]; then
- echo "The port $port has already been used"
- exit 1
- fi
+ local url="$1"
+ local port
+ local state
+
+ port=$(parse_port_from_url "$url") || return 0
+
+ state=$(port_listen_state "$port")
+ case "$state" in
+ busy)
+ echo "The port $port has already been used"
+ exit 1
+ ;;
+ unknown)
+ echo "WARN: could not determine whether port $port is free;" \
+ "continuing and letting the server bind decide." >&2
+ ;;
+ esac
+
+ return 0
}
function crontab_append() {
@@ -167,14 +347,15 @@
local server_url="$3"
local timeout_s="$4"
- local now_s=$(date '+%s')
+ local now_s
+ now_s=$(date '+%s')
local stop_s=$((now_s + timeout_s))
local status
local error_file_name="startup_error.txt"
echo -n "Connecting to $server_name ($server_url)"
- while [ "$now_s" -le $stop_s ]; do
+ while [ "$now_s" -lt "$stop_s" ]; do
echo -n .
process_status "$server_name" "$pid" >/dev/null
if [ $? -eq 1 ]; then
@@ -185,8 +366,18 @@
return 1
fi
- status=$(curl -I -sS -k -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name")
- if [[ $status -eq 200 || $status -eq 401 ]]; then
+ # Bound each probe by the time left in the overall deadline: without
+ # --max-time a single blackholed request blocks past ${timeout_s}s.
+ # Refresh the clock after the process check and immediately before the
+ # probe. A stale loop-entry time must not grant curl a new one-second
+ # budget when the overall deadline has already been reached.
+ now_s=$(date '+%s')
+ local remain_s=$((stop_s - now_s))
+ [ "$remain_s" -le 0 ] && break
+ local connect_s=$((remain_s < 5 ? remain_s : 5))
+ status=$(curl -I -sS -k --connect-timeout "$connect_s" --max-time "$remain_s" \
+ -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name")
+ if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then
echo "OK"
echo "Started [pid $pid]"
if [ -e "$error_file_name" ]; then
@@ -194,13 +385,24 @@
fi
return 0
fi
- sleep 2
+
+ # Pause for the retry interval, but never past the deadline: sleeping a
+ # fixed 2s and only then re-reading the clock made a 1s timeout take 2s.
+ # Nothing left to wait for means the deadline is spent, so stop here
+ # rather than spin until the clock ticks past it.
+ now_s=$(date '+%s')
+ local sleep_s=$((stop_s - now_s))
+ [ "$sleep_s" -le 0 ] && break
+ [ "$sleep_s" -gt 2 ] && sleep_s=2
+ sleep "$sleep_s"
now_s=$(date '+%s')
done
echo ""
- cat "$error_file_name"
- rm "$error_file_name"
+ if [ -e "$error_file_name" ]; then
+ cat "$error_file_name"
+ rm "$error_file_name"
+ fi
echo "The operation timed out(${timeout_s}s) when attempting to connect to $server_url" >&2
return 1
}
@@ -318,17 +520,43 @@
function download() {
local path=$1
local download_url=$2
+
+ if [ ! -d "$path" ]; then
+ mkdir -p "$path" || {
+ echo "Failed to create directory: $path"
+ exit 1
+ }
+ fi
+
+ # Strip query/fragment so the on-disk name matches the server-side artifact.
+ local filename tmp
+ filename=$(basename "${download_url%%[?#]*}")
+ local dest="${path}/${filename}"
+ # mktemp, not $$: a PID is shared by concurrent background subshells.
+ tmp=$(mktemp -- "${path}/.${filename}.XXXXXX") || {
+ echo "Failed to create a temporary file in $path"
+ return 1
+ }
+
if command_available "curl"; then
- if [ ! -d "$path" ]; then
- mkdir -p "$path" || {
- echo "Failed to create directory: $path"
- exit 1
- }
+ # -o must appear before -- so it is parsed as an option, not an extra URL.
+ if curl -fL -o "$tmp" -- "${download_url}"; then
+ mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; }
+ else
+ rm -f -- "$tmp"
+ return 1
fi
- curl -L "${download_url}" -o "${path}/$(basename "${download_url}")"
elif command_available "wget"; then
- wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt=""
- wget "${download_url}" -P "${path}" $progress_opt
+ local -a progress_opt=()
+ if wget --help 2>&1 | grep -q -- '--show-progress'; then
+ progress_opt=(-q --show-progress)
+ fi
+ if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${download_url}"; then
+ mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; }
+ else
+ rm -f -- "$tmp"
+ return 1
+ fi
else
echo "Required curl or wget but they are unavailable"
exit 1
@@ -341,19 +569,17 @@
local tar=$3
local link=$4
- if [ ! -d "${path}"/"${dir}" ]; then
- if [ ! -f "${path}"/"${tar}" ]; then
+ if [ ! -d "${path}/${dir}" ]; then
+ if [ ! -f "${path}/${tar}" ]; then
echo "Downloading the compressed package '${tar}'"
- download "${path}" "${link}"
- if [ $? -ne 0 ]; then
+ if ! download "${path}" "${link}"; then
echo "Failed to download, please ensure the network is available and link is valid"
exit 1
fi
echo "[OK] Finished download"
fi
echo "Unzip the compressed package '$tar'"
- tar zxvf "${path}"/"${tar}" -C "${path}" >/dev/null 2>&1
- if [ $? -ne 0 ]; then
+ if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then
echo "Failed to unzip, please check the compressed package"
exit 1
fi
@@ -363,6 +589,10 @@
###########################################################################
+# TODO(wait_for_shutdown): this loop has the same fixed-2s-then-check-the-clock
+# shape wait_for_startup had, so it can also overrun its timeout by roughly one
+# sleep interval. Left alone here to keep this PR to the port preflight; the
+# fix is the same remaining-deadline cap used above.
function wait_for_shutdown() {
local process_name="$1"
local pid="$2"
diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh
new file mode 100755
index 0000000..e895035
--- /dev/null
+++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh
@@ -0,0 +1,677 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Contract tests for the startup port preflight in bin/util.sh.
+#
+# The preflight is best effort: the server's own bind is authoritative. These
+# tests pin the three-state contract (busy / free / unknown) rather than the
+# internals of any one probe.
+#
+# TODO(test-check-port): the Linux and BSD detection branches are both driven
+# by mocked tool output, so on any one runner only the host's own branch is
+# ever exercised against a real kernel. The single real-listener case covers
+# whichever OS the job runs on. Closing this needs the suite to run on both
+# a Linux and a macOS runner, which CI already does for the server job.
+
+set -u
+
+STATIC_DIR="${1:-hugegraph-server/hugegraph-dist/src/assembly/static}"
+UTIL_SH="$STATIC_DIR/bin/util.sh"
+STORE_UTIL_SH="${2:-}"
+
+if [[ ! -f "$UTIL_SH" ]]; then
+ if [[ $# -gt 0 ]]; then
+ echo "ERROR: required util.sh not found at $UTIL_SH" >&2
+ exit 1
+ fi
+ # With no argument this script may be discovered outside the repository by
+ # an optional test runner. CI always supplies STATIC_DIR explicitly, so a
+ # bad workflow path takes the required failure branch above.
+ echo "SKIP: default util.sh not found at $UTIL_SH"
+ exit 0
+fi
+
+if [[ -n "$STORE_UTIL_SH" && ! -f "$STORE_UTIL_SH" ]]; then
+ echo "ERROR: required Store util.sh not found at $STORE_UTIL_SH" >&2
+ exit 1
+fi
+
+# shellcheck source=/dev/null
+source "$UTIL_SH"
+
+# Sections run in subshells so their command overrides stay isolated, which
+# means results have to be tallied through a file rather than a variable.
+RESULTS=$(mktemp)
+# Clean the tally on any exit, not only the normal one. Section 5 runs outside
+# a subshell and has to extend this trap, so it restores this handler rather
+# than clearing it.
+trap 'rm -f "$RESULTS"' EXIT
+
+pass() {
+ echo " PASS $1"
+ echo "P" >> "$RESULTS"
+}
+
+fail() {
+ echo " FAIL $1"
+ echo " $2"
+ echo "F" >> "$RESULTS"
+}
+
+expect() {
+ # expect <name> <expected> <actual>
+ if [[ "$2" == "$3" ]]; then
+ pass "$1"
+ else
+ fail "$1" "expected '$2', got '$3'"
+ fi
+}
+
+echo ""
+echo "check_port contract tests ($UTIL_SH)"
+
+# --------------------------------------------------------------------------
+echo ""
+echo "1. URL to port"
+# "<url>|<expected>", where SKIP means "no usable port, preflight is skipped".
+url_cases=(
+ 'http://127.0.0.1:8080|8080'
+ 'HTTP://127.0.0.1:8080|8080'
+ 'http://127.0.0.1|80'
+ 'https://127.0.0.1|443'
+ '127.0.0.1:8080|8080'
+ 'http://127.0.0.1:8080/path:9090|8080'
+ 'http://127.0.0.1:8080?probe=x|8080'
+ 'http://127.0.0.1:8080#frag|8080'
+ 'http://user:pass@127.0.0.1:8080|8080'
+ 'http://user@127.0.0.1:8080|8080'
+ 'http://[::1]:8080|8080'
+ 'http://user:pass@[::1]:8080|8080'
+ '[::1]:8080|8080'
+ 'https://[::1]|443'
+ 'http://127.0.0.1:08080|8080'
+ '::1:8080|SKIP'
+ 'http://127.0.0.1:abc|SKIP'
+ 'http://127.0.0.1:0|SKIP'
+ 'http://127.0.0.1:70000|SKIP'
+ '127.0.0.1|SKIP'
+ # Oversized values must be rejected on their digits, not after a 64-bit
+ # arithmetic conversion that would wrap them back into range.
+ 'http://127.0.0.1:18446744073709551617|SKIP'
+ 'http://127.0.0.1:18446744073709559616|SKIP'
+ 'http://127.0.0.1:0000000000000008080|8080'
+ 'http://127.0.0.1:00000|SKIP'
+)
+for case in "${url_cases[@]}"; do
+ url="${case%|*}"
+ want="${case##*|}"
+ got=$(parse_port_from_url " $url " 2>/dev/null) || got="SKIP"
+ expect "$url" "$want" "$got"
+done
+
+# --------------------------------------------------------------------------
+echo ""
+echo "2. Linux: ss busy / free / failure, and the netstat fallback"
+(
+ uname() { echo "Linux"; }
+ command_available() { [[ "$1" == "ss" ]]; }
+ WORK=$(mktemp -d)
+ trap 'rm -rf "$WORK"' EXIT
+ SS_ARGS_FILE="$WORK/ss-args"
+ SS_OUT=""
+ SS_RC=0
+ SS_FILTER_LISTEN=0
+ ss() {
+ echo "$*" > "$SS_ARGS_FILE"
+ [[ $# -eq 2 && "$1" == "-H" && "$2" == "-ltn" ]] || return 64
+ if [[ "$SS_FILTER_LISTEN" -eq 1 ]]; then
+ echo "$SS_OUT" | awk '$1 == "LISTEN"'
+ else
+ printf '%s' "$SS_OUT"
+ fi
+ return "$SS_RC"
+ }
+
+ SS_OUT='LISTEN 0 4096 0.0.0.0:8080 0.0.0.0:*'
+ expect "listener on 8080 is busy" "busy" "$(port_listen_state 8080)"
+ expect "ss receives the exact listener-table arguments" \
+ "-H -ltn" "$(cat "$SS_ARGS_FILE")"
+
+ # Model the filtering done by `ss -l`: an established connection may use
+ # the target as its local ephemeral port, but it is not a listener. The
+ # mock refuses changed argv, so dropping -l makes this unknown, not free.
+ SS_FILTER_LISTEN=1
+ SS_OUT='ESTAB 0 0 127.0.0.1:8080 127.0.0.1:443'
+ expect "Linux non-LISTEN local port is free" "free" "$(port_listen_state 8080)"
+ SS_FILTER_LISTEN=0
+
+ SS_OUT='LISTEN 0 4096 0.0.0.0:22 0.0.0.0:*'
+ expect "no listener on 8080 is free" "free" "$(port_listen_state 8080)"
+
+ # The port must come from the local-address field, not an IPv6 hextet.
+ SS_OUT='LISTEN 0 128 [2001:db8:8080::1]:9090 [::]:*'
+ expect "IPv6 hextet 8080 is not the port" "free" "$(port_listen_state 8080)"
+
+ SS_OUT='LISTEN 0 128 [::]:8080 [::]:*'
+ expect "IPv6 wildcard listener is busy" "busy" "$(port_listen_state 8080)"
+
+ # A tool that runs but fails proves nothing.
+ SS_OUT=''
+ SS_RC=1
+ expect "ss failure is unknown" "unknown" "$(port_listen_state 8080)"
+
+ # `-H` prints no header, so a zero exit with no output is a positive "this
+ # host has no TCP listeners" - the state a fresh container starts in.
+ SS_RC=0
+ SS_OUT=''
+ expect "ss empty output is free" "free" "$(port_listen_state 8080)"
+
+ # Success with an unusable table proves nothing.
+ SS_OUT='some diagnostic banner'
+ expect "ss unparseable output is unknown" "unknown" "$(port_listen_state 8080)"
+
+ # ...and it must not end the search either: netstat may still have a
+ # readable table. Returning on the first probe left the port "unknown"
+ # even when the fallback could have answered.
+ command_available() { [[ "$1" == "ss" || "$1" == "netstat" ]]; }
+ netstat() {
+ echo 'Active Internet connections (only servers)'
+ echo 'Proto Recv-Q Send-Q Local Address Foreign Address State'
+ echo 'tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN'
+ }
+
+ SS_OUT='some diagnostic banner'
+ expect "unparseable ss falls back to netstat" "busy" "$(port_listen_state 8080)"
+ expect "netstat fallback can also report free" "free" "$(port_listen_state 9999)"
+
+ SS_RC=1
+ SS_OUT=''
+ expect "failed ss falls back to netstat" "busy" "$(port_listen_state 8080)"
+
+)
+
+# --------------------------------------------------------------------------
+echo ""
+echo "3. macOS/BSD: LISTEN versus ESTABLISHED"
+(
+ uname() { echo "Darwin"; }
+ command_available() { [[ "$1" == "netstat" ]]; }
+ NS_OUT=""
+ netstat() { printf '%s' "$NS_OUT"; }
+
+ NS_OUT='Active Internet connections (including servers)
+Proto Recv-Q Send-Q Local Address Foreign Address (state)
+tcp4 0 0 *.8080 *.* LISTEN'
+ expect "BSD LISTEN on 8080 is busy" "busy" "$(port_listen_state 8080)"
+
+ # An outbound connection to :443 is not a local listener on 443. This is
+ # the false positive that blocked startup on macOS.
+ NS_OUT='Active Internet connections (including servers)
+Proto Recv-Q Send-Q Local Address Foreign Address (state)
+tcp4 0 0 192.168.1.3.60320 44.195.16.138.443 ESTABLISHED
+tcp4 0 0 *.22 *.* LISTEN'
+ expect "BSD ESTABLISHED peer :443 is free" "free" "$(port_listen_state 443)"
+
+ # A socket lingering in TIME_WAIT holds the port in its *local* address but
+ # is not a listener, so only the connection state can tell them apart.
+ NS_OUT='Proto Recv-Q Send-Q Local Address Foreign Address (state)
+tcp4 0 0 127.0.0.1.8080 127.0.0.1.51000 TIME_WAIT
+tcp4 0 0 *.22 *.* LISTEN'
+ expect "BSD TIME_WAIT on local 8080 is free" "free" "$(port_listen_state 8080)"
+
+ NS_OUT='Proto Recv-Q Send-Q Local Address Foreign Address (state)
+tcp6 0 0 ::1.8080 *.* LISTEN'
+ expect "BSD IPv6 LISTEN is busy" "busy" "$(port_listen_state 8080)"
+
+ NS_OUT='Proto Recv-Q Send-Q Local Address Foreign Address (state)'
+ expect "BSD header only is unknown" "unknown" "$(port_listen_state 8080)"
+
+)
+
+# --------------------------------------------------------------------------
+echo ""
+echo "4. Unknown never blocks startup"
+(
+ uname() { echo "Linux"; }
+ command_available() { return 1; }
+
+ expect "no probe tool is unknown" "unknown" "$(port_listen_state 8080)"
+
+ # check_port must warn and let the server perform the authoritative bind.
+ err=$( (check_port "http://127.0.0.1:8080") 2>&1 >/dev/null )
+ rc=$?
+ expect "unknown exits 0" "0" "$rc"
+ if [[ "$err" == *"could not determine"* ]]; then
+ pass "unknown warns on stderr"
+ else
+ fail "unknown warns on stderr" "stderr was: $err"
+ fi
+
+)
+
+# --------------------------------------------------------------------------
+echo ""
+echo "5. Real listener on an ephemeral port"
+if command -v python3 >/dev/null 2>&1; then
+ PORT_FILE=$(mktemp)
+ # Cover the temp file from the moment it exists, then widen the trap to the
+ # child as soon as there is a pid; registering only after the fork leaves a
+ # window where an interrupt orphans the listener for its full 30s sleep.
+ trap 'rm -f "$PORT_FILE" "$RESULTS"' EXIT
+ python3 -c '
+import socket, sys, time
+s = socket.socket()
+s.bind(("127.0.0.1", 0))
+s.listen(1)
+print(s.getsockname()[1])
+sys.stdout.flush()
+time.sleep(30)
+' > "$PORT_FILE" &
+ PY_PID=$!
+ trap 'kill "$PY_PID" 2>/dev/null; rm -f "$PORT_FILE" "$RESULTS"' EXIT
+
+ BOUND=""
+ for _ in 1 2 3 4 5 6 7 8 9 10; do
+ BOUND=$(head -1 "$PORT_FILE" 2>/dev/null)
+ [[ -n "$BOUND" ]] && break
+ # Whole seconds only: fractional sleep is not POSIX, and this suite runs
+ # with `set -u` but no `-e`, so a busybox sleep would fail silently and
+ # spin all ten iterations instantly.
+ sleep 1
+ done
+
+ if [[ -z "$BOUND" ]] || ! kill -0 "$PY_PID" 2>/dev/null; then
+ fail "listener bound an ephemeral port" "child did not report a port"
+ else
+ pass "listener bound an ephemeral port ($BOUND)"
+ expect "bound port is busy" "busy" "$(port_listen_state "$BOUND")"
+ (check_port "http://127.0.0.1:$BOUND" >/dev/null 2>&1)
+ expect "check_port exits 1 on the bound port" "1" "$?"
+
+ # A port that was bound and released must not read as busy.
+ FREE=$(python3 -c 'import socket
+s = socket.socket()
+s.bind(("127.0.0.1", 0))
+p = s.getsockname()[1]
+s.close()
+print(p)')
+ expect "released port is not busy" "0" "$([[ "$(port_listen_state "$FREE")" == "busy" ]] && echo 1 || echo 0)"
+ fi
+
+ kill "$PY_PID" 2>/dev/null
+ wait "$PY_PID" 2>/dev/null
+ rm -f "$PORT_FILE"
+ # Restore the script-level handler rather than clearing it, so the tally
+ # file stays covered for the remaining sections.
+ trap 'rm -f "$RESULTS"' EXIT
+else
+ echo " SKIP real listener test: python3 not available"
+fi
+
+# --------------------------------------------------------------------------
+echo ""
+echo "6. download() rename failure is reported"
+(
+ WORK=$(mktemp -d)
+ trap 'rm -rf "$WORK"' EXIT
+
+ command_available() { [[ "$1" == "curl" ]]; }
+ curl() {
+ # Write to the -o destination so a temp file exists to rename.
+ local out=""
+ while [[ $# -gt 0 ]]; do
+ [[ "$1" == "-o" ]] && { out="$2"; shift; }
+ shift
+ done
+ echo "payload" > "$out"
+ return 0
+ }
+ # Simulate a rename that fails (read-only target, cross-device, ...).
+ mv() { return 1; }
+
+ if download "$WORK" "https://example.com/pkg.tar.gz"; then
+ fail "download reports rename failure" "download returned 0 after mv failed"
+ else
+ pass "download reports rename failure"
+ fi
+
+ if [[ -e "$WORK/pkg.tar.gz" ]]; then
+ fail "failed rename leaves no destination" "$WORK/pkg.tar.gz exists"
+ else
+ pass "failed rename leaves no destination"
+ fi
+
+ leftover=$(find "$WORK" -name '.pkg.tar.gz.*' 2>/dev/null | wc -l | tr -d ' ')
+ expect "failed rename cleans its temp file" "0" "$leftover"
+
+)
+
+# --------------------------------------------------------------------------
+echo ""
+echo "7. Startup probe is bounded"
+(
+ WORK=$(mktemp -d)
+ cd "$WORK" || exit 1
+ trap 'cd /; rm -rf "$WORK"' EXIT
+
+ CLOCK_FILE="$WORK/clock"
+ ARGS_FILE="$WORK/curl-args"
+ SLEEP_FILE="$WORK/sleep-args"
+ echo 100 > "$CLOCK_FILE"
+ : > "$ARGS_FILE"
+ : > "$SLEEP_FILE"
+ curl() { echo "$*" >> "$ARGS_FILE"; echo "000"; return 28; }
+ # Keep the contract checks deterministic across whole-second boundaries.
+ # The retry pause consumes the remaining simulated second without making
+ # this half of the test depend on scheduler timing.
+ date() { cat "$CLOCK_FILE"; }
+ sleep() {
+ echo "$1" >> "$SLEEP_FILE"
+ echo 101 > "$CLOCK_FILE"
+ }
+ process_status() { return 0; }
+ ps() { return 0; }
+
+ TIMEOUT_S=1
+ if wait_for_startup "$$" "test-server" "http://127.0.0.1:1" "$TIMEOUT_S" \
+ >/dev/null 2>&1; then
+ WAIT_STATUS=0
+ else
+ WAIT_STATUS=$?
+ fi
+
+ expect "failed startup probe returns failure" "1" "$WAIT_STATUS"
+
+ # The deterministic half: bounding each curl does not bound the loop, which
+ # used to sleep a flat 2s and only then re-read the clock. A 1s timeout
+ # must never ask for more than 1s of sleeping.
+ TOTAL_SLEEP_S=$(awk '{ total += $1 } END { print total + 0 }' "$SLEEP_FILE")
+ if [[ "$TOTAL_SLEEP_S" -le "$TIMEOUT_S" ]]; then
+ pass "retry sleeps stay inside the ${TIMEOUT_S}s deadline"
+ else
+ fail "retry sleeps stay inside the ${TIMEOUT_S}s deadline" \
+ "slept ${TOTAL_SLEEP_S}s in total: $(tr '\n' ' ' < "$SLEEP_FILE")"
+ fi
+
+ # With a one-second overall deadline, every positive remaining budget is
+ # exactly one second. Parse each curl call so option presence, values, and
+ # the deadline bound are all part of the contract rather than string-grep
+ # smoke checks.
+ CURL_CALLS=0
+ TIMEOUT_VALUES_VALID=1
+ TIMEOUT_ERROR=""
+ while IFS= read -r curl_args; do
+ [[ -z "$curl_args" ]] && continue
+ CURL_CALLS=$((CURL_CALLS + 1))
+ CONNECT_TIMEOUT=""
+ MAX_TIME=""
+ set -- $curl_args
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --connect-timeout)
+ shift
+ [[ $# -gt 0 ]] && CONNECT_TIMEOUT="$1"
+ ;;
+ --max-time)
+ shift
+ [[ $# -gt 0 ]] && MAX_TIME="$1"
+ ;;
+ esac
+ [[ $# -gt 0 ]] && shift
+ done
+
+ if [[ ! "$CONNECT_TIMEOUT" =~ ^[0-9]+$ ||
+ ! "$MAX_TIME" =~ ^[0-9]+$ ||
+ "$CONNECT_TIMEOUT" -le 0 || "$CONNECT_TIMEOUT" -gt "$TIMEOUT_S" ||
+ "$MAX_TIME" -le 0 || "$MAX_TIME" -gt "$TIMEOUT_S" ]]; then
+ TIMEOUT_VALUES_VALID=0
+ TIMEOUT_ERROR="invalid timeouts connect='$CONNECT_TIMEOUT' max='$MAX_TIME' in: $curl_args"
+ break
+ fi
+ done < "$ARGS_FILE"
+
+ if [[ "$CURL_CALLS" -gt 0 && "$TIMEOUT_VALUES_VALID" -eq 1 ]]; then
+ pass "startup probe timeouts are positive and within the remaining deadline"
+ else
+ [[ -n "$TIMEOUT_ERROR" ]] || TIMEOUT_ERROR="no curl calls recorded"
+ fail "startup probe timeouts are positive and within the remaining deadline" \
+ "$TIMEOUT_ERROR"
+ fi
+
+ # A probe may consume all of its assigned budget. Refreshing the clock
+ # after curl must then suppress the retry pause rather than sleep beyond
+ # the overall deadline.
+ EXHAUST_SLEEP_FILE="$WORK/exhausted-sleep-calls"
+ echo 100 > "$CLOCK_FILE"
+ : > "$EXHAUST_SLEEP_FILE"
+ curl() {
+ echo 101 > "$CLOCK_FILE"
+ echo "000"
+ return 28
+ }
+ sleep() { echo "$1" >> "$EXHAUST_SLEEP_FILE"; }
+ if wait_for_startup "$$" "budget-server" "http://127.0.0.1:1" \
+ "$TIMEOUT_S" >/dev/null 2>&1; then
+ EXHAUST_STATUS=0
+ else
+ EXHAUST_STATUS=$?
+ fi
+
+ expect "budget-exhausting probe returns failure" "1" "$EXHAUST_STATUS"
+ EXHAUST_SLEEP_CALLS=$(wc -l < "$EXHAUST_SLEEP_FILE" | tr -d ' ')
+ expect "budget-exhausting probe starts no retry sleep" \
+ "0" "$EXHAUST_SLEEP_CALLS"
+
+ # Keep a small real-clock smoke test for the end-to-end deadline. Crossing
+ # an integer-second boundary between the initial and pre-probe clock reads
+ # may legitimately produce zero curl calls, so this half checks only the
+ # return status and wall-clock bound and does not require an args file.
+ date() { command date "$@"; }
+ curl() { echo "000"; return 28; }
+ sleep() { command sleep "$1"; }
+ START_S=$(date '+%s')
+ if wait_for_startup "$$" "wall-clock-server" "http://127.0.0.1:1" \
+ "$TIMEOUT_S" >/dev/null 2>&1; then
+ WALL_STATUS=0
+ else
+ WALL_STATUS=$?
+ fi
+ ELAPSED_S=$(( $(date '+%s') - START_S ))
+
+ expect "real-clock failed startup probe returns failure" "1" "$WALL_STATUS"
+ # One second of slack accounts for the granularity of date '+%s'.
+ if [[ "$ELAPSED_S" -le $((TIMEOUT_S + 1)) ]]; then
+ pass "wait_for_startup returns within the deadline (${ELAPSED_S}s)"
+ else
+ fail "wait_for_startup returns within the deadline" \
+ "took ${ELAPSED_S}s for a ${TIMEOUT_S}s timeout"
+ fi
+
+)
+
+# --------------------------------------------------------------------------
+echo ""
+echo "8. Deadline boundary starts no probe"
+(
+ WORK=$(mktemp -d)
+ cd "$WORK" || exit 1
+ trap 'cd /; rm -rf "$WORK"' EXIT
+
+ CLOCK_FILE="$WORK/clock-calls"
+ CURL_FILE="$WORK/curl-calls"
+ echo 0 > "$CLOCK_FILE"
+ : > "$CURL_FILE"
+
+ # The first read establishes [100, 101) as the allowed interval. The
+ # process check consumes that interval, so the mandatory pre-curl refresh
+ # observes the deadline exactly. File-backed state works through the
+ # command substitutions used by wait_for_startup, including Bash 3.2.
+ date() {
+ local calls
+ calls=$(cat "$CLOCK_FILE")
+ calls=$((calls + 1))
+ echo "$calls" > "$CLOCK_FILE"
+ if [[ "$calls" -eq 1 ]]; then
+ echo 100
+ else
+ echo 101
+ fi
+ }
+ process_status() { return 0; }
+ curl() { echo called >> "$CURL_FILE"; echo "000"; return 28; }
+ sleep() { echo called >> "$WORK/sleep-calls"; }
+
+ wait_for_startup "$$" "boundary-server" "http://127.0.0.1:1" 1 \
+ >/dev/null 2>&1
+
+ CLOCK_CALLS=$(cat "$CLOCK_FILE")
+ if [[ "$CLOCK_CALLS" -ge 2 ]]; then
+ pass "startup refreshes the clock at the probe boundary"
+ else
+ fail "startup refreshes the clock at the probe boundary" \
+ "clock was read only ${CLOCK_CALLS} time(s)"
+ fi
+
+ PROBE_CALLS=$(wc -l < "$CURL_FILE" | tr -d ' ')
+ expect "no startup probe begins at the deadline" "0" "$PROBE_CALLS"
+
+)
+
+if [[ -n "$STORE_UTIL_SH" ]]; then
+ # ----------------------------------------------------------------------
+ echo ""
+ echo "9. Store verified replacement survives a stale concurrent caller"
+ (
+ # Keep the Store helper overrides isolated from the server helpers used
+ # by every other section in this suite.
+ # shellcheck source=/dev/null
+ source "$STORE_UTIL_SH"
+
+ WORK=$(mktemp -d)
+ DEST="$WORK/lib.so"
+ HASHED="$WORK/stale-hashed"
+ RELEASE="$WORK/release-stale"
+ STALE_DOWNLOAD="$WORK/stale-download"
+ STALE_PID=""
+ trap '
+ touch "$RELEASE"
+ if [[ -n "$STALE_PID" ]]; then
+ kill "$STALE_PID" 2>/dev/null
+ wait "$STALE_PID" 2>/dev/null
+ fi
+ rm -rf "$WORK"
+ ' EXIT
+
+ echo "invalid" > "$DEST"
+
+ wait_for_file() {
+ local path="$1"
+ local _
+ for _ in 1 2 3 4 5 6 7 8 9 10; do
+ [[ -e "$path" ]] && return 0
+ command sleep 1
+ done
+ return 1
+ }
+
+ # Both callers first observe the invalid destination. The stale caller
+ # then holds that checksum until the installer has atomically published
+ # valid bytes. Reintroducing the old pre-download rm makes the stale
+ # caller delete that valid replacement before its own download fails.
+ md5sum() {
+ local target="${@: -1}"
+ if [[ "$target" == "$DEST" ]]; then
+ if [[ "$CALLER" == "stale" ]]; then
+ touch "$HASHED"
+ wait_for_file "$RELEASE" || return 1
+ fi
+ echo "bad $target"
+ else
+ echo "good $target"
+ fi
+ }
+
+ curl() {
+ [[ "$1" == "-fL" && "$2" == "-o" && "$4" == "--" ]] || return 64
+ if [[ "$CALLER" == "installer" ]]; then
+ echo "valid" > "$3"
+ return 0
+ fi
+ touch "$STALE_DOWNLOAD"
+ return 22
+ }
+
+ (
+ CALLER="stale"
+ download_and_verify "https://example.com/lib.so" "$DEST" "good" \
+ >/dev/null 2>&1
+ ) &
+ STALE_PID=$!
+
+ if ! wait_for_file "$HASHED"; then
+ fail "stale caller captured the original checksum" \
+ "caller did not reach the checksum barrier"
+ exit 0
+ fi
+ pass "stale caller captured the original checksum"
+
+ CALLER="installer"
+ if download_and_verify "https://example.com/lib.so" "$DEST" "good" \
+ >/dev/null 2>&1; then
+ pass "concurrent installer succeeds"
+ else
+ fail "concurrent installer succeeds" "installer returned nonzero"
+ fi
+
+ touch "$RELEASE"
+ wait "$STALE_PID" 2>/dev/null
+ STALE_WAIT_RC=$?
+ STALE_PID=""
+ if [[ "$STALE_WAIT_RC" -ne 0 && -e "$STALE_DOWNLOAD" ]]; then
+ pass "stale caller reports its failed replacement download"
+ else
+ CURL_REACHED=$([[ -e "$STALE_DOWNLOAD" ]] && echo yes || echo no)
+ fail "stale caller reports its failed replacement download" \
+ "rc=$STALE_WAIT_RC, curl reached=$CURL_REACHED"
+ fi
+
+ if [[ -f "$DEST" && "$(cat "$DEST")" == "valid" ]]; then
+ pass "failed stale caller preserves the valid replacement"
+ else
+ fail "failed stale caller preserves the valid replacement" \
+ "destination is missing or no longer contains valid bytes"
+ fi
+
+ LEFTOVERS=$(find "$WORK" -name 'lib.so.*' -type f | wc -l | tr -d ' ')
+ expect "concurrent replacement cleans private temp files" "0" "$LEFTOVERS"
+ )
+fi
+
+# --------------------------------------------------------------------------
+echo ""
+echo "----------------------------------------"
+# grep -c prints 0 and exits 1 when there are no matches; the count is what
+# matters here, so the exit status is deliberately ignored.
+PASSED=$(grep -c '^P' "$RESULTS" 2>/dev/null)
+FAILED=$(grep -c '^F' "$RESULTS" 2>/dev/null)
+rm -f "$RESULTS"
+echo "passed: $PASSED failed: $FAILED"
+if [[ "$FAILED" -gt 0 ]]; then
+ exit 1
+fi
+exit 0
diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh
index 260e7f5..ab73255 100755
--- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh
+++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh
@@ -30,6 +30,7 @@
PD_ROOT="${1:-$(pwd)}"
BIN="$PD_ROOT/bin"
START_SCRIPT="$BIN/start-hugegraph-pd.sh"
+STOP_SCRIPT="$BIN/stop-hugegraph-pd.sh"
PID_FILE="$BIN/pid"
PD_URL="http://localhost:8620"
STARTUP_WAIT=60 # seconds to wait for PD HTTP to respond
@@ -67,14 +68,30 @@
cleanup() {
info "Cleaning up..."
+ "$STOP_SCRIPT" >/dev/null 2>&1 || true
if [[ -s "$PID_FILE" ]]; then
kill "$(cat "$PID_FILE")" 2>/dev/null || true
fi
rm -f "$PID_FILE"
rm -rf "$PD_ROOT/logs/"
- # kill anything still holding the PD port
- lsof -ti :8620 | xargs kill -9 2>/dev/null || true
- lsof -ti :8686 | xargs kill -9 2>/dev/null || true
+ # kill anything still holding the PD port: fuser on Linux, lsof on macOS,
+ # which ships no fuser. Only the Linux path had to lose its lsof dependency.
+ if [[ "$(uname)" == "Darwin" ]]; then
+ local port pids pid
+ for port in 8620 8686; do
+ pids=$(lsof -ti ":$port" 2>/dev/null)
+ if [[ -n "$pids" ]]; then
+ for pid in $pids; do
+ kill -9 "$pid" 2>/dev/null || true
+ done
+ fi
+ done
+ elif command -v fuser >/dev/null 2>&1; then
+ fuser -k 8620/tcp 2>/dev/null || true
+ fuser -k 8686/tcp 2>/dev/null || true
+ else
+ echo "Warning: Neither lsof nor fuser available for PD port cleanup"
+ fi
sleep 3
}
@@ -129,7 +146,12 @@
exit 1
fi
-for tool in lsof curl java; do
+if [[ "$(uname)" == "Darwin" ]]; then
+ _prereq_tools="lsof curl java"
+else
+ _prereq_tools="fuser curl java"
+fi
+for tool in $_prereq_tools; do
if ! command -v "$tool" >/dev/null 2>&1; then
echo "SKIP: required tool '$tool' not found — skipping test suite"
exit 77
diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh
index 7e11da2..9ff324a 100755
--- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh
+++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh
@@ -51,15 +51,30 @@
cleanup() {
info "Cleaning up..."
+ "$STOP_SCRIPT" >/dev/null 2>&1 || true
if [[ -s "$PID_FILE" ]]; then
kill "$(cat "$PID_FILE")" 2>/dev/null || true
fi
rm -f "$PID_FILE"
rm -rf "$STORE_ROOT/logs/"
# kill anything holding Store ports (8520 REST, 8510 raft, 8500 gRPC)
- lsof -ti :8520 | xargs kill -9 2>/dev/null || true
- lsof -ti :8510 | xargs kill -9 2>/dev/null || true
- lsof -ti :8500 | xargs kill -9 2>/dev/null || true
+ if [[ "$(uname)" == "Darwin" ]]; then
+ local port pids pid
+ for port in 8520 8510 8500; do
+ pids=$(lsof -ti ":$port" 2>/dev/null)
+ if [[ -n "$pids" ]]; then
+ for pid in $pids; do
+ kill -9 "$pid" 2>/dev/null || true
+ done
+ fi
+ done
+ elif command -v fuser >/dev/null 2>&1; then
+ fuser -k 8520/tcp 2>/dev/null || true
+ fuser -k 8510/tcp 2>/dev/null || true
+ fuser -k 8500/tcp 2>/dev/null || true
+ else
+ echo "Warning: Neither lsof nor fuser available for Store port cleanup"
+ fi
sleep 3
}
@@ -110,7 +125,12 @@
exit 1
fi
-for tool in lsof curl java; do
+if [[ "$(uname)" == "Darwin" ]]; then
+ _prereq_tools="lsof curl java"
+else
+ _prereq_tools="fuser curl java"
+fi
+for tool in $_prereq_tools; do
if ! command -v "$tool" >/dev/null 2>&1; then
echo "SKIP: required tool '$tool' not found — skipping test suite"
exit 77
diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh
index 353cab6..9f0bcfa 100755
--- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh
+++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh
@@ -76,10 +76,27 @@
kill "$(cat "$PID_FILE")" 2>/dev/null || true
fi
rm -f "$PID_FILE"
- # kill anything still holding server ports so check_port doesn't fail next test
- lsof -ti :8080 | xargs kill -9 2>/dev/null || true
- lsof -ti :8182 | xargs kill -9 2>/dev/null || true
- lsof -ti :8088 | xargs kill -9 2>/dev/null || true
+ # kill anything still holding server ports: fuser on Linux, lsof on macOS,
+ # which ships no fuser. Only the Linux path had to lose its lsof dependency.
+ if [[ "$(uname)" == "Darwin" ]]; then
+ local port pids pid
+ for port in 8080 8182 8088; do
+ pids=$(lsof -ti ":$port" 2>/dev/null)
+ if [[ -n "$pids" ]]; then
+ for pid in $pids; do
+ kill -9 "$pid" 2>/dev/null || true
+ done
+ fi
+ done
+ elif command -v fuser >/dev/null 2>&1; then
+ # fuser exists and should work on Linux; suppress failures but log if debug needed
+ fuser -k 8080/tcp 2>/dev/null || true
+ fuser -k 8182/tcp 2>/dev/null || true
+ fuser -k 8088/tcp 2>/dev/null || true
+ else
+ # Neither lsof nor fuser available; ports may remain occupied
+ echo "Warning: Neither lsof nor fuser available for port cleanup"
+ fi
sleep 3
crontab -l 2>/dev/null | grep -v monitor-hugegraph | crontab - 2>/dev/null || true
}
@@ -145,7 +162,12 @@
exit 1
fi
-for tool in lsof crontab curl java; do
+if [[ "$(uname)" == "Darwin" ]]; then
+ _prereq_tools="lsof crontab curl java"
+else
+ _prereq_tools="fuser crontab curl java"
+fi
+for tool in $_prereq_tools; do
if ! command -v "$tool" >/dev/null 2>&1; then
echo "SKIP: required tool '$tool' not found — skipping test suite"
exit 77
@@ -155,7 +177,8 @@
# start-monitor.sh requires JAVA_HOME
if [[ -z "${JAVA_HOME:-}" ]]; then
if command -v /usr/libexec/java_home >/dev/null 2>&1; then
- export JAVA_HOME="$(/usr/libexec/java_home 2>/dev/null)"
+ JAVA_HOME="$(/usr/libexec/java_home 2>/dev/null)"
+ export JAVA_HOME
fi
fi
diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile
index 49d41f6..f1b9db5 100644
--- a/hugegraph-store/Dockerfile
+++ b/hugegraph-store/Dockerfile
@@ -45,12 +45,14 @@
WORKDIR /hugegraph-store/
# 1. Install runtime dependencies
+# Note: lsof is gone because its only consumer was the dead check_port removed
+# from hg-store-dist bin/util.sh. Store runs no port preflight, so unlike the
+# server images this needs no socket-table tool in its place.
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends --no-install-suggests \
dumb-init \
procps \
curl \
- lsof \
vim \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh
index 5a11eea..88165e4 100755
--- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh
+++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh
@@ -46,8 +46,8 @@
lib_file="$TOP/bin/libjemalloc_aarch64.so"
download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so"
expected_md5="2a631d2f81837f9d5864586761c5e380"
- if download_and_verify $download_url $lib_file $expected_md5; then
- export LD_PRELOAD=$lib_file
+ if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then
+ export LD_PRELOAD="$lib_file"
else
echo "Failed to verify or download $lib_file, skip it"
fi
@@ -55,8 +55,8 @@
lib_file="$TOP/bin/libjemalloc.so"
download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so"
expected_md5="fd61765eec3bfea961b646c269f298df"
- if download_and_verify $download_url $lib_file $expected_md5; then
- export LD_PRELOAD=$lib_file
+ if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then
+ export LD_PRELOAD="$lib_file"
else
echo "Failed to verify or download $lib_file, skip it"
fi
@@ -73,7 +73,8 @@
#export FILE_LIMITN=1024000
function check_evn_limit() {
- local limit_check=$(ulimit -n)
+ local limit_check
+ limit_check=$(ulimit -n)
if [[ ${limit_check} != "unlimited" && ${limit_check} -lt ${FILE_LIMITN} ]]; then
echo -e "${BASH_SOURCE[0]##*/}:${LINENO}:\E[1;32m ulimit -n can open too few maximum file descriptors, need (${FILE_LIMITN})!! \E[0m"
return 1
@@ -218,7 +219,7 @@
# JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:${LIB}/jmx_prometheus_javaagent-0.16.1.jar=${JMX_EXPORT_PORT}:${CONF}/jmx_exporter.yml"
#fi
-if [ $(ps -ef|grep -v grep| grep java|grep -cE ${CONF}) -ne 0 ]; then
+if [ "$(ps -ef | grep -v grep | grep java | grep -cE "${CONF}")" -ne 0 ]; then
echo "HugeGraphStoreServer is already running..."
exit 0
fi
diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh
index 3b3d660..e45249a 100644
--- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh
+++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh
@@ -19,20 +19,21 @@
function command_available() {
local cmd=$1
- if [ `command -v $cmd >/dev/null 2>&1` ]; then
- return 1
- else
+ if command -v "$cmd" >/dev/null 2>&1; then
return 0
fi
+ return 1
}
# read a property from .properties file
function read_property() {
# file path
+ local file_name
+ local property_name
file_name=$1
# replace "." to "\."
- property_name=`echo $2 | sed 's/\./\\\./g'`
- cat $file_name | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1
+ property_name=$(echo "$2" | sed 's/\./\\\./g')
+ cat "$file_name" | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1
}
function write_property() {
@@ -40,7 +41,7 @@
local key=$2
local value=$3
- local os=`uname`
+ local os=$(uname)
case $os in
# Note: in mac os should use sed -i '' "xxx" to replace string,
# otherwise prompt 'command c expects \ followed by text'.
@@ -55,7 +56,7 @@
local version=$2
local module=$3
- cat $file | tr -d '\n {}'| awk -F',+|:' '''{
+ cat "$file" | tr -d '\n {}'| awk -F',+|:' '''{
pre="";
for(i=1; i<=NF; ) {
if(match($i, /version/)) {
@@ -71,27 +72,13 @@
}
function process_num() {
- num=`ps -ef | grep $1 | grep -v grep | wc -l`
- return $num
+ num=$(ps -ef | grep "$1" | grep -v grep | wc -l)
+ return "$num"
}
function process_id() {
- pid=`ps -ef | grep $1 | grep -v grep | awk '{print $2}'`
- return $pid
-}
-
-# check the port of rest server is occupied
-function check_port() {
- local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||')
- if ! command_available "lsof"; then
- echo "Required lsof but it is unavailable"
- exit 1
- fi
- lsof -i :$port >/dev/null
- if [ $? -eq 0 ]; then
- echo "The port $port has already been used"
- exit 1
- fi
+ pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}')
+ return "$pid"
}
function crontab_append() {
@@ -129,13 +116,14 @@
local server_url="$3"
local timeout_s="$4"
- local now_s=`date '+%s'`
- local stop_s=$(( $now_s + $timeout_s ))
+ local now_s
+ now_s=$(date '+%s')
+ local stop_s=$(( now_s + timeout_s ))
local status
echo -n "Connecting to $server_name ($server_url)"
- while [ $now_s -le $stop_s ]; do
+ while [ "$now_s" -le "$stop_s" ]; do
echo -n .
process_status "$server_name" "$pid" >/dev/null
if [ $? -eq 1 ]; then
@@ -143,14 +131,14 @@
return 1
fi
- status=`curl -o /dev/null -s -k -w %{http_code} $server_url`
- if [[ $status -eq 200 || $status -eq 401 ]]; then
+ status=$(curl -o /dev/null -s -k -w '%{http_code}' "$server_url")
+ if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then
echo "OK"
echo "Started [pid $pid]"
return 0
fi
sleep 2
- now_s=`date '+%s'`
+ now_s=$(date '+%s')
done
echo "The operation timed out when attempting to connect to $server_url" >&2
@@ -159,22 +147,28 @@
function free_memory() {
local free=""
- local os=`uname`
+ local os=$(uname)
if [ "$os" == "Linux" ]; then
- local mem_free=`cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}'`
- local mem_buffer=`cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}'`
- local mem_cached=`cat /proc/meminfo | grep -w "Cached" | awk '{print $2}'`
+ local mem_free=$(cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}')
+ local mem_buffer=$(cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}')
+ local mem_cached=$(cat /proc/meminfo | grep -w "Cached" | awk '{print $2}')
if [[ "$mem_free" == "" || "$mem_buffer" == "" || "$mem_cached" == "" ]]; then
echo "Failed to get free memory"
exit 1
fi
- free=`expr $mem_free + $mem_buffer + $mem_cached`
- free=`expr $free / 1024`
+ free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached")
+ free=$(expr "$free" / 1024)
elif [ "$os" == "Darwin" ]; then
- local pages_free=`vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "`
- local pages_inactive=`vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "`
- local pages_available=`expr $pages_free + $pages_inactive`
- free=`expr $pages_available \* 4096 / 1024 / 1024`
+ local pages_free pages_inactive
+ pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ")
+ pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ")
+ if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then
+ echo "Failed to get free memory"
+ exit 1
+ fi
+ local pages_available
+ pages_available=$(expr "$pages_free" + "$pages_inactive")
+ free=$(expr "$pages_available" \* 4096 / 1024 / 1024)
else
echo "Unsupported operating system $os"
exit 1
@@ -186,8 +180,8 @@
local min_mem=$1
local max_mem=$2
# Get machine available memory
- local free=`free_memory`
- local half_free=$[free/2]
+ local free=$(free_memory)
+ local half_free=$((free/2))
local xmx=$min_mem
if [[ "$free" -lt "$min_mem" ]]; then
@@ -235,47 +229,78 @@
}
function get_ip() {
- local os=`uname`
+ local os=$(uname)
local loopback="127.0.0.1"
local ip=""
case $os in
Linux)
if command_available "ifconfig"; then
- ip=`ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}'`
+ ip=$(ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}')
elif command_available "ip"; then
- ip=`ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}'`
+ ip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}')
else
ip=$loopback
fi
;;
FreeBSD|OpenBSD|Darwin)
if command_available "ifconfig"; then
- ip=`ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}'`
+ ip=$(ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}')
else
ip=$loopback
fi
;;
SunOS)
if command_available "ifconfig"; then
- ip=`ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} '`
+ ip=$(ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} ')
else
ip=$loopback
fi
;;
*) ip=$loopback;;
esac
- echo $ip
+ [[ -z "$ip" ]] && ip=$loopback
+ echo "$ip"
}
function download() {
local path=$1
local link_url=$2
- if command_available "wget"; then
- wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt=""
- wget ${link_url} -P ${path} $progress_opt
- elif command_available "curl"; then
- curl ${link_url} -o ${path}/${link_url}
+ if [ ! -d "$path" ]; then
+ mkdir -p "$path" || {
+ echo "Failed to create directory: $path"
+ exit 1
+ }
+ fi
+
+ local filename tmp
+ filename=$(basename "${link_url%%[?#]*}")
+ local dest="${path}/${filename}"
+ # mktemp, not $$: a PID is shared by concurrent background subshells.
+ tmp=$(mktemp -- "${path}/.${filename}.XXXXXX") || {
+ echo "Failed to create a temporary file in $path"
+ return 1
+ }
+
+ if command_available "curl"; then
+ # -o must appear before -- so it is parsed as an option, not an extra URL.
+ if curl -fL -o "$tmp" -- "${link_url}"; then
+ mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; }
+ else
+ rm -f -- "$tmp"
+ return 1
+ fi
+ elif command_available "wget"; then
+ local -a progress_opt=()
+ if wget --help 2>&1 | grep -q -- '--show-progress'; then
+ progress_opt=(-q --show-progress)
+ fi
+ if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then
+ mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; }
+ else
+ rm -f -- "$tmp"
+ return 1
+ fi
else
echo "Required wget or curl but they are unavailable"
exit 1
@@ -286,14 +311,15 @@
local url=$1
local filepath=$2
local expected_md5=$3
+ local actual_md5
- if [[ -f $filepath ]]; then
+ if [[ -f "$filepath" ]]; then
echo "File $filepath exists. Verifying MD5 checksum..."
- actual_md5=$(md5sum $filepath | awk '{ print $1 }')
- if [[ $actual_md5 != $expected_md5 ]]; then
+ actual_md5=$(md5sum -- "$filepath" | awk '{ print $1 }')
+ if [[ "$actual_md5" != "$expected_md5" ]]; then
echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5"
- echo "Deleting $filepath..."
- rm -f $filepath
+ # Keep the shared destination until a verified temporary file is
+ # ready; a stale reader must not delete another caller's replacement.
else
echo "MD5 checksum verification succeeded for $filepath."
return 0
@@ -301,11 +327,21 @@
fi
echo "Downloading $filepath..."
- curl -L -o $filepath $url
-
- actual_md5=$(md5sum $filepath | awk '{ print $1 }')
- if [[ $actual_md5 != $expected_md5 ]]; then
- echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5"
+ local tmp
+ # mktemp, not $$: a PID is shared by concurrent background subshells.
+ tmp=$(mktemp -- "${filepath}.XXXXXX") || return 1
+ if curl -fL -o "$tmp" -- "$url"; then
+ actual_md5=$(md5sum -- "$tmp" | awk '{ print $1 }')
+ if [[ "$actual_md5" != "$expected_md5" ]]; then
+ echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5"
+ rm -f -- "$tmp"
+ return 1
+ fi
+ # A failed rename must not report success: callers export LD_PRELOAD
+ # from $filepath and would otherwise use a library that never installed.
+ mv -f -- "$tmp" "$filepath" || { rm -f -- "$tmp"; return 1; }
+ else
+ rm -f -- "$tmp"
return 1
fi
@@ -318,19 +354,17 @@
local tar=$3
local link=$4
- if [ ! -d ${path}/${dir} ]; then
- if [ ! -f ${path}/${tar} ]; then
+ if [ ! -d "${path}/${dir}" ]; then
+ if [ ! -f "${path}/${tar}" ]; then
echo "Downloading the compressed package '${tar}'"
- download ${path} ${link}
- if [ $? -ne 0 ]; then
+ if ! download "${path}" "${link}"; then
echo "Failed to download, please ensure the network is available and link is valid"
exit 1
fi
echo "[OK] Finished download"
fi
echo "Unzip the compressed package '$tar'"
- tar -zxvf ${path}/${tar} -C ${path} >/dev/null 2>&1
- if [ $? -ne 0 ]; then
+ if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then
echo "Failed to unzip, please check the compressed package"
exit 1
fi
@@ -345,7 +379,7 @@
local pid="$2"
local timeout_s="$3"
- local now_s=`date '+%s'`
+ local now_s=$(date '+%s')
local stop_s=$(( $now_s + $timeout_s ))
echo -n "Killing $process_name(pid $pid)" >&2
@@ -357,7 +391,7 @@
return 0
fi
sleep 2
- now_s=`date '+%s'`
+ now_s=$(date '+%s')
done
echo "$process_name shutdown timeout(exceeded $timeout_s seconds)" >&2
return 1
@@ -386,7 +420,7 @@
return 0
fi
- case "`uname`" in
+ case "$(uname)" in
CYGWIN*) taskkill /F /PID "$pid" ;;
*) kill "$pid" ;;
esac
