fix: pack without libaio (#676)
diff --git a/.github/workflows/ci-pull-request.yaml b/.github/workflows/ci-pull-request.yaml
index ee3c3fb..e6e3109 100644
--- a/.github/workflows/ci-pull-request.yaml
+++ b/.github/workflows/ci-pull-request.yaml
@@ -17,10 +17,7 @@
paths:
- 'rdsn/**'
- 'src/**'
- - '.github/workflows/ci-pull-request.yml'
- push:
- paths:
- - '.github/workflows/ci-pull-request.yml'
+ - '.github/workflows/ci-pull-request.yaml'
# for manually triggering workflow
workflow_dispatch:
@@ -50,18 +47,21 @@
needs: lint
runs-on: ubuntu-latest
container:
- image: apachepegasus/ci-env
+ image: ghcr.io/pegasus-kv/thirdparties-bin:ubuntu1804
defaults:
run:
working-directory: /root/incubator-pegasus
+ shell: bash
steps:
- name: Clone Apache Pegasus Source
working-directory: /root
run: |
git clone --recursive --depth=1 https://github.com/apache/incubator-pegasus.git
- name: Unpack prebuilt third-parties
- run: unzip /root/pegasus-thirdparty-output.zip -d ./rdsn/thirdparty
+ run: unzip /root/thirdparties-bin.zip -d ./rdsn/thirdparty
- name: Compilation
run: ./run.sh build -c --skip_thirdparty
- name: Unit Testing
- run: ./run.sh test --on_travis
+ run: |
+ source ./config_hdfs.sh
+ ./run.sh test --on_travis
diff --git a/rdsn b/rdsn
index d8cbbe5..c057d35 160000
--- a/rdsn
+++ b/rdsn
@@ -1 +1 @@
-Subproject commit d8cbbe59854fd382d10954349f24c058ffa1b107
+Subproject commit c057d35c93ff9d895d5eefef343b6c7afd74812d
diff --git a/scripts/pack_common.sh b/scripts/pack_common.sh
index 532ded2..48d3221 100755
--- a/scripts/pack_common.sh
+++ b/scripts/pack_common.sh
@@ -100,3 +100,17 @@
fi
}
+function pack_system_lib()
+{
+ local package_path=$1
+ local package_type=$2
+ local lib_name=$3
+
+ SYS_LIB_PATH=$(get_system_lib "${package_type}" "${lib_name}")
+ if [ -z "${SYS_LIB_PATH}" ]; then
+ echo "ERROR: library ${lib_name} is missing on your system"
+ exit 1
+ fi
+ SYS_LIB_NAME=$(get_system_libname "${package_type}" "${lib_name}")
+ copy_file "${SYS_LIB_PATH}" "${package_path}/${SYS_LIB_NAME}"
+}
diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh
index 94b7f72..ff97858 100755
--- a/scripts/pack_server.sh
+++ b/scripts/pack_server.sh
@@ -6,9 +6,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,8 +18,7 @@
source $(dirname $0)/pack_common.sh
-function usage()
-{
+function usage() {
echo "Options for subcommand 'pack_server':"
echo " -h"
echo " -p|--update-package-template <minos-package-template-file-path>"
@@ -28,55 +27,49 @@
exit 0
}
-pwd="$( cd "$( dirname "$0" )" && pwd )"
-shell_dir="$( cd $pwd/.. && pwd )"
-cd $shell_dir
+pwd="$(cd "$(dirname "$0")" && pwd)"
+shell_dir="$(cd $pwd/.. && pwd)"
+cd "$shell_dir" || exit 1
-if [ ! -f src/include/pegasus/git_commit.h ]
-then
+if [ ! -f src/include/pegasus/git_commit.h ]; then
echo "ERROR: src/include/pegasus/git_commit.h not found"
exit 1
fi
-if [ ! -f DSN_ROOT/bin/pegasus_server/pegasus_server ]
-then
+if [ ! -f DSN_ROOT/bin/pegasus_server/pegasus_server ]; then
echo "ERROR: DSN_ROOT/bin/pegasus_server/pegasus_server not found"
exit 1
fi
-if [ ! -f src/builder/CMAKE_OPTIONS ]
-then
+if [ ! -f src/builder/CMAKE_OPTIONS ]; then
echo "ERROR: src/builder/CMAKE_OPTIONS not found"
exit 1
fi
-if grep -q Debug src/builder/CMAKE_OPTIONS
-then
+if grep -q Debug src/builder/CMAKE_OPTIONS; then
build_type=debug
else
build_type=release
fi
-version=`grep "VERSION" src/include/pegasus/version.h | cut -d "\"" -f 2`
-commit_id=`grep "GIT_COMMIT" src/include/pegasus/git_commit.h | cut -d "\"" -f 2`
-glibc_ver=`ldd --version | grep ldd | grep -Eo "[0-9]+.[0-9]+$"`
+version=$(grep "VERSION" src/include/pegasus/version.h | cut -d "\"" -f 2)
+commit_id=$(grep "GIT_COMMIT" src/include/pegasus/git_commit.h | cut -d "\"" -f 2)
+glibc_ver=$(ldd --version | grep ldd | grep -Eo "[0-9]+.[0-9]+$")
echo "Packaging pegasus server $version ($commit_id) glibc-$glibc_ver $build_type ..."
pack_version=server-$version-${commit_id:0:7}-glibc${glibc_ver}-${build_type}
pack=pegasus-$pack_version
-if [ -f ${pack}.tar.gz ]
-then
+if [ -f ${pack}.tar.gz ]; then
rm -f ${pack}.tar.gz
fi
-if [ -d ${pack} ]
-then
+if [ -d ${pack} ]; then
rm -rf ${pack}
fi
pack_template=""
if [ -n "$MINOS_CONFIG_FILE" ]; then
- pack_template=`dirname $MINOS_CONFIG_FILE`/xiaomi-config/package/pegasus.yaml
+ pack_template=$(dirname $MINOS_CONFIG_FILE)/xiaomi-config/package/pegasus.yaml
fi
custom_gcc="false"
@@ -85,20 +78,20 @@
while [[ $# > 0 ]]; do
option_key="$1"
case $option_key in
- -p|--update-package-template)
- pack_template="$2"
- shift
- ;;
- -g|--custom-gcc)
- custom_gcc="true"
- ;;
- -h|--help)
- usage
- ;;
- -k|--keytab-file)
- keytab_file="$2"
- shift
- ;;
+ -p | --update-package-template)
+ pack_template="$2"
+ shift
+ ;;
+ -g | --custom-gcc)
+ custom_gcc="true"
+ ;;
+ -h | --help)
+ usage
+ ;;
+ -k | --keytab-file)
+ keytab_file="$2"
+ shift
+ ;;
esac
shift
done
@@ -121,14 +114,20 @@
copy_file ./rdsn/thirdparty/output/lib/sasl2 ${pack}/bin
copy_file ./scripts/sendmail.sh ${pack}/bin
copy_file ./src/server/config.ini ${pack}/bin
+copy_file ./src/server/config.min.ini ${pack}/bin
+copy_file ./config_hdfs.sh ${pack}/bin
-copy_file `get_stdcpp_lib $custom_gcc` ${pack}/bin
-copy_file `get_system_lib server snappy` ${pack}/bin/`get_system_libname server snappy`
-copy_file `get_system_lib server crypto` ${pack}/bin/`get_system_libname server crypto`
-copy_file `get_system_lib server ssl` ${pack}/bin/`get_system_libname server ssl`
-copy_file `get_system_lib server aio` ${pack}/bin/`get_system_libname server aio`
-copy_file `get_system_lib server zstd` ${pack}/bin/`get_system_libname server zstd`
-copy_file `get_system_lib server lz4` ${pack}/bin/`get_system_libname server lz4`
+copy_file "$(get_stdcpp_lib $custom_gcc)" "${pack}/bin"
+
+pack_server_lib() {
+ pack_system_lib "${pack}/bin" server "$1"
+}
+
+pack_server_lib snappy
+pack_server_lib crypto
+pack_server_lib ssl
+pack_server_lib zstd
+pack_server_lib lz4
# Pack hadoop-related files.
# If you want to use hdfs service to backup/restore/bulkload pegasus tables,
@@ -160,12 +159,12 @@
DISTRIB_ID=$(cat /etc/*-release | grep DISTRIB_ID | awk -F'=' '{print $2}')
DISTRIB_RELEASE=$(cat /etc/*-release | grep DISTRIB_RELEASE | awk -F'=' '{print $2}')
if [ -n "$DISTRIB_ID" ] && [ -n "$DISTRIB_RELEASE" ]; then
- if [ "$DISTRIB_ID" == "Ubuntu" ] && [ "$DISTRIB_RELEASE" == "18.04" ]; then
- copy_file "$(get_system_lib server icui18n)" "$pack/bin/$(get_system_libname server icui18n)"
- copy_file "$(get_system_lib server icuuc)" "$pack/bin/$(get_system_libname server icuuc)"
- copy_file "$(get_system_lib server icudata)" "$pack/bin/$(get_system_libname server icudata)"
- fi
- # more cases can be added here.
+ if [ "$DISTRIB_ID" == "Ubuntu" ] && [ "$DISTRIB_RELEASE" == "18.04" ]; then
+ pack_system_lib icui18n
+ pack_system_lib icuuc
+ pack_system_lib icudata
+ fi
+ # more cases can be added here.
fi
chmod +x ${pack}/bin/pegasus_* ${pack}/bin/*.sh
@@ -182,6 +181,6 @@
sed -i "/^source:/c source: \"$PEGASUS_ROOT\"" $pack_template
fi
-echo ${pack} > PACKAGE
+echo ${pack} >PACKAGE
echo "Done"
diff --git a/scripts/pack_tools.sh b/scripts/pack_tools.sh
index 0bc80e1..9f45b0d 100755
--- a/scripts/pack_tools.sh
+++ b/scripts/pack_tools.sh
@@ -114,12 +114,17 @@
copy_file ./rdsn/thirdparty/output/lib/libtcmalloc_and_profiler.so.4 ${pack}/DSN_ROOT/lib/
copy_file ./rdsn/thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/DSN_ROOT/lib/
copy_file `get_stdcpp_lib $custom_gcc` ${pack}/DSN_ROOT/lib/
-copy_file `get_system_lib shell snappy` ${pack}/DSN_ROOT/lib/`get_system_libname shell snappy`
-copy_file `get_system_lib shell crypto` ${pack}/DSN_ROOT/lib/`get_system_libname shell crypto`
-copy_file `get_system_lib shell ssl` ${pack}/DSN_ROOT/lib/`get_system_libname shell ssl`
-copy_file `get_system_lib shell aio` ${pack}/DSN_ROOT/lib/`get_system_libname shell aio`
-copy_file `get_system_lib shell zstd` ${pack}/DSN_ROOT/lib/`get_system_libname shell zstd`
-copy_file `get_system_lib shell lz4` ${pack}/DSN_ROOT/lib/`get_system_libname shell lz4`
+
+pack_tools_lib() {
+ pack_system_lib "${pack}/DSN_ROOT/lib" shell "$1"
+}
+
+pack_tools_lib snappy
+pack_tools_lib crypto
+pack_tools_lib ssl
+pack_tools_lib zstd
+pack_tools_lib lz4
+
chmod -x ${pack}/DSN_ROOT/lib/*
mkdir -p ${pack}/scripts
diff --git a/src/base/test/CMakeLists.txt b/src/base/test/CMakeLists.txt
index 651e0c0..b3e4e1a 100644
--- a/src/base/test/CMakeLists.txt
+++ b/src/base/test/CMakeLists.txt
@@ -29,6 +29,7 @@
set(MY_PROJ_LIBS
dsn_runtime
+ dsn_aio
dsn_utils
pegasus_base
gtest)
diff --git a/src/client_lib/CMakeLists.txt b/src/client_lib/CMakeLists.txt
index f2a9986..5b1f96f 100644
--- a/src/client_lib/CMakeLists.txt
+++ b/src/client_lib/CMakeLists.txt
@@ -46,6 +46,7 @@
set_target_properties(pegasus_client_static
PROPERTIES
IMPORTED_LOCATION ${pegasus_client_static_lib})
+target_link_libraries(pegasus_client_static INTERFACE dsn_aio)
install(FILES ${pegasus_client_static_lib} DESTINATION "lib")
# link the shared lib of pegasus client
diff --git a/src/server/config.ini b/src/server/config.ini
index eb6e560..4b52ffd 100644
--- a/src/server/config.ini
+++ b/src/server/config.ini
@@ -200,7 +200,6 @@
verbose_commit_log_on_start = false
delay_for_fd_timeout_on_start = false
empty_write_disabled = false
- allow_non_idempotent_write = false
prepare_timeout_ms_for_secondaries = 3000
prepare_timeout_ms_for_potential_secondaries = 5000
diff --git a/src/server/config.min.ini b/src/server/config.min.ini
index 76612b4..5b9a7b1 100644
--- a/src/server/config.min.ini
+++ b/src/server/config.min.ini
@@ -111,12 +111,9 @@
recover_from_replica_server = false
[replication]
- allow_non_idempotent_write = true
mutation_2pc_min_replica_count = 1
cold_backup_root = onebox
- duplication_disabled = true
cluster_name = onebox
- cold_backup_checkpoint_reserve_minutes = 10
[meta_server.apps.@APP_NAME@]
app_name = @APP_NAME@
@@ -134,7 +131,7 @@
# Options:
# - falcon
# - prometheus
- perf_counter_sink =
+ perf_counter_sink = prometheus
# The HTTP port exposed to Prometheus for pulling metrics from pegasus server.
prometheus_port = @PROMETHEUS_PORT@
diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h
index 65403e7..d345ad6 100644
--- a/src/shell/command_helper.h
+++ b/src/shell/command_helper.h
@@ -1093,3 +1093,14 @@
st_stat.timestamp = buf;
return true;
}
+
+inline configuration_proposal_action new_proposal_action(const dsn::rpc_address &target,
+ const dsn::rpc_address &node,
+ config_type::type type)
+{
+ configuration_proposal_action act;
+ act.__set_target(target);
+ act.__set_node(node);
+ act.__set_type(type);
+ return act;
+}
diff --git a/src/shell/commands/rebalance.cpp b/src/shell/commands/rebalance.cpp
index b326d23..ad3dc38 100644
--- a/src/shell/commands/rebalance.cpp
+++ b/src/shell/commands/rebalance.cpp
@@ -110,7 +110,7 @@
type_from_string(_config_type_VALUES_TO_NAMES, proposal_type, config_type::CT_INVALID);
verify_logged(
tp != config_type::CT_INVALID, "parse %s as config_type failed.\n", proposal_type.c_str());
- request.action_list = {configuration_proposal_action{target, node, tp}};
+ request.action_list = {new_proposal_action(target, node, tp)};
dsn::error_code err = sc->ddl_client->send_balancer_proposal(request);
std::cout << "send proposal response: " << err.to_string() << std::endl;
return true;
@@ -172,21 +172,18 @@
actions.reserve(4);
if (balance_type == "move_pri") {
actions.emplace_back(
- configuration_proposal_action{from, from, config_type::CT_DOWNGRADE_TO_SECONDARY});
- actions.emplace_back(
- configuration_proposal_action{to, to, config_type::CT_UPGRADE_TO_PRIMARY});
+ new_proposal_action(from, from, config_type::CT_DOWNGRADE_TO_SECONDARY));
+ actions.emplace_back(new_proposal_action(to, to, config_type::CT_UPGRADE_TO_PRIMARY));
} else if (balance_type == "copy_pri") {
+ actions.emplace_back(new_proposal_action(from, to, config_type::CT_ADD_SECONDARY_FOR_LB));
actions.emplace_back(
- configuration_proposal_action{from, to, config_type::CT_ADD_SECONDARY_FOR_LB});
- actions.emplace_back(
- configuration_proposal_action{from, from, config_type::CT_DOWNGRADE_TO_SECONDARY});
- actions.emplace_back(
- configuration_proposal_action{to, to, config_type::CT_UPGRADE_TO_PRIMARY});
+ new_proposal_action(from, from, config_type::CT_DOWNGRADE_TO_SECONDARY));
+ actions.emplace_back(new_proposal_action(to, to, config_type::CT_UPGRADE_TO_PRIMARY));
} else if (balance_type == "copy_sec") {
- actions.emplace_back(configuration_proposal_action{
- dsn::rpc_address(), to, config_type::CT_ADD_SECONDARY_FOR_LB});
- actions.emplace_back(configuration_proposal_action{
- dsn::rpc_address(), from, config_type::CT_DOWNGRADE_TO_INACTIVE});
+ actions.emplace_back(
+ new_proposal_action(dsn::rpc_address(), to, config_type::CT_ADD_SECONDARY_FOR_LB));
+ actions.emplace_back(
+ new_proposal_action(dsn::rpc_address(), from, config_type::CT_DOWNGRADE_TO_INACTIVE));
} else {
fprintf(stderr, "parse %s as a balance type failed\n", balance_type.c_str());
return false;
diff --git a/src/shell/commands/recovery.cpp b/src/shell/commands/recovery.cpp
index 2b58a3e..b1f539b 100644
--- a/src/shell/commands/recovery.cpp
+++ b/src/shell/commands/recovery.cpp
@@ -350,8 +350,8 @@
if (!primary.is_invalid() && !skip_this) {
dsn::replication::configuration_balancer_request request;
request.gpid = pinfo.config.pid;
- request.action_list = {configuration_proposal_action{
- primary, primary, config_type::CT_ASSIGN_PRIMARY}};
+ request.action_list = {
+ new_proposal_action(primary, primary, config_type::CT_ASSIGN_PRIMARY)};
request.force = false;
dsn::error_code err = sc->ddl_client->send_balancer_proposal(request);
out << " propose_request: propose -g " << request.gpid.to_string()
