install: add support for building psol from source and setting up for development (#1313)
* adds support for `--psol-from-source` so you don't need binary modules, and `--devel` so you can run our tests without going and getting all our dependencies
* adds submodules for testing: mod_pagespeed, ngx_cache_purge etc
* adds support for running as:
```
git clone git@github.com:pagespeed/ngx_pagespeed.git
cd ngx_pagespeed/
git checkout <branch>
scripts/build_ngx_pagespeed.sh [options]
```
* depends on the scripts @hillsp is working on so that we can just check out mod_pagespeed and ask it to build and rebuild itself
* adds colors to output to make it easier to read
diff --git a/.gitignore b/.gitignore
index c1ddac9..f03b88d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
psol/
psol-*.tar.gz
*.*.*.*.tar.gz
+nginx
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..d38a4bc
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,18 @@
+[submodule "testing-dependencies/mod_pagespeed"]
+ path = testing-dependencies/mod_pagespeed
+ url = https://github.com/pagespeed/mod_pagespeed.git
+[submodule "testing-dependencies/ngx_cache_purge"]
+ path = testing-dependencies/ngx_cache_purge
+ url = https://github.com/FRiCKLE/ngx_cache_purge.git
+[submodule "testing-dependencies/nginx"]
+ path = testing-dependencies/nginx
+ url = https://github.com/nginx/nginx.git
+[submodule "testing-dependencies/set-misc-nginx-module"]
+ path = testing-dependencies/set-misc-nginx-module
+ url = https://github.com/openresty/set-misc-nginx-module
+[submodule "testing-dependencies/ngx_devel_kit"]
+ path = testing-dependencies/ngx_devel_kit
+ url = https://github.com/simpl/ngx_devel_kit
+[submodule "testing-dependencies/headers-more-nginx-module"]
+ path = testing-dependencies/headers-more-nginx-module
+ url = https://github.com/openresty/headers-more-nginx-module
diff --git a/.travis.yml b/.travis.yml
index 793dc9f..b722a04 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,50 +1,29 @@
language: c++
+env:
+ global:
+ - MAKEFLAGS=-j3
+# By default travis loads submodules serially, but we can load them in parallel
+# if we install an updated git and use --jobs. Some timing numbers:
+# serial: 257s
+# jobs=4: 182s (29s to install new git, 153s to run the downloads)
+# jobs=8: 179s (29s to install new git, 150s to run the downloads)
+# We can't use --depth=1, though, because github doesn't have
+# allowReachableSHA1InWant set.
+git:
+ submodules: false
+before_install:
+ - sudo add-apt-repository --yes ppa:git-core/ppa
+ - sudo apt-get update
+ - sudo apt-get install git
+ - git submodule update --init --recursive --jobs=8
install:
- - mv $TRAVIS_BUILD_DIR ~/ngxpagespeed
- - sudo apt-get install build-essential zlib1g-dev libpcre3 libpcre3-dev unzip g++ python gperf make devscripts fakeroot git curl netcat-traditional gcc-mozilla clang-3.4 2>&1 > /dev/null
- - export PATH=/usr/lib/gcc-mozilla/bin:$PATH
- - sudo ln -sf /usr/lib/gcc-mozilla/lib/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so.6
-# - sudo sh -c 'echo "image/webp webp" >> /etc/mime.types'
-# - mkdir -p ~/bin
-# - cd ~/bin
-# - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
- - mkdir ~/mod_pagespeed
- - cd ~/mod_pagespeed
- - git clone --recursive https://github.com/pagespeed/mod_pagespeed.git src
- - cd src
- - python build/gyp_chromium --depth=.
- - cd ~/mod_pagespeed/src/pagespeed/automatic
- - make BUILDTYPE=Release -C ../../pagespeed/automatic all
- - cd ~
- - git clone https://github.com/FRiCKLE/ngx_cache_purge.git
- - NGX_CACHE_PURGE=$PWD/ngx_cache_purge
- - wget https://openresty.org/download/ngx_openresty-1.9.7.2.tar.gz
- - tar xzf ngx_openresty-*.tar.gz
- - cd ngx_openresty-*/
- - ./configure --with-luajit
- - make
- - NGX_DEVEL_KIT=$(echo $HOME/ngx_openresty-*/build/ngx_devel_kit-*/)
- - SET_MISC_MODULE=$(echo $HOME/ngx_openresty-*/build/set-misc-nginx-module-*/)
- - HEADERS_MORE_MODULE=$(echo $HOME/ngx_openresty-*/build/headers-more-nginx-module-*/)
- - cd ~
- - wget https://github.com/nginx/nginx/archive/branches/default.zip
- - unzip default.zip
- - cd nginx-branches-default
- - MOD_PAGESPEED_DIR="$HOME/mod_pagespeed/src" ./auto/configure --add-module=$HOME/ngxpagespeed --add-module="$NGX_CACHE_PURGE" --add-module="$NGX_DEVEL_KIT" --add-module="$SET_MISC_MODULE" --add-module="$HEADERS_MORE_MODULE" --with-ipv6
- - make
- - sudo make install
+ scripts/build_ngx_pagespeed.sh --devel --assume-yes
script:
- - echo "build successful"
- - echo "cd ~/ngxpagespeed"
- - echo "sudo ./test/run_tests.sh $HOME/mod_pagespeed $HOME/nginx-branches-default/objs/nginx"
+ test/run_tests.sh $PWD/testing-dependencies/mod_pagespeed/ \
+ $PWD/nginx/sbin/nginx
sudo: required
compiler:
- gcc
notifications:
email:
- - cheesy@google.com
- jefftk@google.com
- - morlovich@google.com
- - jmarantz@google.com
- - huibao@google.com
- - jcrowell@google.com
diff --git a/scripts/build_ngx_pagespeed.sh b/scripts/build_ngx_pagespeed.sh
index edf3314..555b1dd 100755
--- a/scripts/build_ngx_pagespeed.sh
+++ b/scripts/build_ngx_pagespeed.sh
@@ -5,7 +5,16 @@
Usage: build_ngx_pagespeed.sh [options]
Installs ngx_pagespeed and its dependencies. Can optionally build and install
- nginx as well.
+ nginx as well. Can be run either as:
+
+ bash <(curl -f -L -sS https://ngxpagespeed.com/install) [options]
+
+ Or:
+
+ git clone git@github.com:pagespeed/ngx_pagespeed.git
+ cd ngx_pagespeed/
+ git checkout <branch>
+ scripts/build_ngx_pagespeed.sh [options]
Options:
-v, --ngx-pagespeed-version <ngx_pagespeed version>
@@ -14,7 +23,11 @@
* latest-stable
* a version number, such as 1.11.33.4
- If you don't specify a version, defaults to latest-stable.
+ If you don't specify a version, defaults to latest-stable unless --devel
+ is specified, in which case it defaults to trunk-tracking.
+
+ This option doesn't make sense if we're running within an existing
+ ngx_pagespeed checkout.
-n, --nginx-version <nginx version>
What version of nginx to build. If not set, this script only prepares the
@@ -36,6 +49,23 @@
non-deb non-rpm system, this won't work. In that case, install the
dependencies yourself and pass --no-deps-check.
+ -s, --psol-from-source
+ Build PSOL from source instead of downloading a pre-built binary module.
+
+ -l, --devel
+ Sets up a development environment in ngx_pagespeed/nginx, building with
+ testing-only dependencies. Includes --psol-from-source, conflicts with
+ --nginx-version. Uses a 'git clone' checkout for ngx_pagespeed and nginx
+ instead of downloading a tarball.
+
+ -t, --build-type
+ When building PSOL from source, what to tell it for BUILD_TYPE. Defaults
+ to 'Release' unless --devel is set in which case it defaults to 'Debug'.
+
+ -y, --assume-yes
+ Assume the answer to all prompts is 'yes, please continue'. Intended for
+ automated usage, such as buildbots.
+
-d, --dryrun
Don't make any changes to the system, just print what changes you
would have made.
@@ -44,10 +74,34 @@
Print this message and exit."
}
+RED=31
+GREEN=32
+YELLOW=33
+function begin_color() {
+ color="$1"
+ echo -e -n "\e[${color}m"
+}
+function end_color() {
+ echo -e -n "\e[0m"
+}
+
+function echo_color() {
+ color="$1"
+ shift
+ begin_color "$color"
+ echo "$@"
+ end_color
+}
+
+function error() {
+ local error_message="$@"
+ echo_color "$RED" -n "Error: " >&2
+ echo "$@" >&2
+}
+
# Prints an error message and exits with an error code.
function fail() {
- local error_message="$@"
- echo "$@" >&2
+ error "$@"
# Normally I'd use $0 in "usage" here, but since most people will be running
# this via curl, that wouldn't actually give something useful.
@@ -56,6 +110,11 @@
exit 1
}
+
+function status() {
+ echo_color "$GREEN" "$@"
+}
+
# Intended to be called as:
# bash <(curl dl.google.com/.../build_ngx_pagespeed.sh) <args>
@@ -64,12 +123,19 @@
# The run function handles exit-status checking for system-changing commands.
# Additionally, this allows us to easily have a dryrun mode where we don't
# actually make any changes.
+INITIAL_ENV=$(printenv | sort)
function run() {
if "$DRYRUN"; then
- echo "would run $@"
+ echo_color "$YELLOW" -n "would run"
+ echo " $@"
+ env_differences=$(comm -13 <(echo "$INITIAL_ENV") <(printenv | sort))
+ if [ -n "$env_differences" ]; then
+ echo " with the following additional environment variables:"
+ echo "$env_differences" | sed 's/^/ /'
+ fi
else
if ! "$@"; then
- echo "Failure running $@, exiting."
+ error "Failure running '$@', exiting."
exit 1
fi
fi
@@ -155,9 +221,9 @@
fi
done
if [ -n "$missing_dependencies" ]; then
- echo "Detected that we're missing the following depencencies:"
+ status "Detected that we're missing the following depencencies:"
echo " $missing_dependencies"
- echo "Installing them:"
+ status "Installing them:"
run sudo $install_pkg_cmd $missing_dependencies
fi
}
@@ -176,8 +242,13 @@
}
function continue_or_exit() {
+ if "$ASSUME_YES"; then
+ return
+ fi
+
local prompt="$1"
- read -p "$prompt [Y/n] " yn
+ echo_color "$YELLOW" -n "$prompt"
+ read -p " [Y/n] " yn
if [[ "$yn" == N* || "$yn" == n* ]]; then
echo "Cancelled."
exit 0
@@ -217,9 +288,10 @@
fail "Your version of getopt is too old. Exiting with no changes made."
fi
- opts=$(getopt -o v:n:mb:pdh \
+ opts=$(getopt -o v:n:mb:pslt:ydh \
--longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \
- --longoptions buildir:,no-deps-check,dryrun,help \
+ --longoptions buildir:,no-deps-check,psol-from-source,devel,build-type: \
+ --longoptions assume-yes,dryrun,help \
-n "$(basename "$0")" -- "$@")
if [ $? != 0 ]; then
usage
@@ -227,10 +299,14 @@
fi
eval set -- "$opts"
- NPS_VERSION="latest-stable"
+ NPS_VERSION="DEFAULT"
NGINX_VERSION=""
BUILDDIR="$HOME"
DO_DEPS_CHECK=true
+ PSOL_FROM_SOURCE=false
+ DEVEL=false
+ BUILD_TYPE=""
+ ASSUME_YES=false
DRYRUN=false
DYNAMIC_MODULE=false
while true; do
@@ -253,6 +329,19 @@
-p | --no-deps-check) shift
DO_DEPS_CHECK=false
;;
+ -s | --psol-from-source) shift
+ PSOL_FROM_SOURCE=true
+ ;;
+ -l | --devel) shift
+ DEVEL=true
+ ;;
+ -t | --build-type) shift
+ BUILD_TYPE="$1"
+ shift
+ ;;
+ -y | --assume-yes) shift
+ ASSUME_YES="true"
+ ;;
-d | --dryrun) shift
DRYRUN="true"
;;
@@ -271,10 +360,58 @@
esac
done
+ USE_GIT_CHECKOUT="$DEVEL"
+ ALREADY_CHECKED_OUT=false
+ if [ -e PSOL_BINARY_URL ]; then
+ status "Detected that we're running in an existing ngx_pagespeed checkout."
+ USE_GIT_CHECKOUT=true
+ ALREADY_CHECKED_OUT=true
+ fi
+
+ if "$ALREADY_CHECKED_OUT"; then
+ if [ "$NPS_VERSION" != "DEFAULT" ]; then
+ fail \
+"The --ngx-pagespeed-version argument doesn't make sense when running within an existing checkout."
+ fi
+ elif [ "$NPS_VERSION" = "DEFAULT" ]; then
+ if "$DEVEL"; then
+ NPS_VERSION="trunk-tracking"
+ else
+ NPS_VERSION="latest-stable"
+ fi
+ fi
+
if [ ! -d "$BUILDDIR" ]; then
fail "Told to build in $BUILDDIR, but that directory doesn't exist."
fi
+ BUILD_NGINX=false
+ if [ -n "$NGINX_VERSION" ]; then
+ BUILD_NGINX=true
+ fi
+
+ if "$DEVEL"; then
+ PSOL_FROM_SOURCE=true
+ BUILD_NGINX=true
+ if [ -n "$NGINX_VERSION" ]; then
+ fail \
+"The --devel argument conflicts with --nginx. In devel mode we use the version of nginx that's included as a submodule."
+ fi
+ if "$DYNAMIC_MODULE"; then
+ fail "Can't currently build a dynamic module in --devel mode."
+ fi
+ fi
+
+ if "$PSOL_FROM_SOURCE" && [ -z "$BUILD_TYPE" ]; then
+ if "$DEVEL"; then
+ BUILD_TYPE="Debug"
+ else
+ BUILD_TYPE="Release"
+ fi
+ elif [ -n "$BUILD_TYPE" ]; then
+ fail "Setting --build-type requires --psol-from-source or --devel."
+ fi
+
if [ "$NGINX_VERSION" = "latest" ]; then
# When this function fails it prints the debugging information needed first
# to stderr.
@@ -314,16 +451,16 @@
# Now make sure our dependencies are installed.
if "$DO_DEPS_CHECK"; then
if [ -f /etc/debian_version ]; then
- echo "Detected debian-based distro."
+ status "Detected debian-based distro."
install_dependencies "apt-get install" debian_is_installed \
"build-essential zlib1g-dev libpcre3 libpcre3-dev unzip"
if gcc_too_old; then
if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then
- echo "Detected that gcc is older than 4.8. Installing gcc-mozilla"
- echo "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
- echo "affect your global gcc installation."
+ status "Detected that gcc is older than 4.8. Installing gcc-mozilla"
+ status "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
+ status "affect your global gcc installation."
run sudo apt-get install gcc-mozilla
fi
@@ -332,7 +469,7 @@
fi
elif [ -f /etc/redhat-release ]; then
- echo "Detected redhat-based distro."
+ status "Detected redhat-based distro."
install_dependencies "yum install" redhat_is_installed \
"gcc-c++ pcre-devel zlib-devel make unzip wget"
@@ -350,9 +487,9 @@
$(cat /etc/redhat-release) Expected 5 or 6."
fi
- echo "Detected that gcc is older than 4.8. Scientific Linux provides"
- echo "a gcc package that installs gcc-4.8 into /opt/ and doesn't"
- echo "affect your global gcc installation."
+ status "Detected that gcc is older than 4.8. Scientific Linux"
+ status "provides a gcc package that installs gcc-4.8 into /opt/ and"
+ status "doesn't affect your global gcc installation."
slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/"
slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern"
slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern"
@@ -378,9 +515,9 @@
be able to install dependencies. Please install dependencies manually and rerun
with --no-deps-check."
fi
- echo "Dependencies are all set."
+ status "Operating system dependencies are all set."
else
- echo "Not checking whether dependencies are installed."
+ status "Not checking whether operating system dependencies are installed."
fi
function delete_if_already_exists() {
@@ -391,7 +528,6 @@
if [ ${#directory} -lt 8 ]; then
fail "
Not deleting $directory; name is suspiciously short. Something is wrong."
- exit 1
fi
continue_or_exit "OK to delete $directory?"
@@ -399,53 +535,106 @@
fi
}
- nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
# In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo,
- # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION.
+ # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION
if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then
- # We've been given a numeric version number. This has an associated tag in
- # the form vVERSION-beta.
- nps_url_fname="v${NPS_VERSION}-beta"
+ # We've been given a numeric version number. This has an associated tag
+ # in the form vVERSION-beta.
+ tag_name="v${NPS_VERSION}-beta"
nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta"
else
# We've been given a tag name, like latest-beta. Download that directly.
- nps_url_fname="$NPS_VERSION"
+ tag_name="$NPS_VERSION"
nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}"
fi
- nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
- run wget "$nps_baseurl/$nps_url_fname.zip" -O "$nps_downloaded"
- nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
- delete_if_already_exists "$nps_module_dir"
- echo "Extracting ngx_pagespeed..."
- run unzip -q "$nps_downloaded" -d "$BUILDDIR"
- run cd "$nps_module_dir"
-
- # Now we need to figure out what precompiled version of PSOL to build
- # ngx_pagespeed against.
- if "$DRYRUN"; then
- psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
- elif [ -e PSOL_BINARY_URL ]; then
- # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
- # where to look.
- psol_url="$(cat PSOL_BINARY_URL)"
- if [[ "$psol_url" != https://* ]]; then
- fail "Got bad psol binary location information: $psol_url"
+ install_dir="this-only-makes-sense-in-devel-mode"
+ if "$USE_GIT_CHECKOUT"; then
+ # We're either doing a --devel build, or someone is running us from an
+ # existing git checkout.
+ nps_module_dir="$PWD"
+ install_dir="$nps_module_dir"
+ if "$ALREADY_CHECKED_OUT"; then
+ run cd "$nps_module_dir"
+ else
+ status "Checking out ngx_pagespeed..."
+ run git clone "git@github.com:pagespeed/ngx_pagespeed.git" \
+ "$nps_module_dir"
+ run cd "$nps_module_dir"
+ run git checkout "$tag_name"
+ fi
+ submodules_dir="$nps_module_dir/testing-dependencies"
+ if "$DEVEL"; then
+ status "Downloading dependencies..."
+ run git submodule update --init --recursive
+ if [[ "$CONTINUOUS_INTEGRATION" != true ]]; then
+ status "Switching submodules over to git protocol."
+ # This lets us push to github by public key.
+ for config in $(find .git/ -name config) ; do
+ run sed -i s~https://github.com/~git@github.com:~ $config ;
+ done
+ fi
fi
else
- # For past releases we have to grep it from the config file. The url has
- # always looked like this, and the config file has contained it since before
- # we started tagging our ngx_pagespeed releases.
- psol_url="$(
- grep -o "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
- if [ -z "$psol_url" ]; then
- fail "Couldn't find PSOL url in $PWD/config"
- fi
+ nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
+ nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
+ status "Downloading ngx_pagespeed..."
+ run wget "$nps_baseurl/$tag_name.zip" -O "$nps_downloaded"
+ nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
+ delete_if_already_exists "$nps_module_dir"
+ status "Extracting ngx_pagespeed..."
+ run unzip -q "$nps_downloaded" -d "$BUILDDIR"
+ run cd "$nps_module_dir"
fi
- run wget "$psol_url"
- echo "Extracting PSOL..."
- run tar -xzf $(basename "$psol_url") # extracts to psol/
+ MOD_PAGESPEED_DIR=""
+ PSOL_BINARY=""
+ if "$PSOL_FROM_SOURCE"; then
+ MOD_PAGESPEED_DIR="$PWD/testing-dependencies/mod_pagespeed"
+ git submodule update --init --recursive -- "$MOD_PAGESPEED_DIR"
+ run pushd "$MOD_PAGESPEED_DIR"
+
+ if "$DEVEL"; then
+ if [ ! -d "$HOME/apache2" ]; then
+ run install/build_development_apache.sh 2.2 prefork
+ fi
+ cd devel
+ run make apache_debug_psol
+ PSOL_BINARY="$MOD_PAGESPEED_DIR/out/$BUILD_TYPE/pagespeed_automatic.a"
+ else
+ run install/build_psol.sh --skip_tests --skip_packaging
+ PSOL_BINARY="$MOD_PAGESPEED_DIR/pagespeed/automatic/pagespeed_automatic.a"
+ fi
+ run popd
+ else
+ # Now we need to figure out what precompiled version of PSOL to build
+ # ngx_pagespeed against.
+ if "$DRYRUN"; then
+ psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
+ elif [ -e PSOL_BINARY_URL ]; then
+ # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
+ # where to look.
+ psol_url="$(cat PSOL_BINARY_URL)"
+ if [[ "$psol_url" != https://* ]]; then
+ fail "Got bad psol binary location information: $psol_url"
+ fi
+ else
+ # For past releases we have to grep it from the config file. The url has
+ # always looked like this, and the config file has contained it since
+ # before we started tagging our ngx_pagespeed releases.
+ psol_url="$(grep -o \
+ "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
+ if [ -z "$psol_url" ]; then
+ fail "Couldn't find PSOL url in $PWD/config"
+ fi
+ fi
+
+ status "Downloading PSOL binary..."
+ run wget "$psol_url"
+
+ status "Extracting PSOL..."
+ run tar -xzf $(basename "$psol_url") # extracts to psol/
+ fi
if "$DYNAMIC_MODULE"; then
add_module="--add-dynamic-module=$nps_module_dir"
@@ -454,17 +643,37 @@
fi
configure_args=("$add_module" "${extra_flags[@]}")
+ if "$DEVEL"; then
+ configure_args=("${configure_args[@]}"
+ "--prefix=$install_dir/nginx"
+ "--add-module=$submodules_dir/ngx_cache_purge"
+ "--add-module=$submodules_dir/ngx_devel_kit"
+ "--add-module=$submodules_dir/set-misc-nginx-module"
+ "--add-module=$submodules_dir/headers-more-nginx-module"
+ "--with-ipv6"
+ "--with-http_v2_module")
+ if [ "$BUILD_TYPE" = "Debug" ]; then
+ configure_args=("${configure_args[@]}" "--with-debug")
+ fi
+ fi
+
echo
- if [ -z "$NGINX_VERSION" ]; then
- # They didn't specify an nginx version, so we're just preparing the
- # module for them to install.
- echo "ngx_pagespeed is ready to be built against nginx."
- echo "When running ./configure pass in:"
- echo " $(quote_arguments "${configure_args[@]}")"
+ if ! "$BUILD_NGINX"; then
+ # Just prepare the module for them to install.
+ status "ngx_pagespeed is ready to be built against nginx."
+ echo "When running ./configure:"
+ if "$PSOL_FROM_SOURCE"; then
+ echo " Set the following environment variables:"
+ echo " MOD_PAGESPEED_DIR=$MOD_PAGESPEED_DIR"
+ echo " PSOL_BINARY=$PSOL_BINARY"
+ fi
+ echo " Give ./configure the following arguments:"
+ echo " $(quote_arguments "${configure_args[@]}")"
+ echo
if [ ${#extra_flags[@]} -eq 0 ]; then
echo "If this is for integration with an already-built nginx, make sure"
- echo "to include any other arguments you originally passed to ./configure"
- echo "You can see these with 'nginx -V'."
+ echo "to include any other arguments you originally passed to"
+ echo "./configure. You can see these with 'nginx -V'."
else
echo "Note: because we need to set $(quote_arguments "${extra_flags[@]}")"
echo "on this platform, if you want to integrate ngx_pagespeed with an"
@@ -472,60 +681,98 @@
echo "those flags set."
fi
else
- # Download and build nginx.
- nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
- nginx_fname="$TEMPDIR/$nginx_leaf"
- run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
- nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
- delete_if_already_exists "$nginx_dir"
- echo "Extracting nginx..."
- run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
- "$DRYRUN" || cd "$nginx_dir"
+ if "$DEVEL"; then
+ # Use the nginx we loaded as a submodule
+ nginx_dir="$submodules_dir/nginx"
+ configure_location="auto"
+ else
+ # Download and build the specified nginx version.
+ nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
+ nginx_fname="$TEMPDIR/$nginx_leaf"
+ status "Downloading nginx..."
+ run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
+ nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
+ delete_if_already_exists "$nginx_dir"
+ status "Extracting nginx..."
+ run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
+ configure_location="."
+ fi
+ run cd "$nginx_dir"
- configure=("./configure" "${configure_args[@]}")
- echo "About to build nginx. Do you have any additional ./configure"
- echo "arguments you would like to set? For example, if you would like"
- echo "to build nginx with https support give --with-http_ssl_module"
- echo "If you don't have any, just press enter."
- read -p "> " additional_configure_args
- if [ -n "$additional_configure_args" ]; then
- # Split additional_configure_args respecting any internal quotation.
- # Otherwise things like --with-cc-opt='-foo -bar' won't work.
- eval additional_configure_args=("$additional_configure_args")
- configure=("${configure[@]}" "${additional_configure_args[@]}")
+ configure=("$configure_location/configure" "${configure_args[@]}")
+ if ! "$ASSUME_YES"; then
+ echo "About to build nginx. Do you have any additional ./configure"
+ echo "arguments you would like to set? For example, if you would like"
+ echo "to build nginx with https support give --with-http_ssl_module"
+ echo "If you don't have any, just press enter."
+ read -p "> " additional_configure_args
+ if [ -n "$additional_configure_args" ]; then
+ # Split additional_configure_args respecting any internal quotation.
+ # Otherwise things like --with-cc-opt='-foo -bar' won't work.
+ eval additional_configure_args=("$additional_configure_args")
+ configure=("${configure[@]}" "${additional_configure_args[@]}")
+ fi
fi
echo "About to configure nginx with:"
echo " $(quote_arguments "${configure[@]}")"
continue_or_exit "Does this look right?"
- run "${configure[@]}"
+ MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
+ PSOL_BINARY="$PSOL_BINARY" \
+ run "${configure[@]}"
- continue_or_exit "Build nginx?"
+ if ! "$DEVEL"; then
+ continue_or_exit "Build nginx?"
+ fi
run make
- continue_or_exit "Install nginx?"
- run sudo make install
+ if "$DEVEL"; then
+ run make install
- echo
- if "$DYNAMIC_MODULE"; then
- echo "Nginx installed with ngx_pagespeed support available as a"
- echo "loadable module."
+ status "Nginx installed with ngx_pagespeed, and set up for testing."
+ # TODO(jefftk): pull these out into separate scripts.
+ echo "To run tests, pick a pair of ports like 8050 and 8051 and run:"
+ echo " cd $nps_module_dir"
+ echo " RUN_TESTS=true \\"
+ echo " USE_VALGRIND=false \\"
+ echo " TEST_NATIVE_FETCHER=false \\"
+ echo " TEST_SERF_FETCHER=true \\"
+ echo " test/run_tests.sh \\"
+ echo " $MOD_PAGESPEED_DIR \\"
+ echo " $install_dir/nginx/sbin/nginx"
echo
- echo "To load the ngx_pagespeed module, you'll need to add:"
- echo " load_module \"modules/ngx_pagespeed.so\";"
- echo "at the top of your main nginx configuration file."
+ echo "To rebuild after changes:"
+ echo " First, if you change things in PSOL or update it:"
+ echo " cd $MOD_PAGESPEED_DIR/devel"
+ echo " make apache_debug_psol"
+ echo " Then, whether or not you updated PSOL, rebuild nginx:"
+ echo " cd $install_dir/nginx"
+ echo " make && make install"
else
- echo "Nginx installed with ngx_pagespeed support compiled-in."
+ continue_or_exit "Install nginx?"
+ run sudo make install
+
+ echo
+ if "$DYNAMIC_MODULE"; then
+ echo "Nginx installed with ngx_pagespeed support available as a"
+ echo "loadable module."
+ echo
+ echo "To load the ngx_pagespeed module, you'll need to add:"
+ echo " load_module \"modules/ngx_pagespeed.so\";"
+ echo "at the top of your main nginx configuration file."
+ else
+ echo "Nginx installed with ngx_pagespeed support compiled-in."
+ fi
+ echo
+ echo "If this is a new installation you probably need an init script to"
+ echo "manage starting and stopping the nginx service. See:"
+ echo " http://wiki.nginx.org/InitScripts"
+ echo
+ echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
+ echo " https://developers.google.com/speed/pagespeed/module/configuration"
fi
- echo
- echo "If this is a new installation you probably need an init script to"
- echo "manage starting and stopping the nginx service. See:"
- echo " http://wiki.nginx.org/InitScripts"
- echo
- echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
- echo " https://developers.google.com/speed/pagespeed/module/configuration"
fi
if "$DRYRUN"; then
- echo "[this was a dry run; your system is unchanged]"
+ echo_color "$YELLOW" "[this was a dry run; your system is unchanged]"
fi
}
diff --git a/test/nginx_system_test.sh b/test/nginx_system_test.sh
index d10d50d..f1d7bd0 100644
--- a/test/nginx_system_test.sh
+++ b/test/nginx_system_test.sh
@@ -49,7 +49,11 @@
mkdir -p "$TEST_TMP"
echo TEST_TMP=$TEST_TMP
-APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/src/install/"
+if [ -d "$MOD_PAGESPEED_DIR/src" ]; then
+ MOD_PAGESPEED_DIR+="/src"
+fi
+
+APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/install/"
SERVER_ROOT="$TEST_TMP/root"
echo SERVER_ROOT=$SERVER_ROOT
rm -rf "$SERVER_ROOT"
@@ -311,7 +315,7 @@
RUN_CONTROLLER_TEST=${RUN_CONTROLLER_TEST:-off}
# run generic system tests
-PAGESPEED_DIR="$MOD_PAGESPEED_DIR/src/pagespeed"
+PAGESPEED_DIR="$MOD_PAGESPEED_DIR/pagespeed"
SYSTEM_TEST_FILE="$PAGESPEED_DIR/system/system_test.sh"
REMOTE_CONFIG_TEST_FILE="$PAGESPEED_DIR/system/remote_config_test.sh"
@@ -1420,7 +1424,7 @@
| grep -v "\\[error\\].*forbidden.example.com*" \
| grep -v "\\[error\\].*custom-paths.example.com*" \
| grep -v "\\[error\\].*bogus_format*" \
- | grep -v "\\[error\\].*src/install/foo*" \
+ | grep -v "\\[error\\].*/install/foo*" \
| grep -v "\\[error\\].*recv() failed*" \
| grep -v "\\[error\\].*send() failed*" \
| grep -v "\\[error\\].*Invalid url requested: js_defer.js.*" \
diff --git a/test/run_tests.sh b/test/run_tests.sh
index f03105d..266aa84 100755
--- a/test/run_tests.sh
+++ b/test/run_tests.sh
@@ -58,10 +58,12 @@
: ${CONTROLLER_PORT:=8053}
: ${RCPORT:=9991}
: ${PAGESPEED_TEST_HOST:=selfsigned.modpagespeed.com}
+: ${PHP_PORT:=9000}
this_dir="$( cd $(dirname "$0") && pwd)"
function run_test_checking_failure() {
+ "$MOD_PAGESPEED_DIR/install/start_php.sh" "$PHP_PORT"
USE_VALGRIND="$USE_VALGRIND" \
PRIMARY_PORT="$PRIMARY_PORT" \
SECONDARY_PORT="$SECONDARY_PORT" \
diff --git a/testing-dependencies/headers-more-nginx-module b/testing-dependencies/headers-more-nginx-module
new file mode 160000
index 0000000..30fb259
--- /dev/null
+++ b/testing-dependencies/headers-more-nginx-module
@@ -0,0 +1 @@
+Subproject commit 30fb25901cad32b342f53cb174730c6e00fb66eb
diff --git a/testing-dependencies/mod_pagespeed b/testing-dependencies/mod_pagespeed
new file mode 160000
index 0000000..91662d0
--- /dev/null
+++ b/testing-dependencies/mod_pagespeed
@@ -0,0 +1 @@
+Subproject commit 91662d08ae6653b459372daff69bd7cd3bb0589f
diff --git a/testing-dependencies/nginx b/testing-dependencies/nginx
new file mode 160000
index 0000000..6917d29
--- /dev/null
+++ b/testing-dependencies/nginx
@@ -0,0 +1 @@
+Subproject commit 6917d29d409b187d440d8e5012a1353398e3a365
diff --git a/testing-dependencies/ngx_cache_purge b/testing-dependencies/ngx_cache_purge
new file mode 160000
index 0000000..331fe43
--- /dev/null
+++ b/testing-dependencies/ngx_cache_purge
@@ -0,0 +1 @@
+Subproject commit 331fe43e8d9a3d1fa5e0c9fec7d3201d431a9177
diff --git a/testing-dependencies/ngx_devel_kit b/testing-dependencies/ngx_devel_kit
new file mode 160000
index 0000000..e443262
--- /dev/null
+++ b/testing-dependencies/ngx_devel_kit
@@ -0,0 +1 @@
+Subproject commit e443262071e759c047492be60ec7e2d73c5b57ec
diff --git a/testing-dependencies/set-misc-nginx-module b/testing-dependencies/set-misc-nginx-module
new file mode 160000
index 0000000..72be651
--- /dev/null
+++ b/testing-dependencies/set-misc-nginx-module
@@ -0,0 +1 @@
+Subproject commit 72be6512cfe330af41b44f273b26f738e6b77864
