src/install/apache_downstream_caching_test.sh

#!/bin/bash
#
# Copyright 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: anupama@google.com (Anupama Dutta)
#
# Runs all Apache-specific downstream caching tests.
#
# Environment variables that are used by this test:
# 1) VARNISH_SERVER should be set to the varnish caching layer
# host:port for testing to be complete. If VARNISH_SERVER is empty,
# minimal testing of the feature is done.
# 2) MUST_BACKUP_DEFAULT_VCL should be set to 1 or 0 depending on
# whether we want the existing default.vcl to be backed up and restored
# before and after the test or it can be overwritten by the updated
# debug_conf.v3.vcl. By default, it is assumed to have the value 1.

this_dir="$( dirname "${BASH_SOURCE[0]}" )"
PAGESPEED_CODE_DIR="$this_dir/../../../third_party/pagespeed"
if [ ! -e "$PAGESPEED_CODE_DIR" ] ; then
  PAGESPEED_CODE_DIR="$this_dir/../pagespeed"
fi
SERVER_NAME=apache
source "$PAGESPEED_CODE_DIR/automatic/system_test_helpers.sh" || exit 1

DEFAULT_VCL="/etc/varnish/default.vcl"
BACKUP_DEFAULT_VCL=$TESTTMP"/default.vcl.bak"
DEBUG_CONF_VCL="$this_dir/debug_conf.v3.vcl"
TMP_DEBUG_CONF_VCL=$TESTTMP"/debug_conf.v3.vcl"

# Environment variables.
# MUST_BACKUP_DEFAULT_VCL is 1 by default because we would not like to overwrite
# the file without explicit permission.
: ${MUST_BACKUP_DEFAULT_VCL:=1}

# Helper method to print out varnish setup instructions.
print_varnish_setup_instructions() {
  echo "*** Please follow these instructions to install and start varnish"
  echo "*** on your system, so that apache_downstream_caching_test.sh can"
  echo "*** run successfully."
  echo "*** 1) sudo apt-get install varnish"
  echo "*** 2) sudo tee -a /etc/default/varnish <<EOF"
  echo "  DAEMON_OPTS=\"-a :8020 \ "
  echo "        -T localhost:6082 \ "
  echo "        -f /etc/varnish/default.vcl \ "
  echo "        -S /etc/varnish/secret \ "
  echo "        -s file,/var/lib/varnish/\$INSTANCE/varnish_storage.bin,1G\" "
  echo "EOF"
  echo "*** 3) sudo cp $DEBUG_CONF_VCL $DEFAULT_VCL"
  echo "*** 4) sudo service varnish restart"
  echo "*** 5) export VARNISH_SERVER=\"localhost:8020\""
  echo "*** 6) Rerun apache_downstream_caching_tests.sh"
}

OUT_CONTENTS_FILE="$OUTDIR/gzipped.html"
OUT_HEADERS_FILE="$OUTDIR/headers.html"
GZIP_WGET_ARGS="-q -S --header=Accept-Encoding:gzip -o $OUT_HEADERS_FILE -O - "

# Helper method that does a wget and verifies that the rewriting status matches
# the $1 argument that is passed to this method.
check_rewriting_status() {
  $WGET $WGET_ARGS $CACHABLE_HTML_LOC > $OUT_CONTENTS_FILE
  if $1; then
    check zgrep -q "pagespeed.ic" $OUT_CONTENTS_FILE
  else
    check_not zgrep -q "pagespeed.ic" $OUT_CONTENTS_FILE
  fi
  # Reset WGET_ARGS.
  WGET_ARGS=""
}

# Helper method that obtains a gzipped response and verifies that rewriting
# has happened. Also takes an extra parameter that identifies extra headers
# to be added during wget.
check_for_rewriting() {
  WGET_ARGS="$GZIP_WGET_ARGS $1"
  check_rewriting_status true
}

# Helper method that obtains a gzipped response and verifies that no rewriting
# has happened.
check_for_no_rewriting() {
  WGET_ARGS="$GZIP_WGET_ARGS"
  check_rewriting_status false
}

# Helper method to check that a variable in the statistics file has the expected
# value.
check_statistic_value() {
  check_from "$CURRENT_STATS" egrep -q "$1:[[:space:]]*$2"
}

check_num_downstream_cache_purge_attempts() {
  check_statistic_value $ATTEMPTS_VAR $1
}

check_num_successful_downstream_cache_purges() {
  check_statistic_value $SUCCESS_VAR $1
}

restore_default_vcl_from_backup() {
  sudo mv -f $BACKUP_DEFAULT_VCL $DEFAULT_VCL
  sudo service varnish restart
}

# Portions of the below test will be skipped if no VARNISH_SERVER is specified.
have_varnish_downstream_cache="1"
if [ -z ${VARNISH_SERVER:-} ]; then
  have_varnish_downstream_cache="0"
  echo "*** Skipping parts of the test because varnish server host:port has"
  echo "*** not been specified. If you'd like to run all parts of this test,"
  echo "*** please follow these instructions:"
  print_varnish_setup_instructions
  CACHABLE_HTML_HOST_PORT="http://${HOSTNAME}"
else
  CACHABLE_HTML_HOST_PORT="http://$VARNISH_SERVER"
  # Check for the presence of $DEFAULT_VCL file to confirm that varnish is
  # installed on the system. If varnish is not installed, print out
  # instructions for it.
  if [ ! -f $DEFAULT_VCL ]; then
    print_varnish_setup_instructions
    exit 1
  fi
  # Check whether the default.vcl being used by varnish is different from
  # debug_conf.v3.vcl.
  # a) If there are no differences, we assume that varnish has been restarted
  # after debug_conf.v3.vcl contents were copied over to default.vcl and
  # continue with the tests.
  cp -f $DEBUG_CONF_VCL $TMP_DEBUG_CONF_VCL
  if ! cmp -s $DEFAULT_VCL $TMP_DEBUG_CONF_VCL; then
    # Copy over the permissions and ownership attributes for $DEFAULT_VCL onto
    # $TMP_DEBUG_CONF_VCL.
    sudo chmod --reference=$DEFAULT_VCL $TMP_DEBUG_CONF_VCL
    sudo chown --reference=$DEFAULT_VCL $TMP_DEBUG_CONF_VCL
    if [ "$MUST_BACKUP_DEFAULT_VCL" = "1" ]; then
      # b) If there are differences, and MUST_BACKUP_DEFAULT_VCL is set to true,
      # we backup the default vcl.
      sudo mv $DEFAULT_VCL $BACKUP_DEFAULT_VCL
      trap restore_default_vcl_from_backup 0
    else
      # c) If there are differences, and MUST_BACKUP_DEFAULT_VCL is set to
      # false, we assume that the user would like to permanently copy over
      # debug_conf.v3.vcl into default.vcl for continuous testing purposes.
      echo "*** Overwriting /etc/varnish/default.vcl with the latest version"
      echo "*** of debug_conf.v3.vcl and restarting varnish."
      echo "*** You only need to do this once for every update to"
      echo "*** debug_conf.v3.vcl, which should not be very frequent."
    fi
    sudo cp -fp $TMP_DEBUG_CONF_VCL $DEFAULT_VCL
  fi
  # Restart varnish to clear its cache.
  sudo service varnish restart
fi

CACHABLE_HTML_LOC="$CACHABLE_HTML_HOST_PORT/mod_pagespeed_test"
CACHABLE_HTML_LOC+="/cachable_rewritten_html/downstream_caching.html"

STATS_URL="${HOSTNAME}/mod_pagespeed_statistics"
ATTEMPTS_VAR="downstream_cache_purge_attempts"
SUCCESS_VAR="successful_downstream_cache_purges"

# Number of downstream cache purges should be 0 here.
start_test Check that downstream cache purges are 0 initially.
CURRENT_STATS=$($WGET_DUMP $STATS_URL)
check_num_downstream_cache_purge_attempts 0
check_num_successful_downstream_cache_purges 0

# Output should not be rewritten and 1 successful purge should have
# occurred here.
start_test Check for case where rewritten cache should get purged.
check_for_no_rewriting
# Fetch until the purge happens.
fetch_until $STATS_URL "grep -c $ATTEMPTS_VAR:[[:space:]]*1" 1
if [ $have_varnish_downstream_cache = "1" ]; then
  CURRENT_STATS=$($WGET_DUMP $STATS_URL)
  check_num_successful_downstream_cache_purges 1
  check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
  fi

# Output should be fully rewritten here.
start_test Check for case where rewritten cache should not get purged.
check_for_rewriting "--header=X-PSA-Blocking-Rewrite:psatest"
# Number of downstream cache purges should still be 1.
CURRENT_STATS=$($WGET_DUMP $STATS_URL)
check_num_downstream_cache_purge_attempts 1
if [ $have_varnish_downstream_cache = "1" ]; then
  check_num_successful_downstream_cache_purges 1
  check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
fi

# Output should be fully rewritten here and we should have a HIT.
start_test Check for case when there should be a varnish cache hit.
check_for_rewriting ""
# Number of downstream cache purges should still be 1.
CURRENT_STATS=$($WGET_DUMP $STATS_URL)
check_num_downstream_cache_purge_attempts 1
if [ $have_varnish_downstream_cache = "1" ]; then
  check_num_successful_downstream_cache_purges 1
  check egrep -q "X-Cache: HIT" $OUT_HEADERS_FILE
fi

if [ $have_varnish_downstream_cache = "1" ]; then
  # Enable one of the beaconing dependent filters and verify interaction
  # between beaconing and downstream caching logic, by verifying that
  # whenever beaconing code is present in the rewritten page, the
  # output is also marked as a cache-miss, indicating that the instrumentation
  # was done by the backend.
  start_test Check whether beaconing is accompanied by a MISS always.
  WGET_ARGS="-S"
  CACHABLE_HTML_LOC+="?ModPagespeedFilters=lazyload_images"
  fetch_until -gzip $CACHABLE_HTML_LOC \
      "zgrep -c \"pagespeed\.CriticalImages\.Run\"" 1
  check fgrep -q 'X-Cache: MISS' $WGET_OUTPUT
  check fgrep -q 'Cache-Control: no-cache, max-age=0' $WGET_OUTPUT
fi


check_failures_and_exit