/*
 * Copyright 2011 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Authors: nforman@google.com (Naomi Forman)
//          jefftk@google.com (Jeff Kaufman)
//
// Implements the insert_ga_snippet filter, which inserts the Google Analytics
// tracking snippet into html pages.  When experiments are enabled, also inserts
// snippets to report experiment status back.

#include "net/instaweb/rewriter/public/insert_ga_filter.h"

#include "base/logging.h"
#include "net/instaweb/rewriter/public/experiment_util.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/statistics.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/html/html_element.h"
#include "pagespeed/kernel/html/html_name.h"
#include "pagespeed/kernel/html/html_node.h"
#include "pagespeed/kernel/http/google_url.h"
#include "pagespeed/kernel/js/js_keywords.h"
#include "pagespeed/kernel/js/js_tokenizer.h"

namespace {

// Name for statistics variable.
const char kInsertedGaSnippets[] = "inserted_ga_snippets";

}  // namespace

namespace net_instaweb {

// This filter primarily exists to support PageSpeed experiments that report
// back to Google Analytics for reporting.  You can also use it just to insert
// the Google Analytics tracking snippet, though.
//
// GA had a rewrite recently, switching from ga.js to analytics.js with a new
// API.  They also released support for content experiments.  The older style of
// reporting is to use a custom variable.  This filter can report to a content
// experiment with either ga.js or analytics.js; with ga.js reporting to a
// custom variable is still supported.
//
// If no GA snippet is present on the page then PageSpeed will insert one.
// Additionally, if you're running an experiment then PageSpeed will insert the
// JS necessary to report details back to GA.  This can look like any of these
// three things:
//
// ga.js + custom variables:
//   <script>kGAExperimentSnippet
//           kGAJsSnippet</script> [ possibly existing ]
//
// ga.js + content experiments:
//   <script src="kContentExperimentsJsClientUrl"></script>
//   <script>kContentExperimentsSetChosenVariationSnippet
//           kGAJsSnippet</script> [ possibly existing ]
//
// analytics.js + content experiments:
//   <script>kAnalyticsJsSnippet</script> [ possibly existing ]
//   kContentExperimentsSetExpAndVariantSnippet goes inside the analytics js
//   snippet, just before the ga(send, pageview) call.

// Google Analytics snippet for setting experiment related variables.  Use with
// old ga.js and custom variable experiment reporting. Arguments are:
//   %s: Optional snippet to increase site speed tracking.
//   %u: Which ga.js custom variable to support to.
//   %s: Experiment spec string, shown in the GA UI.
extern const char kGAExperimentSnippet[] =
    "var _gaq = _gaq || [];"
    "%s"
    "_gaq.push(['_setCustomVar', %u, 'ExperimentState', '%s'"
    "]);";

// Google Analytics async snippet along with the _trackPageView call.
extern const char kGAJsSnippet[] =
    "if (window.parent == window) {"
    "var _gaq = _gaq || [];"
    "_gaq.push(['_setAccount', '%s']);"  // %s is the GA account number.
    "_gaq.push(['_setDomainName', '%s']);"  // %s is the domain name
    "_gaq.push(['_setAllowLinker', true]);"
    "%s"  // Optional snippet to increase site speed tracking.
    "_gaq.push(['_trackPageview']);"
    "(function() {"
    "var ga = document.createElement('script'); ga.type = 'text/javascript';"
    "ga.async = true;"
    "ga.src = 'https://ssl.google-analytics.com/ga.js';"
    "var s = document.getElementsByTagName('script')[0];"
    "s.parentNode.insertBefore(ga, s);"
    "})();"
    "}";

// Google Universal analytics snippet.  First argument is the GA account number,
// second is kContentExperimentsSetExpAndVariantSnippet or nothing.
extern const char kAnalyticsJsSnippet[] =
    "if (window.parent == window) {"
    "(function(i,s,o,g,r,a,m){"
    "i['GoogleAnalyticsObject']=r;"
    "i[r]=i[r]||function(){"
    "(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();"
    "a=s.createElement(o), m=s.getElementsByTagName(o)[0];"
    "a.async=1;a.src=g;m.parentNode.insertBefore(a,m)"
    "})(window,document,'script',"
    "'//www.google-analytics.com/analytics.js','ga');"
    "ga('create', '%s', 'auto'%s);"
    "%s"
    "ga('send', 'pageview');"
    "}";

// Increase site speed tracking to 100% when using analytics.js
// Use the first one if we're inserting the snippet, or if the site we're
// modifying isn't already using a fields object with ga('create'), the second
// one if there is an existing snippet with a fields object.
extern const char kAnalyticsJsIncreaseSiteSpeedTracking[] =
    ", {'siteSpeedSampleRate': 100}";
extern const char kAnalyticsJsIncreaseSiteSpeedTrackingMinimal[] =
    "'siteSpeedSampleRate': 100,";

// When using content experiments with ga.js you need to do a sychronous load
// of /cx/api.js first.
extern const char kContentExperimentsJsClientUrl[] =
    "//www.google-analytics.com/cx/api.js";

// When using content experiments with ga.js, after /cx/api.js has loaded and
// before ga.js loads you need to call this.  The first argument is the
// variant id, the second is the experiment id.
extern const char kContentExperimentsSetChosenVariationSnippet[] =
    "cxApi.setChosenVariation(%d, '%s');";

// When using content experiments with ga.js, the variant ID must be numeric.
// If the user requests a non-numeric variant with ga.js, we inject this
// comment. The string is bracketed with newlines because otherwise it's
// invisible in a wall of JavaScript.
extern const char kContentExperimentsNonNumericVariantComment[] =
    "\n/* mod_pagespeed cannot inject experiment variant '%s' "
    "because it's not a number */\n";

// When using content experiments with analytics.js, after ga('create', ..._)
// and before ga('[...].send', 'pageview'), we need to insert:
extern const char kContentExperimentsSetExpAndVariantSnippet[] =
    "ga('set', 'expId', '%s');"
    "ga('set', 'expVar', '%s');";

// Set the sample rate to 100%.
// TODO(nforman): Allow this to be configurable through RewriteOptions.
extern const char kGASpeedTracking[] =
    "_gaq.push(['_setSiteSpeedSampleRate', 100]);";

InsertGAFilter::InsertGAFilter(RewriteDriver* rewrite_driver)
    : CommonFilter(rewrite_driver),
      script_element_(NULL),
      added_analytics_js_(false),
      added_experiment_snippet_(false),
      ga_id_(rewrite_driver->options()->ga_id()),
      found_snippet_(false),
      increase_speed_tracking_(
          rewrite_driver->options()->increase_speed_tracking()),
      seen_sync_ga_js_(false) {
  Statistics* stats = driver()->statistics();
  inserted_ga_snippets_count_ = stats->GetVariable(kInsertedGaSnippets);
  DCHECK(!ga_id_.empty()) << "Enabled ga insertion, but did not provide ga id.";
}

void InsertGAFilter::InitStats(Statistics* stats) {
  stats->AddVariable(kInsertedGaSnippets);
}

InsertGAFilter::~InsertGAFilter() {}

bool InsertGAFilter::StringLiteralMatches(StringPiece literal,
                                          StringPiece desired) {
  // Literal includes the beginning and ending quotes; need to exclude them.
  return literal.substr(1, literal.size() - 2) == desired;
}

bool InsertGAFilter::StringLiteralEndsWith(StringPiece literal,
                                           StringPiece desired) {
  // Literal includes the beginning and ending quotes; need to exclude them.
  return literal.substr(1, literal.size() - 2).ends_with(desired);
}

void InsertGAFilter::StartDocumentImpl() {
  found_snippet_ = false;
  script_element_ = NULL;
  added_analytics_js_ = false;
  added_experiment_snippet_ = false;
  if (driver()->options()->running_experiment()) {
    driver()->message_handler()->Message(
        kInfo, "run_experiment: %s",
        driver()->options()->ToExperimentDebugString().c_str());
  }
}

// Start looking for ga snippet.
void InsertGAFilter::StartElementImpl(HtmlElement* element) {
  if (!ga_id_.empty() &&
      !found_snippet_ &&
      element->keyword() == HtmlName::kScript &&
      script_element_ == NULL) {
    script_element_ = element;
  }
}

// This isn't perfect but matches all the cases we've found.  It's ok if it has
// some false positives; the later check is more thorough.
InsertGAFilter::AnalyticsStatus InsertGAFilter::FindSnippetInScript(
    const GoogleString& s) {
  // dc.js is a synonym for old-style ga.js
  if (!seen_sync_ga_js_ &&
      (s.find("google-analytics.com/ga.js") != GoogleString::npos ||
       s.find("stats.g.doubleclick.net/dc.js") != GoogleString::npos)) {
    // The synchronous snippet has two parts: first one with
    // [google-analytics.com/ga.js] (no initial dot) and then a later one with
    // ga_id, _getTracker, and _trackPageview.  Track that we've seen what is
    // probably the first snippet, and then if we later get what could be the
    // second snippet we'll check below.
    seen_sync_ga_js_ = true;
  }
  if (s.find(StrCat("'", ga_id_, "'")) == GoogleString::npos &&
      s.find(StrCat("\"", ga_id_, "\"")) == GoogleString::npos) {
    return kNoSnippetFound;
  }
  if (s.find(".google-analytics.com/urchin.js") != GoogleString::npos) {
    return kUnusableSnippetFound;  // urchin.js is too old.
  } else if (s.find(".google-analytics.com/ga.js") != GoogleString::npos ||
             s.find("stats.g.doubleclick.net/dc.js") != GoogleString::npos) {
    // With the async snippet there is one part that first loads ga.js
    // (using [.google-analytics.com/ga.js], with initial dot) and then has the
    // ga_id (which we checked for above).
    return kGaJs;  // Asynchronous ga.js
  } else if (seen_sync_ga_js_ &&
             s.find("_getTracker") != GoogleString::npos &&
             s.find("_trackPageview") != GoogleString::npos) {
    // Synchronous ga.js was split over two script tags: first one to do the
    // loading then one to do the initialization and page tracking.  We want to
    // process the second one.
    return kGaJs;  // Syncronous ga.js
  } else if (s.find(".google-analytics.com/analytics.js")) {
    return kAnalyticsJs;
  }
  return kUnusableSnippetFound;
}

GoogleString InsertGAFilter::AnalyticsJsExperimentSnippet() const {
  return StringPrintf(
      kContentExperimentsSetExpAndVariantSnippet,
      driver()->options()->content_experiment_id().c_str(),
      driver()->options()->content_experiment_variant_id().c_str());
}

GoogleString InsertGAFilter::GaJsExperimentSnippet() const {
  // ga.js requires a numeric variant id. Attempt to convert the string
  // variant ID to int and use that.
  const char* variant_id =
      driver()->options()->content_experiment_variant_id().c_str();
  int numeric_variant_id;
  if (StringToInt(variant_id, &numeric_variant_id)) {
    return StringPrintf(
        kContentExperimentsSetChosenVariationSnippet, numeric_variant_id,
        driver()->options()->content_experiment_id().c_str());
  } else {
    // Variant ID was non-numeric, so inject a warning.
    return StringPrintf(kContentExperimentsNonNumericVariantComment,
                        variant_id);
  }
}

// * If we've already inserted any GA snippet or if we found a GA snippet in the
//   original page, don't do anything.
// * If we haven't found anything, and haven't inserted anything yet, insert the
//   GA js snippet.
//
// Caveat: The snippet should ideally be placed in <head> for accurate
// collection of data (e.g. pageviews etc.). We place it at the end of the
// document so that we won't add duplicate analytics js code for any page.
//
// For pages which don't already have analytics js, this might result in some
// data being lost.
void InsertGAFilter::EndDocument() {
  if (found_snippet_ || added_analytics_js_ || ga_id_.empty()) {
    return;
  }

  // No snippets have been found, and we haven't added any snippets yet, so add
  // one now.  Include experiment setup if experiments are on.

  GoogleString js_text;
  GoogleString experiment_snippet;
  const char* speed_tracking = "";
  if (driver()->options()->use_analytics_js()) {
    if (increase_speed_tracking_) {
      speed_tracking = kAnalyticsJsIncreaseSiteSpeedTracking;
    }
    if (ShouldInsertExperimentTracking(true /* analytics.js */)) {
      experiment_snippet = AnalyticsJsExperimentSnippet();
    }
    js_text = StringPrintf(
        kAnalyticsJsSnippet,
        ga_id_.c_str(),
        speed_tracking,
        experiment_snippet.c_str());
  } else {
    if (ShouldInsertExperimentTracking(false /* ga.js */)) {
      if (driver()->options()->is_content_experiment()) {
        HtmlElement* cxapi = driver()->NewElement(NULL, HtmlName::kScript);
        driver()->AddAttribute(
            cxapi, HtmlName::kSrc, kContentExperimentsJsClientUrl);
        InsertNodeAtBodyEnd(cxapi);
        experiment_snippet = GaJsExperimentSnippet();
      } else {
        experiment_snippet = StringPrintf(
            kGAExperimentSnippet,
            "" /* don't change speed tracking here, we add it below */,
            driver()->options()->experiment_ga_slot(),
            driver()->options()->ToExperimentString().c_str());
      }
    }

    // Domain for this html page.
    GoogleString domain = driver()->google_url().Host().as_string();
    if (increase_speed_tracking_) {
      speed_tracking = kGASpeedTracking;
    }
    js_text = StrCat(experiment_snippet,
                     StringPrintf(kGAJsSnippet,
                                  ga_id_.c_str(),
                                  domain.c_str(),
                                  speed_tracking));
  }

  HtmlElement* script_element = driver()->NewElement(NULL, HtmlName::kScript);
  InsertNodeAtBodyEnd(script_element);
  HtmlNode* snippet = driver()->NewCharactersNode(script_element, js_text);
  driver()->AppendChild(script_element, snippet);

  added_analytics_js_ = true;
  inserted_ga_snippets_count_->Add(1);
}

bool InsertGAFilter::ShouldInsertExperimentTracking(bool is_analytics_js) {
  if (driver()->options()->running_experiment()) {
    if (is_analytics_js && !driver()->options()->is_content_experiment()) {
      driver()->WarningHere("Experiment framework requires a content experiment"
                            " when used with analytics.js.");
      return false;
    }

    int experiment_state = driver()->options()->experiment_id();
    if (experiment_state != experiment::kExperimentNotSet &&
        experiment_state != experiment::kNoExperiment) {
      return true;
    }
  }
  return false;
}

void InsertGAFilter::RewriteInlineScript(HtmlCharactersNode* characters) {
  AnalyticsStatus analytics_status =
      FindSnippetInScript(characters->contents());
  if (analytics_status == kNoSnippetFound) {
    return;  // This inline script isn't for GA; nothing to change.
  }

  found_snippet_ = true;

  if (!ShouldInsertExperimentTracking(analytics_status == kAnalyticsJs)) {
    return;  // GA script found, but we don't need to change it.
  }

  if (analytics_status == kUnusableSnippetFound) {
    driver()->InfoHere("Page contains unusual Google Analytics snippet that"
                       " we're not able to modify to add experiment tracking.");
    return;
  }

  if (analytics_status == kAnalyticsJs) {
    GoogleString rewritten;
    StringPiece token;
    pagespeed::JsKeywords::Type token_type;
    pagespeed::js::JsTokenizer tokenizer(
        server_context()->js_tokenizer_patterns(), characters->contents());
    ParseState state = kInitial;

    // Go through the tokens, appending them to rewritten.  We need to find the
    // ga(create) call so we can increase speed tracking.  Then we need to find
    // the ga(send pageview) call so we can insert our experiment snippet.

    // When we find a ga(send pageview) call it won't be obvious what we've
    // found until we're several tokens along.  So save the offset of each ga
    // function call when we find it so we can later insert before if need be.
    int ga_send_pageview_offset = -1;
    bool inserted_speed_tracking = false;

    while ((token_type = tokenizer.NextToken(&token)) !=
           pagespeed::JsKeywords::kEndOfInput) {
      if (token_type == pagespeed::JsKeywords::kError) {
        driver()->InfoHere("Got invalid js in Google Analytics snippet");
        return;
      }
      if (token_type == pagespeed::JsKeywords::kComment ||
          token_type == pagespeed::JsKeywords::kWhitespace ||
          token_type == pagespeed::JsKeywords::kLineSeparator) {
        // All states allow these, so stay in the same state.  kLineSeparator is
        // specifically for newlines that don't trigger semicolon insertion.
      } else if (state == kInitial &&
                 token_type == pagespeed::JsKeywords::kIdentifier &&
                 token == "ga") {
        ga_send_pageview_offset = rewritten.size();
        state = kGotGa;
      } else if (state == kGotGa &&
                 token_type == pagespeed::JsKeywords::kOperator &&
                 token == "(") {
        state = kGotGaFuncCall;
      } else if (state == kGotGaFuncCall &&
                 token_type == pagespeed::JsKeywords::kStringLiteral &&
                 StringLiteralMatches(token, "create")) {
        state = kGotGaCreate;
      } else if (state == kGotGaFuncCall &&
                 token_type == pagespeed::JsKeywords::kStringLiteral &&
                 (StringLiteralMatches(token, "send") ||
                  StringLiteralEndsWith(token, ".send"))) {
        state = kGotGaSend;
      } else if (state == kGotGaCreate &&
                 token_type == pagespeed::JsKeywords::kOperator &&
                 token == ",") {
        state = kGotGaCreateComma;
      } else if (state == kGotGaCreate &&
                 token_type == pagespeed::JsKeywords::kOperator &&
                 token == ")") {
        // Saw end of function call without any fields object.  Insert
        // standard speed tracking here.
        if (increase_speed_tracking_) {
          rewritten.append(kAnalyticsJsIncreaseSiteSpeedTracking);
          inserted_speed_tracking = true;
        }
        state = kInitial;
      } else if (state == kGotGaCreateComma &&
                 token_type == pagespeed::JsKeywords::kStringLiteral) {
        // Ignore any string arguments after create, just let them pass.
        state = kGotGaCreate;
      } else if (state == kGotGaCreateComma &&
                 token_type == pagespeed::JsKeywords::kOperator &&
                 token == "{") {
        state = kGotFieldsObject;
      } else if (state == kGotFieldsObject) {
        // Add our field setting before any of the others.
        if (increase_speed_tracking_) {
          rewritten.append(kAnalyticsJsIncreaseSiteSpeedTrackingMinimal);
          inserted_speed_tracking = true;
        }
        state = kInitial;
      } else if (state == kGotGaSend &&
                 token_type == pagespeed::JsKeywords::kOperator &&
                 token == ",") {
        state = kGotGaSendComma;
      } else if (state == kGotGaSendComma &&
                 token_type == pagespeed::JsKeywords::kStringLiteral &&
                 StringLiteralMatches(token, "pageview")) {
        state = kGotGaSendPageview;
      } else if (state == kGotGaSendPageview &&
                 token_type == pagespeed::JsKeywords::kOperator &&
                 (token == "," || token == ")")) {
        CHECK(ga_send_pageview_offset != -1);
        rewritten.insert(ga_send_pageview_offset,
                         AnalyticsJsExperimentSnippet());
        state = kSuccess;
      } else if (state == kSuccess) {
        // Pass the remaining tokens through, we already made our changes.
      } else {
        // Any token we weren't expecting puts us back into looking for "ga".
        state = kInitial;
      }

      rewritten.append(token.as_string());
    }
    if (state == kSuccess) {
      (*characters->mutable_contents()) = rewritten;
      added_experiment_snippet_ = true;

      if (increase_speed_tracking_ && !inserted_speed_tracking) {
        driver()->InfoHere("Failed to increase site speed tracking.");
      }
    } else {
      driver()->InfoHere(
          "Failed to add experiment tracking to existing snippet.");
    }
  } else {
    DCHECK(analytics_status == kGaJs);

    if (driver()->options()->is_content_experiment()) {
      // The API for content experiments with ga.js unfortunately requires a
      // synchronous script load first.  Ideally people would switch to
      // analytics.js, which doesn't have this problem, but we need to support
      // people who haven't switched as well.
      //
      // We can't do InsertBeforeCurrent here, because we could be in the
      // horrible case where "<script>" has been flushed and now we're
      // rewriting the script body.  So the best we can do is:
      // * Blank out this script.
      // * Append the blocking external script load.
      // * Append the edited body of the original script tag as a new
      //   inline script.
      postponed_script_body_ = characters->contents();
      characters->mutable_contents()->clear();
    } else {
      const char* speed_tracking =
          increase_speed_tracking_ ? kGASpeedTracking : "";
      GoogleString snippet_text = StringPrintf(
          kGAExperimentSnippet,
          speed_tracking,
          driver()->options()->experiment_ga_slot(),
          driver()->options()->ToExperimentString().c_str());
      GoogleString* script = characters->mutable_contents();
      // Prepend snippet_text to the script block.
      script->insert(0, snippet_text);
      added_experiment_snippet_ = true;
    }
  }
}

// If RewriteInlineScript decided to insert any new script nodes, do that
// insertion here.
void InsertGAFilter::HandleEndScript(HtmlElement* script) {
  if (!postponed_script_body_.empty()) {
    DCHECK(script == script_element_);
    driver()->InsertScriptAfterCurrent(
        kContentExperimentsJsClientUrl, true /* external */);
    driver()->InsertScriptAfterCurrent(
        StrCat(GaJsExperimentSnippet(), postponed_script_body_),
        false /* inline */);
    added_experiment_snippet_ = true;
    postponed_script_body_.clear();
  }
  script_element_ = NULL;
}

void InsertGAFilter::EndElementImpl(HtmlElement* element) {
  if (ga_id_.empty()) {
    // We only DCHECK that it's non-empty above, but there's nothing useful we
    // can do if it hasn't been set.  Checking here means we'll make no changes.
    return;
  }
  if (element->keyword() == HtmlName::kScript) {
    HandleEndScript(element);
  }
}

void InsertGAFilter::Characters(HtmlCharactersNode* characters) {
  if (script_element_ != NULL && !found_snippet_ &&
      !added_experiment_snippet_) {
    RewriteInlineScript(characters);
  }
}

}  // namespace net_instaweb