#!/usr/bin/python

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import time
import ConnectorHelpers
import sqatools
from wintools import sqa_domain_info
from sqatools import LicenseMakerClient
import TestDocs

sys.path.append("/usr/lib/metacarta")

import MetaCartaVersion

# Method to add a document to the meridio repository
def add_document(docurl, recurl, domainuser, password, folder, filepath, filename, filetitle, category="MetacartaSearchable"):
    """Add a document to the repository"""
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(folder),
            ConnectorHelpers.process_argument(filepath),
            ConnectorHelpers.process_argument(filename),
            ConnectorHelpers.process_argument(filetitle),
            ConnectorHelpers.process_argument(category) ]
    stringval = ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.AddDoc", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )
    stringarry = stringval.split("\n")
    # Take the last line
    return stringarry[len(stringarry)-1]

# Method to add multiple documents to the repository
def add_documents_multiple(docurl, recurl, domainuser, password, folder, basefilepath, basetitle, levelcount, levelmax, category="MetacartaSearchable"):
    """Add a document to the repository"""
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(folder),
            ConnectorHelpers.process_argument(basefilepath),
            ConnectorHelpers.process_argument(basetitle),
            str(levelcount),
            str(levelmax),
            ConnectorHelpers.process_argument(category) ]
    stringval = ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.AddDocMultiple", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )
    stringarry = stringval.split("\n")

# Method to add a record to the meridio repository
def add_record(docurl, recurl, domainuser, password, folder, filepath, filename, filetitle, category="MetacartaSearchable"):
    """Add a document to the repository"""
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(folder),
            ConnectorHelpers.process_argument(filepath),
            ConnectorHelpers.process_argument(filename),
            ConnectorHelpers.process_argument(filetitle),
            ConnectorHelpers.process_argument(category) ]
    stringval = ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.AddRec", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )
    stringarry = stringval.split("\n")
    # Take the last line
    return stringarry[len(stringarry)-1]

# Method to add multiple documents to the repository
def add_records_multiple(docurl, recurl, domainuser, password, folder, basefilepath, basetitle, levelcount, levelmax, category="MetacartaSearchable"):
    """Add a document to the repository"""
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(folder),
            ConnectorHelpers.process_argument(basefilepath),
            ConnectorHelpers.process_argument(basetitle),
            str(levelcount),
            str(levelmax),
            ConnectorHelpers.process_argument(category) ]
    ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.AddRecMultiple", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )

# Method to remove a document from the meridio repository
def remove_document(docurl, recurl, domainuser, password, docid):
    """Remove a document from the repository"""
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(docid) ]
    try:
        ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.DeleteDoc", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )
    except Exception, e:
        print "Warning: Error deleting document: %s" % str(e)

# Method to remove a record from the meridio repository
def remove_record(docurl, recurl, domainuser, password, docid):
    """Remove a document from the repository"""
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(docid) ]
    try:
        ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.DeleteRec", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )
    except Exception, e:
        print "Warning: Error deleting record: %s" % str(e)

# Method to lookup a document in the meridio repository
def lookup_document(docurl, recurl, domainuser, password, folder, title):
    """Lookup a document in the repository"""
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(folder),
            ConnectorHelpers.process_argument(title) ]
    stringval = ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.LookupDoc", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )
    stringarry = stringval.split("\n")
    # Take the last line
    return stringarry[len(stringarry)-1]

# Method to lookup a record in the meridio repository
def lookup_record(docurl, recurl, domainuser, password, folder, title):
    """Lookup a record in the repository"""
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(folder),
            ConnectorHelpers.process_argument(title) ]
    stringval = ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.LookupRec", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )
    stringarry = stringval.split("\n")
    # Take the last line
    return stringarry[len(stringarry)-1]

# Method to update a document in the meridio repository
def update_document(docurl, recurl, domainuser, password, docid, folder, filename):
    """Create a new version of an existing document"""
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(docid),
            ConnectorHelpers.process_argument(folder),
            ConnectorHelpers.process_argument(filename),
            ConnectorHelpers.process_argument("Updated for testing") ]
    ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.UpdateDoc", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )

# Method to add a read acl to a document
def add_acl_to_document(docurl, recurl, domainuser, password, docid, read_username):
    """ Add read permission for the given user to the document """
    listparams = [ ConnectorHelpers.process_argument(docurl),
            ConnectorHelpers.process_argument(recurl),
            ConnectorHelpers.process_argument(domainuser),
            ConnectorHelpers.process_argument(password),
            ConnectorHelpers.process_argument(docid),
            ConnectorHelpers.process_argument(read_username),
            ConnectorHelpers.process_argument("READ") ]
    ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.meridio.AddDocAcl", 
                                    argument_list=listparams,
                                    additional_switches=["-Djava.security.egd=file:///dev/urandom"],
                                    additional_classpath="meridio-testing-package/metacarta-meridioconnector-test.jar" )


# Method to wait whatever time is needed after changing meridio documents
# for them to be noted as changed.
def wait_for_meridio(docurl, recurl, domainuser, password):
    """Nothing needed"""
    pass

# Protocol
meridioProtocol = "http"
# Server name to talk to
meridioServerName = "w2k3-mer-76-1.qa-ad-76.metacarta.com"
# Server port to talk to
meridioServerPort = ""
# User
meridioUser = "meridioServiceUser"
# Password
meridioPassword = "password"
# Domain
meridioDomain = "QA-AD-76.METACARTA.COM"
# Document management location
documentLocation = "/DMWS/MeridioDMWS.asmx";
# Record management location
recordLocation = "/RMWS/MeridioRMWS.asmx";
# Web client location
webClientLocation = "/meridio/browse/downloadcontent.aspx";
# Document path for this test
docpath = "/Testing/LargeDocs"

# These are the calculated URLs for talking with Meridio
meridioDocURL = None
if len(meridioServerPort) > 0:
    meridioDocURL = "%s://%s:%s%s" % (meridioProtocol,meridioServerName,meridioServerPort,documentLocation)
else:
    meridioDocURL = "%s://%s%s" % (meridioProtocol,meridioServerName,documentLocation)

meridioRecURL = None
if len(meridioServerPort) > 0:
    meridioRecURL = "%s://%s:%s%s" % (meridioProtocol,meridioServerName,meridioServerPort,recordLocation)
else:
    meridioRecURL = "%s://%s%s" % (meridioProtocol,meridioServerName,recordLocation)

meridioDomainUser = "%s\%s" % (meridioDomain,meridioUser)

# This makes a URL of the kind that will be ingested
def make_meridio_url(document_id):
    # Note: TestDocs results have protocol stripped off.
    if meridioServerPort == "":
        return "%s%s?launchMode=1&launchAs=0&documentId=%s" % (meridioServerName,webClientLocation,document_id)
    else:
        return "%s:%s/%s?launchMode=1&launchAs=0&documentId=%s" % (meridioServerName,meridioServerPort,webClientLocation,document_id)

# Crawl user credentials
username = "testingest"
password = "testingest"

def preclean( print_errors=True ):
    ''' Clean up everything we might have done during the execution of this test.
        This will include all jobs and ingested documents. '''

    try:
        ConnectorHelpers.reset_all()
    except Exception, e:
        if print_errors:
            print "Error resetting all jobs"
            print e

    try:
        ConnectorHelpers.delete_crawler_user( username )
    except Exception, e:
        if print_errors:
            print "Error deleting crawl user"
            print e

    try:
        LicenseMakerClient.revoke_license()
    except Exception, e:
        if print_errors:
            print "Error cleaning up old license"
            print e

    try:
        ConnectorHelpers.teardown_connector_environment( )
    except Exception, e:
        if print_errors:
            print "Error cleaning up debs"
            print e


# Main
if __name__ == '__main__':

    print "Precleaning!"
    preclean( print_errors=False )

    print "Setup Connector Environment."
    ConnectorHelpers.setup_connector_environment()

    # Look for the last document
    id = lookup_document(meridioDocURL,meridioRecURL,meridioDomainUser,meridioPassword,docpath,"lf009-9-9-9")
    if id == None or id == "":
        # Presume that we have to set up the server
        print "Initializing test documents."
        # We need at least 10,000 documents.  We'll get this by
        # having 10 documents each with 1000 different names at the same level in the hierarchy.
        # We do it this way because we have no programmatic way of creating folders at this time.
        add_documents_multiple(meridioDocURL,meridioRecURL,meridioDomainUser,meridioPassword,docpath,"/root/largefiles","lf",3,10)
        print "Done creating test documents on server."

    # The documents are already on the Meridio box, under the LargeDocs folder.  We
    # just need exemplars of each one, so we can search for them.

    # We need at least 10,000 documents.  We'll get this by
    # having 10 documents each with 1000 different names at the same level in the hierarchy.
    # We do it this way because we have no programmatic way of creating folders at this time.
    doc_id_array = []
    level0 = 0
    while level0 < 10:
        meridio_title = "lf00%d-%d-%d-%d" % (level0,0,0,0)
        id = lookup_document(meridioDocURL,meridioRecURL,meridioDomainUser,meridioPassword,docpath,meridio_title)
        if id == None:
            raise Exception("Couldn't find %s in repository" % (meridio_title) )
        doc_id_array.append( id )
        level0 += 1

    print "Setting up license."
    sqatools.appliance.install_license(extra_services=["meridioConnector"], detect_gdms=True)


    # Set up the ingestion user.
    ConnectorHelpers.create_crawler_user( username, password )
    ConnectorHelpers.define_gts_outputconnection( )

    # PHASE 1: Ingestion


    print "Load Test."

    # In case there is clock skew, sleep a minute
    wait_for_meridio(meridioDocURL,meridioRecURL,meridioDomainUser,meridioPassword)

    ConnectorHelpers.define_repositoryconnection( "MeridioConnection",
                                 "Meridio Connection",
                                 "com.metacarta.crawler.connectors.meridio.MeridioConnector",
                                 configparams = [ "DMWSServerProtocol="+meridioProtocol,
                                                  "DMWSServerName="+meridioServerName,
                                                  "DMWSLocation=/DMWS/MeridioDMWS.asmx",
                                                  "RMWSServerProtocol="+meridioProtocol,
                                                  "RMWSServerName="+meridioServerName,
                                                  "RMWSLocation=/RMWS/MeridioRMWS.asmx",
                                                  "MeridioWebClientProtocol="+meridioProtocol,
                                                  "MeridioWebClientServerName="+meridioServerName,
                                                  "MeridioWebClientDocDownloadLocation=/meridio/browse/downloadcontent.aspx",
                                                  "UserName="+meridioDomain+"\\"+meridioUser,
                                                  "Password="+meridioPassword ] )

    # Define job
    doc_spec_xml = '<?xml version="1.0" encoding="UTF-8"?><specification><SearchPath path="/Testing/LargeDocs"/><SearchCategory category="MetacartaSearchable"/><MIMEType type="text/html"/><security value="off"/></specification>'
    job_id = ConnectorHelpers.define_job( "Meridio test job",
                             "MeridioConnection",
                             doc_spec_xml )
    # Run the job to completion
    ConnectorHelpers.start_job( job_id )
    # Wait until it's really working, then cycle the service, to be sure it is shutting down cleanly
    time.sleep(120)
    ConnectorHelpers.restart_agents( )

    ConnectorHelpers.wait_job_complete( job_id )

    # Wait until ingest has caught up
    ConnectorHelpers.wait_for_ingest( )

    # See if we can find the documents we just ingested
    ConnectorHelpers.search_exists_check( [ "divestment url:http://" + make_meridio_url(doc_id_array[0]) ], None, ConnectorHelpers.regexp_encode(make_meridio_url(doc_id_array[0])) )
    ConnectorHelpers.search_exists_check( [ "visitors url:http://" + make_meridio_url(doc_id_array[2]) ], None, ConnectorHelpers.regexp_encode(make_meridio_url(doc_id_array[2])) )

    # Success: done
    print "Done load test."

    # PHASE 5: Delete Job

    print "Job Delete Test."
    ConnectorHelpers.delete_job( job_id )
    print "...job delete request sent"
    ConnectorHelpers.wait_job_deleted( job_id )
    print "...job has vanished"

    # Make sure the documents all went away
    ConnectorHelpers.search_nonexists_check( [ "divestment url:http://" + make_meridio_url(doc_id_array[0]) ], None, ConnectorHelpers.regexp_encode(make_meridio_url(doc_id_array[0])) )
    ConnectorHelpers.search_nonexists_check( [ "visitors url:http://" + make_meridio_url(doc_id_array[2]) ], None, ConnectorHelpers.regexp_encode(make_meridio_url(doc_id_array[2])) )

    print "Done Job Delete Test."

    ConnectorHelpers.delete_repositoryconnection( "MeridioConnection" )

    ConnectorHelpers.delete_gts_outputconnection( )
    ConnectorHelpers.delete_crawler_user( username )

    LicenseMakerClient.revoke_license

    ConnectorHelpers.teardown_connector_environment( )

    print "MeridioConnector load tests PASSED"