#!/usr/bin/python

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import time
import ConnectorHelpers
import sqatools
from wintools import sqa_domain_info
from sqatools import LicenseMakerClient
import VirtualBrowser

# Set up SSL mode
def setup_ssl_mode( file_name ):
    ConnectorHelpers.invoke_root_script( [ "/usr/bin/memexconnector_control", "installcert", file_name ] )

# Tear down SSL mode
def teardown_ssl_mode( ):
    ConnectorHelpers.invoke_root_script( [ "/usr/bin/memexconnector_control", "removecert" ] )

# Copy a folder to a (new) area
def copy_folder( source, target ):
    ConnectorHelpers.invoke_root_script( [ "mkdir", "-p", target ] )
    ConnectorHelpers.invoke_root_script( [ "cp", "-r", source, target ] )

# Remove a folder
def delete_folder( target ):
    ConnectorHelpers.invoke_root_script( [ "rm", "-rf", target ] )

# Run a remote memex command
def remote_memex_command( command_string, host_name ):
    """ Run a remote memex command """
    ConnectorHelpers.invoke_root_script(["chmod", "600", "memex_keyfile"])
    ConnectorHelpers.invoke_root_script(["ssh", "-o", "CheckHostIP=no",
                                                "-o", "StrictHostKeyChecking=no",
                                                "-o", "UserKnownHostsFile=/dev/null",
                                                "-o", "PasswordAuthentication=no",
                                                "-i", "memex_keyfile",
                                                "-F", "/dev/null",
                                                "mxadmin@%s" % host_name,
                                                command_string] ) 

# Restore memex audit history
def restore_memex_server(host_name):
    """ Restore audit history for memex server. """
    remote_memex_command("set -x; if [ -f audit_backup.tar ] ; then rm -rf /opt/memex/im/CS/mxAudit001/* ; ( cd /opt/memex/im/CS/mxAudit001 ; tar xf ~/audit_backup.tar ) ; rm audit_backup.tar ; fi", host_name)

# Save memex audit history for this test run
def snapshot_memex_server(host_name):
    """ Create a snapshot of the memex server audit history. """
    remote_memex_command("( cd /opt/memex/im/CS/mxAudit001 ; tar cf ~/audit_backup.tar . )", host_name)

# Create a memex repository connection via the UI
def define_memex_authority_connection_ui( username,
                                        password,
                                        connection_name,
                                        connection_description,
                                        server_name,
                                        server_port,
                                        mmxuser_name,
                                        mmxuser_password,
                                        usernameregexp=None,
                                        memexuserexpr=None,
                                        max_connections=2 ):
    # Set up virtual browser instance
    vb = VirtualBrowser.VirtualBrowser( username=username, password=password )

    # First, go to main page
    vb.load_main_window( "http://localhost/crawler/index.jsp" )

    # Find the link for repository connection management and click it
    window = vb.find_window("")
    link = window.find_link("List authorities")
    link.click( )

    # Click "add a connection"
    window = vb.find_window("")
    link = window.find_link("Add new connection")
    link.click( )

    # Find the right form elements and set them
    window = vb.find_window("")
    form = window.find_form("editconnection")

    # "Name" tab
    namefield = form.find_textarea("connname")
    descriptionfield = form.find_textarea("description")
    namefield.set_value( connection_name )
    descriptionfield.set_value( connection_description )

    # "Type" tab
    link = window.find_link("Type tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editconnection")
    connectortypefield = form.find_selectbox("classname")
    connectortypefield.select_value( "com.metacarta.crawler.connectors.memex.MemexAuthority" )
    # Click the "Continue" button
    continue_button = window.find_button("Continue to next page")
    continue_button.click( )

    # "Throttling" tab
    window = vb.find_window("")
    link = window.find_link("Throttling tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editconnection")
    if max_connections != None:
        form.find_textarea("maxconnections").set_value( str(max_connections) )

    # "Memex Server" tab
    window = vb.find_window("")
    link = window.find_link("Memex Server tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editconnection")
    # Set memex-specific stuff
    form.find_textarea("memexservername").set_value( server_name )
    form.find_textarea("memexserverport").set_value( server_port )
    form.find_textarea("crawluser").set_value( mmxuser_name )
    form.find_textarea("crawluserpassword").set_value( mmxuser_password )

    # "User Mapping" tab
    window = vb.find_window("")
    link = window.find_link("User Mapping tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editconnection")
    if usernameregexp != None:
        form.find_textarea("usernameregexp").set_value(usernameregexp)
    if memexuserexpr != None:
        form.find_textarea("memexuserexpr").set_value(memexuserexpr)

    # Now, save this page
    save_button = window.find_button("Save this authority connection")
    save_button.click( )

    # See if the connector saved
    window = vb.find_window("")
    found_connection_name = window.find_match("<!--connection=(.*)-->", 1)
    if found_connection_name != connection_name:
        raise Exception("Created connection doesn't match")

# Create a memex repository connection via the UI
def define_memex_repository_connection_ui( username,
                                        password,
                                        connection_name,
                                        connection_description,
                                        server_name,
                                        server_port,
                                        mmxuser_name,
                                        mmxuser_password,
                                        authority_name=None,
                                        view_protocol=None,
                                        view_server=None,
                                        view_port=None,
                                        max_connections=2 ):
    # Set up virtual browser instance
    vb = VirtualBrowser.VirtualBrowser( username=username, password=password )

    # First, go to main page
    vb.load_main_window( "http://localhost/crawler/index.jsp" )

    # Find the link for repository connection management and click it
    window = vb.find_window("")
    link = window.find_link("List repository connections")
    link.click( )

    # Click "add a connection"
    window = vb.find_window("")
    link = window.find_link("Add a connection")
    link.click( )

    # Find the right form elements and set them
    window = vb.find_window("")
    form = window.find_form("editconnection")

    # "Name" tab
    namefield = form.find_textarea("connname")
    descriptionfield = form.find_textarea("description")
    namefield.set_value( connection_name )
    descriptionfield.set_value( connection_description )

    # "Type" tab
    link = window.find_link("Type tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editconnection")
    connectortypefield = form.find_selectbox("classname")
    connectortypefield.select_value( "com.metacarta.crawler.connectors.memex.MemexConnector" )
    if authority_name != None:
        form.find_selectbox("authorityname").select_value( authority_name )
    # Click the "Continue" button
    continue_button = window.find_button("Continue to next page")
    continue_button.click( )

    # "Throttling" tab
    window = vb.find_window("")
    link = window.find_link("Throttling tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editconnection")
    if max_connections != None:
        form.find_textarea("maxconnections").set_value( str(max_connections) )

    # "Memex Server" tab
    window = vb.find_window("")
    link = window.find_link("Memex Server tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editconnection")
    # Set memex-specific stuff
    form.find_textarea("memexservername").set_value( server_name )
    form.find_textarea("memexserverport").set_value( server_port )
    form.find_textarea("crawluser").set_value( mmxuser_name )
    form.find_textarea("crawluserpassword").set_value( mmxuser_password )

    # "Web Server" tab
    link = window.find_link("Web Server tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editconnection")
    if view_protocol != None:
        form.find_selectbox("webserverprotocol").select_value( view_protocol )
    if view_port != None:
        form.find_textarea("webserverport").set_value( str(view_port) )
    if view_server != None:
        form.find_textarea("webservername").set_value( view_server )

    # Now, save this page
    save_button = window.find_button("Save this connection")
    save_button.click( )

    # See if the connector saved
    window = vb.find_window("")
    found_connection_name = window.find_match("<!--connection=(.*)-->", 1)
    if found_connection_name != connection_name:
        raise Exception("Created connection doesn't match")

# Define a standard livelink job using the UI
def define_memex_job_ui( username,
                password,
                job_name,
                connection_name,
                rules,
                entities,
                security_enabled=False,
                collection_name=None,
                type="specified",
                startmethod="windowbegin",
                recrawlinterval=0 ):
    """connection_name is the name of the memex connection.
       rules is an array of tuples of (virtual_server_name, entity_prefix:entity_description, field_name, operation, field_value), where
         all values may be None.
       entities is an array of tuples of (entity_name, primary_field_list, metadata_field_list), where
         primary_field_list is an array of field names, and metadata_field_list also an array
         of field names.
       security_enabled is True if security should be enabled.
       Legal values for type are: "specified" or "continuous"
       Legal values for start method are: "windowbegin", "windowinside", or "disable".
    """
    # Set up virtual browser instance
    vb = VirtualBrowser.VirtualBrowser( username=username, password=password )

    # First, go to main page
    vb.load_main_window( "http://localhost/crawler/index.jsp" )

    # Find the link for job management and click it
    window = vb.find_window("")
    link = window.find_link("List jobs")
    link.click( )

    # Grab the new window
    window = vb.find_window("")
    # Add a job
    link = window.find_link("Add a job")
    link.click( )
    # Grab the edit window
    window = vb.find_window("")
    form = window.find_form("editjob")

    # "Name" tab
    # textarea for setting description
    form.find_textarea("description").set_value( job_name )

    # "Connection" tab
    link = window.find_link("Connection tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editjob")
    # start method
    if startmethod == "windowbegin":
        startmethod_value = 0
    elif startmethod == "windowinside":
        startmethod_value = 1
    elif startmethod == "disable":
        startmethod_value = 2
    else:
        raise Exception("Illegal start method value: '%s'" % startmethod )
    form.find_selectbox("startmethod").select_value( str(startmethod_value) )
    # connection name
    form.find_selectbox("connectionname").select_value( connection_name )
    form.find_selectbox("outputname").select_value( "GTS" )
    # Click the "Continue" button
    window.find_button("Continue to next screen").click( )
    window = vb.find_window("")
    form = window.find_form("editjob")

    # "Collections" tab
    link = window.find_link("Collections tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editjob")
    # textarea for setting collection
    if collection_name != None:
        form.find_textarea("gts_collectionname").set_value( collection_name )

    # "Scheduling" tab
    link = window.find_link("Scheduling tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editjob")
    # type
    if type == "specified":
        type_value = 1
    elif type == "continuous":
        type_value = 0
    else:
        raise Exception("Illegal type value: '%s'" % type )
    form.find_selectbox("scheduletype").select_value( str(type_value) )
    # Recrawl interval
    if type == "continuous":
        form.find_textarea("recrawlinterval").set_value( str(recrawlinterval * 1000 * 60) )

    # "Record Criteria" tab
    link = window.find_link("Record Criteria tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editjob")
    # Loop through all virtual server specs
    for vs_name, entity, field_name, operation, field_value in rules:
        # Build a rule
        if vs_name != None:
            form.find_selectbox("rulevirtualserverselect").select_value( vs_name )
            window.find_button("Set virtual server in rule").click()
            window = vb.find_window("")
            form = window.find_form("editjob")
            if entity != None:
                form.find_selectbox("ruleentityselect").select_value( entity )
                window.find_button("Set entity in rule").click()
                window = vb.find_window("")
                form = window.find_form("editjob")
                if field_name != None:
                    form.find_selectbox("rulefieldnameselect").select_value( field_name )
                    form.find_selectbox("ruleoperationselect").select_value( operation )
                    form.find_textarea("rulefieldvalueselect").set_value( str(field_value) )
                    window.find_button("Set criteria in rule").click()
        window.find_button("Add rule").click()
        window = vb.find_window("")
        form = window.find_form("editjob")

    # "Entities" tab
    link = window.find_link("Entities tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editjob")
    # Set up includes/excludes
    for entity_name, primary_fields, metadata_fields in entities:
        # We have to locate the correct entity index in the UI table.  The best way to do this is using javascript.
        total_entity_count = int(form.execute_javascript_expression("javascript:editjob.entitytypecount.value").num_value())
        entity_index = None
        for this_index in range(total_entity_count):
            this_entity_type = form.execute_javascript_expression("javascript:editjob.entitytype_%d.value" % this_index).str_value()
            if this_entity_type == entity_name:
                entity_index = this_index
                break
                                
        if entity_index == None:
            raise Exception("Could not find entity %s in displayed entity list!")
                
        # Located the right form elements!
        # Go through the primary fields, selecting each such field in order, and moving it to the primary field box
        for primary_field in primary_fields:
            form.find_selectbox("availablefields_%d" % entity_index).multi_select_value(primary_field)
            # Click the "add to primary fields" button
            window.find_button("Add %d to tagged fields" % entity_index).click()
                
        # Go through the metadata fields, selecting all specified fields, and then we'll click the button at the end
        for metadata_field in metadata_fields:
            form.find_selectbox("availablefields_%d" % entity_index).multi_select_value(metadata_field)
                
        window.find_button("Move %d to metadata fields" % entity_index).click()
        
    # "Security" tab
    link = window.find_link("Security tab")
    link.click()
    window = vb.find_window("")
    form = window.find_form("editjob")
    # Set up security
    if security_enabled:
        form.find_radiobutton("specsecurity","on").select()
    else:
        form.find_radiobutton("specsecurity","off").select()

    # Finally, submit the form
    window.find_button("Save this job").click( )
    window = vb.find_window("")
    jobid = window.find_match("<!--jobid=(.*)-->", 1)
    return jobid

# Method to load the contents of a text file into a string in memory
def load_file( file_name ):
    f = open(file_name, "r")
    try:
        return f.read()
    finally:
        f.close()

# Method to add a record to the memex repository, and return a record number
def add_record(servername, port, user, password, virtual_server, entity_name, field_dict):
    """Add a record to the repository"""
    fields = []
    for field in field_dict.keys():
        value = field_dict[field]
        fields = fields + [ field, value ]
    return ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.memex.AddRecord", 
                                    argument_list=[servername,port,user,password,virtual_server,entity_name] + fields,
                                    additional_classpath="memex-testing-package/metacarta-memexconnector-test.jar")

# Method to locate a document based on field data
def locate_record(servername, port, user, password, virtual_server, entity_name, field_name, field_value):
    """ Locate a record in the repository """
    rval = ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.memex.LookupRecord", 
                                    argument_list=[servername,port,user,password,virtual_server,entity_name,field_name,field_value],
                                    additional_classpath="memex-testing-package/metacarta-memexconnector-test.jar" )
    if rval == "":
        rval = None
    return rval
        
# Method to remove a record from the memex repository
def remove_record(servername, port, user, password, identifier):
    """Remove a record from the repository"""
    ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.memex.RemoveRecord", 
                                    argument_list=[servername,port,user,password,identifier],
                                    additional_classpath="memex-testing-package/metacarta-memexconnector-test.jar" )

# Method to update a record in the memex repository
def version_record(servername, port, user, password, identifier, field_dict):
    """Create a new version of an existing record"""
    fields = []
    for field in field_dict.keys():
        value = field_dict[field]
        fields = fields + [ field, value ]
    ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.memex.ModifyRecord", 
                                    argument_list=[servername,port,user,password,identifier] + fields,
                                    additional_classpath="memex-testing-package/metacarta-memexconnector-test.jar" )

def add_covert_record_rights(servername, port, user, password, identifier, group_or_user_list):
    """Adds the right to see & view contents of the specified record, for the given username"""
    ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.memex.SetRecordSecurity", 
                                    argument_list=[servername,port,user,password,identifier,str(len(group_or_user_list)),"0",""] + group_or_user_list,
                                    additional_classpath="memex-testing-package/metacarta-memexconnector-test.jar" )

def add_protected_record_rights(servername, port, user, password, identifier, group_or_user_list):
    """Adds the right to see & view contents of the specified record, for the given username"""
    ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.memex.SetRecordSecurity", 
                                    argument_list=[servername,port,user,password,identifier,"0",str(len(group_or_user_list)),"Protected!"] + group_or_user_list,
                                    additional_classpath="memex-testing-package/metacarta-memexconnector-test.jar" )

# Method to wait whatever time is needed after changing memex documents
# for them to be noted as changed.
def wait_for_memex(servername, port, user, password):
    """Nothing needed"""
    pass

# Server name to talk to
memexServerName = None
# Server port to talk to
memexServerPort = "9001"
# User
memexUser = "system"
# Password
memexPassword = "test"
# Memex virtual server
memexVirtualServer = "Professional Standards"
# Memex entity name
memexEntityName = "PS"
memexEntityDisplayName = "PSDNom"
# Memex key field name
memexRecordKeyField = "surname"
# Memex data field name
memexDataField = "text"

def build_memex_url( id, port=None, server=None ):
    """Build the url from pieces"""
    server_part = server
    if server_part == None:
        server_part = memexServerName
    if port != None:
        server_part = "%s:%s" % (server_part, port)
    return "%s/search.jsp?urn=%s" % (server_part, id)

# Crawl user credentials
username = "testingest"
password = "testingest"

def preclean( ad_domain_info, perform_legacy_pass, print_errors=True ):
    ''' Clean up everything we might have done during the execution of this test.
        This will include all jobs and ingested documents. '''

    
    # First order of business: synchronize legacy mode, so our state matches the duck state
    ConnectorHelpers.synchronize_legacy_mode( )

    try:
        ConnectorHelpers.reset_all()
    except Exception, e:
        if print_errors:
            print "Error resetting all jobs"
            print e

    # Remove test documents first
    for folder in [ "/root/crawlarea" ]:
        try:
            delete_folder( folder )
        except Exception, e:
            if print_errors:
                print "Error removing %s" % folder
                print e

    # Due to license issues, we have to shut down tomcat and metacarta-agents while we clean up these docs
    ConnectorHelpers.shutdown_tomcat( )
    ConnectorHelpers.shutdown_agents( )
    
    # Remove test documents first
    for docname in [ "TestDocs/f001", "TestDocs/f002", "TestDocs/f003", "TestDocs/f004", "TestDocs/f005",
                     "TestDocs/f006", "TestDocs/f007", "TestDocs/f007", "TestDocs/newfolder/f008",
                     "TestDocs/f009", "TestDocs/f010",
                     "TestDocs/userafolder/f003", "TestDocs/userafolder/f004",
                     "TestDocs/userbfolder/f005", "TestDocs/userbfolder/f006", "TestDocs/usercfolder/f007" ]:
        try:
            while True:
                id = locate_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName,
                        memexRecordKeyField, docname)
                if id != None:
                    remove_record(memexServerName, memexServerPort, memexUser, memexPassword, id )
                else:
                    break
        except Exception, e:
            if print_errors:
                print "Error deleting test record %s" % docname
                print e

    ConnectorHelpers.start_agents( )
    ConnectorHelpers.start_tomcat( )
    
    # Fix up memex server
    restore_memex_server(memexServerName)

    # Turn off memex client SSL
    try:
        teardown_ssl_mode( )
    except Exception, e:
            if print_errors:
                print "Error tearing down SSL mode"
                print e
        
    # Disable ad
    try:
        ConnectorHelpers.turn_off_ad( ad_domain_info )
    except Exception, e:
        if print_errors:
            print "Error disabling AD"
            print e

    try:
        ConnectorHelpers.delete_crawler_user( username )
    except Exception, e:
        if print_errors:
            print "Error deleting crawl user"
            print e

    try:
        LicenseMakerClient.revoke_license()
    except Exception, e:
        if print_errors:
            print "Error cleaning up old license"
            print e

    try:
        ConnectorHelpers.teardown_connector_environment( )
    except Exception, e:
        if print_errors:
            print "Error cleaning up debs"
            print e

    if perform_legacy_pass:
        try:
            ConnectorHelpers.select_legacy_mode(use_legacy_tools=False)
        except Exception, e:
            if print_errors:
                print "Error turning off legacy AD mode"
                print e

def run_ad_test_part( ad_domain_info ):
    """ Perform the ad part of the test.  This may be called twice, if legacy mode testing is requested """

    ad_win_host = ad_domain_info.ie_client_fqdn

    # PHASE 7: Security test with AD and ACLs
    # This particular test involves putting documents into Livelink and setting up specific user rights.
    # In addition to creating a job, an authority connection must also be created.
    # Then, the search is done by specific AD users, which should return documents that have only the
    # right permissions.

    # Now, set up ad with the following users:
    # p_memexusera
    # p_memexuserb
    # p_memexuserc
    # These correspond to users already present in the livelink instance, except for the "p_livelink" prefix.

    # Set up system to use ad
    ConnectorHelpers.configure_ad( ad_domain_info )

    # Enable CF security again
    ConnectorHelpers.enable_connector_framework( )
    
    ConnectorHelpers.conditionally_add_ad_user( ad_domain_info, "p_memexusera", "usera" )
    ConnectorHelpers.conditionally_add_ad_user( ad_domain_info, "p_memexuserb", "userb" )
    ConnectorHelpers.conditionally_add_ad_user( ad_domain_info, "p_memexuserc", "userc" )

    # Add the current set of docs to the repository, and twiddle their security
    id1 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f001", memexDataField:load_file("/root/crawlarea/testfiles/f001.txt")})
    id2 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f002", memexDataField:load_file("/root/crawlarea/testfiles/f002.txt")})
    # TestDocs.userafolder will be visible by llusera only
    id3 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/userafolder/f003", memexDataField:load_file("/root/crawlarea/testfiles/f003.txt")})
    add_covert_record_rights( memexServerName, memexServerPort, memexUser, memexPassword, id3, [memexUser, "usera"] )
    id4 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/userafolder/f004", memexDataField:load_file("/root/crawlarea/testfiles/f004.txt")})
    add_protected_record_rights( memexServerName, memexServerPort, memexUser, memexPassword, id4, [memexUser, "usera"] )
    # TestDocs.userbfolder will be visible by userb only
    id5 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/userbfolder/f005", memexDataField:load_file("/root/crawlarea/testfiles/f005.txt")})
    add_covert_record_rights( memexServerName, memexServerPort, memexUser, memexPassword, id5, [memexUser, "userb"] )
    id6 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/userbfolder/f006", memexDataField:load_file("/root/crawlarea/testfiles/f006.txt")})
    add_protected_record_rights( memexServerName, memexServerPort, memexUser, memexPassword, id6, [memexUser, "userb"] )
    # TestDocs.usercfolder will be visible by userc only
    id7 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/usercfolder/f007", memexDataField:load_file("/root/crawlarea/testfiles/f007.txt")})
    add_covert_record_rights( memexServerName, memexServerPort, memexUser, memexPassword, id7, [memexUser, "userc"] )
    # TestDocs.newfolder will be visible by both llusera and userb, but not userc
    id8 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/newfolder/f008", memexDataField:load_file("/root/crawlarea/testfiles/newfolder/f008.txt")})
    add_covert_record_rights( memexServerName, memexServerPort, memexUser, memexPassword, id8, [ memexUser, "usera", "userb" ] )

    # Define authority connection
    define_memex_authority_connection_ui( username,
                                        password,
                                        "MemexAuthority",
                                        "Memex Authority",
                                        memexServerName,
                                        memexServerPort,
                                        memexUser,
                                        memexPassword,
                                        usernameregexp="^p_memex([^\\@]*).*$",
                                        memexuserexpr="$(1)")

    # Define repository connection
    define_memex_repository_connection_ui( username,
                                        password,
                                        "MemexConnection",
                                        "Memex Connection",
                                        memexServerName,
                                        memexServerPort,
                                        memexUser,
                                        memexPassword,
                                        authority_name="MemexAuthority" )

    # PHASE 6: Ingest documents with the specified connectors.
    # There will be several separate kinds of security as well.

    # Define job
    job_id = define_memex_job_ui( username,
                password,
                "Memex Job",
                "MemexConnection",
                [ (memexVirtualServer,"%s:%s" % (memexEntityName, memexEntityDisplayName),None,None,None) ],
                [ (memexEntityName,[memexDataField],[memexRecordKeyField]) ],
                security_enabled=True )

    # Run the job to completion
    ConnectorHelpers.start_job( job_id )
    ConnectorHelpers.wait_job_complete( job_id )

    # Wait until ingest has caught up
    ConnectorHelpers.wait_for_ingest( )

    # Check for proper visibility of all documents
    # usera
    ConnectorHelpers.search_check( [ "reference" ], None, [ build_memex_url( id1 ) ], username="p_memexusera", password="usera", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "kidneys" ], None, [ build_memex_url( id3 ) ], username="p_memexusera", password="usera", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "city" ], None, [ ], username="p_memexusera", password="usera", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "interesting" ], None, [ ], username="p_memexusera", password="usera", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "smelly" ], None, [ ], username="p_memexusera", password="usera", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "restaurants" ], None, [ build_memex_url( id8 ) ], username="p_memexusera", password="usera", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "good" ], None, [ build_memex_url( id2 ) ], username="p_memexusera", password="usera", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "pub" ], None, [ build_memex_url( id4 ) ], username="p_memexusera", password="usera", win_host=ad_win_host )

    # userb
    ConnectorHelpers.search_check( [ "reference" ], None, [ build_memex_url( id1 ) ], username="p_memexuserb", password="userb", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "kidneys" ], None, [ ], username="p_memexuserb", password="userb", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "city" ], None, [ build_memex_url( id5 ) ], username="p_memexuserb", password="userb", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "interesting" ], None, [ build_memex_url( id6 ) ], username="p_memexuserb", password="userb", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "smelly" ], None, [ ], username="p_memexuserb", password="userb", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "restaurants" ], None, [ build_memex_url( id8 ) ], username="p_memexuserb", password="userb", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "good" ], None, [ build_memex_url( id2 ) ], username="p_memexuserb", password="userb", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "pub" ], None, [ ], username="p_memexuserb", password="userb", win_host=ad_win_host )

    # userc
    ConnectorHelpers.search_check( [ "reference" ], None, [ build_memex_url( id1 ) ], username="p_memexuserc", password="userc", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "kidneys" ], None, [ ], username="p_memexuserc", password="userc", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "city" ], None, [ ], username="p_memexuserc", password="userc", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "interesting" ], None, [ ], username="p_memexuserc", password="userc", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "smelly" ], None, [ build_memex_url( id7 ) ], username="p_memexuserc", password="userc", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "restaurants" ], None, [ ], username="p_memexuserc", password="userc", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "good" ], None, [ build_memex_url( id2 ) ], username="p_memexuserc", password="userc", win_host=ad_win_host )
    ConnectorHelpers.search_check( [ "pub" ], None, [ ], username="p_memexuserc", password="userc", win_host=ad_win_host )


    ConnectorHelpers.delete_job( job_id )
    print "...job delete request sent"
    ConnectorHelpers.wait_job_deleted( job_id )
    print "...job has vanished"

    ConnectorHelpers.delete_repositoryconnection( "MemexConnection" )

    ConnectorHelpers.delete_authorityconnection( "MemexAuthority" )

    # Clean up the documents.
    # But before we do, better restart tomcat so we free up a license or two
    ConnectorHelpers.restart_tomcat()
    
    for docname in [ id1, id2, id3, id4, id5, id6, id7, id8 ]:
        remove_record(memexServerName, memexServerPort, memexUser, memexPassword, docname)

    ConnectorHelpers.turn_off_ad( ad_domain_info )


# Main
if __name__ == '__main__':

    # AD parameters
    ad_group = "76"
    if len(sys.argv) > 1:
        ad_group = sys.argv[1]
    perform_legacy_pass = False
    if len(sys.argv) > 2 and sys.argv[2] == "legacy":
        perform_legacy_pass = True

    ad_domain_info = sqa_domain_info.SQADomainInfo( ad_group )

    memexServerName = getattr( ad_domain_info, "memex_server_fqdn" )

    print "Precleaning!"

    preclean( ad_domain_info, perform_legacy_pass, print_errors=False )

    print "Restarting services (to assure consistency with handle timeouts)"

    ConnectorHelpers.restart_agents()
    ConnectorHelpers.restart_tomcat()

    print "Setting up ssl"
    setup_ssl_mode( "memex-client-cert/cacert.pem" )

    print "Making snapshot of memex audit log."
    snapshot_memex_server(memexServerName)
    
    print "Setup Connector Environment."
    ConnectorHelpers.setup_connector_environment()

    print "Setting up file area."
    copy_folder("/root/testfiles","/root/crawlarea")

    # Enable CF security
    ConnectorHelpers.enable_connector_framework( )


    print "Initializing test documents."
    # These documents are not yet part of the set that gets moved over, but they may be in the future
    o = open( "/root/crawlarea/testfiles/f002a.txt", "w" )
    o.write("Now this document is at 50N 75E, and the keyword is platypus")
    o.close()
    o = open( "/root/crawlarea/testfiles/f004a.txt", "w" )
    o.write("No longer about drinking establishments at 23N 15W")
    o.close()
    o = open( "/root/crawlarea/testfiles/f009.txt", "w" )
    o.write("Now this document is at 50N 75E, and the keyword is albemarle")
    o.close()
    o = open( "/root/crawlarea/testfiles/f010.txt", "w" )
    o.write("No longer about golfcarts at 23N 15W")
    o.close()

    print "Setting up license."
    sqatools.appliance.install_license(extra_services=["memexConnector"], detect_gdms=True)


    ConnectorHelpers.create_crawler_user( username, password )
    ConnectorHelpers.define_gts_outputconnection( )
    
    # PHASE 1: Ingestion

    print "Ingestion Test."

    # MHL
    
    # Add some docs to the repository
    id1 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f001", memexDataField:load_file("/root/crawlarea/testfiles/f001.txt")})
    id2 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f002", memexDataField:load_file("/root/crawlarea/testfiles/f002.txt")})
    id3 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f003", memexDataField:load_file("/root/crawlarea/testfiles/f003.txt")})
    id4 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f004", memexDataField:load_file("/root/crawlarea/testfiles/f004.txt")})
    id5 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f005", memexDataField:load_file("/root/crawlarea/testfiles/f005.txt")})
    id6 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f006", memexDataField:load_file("/root/crawlarea/testfiles/f006.txt")})
    id7 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f007", memexDataField:load_file("/root/crawlarea/testfiles/f007.txt")})
    id8 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/newfolder/f008", memexDataField:load_file("/root/crawlarea/testfiles/newfolder/f008.txt")})

    # In case there is clock skew, sleep a minute
    wait_for_memex(memexServerName, memexServerPort, memexUser, memexPassword)

    # Define repository connection.
    define_memex_repository_connection_ui( username,
                                        password,
                                        "MemexConnection",
                                        "Memex Connection",
                                        memexServerName,
                                        memexServerPort,
                                        memexUser,
                                        memexPassword )

    # Make sure connection is happy
    ConnectorHelpers.view_repository_connection_ui( username, password, "MemexConnection" )

    # Define job
    job_id = define_memex_job_ui( username,
                        password,
                        "Memex test job",
                        "MemexConnection",
                        [ (memexVirtualServer,"%s:%s" % (memexEntityName, memexEntityDisplayName),None,None,None) ],
                        [ (memexEntityName,[memexDataField],[memexRecordKeyField]) ] )

    # Run the job to completion
    ConnectorHelpers.start_job( job_id )
    ConnectorHelpers.wait_job_complete( job_id )

    # Wait until ingest has caught up
    ConnectorHelpers.wait_for_ingest( )

    # See if we can find the documents we just ingested
    ConnectorHelpers.search_check( [ "reference" ], None, [ build_memex_url( id1 ) ] )
    ConnectorHelpers.search_check( [ "good" ], None, [ build_memex_url( id2 ) ] )
    ConnectorHelpers.search_check( [ "kidneys" ], None, [ build_memex_url( id3 ) ] )
    ConnectorHelpers.search_check( [ "pub" ], None, [ build_memex_url( id4 ) ] )
    ConnectorHelpers.search_check( [ "city" ], None, [ build_memex_url( id5 ) ] )
    ConnectorHelpers.search_check( [ "interesting" ], None, [ build_memex_url( id6 ) ] )
    ConnectorHelpers.search_check( [ "smelly" ], None, [ build_memex_url( id7 ) ] )
    ConnectorHelpers.search_check( [ "restaurants" ], None, [ build_memex_url( id8 ) ] )

    # Check if metadata works
    ConnectorHelpers.search_check( [ "reference", "metadata:%s=TestDocs/f001" % memexRecordKeyField ], None, [ build_memex_url( id1 ) ] )

    # Success: done
    print "Done ingestion test."


    # PHASE 2: Document Change Detection

    print "Document Change Test."
    # Modify the documents
    version_record(memexServerName, memexServerPort, memexUser, memexPassword, id2, {memexDataField:load_file("/root/crawlarea/testfiles/f002a.txt")})
    version_record(memexServerName, memexServerPort, memexUser, memexPassword, id4, {memexDataField:load_file("/root/crawlarea/testfiles/f004a.txt")})
    # Sleep, in case there's clock skew
    wait_for_memex(memexServerName, memexServerPort, memexUser, memexPassword)

    # Restart job, which should pick up the changes
    ConnectorHelpers.start_job( job_id )
    ConnectorHelpers.wait_job_complete( job_id )

    # Wait until ingest has caught up
    ConnectorHelpers.wait_for_ingest( )

    # Look for state of index being right
    ConnectorHelpers.search_check( [ "reference" ], None, [ build_memex_url( id1 ) ] )
    ConnectorHelpers.search_check( [ "good" ], None, [ ] )
    ConnectorHelpers.search_check( [ "kidneys" ], None, [ build_memex_url( id3 ) ] )
    ConnectorHelpers.search_check( [ "pub" ], None, [ ] )
    ConnectorHelpers.search_check( [ "city" ], None, [ build_memex_url( id5 ) ] )
    ConnectorHelpers.search_check( [ "interesting" ], None, [ build_memex_url( id6 ) ] )
    ConnectorHelpers.search_check( [ "smelly" ], None, [ build_memex_url( id7 ) ] )
    ConnectorHelpers.search_check( [ "restaurants" ], None, [ build_memex_url( id8 ) ] )
    ConnectorHelpers.search_check( [ "platypus" ], None, [ build_memex_url( id2 ) ] )
    ConnectorHelpers.search_check( [ "establishments" ], None, [ build_memex_url( id4 ) ] )

    print "Done Document Change Test."

    # PHASE 3: Document Delete Detection

    print "Document Delete Test."
    remove_record(memexServerName, memexServerPort, memexUser, memexPassword, id3 )
    remove_record(memexServerName, memexServerPort, memexUser, memexPassword, id5 )
    # Sleep, in case of clock skew
    wait_for_memex(memexServerName, memexServerPort, memexUser, memexPassword)

    # Restart job, which should pick up the changes
    ConnectorHelpers.start_job( job_id )
    ConnectorHelpers.wait_job_complete( job_id )
    ConnectorHelpers.search_check( [ "kidneys" ], None, [ ] )
    ConnectorHelpers.search_check( [ "city" ], None, [ ] )
    print "Done Document Delete Test."

    # PHASE 4: Document Addition Detection

    print "Document Add Test."
    id9 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f009", memexDataField:load_file("/root/crawlarea/testfiles/f009.txt")})
    id10 = add_record(memexServerName, memexServerPort, memexUser, memexPassword, memexVirtualServer, memexEntityName, {memexRecordKeyField:"TestDocs/f010", memexDataField:load_file("/root/crawlarea/testfiles/f010.txt")})
    wait_for_memex(memexServerName, memexServerPort, memexUser, memexPassword)

    # Restart job, which should pick up the changes
    ConnectorHelpers.start_job( job_id )
    ConnectorHelpers.wait_job_complete( job_id )

    # Wait until ingest has caught up
    ConnectorHelpers.wait_for_ingest( )

    ConnectorHelpers.search_check( [ "albemarle" ], None, [ build_memex_url( id9 ) ] )
    ConnectorHelpers.search_check( [ "golfcarts" ], None, [ build_memex_url( id10 ) ] )
    print "Done Document Add Test."

    # PHASE 5: Delete Job

    print "Job Delete Test."
    ConnectorHelpers.delete_job( job_id )
    print "...job delete request sent"
    ConnectorHelpers.wait_job_deleted( job_id )
    print "...job has vanished"
    # Make sure the documents all went away
    ConnectorHelpers.search_check( [ "reference" ], None, [] )
    ConnectorHelpers.search_check( [ "good" ], None, [] )
    ConnectorHelpers.search_check( [ "kidneys" ], None, [] )
    ConnectorHelpers.search_check( [ "pub" ], None, [] )
    ConnectorHelpers.search_check( [ "city" ], None, [] )
    ConnectorHelpers.search_check( [ "interesting" ], None, [] )
    ConnectorHelpers.search_check( [ "smelly" ], None, [] )
    ConnectorHelpers.search_check( [ "restaurants" ], None, [] )
    ConnectorHelpers.search_check( [ "albemarle" ], None, [] )
    ConnectorHelpers.search_check( [ "golfcarts" ], None, [] )
    print "Done Job Delete Test."

    print "Waiting 16 minutes and running a new job"
    
    # This test verifies that the connections can be re-established after being idle for a while.
    time.sleep(16*60)
    # Define job
    job_id = define_memex_job_ui( username,
                        password,
                        "Memex test job",
                        "MemexConnection",
                        [ (memexVirtualServer,"%s:%s" % (memexEntityName, memexEntityDisplayName),None,None,None) ],
                        [ (memexEntityName,[memexDataField],[memexRecordKeyField]) ] )
    # Run job.  If the problem exists, it will never finish.
    ConnectorHelpers.start_job( job_id )
    ConnectorHelpers.wait_job_complete( job_id )
    print "Successfully ran a job after allowing connections to become idle!"
    # Success!
    ConnectorHelpers.delete_job( job_id )
    print "...job delete request sent"
    ConnectorHelpers.wait_job_deleted( job_id )

    ConnectorHelpers.delete_repository_connection_ui( username, password, "MemexConnection" )

    # Clean up the documents we dumped into the folders on memex
    for docname in [ id1, id2, id3, id4, id5, id6, id7, id8, id9, id10 ]:
        remove_record(memexServerName, memexServerPort, memexUser, memexPassword, docname )

    # Run the ad part of the test
    run_ad_test_part( ad_domain_info )
    if perform_legacy_pass:
        ConnectorHelpers.select_legacy_mode()
        run_ad_test_part( ad_domain_info )
        ConnectorHelpers.cancel_legacy_mode()

    ConnectorHelpers.delete_gts_outputconnection( )
    ConnectorHelpers.delete_crawler_user( username )

    # Clean up temporary folder
    delete_folder("/root/crawlarea")
    LicenseMakerClient.revoke_license

    teardown_ssl_mode( )
    
    ConnectorHelpers.teardown_connector_environment( )

    restore_memex_server(memexServerName)
    
    print "Basic MemexConnector tests PASSED"