blob: c1979a67a8f4a765512333586d2b59fdd190ca3c [file] [log] [blame]
#!/usr/bin/python
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import time
import ConnectorHelpers
import sqatools
from wintools import sqa_domain_info
from sqatools import LicenseMakerClient
import TestDocs
sys.path.append("/usr/lib/metacarta")
import MetaCartaVersion
# Method to add a document to the livelink repository
def add_document(servername, port, user, password, llpath, llname, filename):
"""Add a document to the repository"""
listparams = [ ConnectorHelpers.process_argument(servername),
ConnectorHelpers.process_argument(port),
ConnectorHelpers.process_argument(user),
ConnectorHelpers.process_argument(password),
ConnectorHelpers.process_argument(llpath),
ConnectorHelpers.process_argument(llname),
ConnectorHelpers.process_argument(filename) ]
return ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.livelink.AddDoc",
argument_list=listparams,
additional_classpath="livelink-testing-package/metacarta-livelinkconnector-test.jar" )
# Method to lookup a document in the livelink repository
def lookup_document(servername, port, user, password, llpath):
"""Add a document to the repository"""
listparams = [ ConnectorHelpers.process_argument(servername),
ConnectorHelpers.process_argument(port),
ConnectorHelpers.process_argument(user),
ConnectorHelpers.process_argument(password),
ConnectorHelpers.process_argument(llpath) ]
return ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.livelink.LookupDoc",
argument_list=listparams,
additional_classpath="livelink-testing-package/metacarta-livelinkconnector-test.jar" )
# Method to remove a document from the livelink repository
def remove_document(servername, port, user, password, llpath):
"""Remove a document from the repository"""
listparams = [ ConnectorHelpers.process_argument(servername),
ConnectorHelpers.process_argument(port),
ConnectorHelpers.process_argument(user),
ConnectorHelpers.process_argument(password),
ConnectorHelpers.process_argument(llpath) ]
ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.livelink.RemoveDoc",
argument_list=listparams,
additional_classpath="livelink-testing-package/metacarta-livelinkconnector-test.jar" )
# Method to update a document in the livelink repository
def version_document(servername, port, user, password, llpath, filename):
"""Create a new version of an existing document"""
listparams = [ ConnectorHelpers.process_argument(servername),
ConnectorHelpers.process_argument(port),
ConnectorHelpers.process_argument(user),
ConnectorHelpers.process_argument(password),
ConnectorHelpers.process_argument(llpath),
ConnectorHelpers.process_argument(filename) ]
ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.livelink.VersionDoc",
argument_list=listparams,
additional_classpath="livelink-testing-package/metacarta-livelinkconnector-test.jar" )
def add_document_right(servername, port, user, password, llpath, username, domain):
"""Adds the right to see & view contents of the specified path, for the given username"""
listparams = [ ConnectorHelpers.process_argument(servername),
ConnectorHelpers.process_argument(port),
ConnectorHelpers.process_argument(user),
ConnectorHelpers.process_argument(password),
ConnectorHelpers.process_argument(llpath),
ConnectorHelpers.process_argument(username),
ConnectorHelpers.process_argument(domain) ]
ConnectorHelpers.invoke_crawler_command( "com.metacarta.crawler.connectors.livelink.AddDocRights",
argument_list=listparams,
additional_classpath="livelink-testing-package/metacarta-livelinkconnector-test.jar" )
# Method to wait whatever time is needed after changing livelink documents
# for them to be noted as changed.
def wait_for_livelink(servername, port, user, password):
"""Nothing needed"""
pass
# Server name to talk to
llServerName = None
# Server port to talk to
llServerPort = "2099"
# User
llUser = "admin"
# Password
llPassword = "livelink"
fullURLSuffix = "&objAction=download"
def build_livelink_url( id ):
"""Build the url from pieces"""
# No protocol base url
veryBaseURL = llServerName + "/livelink/livelink.exe"
# Base URL
baseURL = "http://"+veryBaseURL
# Full URL
fullURL = veryBaseURL + "?func=ll&objID="
return fullURL + id + fullURLSuffix
# Crawl user credentials
username = "testingest"
password = "testingest"
def preclean( print_errors=True ):
''' Clean up everything we might have done during the execution of this test.
This will include all jobs and ingested documents. '''
try:
ConnectorHelpers.start_agents( )
except Exception, e:
if print_errors:
print "Error starting metacarta-agents"
print e
try:
ConnectorHelpers.reset_all()
except Exception, e:
if print_errors:
print "Error resetting all jobs"
print e
try:
ConnectorHelpers.delete_crawler_user( username )
except Exception, e:
if print_errors:
print "Error deleting crawl user"
print e
try:
LicenseMakerClient.revoke_license()
except Exception, e:
if print_errors:
print "Error cleaning up old license"
print e
try:
ConnectorHelpers.teardown_connector_environment( )
except Exception, e:
if print_errors:
print "Error cleaning up debs"
print e
# Main
if __name__ == '__main__':
llServerName = "livelinksrvr.qa-ad-76.metacarta.com"
if len(sys.argv) > 1:
llServerName = sys.argv[1]
print "Precleaning!"
preclean( print_errors=False )
print "Setup Connector Environment."
ConnectorHelpers.setup_connector_environment()
# Look for the last document
id = lookup_document(llServerName, llServerPort, llUser, llPassword, "LargeDocs/lf009-9-9-9")
if id == None or id == "null":
# Presume that we have to set up the server
print "Initializing test documents."
# We need at least 10,000 documents. We'll get this by
# having 10 documents each with 1000 different names at the same level in the hierarchy.
# We do it this way because we have no programmatic way of creating folders at this time.
doc_id_array = []
level0 = 0
while level0 < 10:
level1 = 0
while level1 < 10:
level2 = 0
while level2 < 10:
level3 = 0
while level3 < 10:
filename = "/root/largefiles/00%d.htm" % level0
llname = "lf00%d-%d-%d-%d" % (level0,level1,level2,level3)
id = add_document(llServerName, llServerPort, llUser, llPassword, "LargeDocs", llname, filename)
if level1 == 0 and level2 == 0 and level3 == 0:
doc_id_array.append( id )
level3 += 1
level2 += 1
level1 += 1
level0 += 1
print "Done creating test documents on server."
# The documents are already on the LLserver box, under the LargeDocs folder. We
# just need exemplars of each one, so we can search for them.
# We need at least 10,000 documents. We'll get this by
# having 10 documents each with 1000 different names at the same level in the hierarchy.
# We do it this way because we have no programmatic way of creating folders at this time.
doc_id_array = []
level0 = 0
while level0 < 10:
llname = "lf00%d-%d-%d-%d" % (level0,0,0,0)
id = lookup_document(llServerName, llServerPort, llUser, llPassword, "LargeDocs/" + llname)
if id == None:
raise Exception("Couldn't find %s in repository" % ("LargeDocs/"+llname) )
doc_id_array.append( id )
level0 += 1
print "Setting up license."
sqatools.appliance.install_license(extra_services=["livelinkConnector"], detect_gdms=True)
# Set up the ingestion user.
ConnectorHelpers.create_crawler_user( username, password )
ConnectorHelpers.define_gts_outputconnection( )
# PHASE 1: Ingestion
print "Load Test."
# In case there is clock skew, sleep a minute
wait_for_livelink(llServerName, llServerPort, llUser, llPassword)
ConnectorHelpers.define_repositoryconnection( "LivelinkConnection",
"Livelink Connection",
"com.metacarta.crawler.connectors.livelink.LivelinkConnector",
configparams = [ "CGI path=/livelink/livelink.exe",
"View CGI path=",
"Server name="+llServerName,
"Server port="+llServerPort,
"Server user name="+llUser,
"Server password="+llPassword,
"NTLM domain=" ] )
# Define job
doc_spec_xml = '<?xml version="1.0" encoding="UTF-8"?><specification><startpoint path="LargeDocs"/><include filespec="*.htm"/><security value="off"/></specification>'
job_id = ConnectorHelpers.define_job( "Livelink test job",
"LivelinkConnection",
doc_spec_xml )
# Run the job
ConnectorHelpers.start_job( job_id )
# Livelink abort test! Abort the job and see how long it takes for it to actually stop doing stuff
# First, wait 1 minute for the job to get really rolling
time.sleep(60)
# Now, abort it
ConnectorHelpers.abort_job( job_id )
# Wait to see how long it actually takes to abort the job
the_time = time.time()
ConnectorHelpers.wait_job_complete( job_id )
elapsed_time = time.time() - the_time;
print "It took %f seconds to abort the job" % elapsed_time
if elapsed_time > 120.0:
raise Exception( "Took too long for job to abort: %f seconds" % elapsed_time )
# Now, start it again and run the job to completion this time
ConnectorHelpers.start_job( job_id )
# Wait until it's really working, then cycle the service, to be sure it is shutting down cleanly
time.sleep(120)
ConnectorHelpers.restart_agents( )
# Stop agents service when a job is running, delicense the connector, start it again, make sure it comes up
print "Delicensing livelink connector test."
time.sleep(120)
ConnectorHelpers.shutdown_agents( )
# Remove license
sqatools.appliance.install_license(extra_services=[ ], detect_gdms=True)
# Start metacarta-agents
ConnectorHelpers.start_agents( )
# If the bug is present, metacarta-agents will come up only briefly and then kill itself. This should be obvious from c_s_h.
time.sleep(60)
# If it makes it this far, the job should have aborted due to the bad license.
job_statuses = ConnectorHelpers.list_job_statuses_api( )
if len(job_statuses) != 1:
raise Exception("Expected one job status, instead found %d" % len(job_statuses))
if job_statuses[0]["status"] != "error":
raise Exception("Expected job status to be 'error', instead found '%s'" % job_statuses[0]["status"])
# Restore license
sqatools.appliance.install_license(extra_services=["livelinkConnector"], detect_gdms=True)
# Start job again
ConnectorHelpers.start_job( job_id )
# Now, wait for job to complete
ConnectorHelpers.wait_job_complete( job_id )
# Wait until ingest has caught up
ConnectorHelpers.wait_for_ingest( )
# See if we can find the documents we just ingested
ConnectorHelpers.search_exists_check( [ "divestment url:http://" + build_livelink_url(doc_id_array[0]) ], None, ConnectorHelpers.regexp_encode(build_livelink_url( doc_id_array[0] )) )
ConnectorHelpers.search_exists_check( [ "visitors url:http://" + build_livelink_url(doc_id_array[2]) ], None, ConnectorHelpers.regexp_encode(build_livelink_url( doc_id_array[2] )) )
# Success: done
print "Done load test."
# PHASE 5: Delete Job
print "Job Delete Test."
ConnectorHelpers.delete_job( job_id )
print "...job delete request sent"
ConnectorHelpers.wait_job_deleted( job_id )
print "...job has vanished"
# Make sure the documents all went away
ConnectorHelpers.search_nonexists_check( [ "divestment url:http://" + build_livelink_url(doc_id_array[0]) ], None, ConnectorHelpers.regexp_encode(build_livelink_url( doc_id_array[0] )) )
ConnectorHelpers.search_nonexists_check( [ "visitors url:http://" + build_livelink_url(doc_id_array[2]) ], None, ConnectorHelpers.regexp_encode(build_livelink_url( doc_id_array[2] )) )
print "Done Job Delete Test."
ConnectorHelpers.delete_repositoryconnection( "LivelinkConnection" )
ConnectorHelpers.delete_gts_outputconnection( )
ConnectorHelpers.delete_crawler_user( username )
LicenseMakerClient.revoke_license
ConnectorHelpers.teardown_connector_environment( )
print "LivelinkConnector load tests PASSED"