| #!/usr/bin/python |
| |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """ This test exercises problems unique to multi-domain Active Directory. |
| It tests searches for documents with various combinations of ACLs from |
| multiple domains, with users who have group memberships that cross domains |
| and memberships in groups whose memberships cross domains, and so on. |
| |
| It is not meant to be a substitute for ad_user_search.py, but only |
| to exercise additional features of multi-domain. """ |
| |
| import re |
| import commands |
| import sys |
| import time |
| import math |
| import pprint |
| import traceback |
| |
| sys.path.append("/usr/lib/metacarta") |
| |
| import SearchCanary |
| import TestDocs |
| import ADTestHelpers |
| import SQAhelpers |
| |
| from wintools import adtools_md as adtools |
| from wintools import filetools |
| from wintools import sqa_domain_info |
| from wintools import sharecrawlertools |
| from wintools import ambassador_client |
| |
| debug = False |
| nojoin = False |
| TestDocs.verbose = False |
| |
| # constants for referencing parts of user, group, doc, or acl structures |
| EXPECTED_SEARCH_RESULTS = 2 |
| ACL_ENTRY_TYPE = 0 |
| ACL_ENTRY_USER = 1 |
| CONTENTS = 0 |
| DOC_ID = 1 |
| DOC_ACL = 2 |
| DOC_PATH = 3 |
| |
| # other constants |
| A_GOOD_NUMBER = 300 |
| A_LOT = 100 |
| |
| def makepswd(username): |
| return username+"paswd" |
| |
| def print_flush( *args ): |
| """ print the args and flush them """ |
| print time.ctime(),' '.join(map(str,args)) |
| sys.stdout.flush() |
| |
| def debug_flush( *args ): |
| """ print the args and flush them if debug is on""" |
| if debug: |
| print time.ctime(),' '.join(map(str,args)) |
| sys.stdout.flush() |
| |
| # Search through the ambassador as the given username. Return |
| # the sorted list of document numbers found |
| def get_user_search_results(username, random_string, domain_info): |
| """ return matches, soap_matches, json_matches soap_matches, json_matches will be None |
| if the other api/ui's match the soap_headnode results. |
| Otherwise they will be the mismatched results for reporting at a higher level """ |
| |
| password = makepswd(username) |
| |
| # Search using soap headnode and json, we used to search the classic webui |
| # but now it's going away so it'll just be these two, the matches variable |
| # is left over, but in the interests of simplicity we're just setting it |
| # to the soap results |
| soap_matches = TestDocs.search_documents( |
| method="soapheadnode-ambassador", |
| keywords=[random_string], |
| win_host=domain_info.ie_client_fqdn + ":" + domain_info.ambassador_port, |
| win_user=username, |
| win_password=password, |
| unique=0) |
| |
| matches = soap_matches |
| |
| if TestDocs.JSON_AVAILABLE: |
| json_matches = TestDocs.search_documents( |
| method="jsonsearchapi-ambassador", |
| keywords=[random_string], |
| win_host=domain_info.ie_client_fqdn + ":" + domain_info.ambassador_port, |
| win_user=username, |
| win_password=password, |
| unique=0) |
| else: |
| # So we don't have to special case the compare below. |
| json_matches = soap_matches |
| |
| # Sort them both for the comparision |
| matches.sort() |
| soap_matches.sort() |
| json_matches.sort() |
| |
| # They are expected, and supposed, to be equal. If so then we're |
| # done here. |
| if matches == soap_matches == json_matches: |
| # get rid of the beginning of the path name so we can treat all the |
| # document URLs as integers for easy comparision against the expected |
| # documents in the callers. Since we know the soap_matches are the same |
| # as the ASCII headnode matches we can ignore them. |
| matches = [match.split("/")[-1] for match in matches] |
| |
| # get rid of any non-numerical (i.e. canary) results and make |
| # it a sorted list of integers |
| for match in matches: |
| if not re.search("^[0-9]+$", match): |
| matches.remove(match) |
| # convert to integers, resort |
| matches = map(int, matches) |
| matches.sort() |
| # returning None for these indicates that they matched |
| json_matches = None |
| soap_matches = None |
| |
| #otherwise we'll just return the raw stuff to be processed above |
| |
| return matches,soap_matches,json_matches |
| |
| def run_searches(domain_info, users, random_strings, documents_destroyed=0, |
| user_slice=0): |
| """ Search through the ambassador as each user in the users list. Check that the |
| results returned match each user's expected results list or, if <documents_destroyed>, |
| that each user gets an empty list. |
| |
| Runs len(users)*len(groups) searches (because len(random_strings) = len(groups)), |
| so this scales as T(d) = (4d^4+16d^3+20d^2+8d+1)*2*(d^3+3d^2+3d) = O(d^7), where |
| d is the number of domains in the forest. |
| """ |
| debug_flush("run_searches") |
| userkeys = users.keys() |
| userkeys = userkeys[user_slice:] + userkeys[:user_slice] |
| mismatches = [] |
| for username in userkeys: |
| print_flush("Searching domain %s with user %s"%(domain_info.ad_domain,username)) |
| if (not users[username]['exists_in_ad']): |
| # don't search with a user who wasn't created/is deleted |
| continue |
| if documents_destroyed and not users[username].get('nosearch'): |
| # nosearch tag is for SuperUser, who has access to all documents, |
| # to speed deletion searches |
| continue |
| if not documents_destroyed and ('expected_search_results' not in users[username]): |
| # or has not expected search results |
| continue |
| |
| expected_ids_sort = users[username].get('expected_search_results',[]) |
| expected_ids_sort.sort() |
| |
| # try 2 times for not getting enough results |
| tries = 2 |
| user_mismatches = [] |
| while tries: |
| matches = [] |
| user_mismatches = [] |
| for r in random_strings: |
| web_matches = [] |
| soap_matches = [] |
| json_matches = [] |
| try: |
| # try 3 times for maybe having a merge cause a disagreement between interfaces |
| search_tries = 3 |
| while search_tries: |
| web_matches,soap_matches,json_matches = get_user_search_results(username,r,domain_info) |
| # intefaces matched no retry, none for both of these indicates this |
| if not (soap_matches or json_matches): |
| break |
| # could be merging, so try again |
| search_tries = search_tries - 1 |
| time.sleep(5) |
| else: |
| # used up our retries, record this mismatch occurrence |
| print_flush("Failed retrying due to merge",username,domain_info.ad_domain) |
| user_mismatches.append((username,r,domain_info.ad_domain,web_matches,soap_matches,json_matches,[],"Different results from interfaces (WEBUI,SOAP,JSON)")) |
| continue |
| except Exception,e: |
| print_flush("Failed due to exception:",traceback.format_exc(),username,domain_info.ad_domain) |
| user_mismatches.append((username,r,domain_info.ad_domain,[],[],[],[],"Exception %s"%(traceback.format_exc()))) |
| continue |
| # record the matches we got |
| matches.extend(web_matches) |
| |
| # if we have enough matches or we have mismatches that make this retry loop meaningless, get out of here |
| if len(matches) >= len(expected_ids_sort) or mismatches: |
| break |
| |
| # could have been stoplist processing, so give it another try |
| print_flush("Retrying because the number of matches was too low") |
| tries = tries - 1 |
| time.sleep(5) |
| |
| # sort the matches for comparison |
| matches.sort() |
| |
| # shouldn't have any matches if there are no documents |
| if documents_destroyed: |
| if matches: |
| user_mismatches.append((username,"",domain_info.ad_domain,matches,[],[],[],"Documents should have been destroyed")) |
| else: |
| # if the sequence expected doesn't match then we didn't find the docs we should have |
| if not expected_ids_sort == matches: |
| user_mismatches.append((username,"",domain_info.ad_domain,matches,[],[],expected_ids_sort,"Results didn't match expected")) |
| |
| # collect this users errors into the big list of errors |
| mismatches.extend(user_mismatches) |
| |
| # get of of here after lets say 5 mismatches |
| if len(mismatches) > 5: |
| break |
| |
| # if there are any summary errors then sort them and report them back with an exception |
| if mismatches: |
| mismatches.sort() |
| raise Exception("Search errors found %s" % (pprint.pformat(mismatches))) |
| |
| class TestDocsIngestion: |
| def __init__(self, ingestion_method): |
| self.ingestion_method = ingestion_method |
| self.ingested_documents = [] |
| |
| def ingest_docs(self, docs, doc_acls, force_public=0, |
| debug_info=None): |
| print_flush( "Ingesting documents using %s" % self.ingestion_method) |
| for doc in docs: |
| # the arguments to ingest_document are contents, name, acl, method |
| # The name of each document is the id string |
| self.ingested_documents.append(str(doc[DOC_ID])) |
| # make sure there is not an existing collection with that name |
| TestDocs.delete_document(str(doc[DOC_ID]), self.ingestion_method) |
| TestDocs.ingest_document(doc[CONTENTS], |
| str(doc[DOC_ID]), |
| xml_acl=doc[DOC_ACL], |
| method=self.ingestion_method) |
| |
| def destroy_docs(self): |
| print_flush( "Destroying documents") |
| for doc_id in self.ingested_documents: |
| TestDocs.delete_document(doc_id, self.ingestion_method) |
| self.ingested_documents = [] |
| |
| class CollectionIngestion(TestDocsIngestion): |
| def __init__(self): |
| self.ingestion_method = "collection" |
| |
| # initialize the base class |
| TestDocsIngestion.__init__(self, self.ingestion_method) |
| |
| def merge_docs(self): |
| print_flush( "Merging collections") |
| original_documents = self.ingested_documents[:] |
| TestDocs.delete_document("merged", self.ingestion_method) |
| self.ingested_documents.append("merged") |
| TestDocs.merge_collections(original_documents, "merged") |
| for doc_id in original_documents: |
| TestDocs.delete_document(doc_id, self.ingestion_method) |
| self.ingested_documents = ["merged"] |
| |
| # Creates documents, gets them onto the duck using the ingestion method |
| # of <ingestion_obj> and waits until the SearchCanary shows up in a |
| # webui search. Returns the id of the canary, which is needed to delete |
| # it later. |
| def create_documents(domain_info, documents, doc_acls, |
| ingestion_obj, force_public=False, debug_info=None): |
| |
| debug_flush("create_documents") |
| ingestion_obj.ingest_docs(documents, doc_acls, |
| force_public=force_public, debug_info=debug_info) |
| |
| # the canary ensures that all the documents are fully ingested |
| # and will now show up in a search for them |
| canaryID = None |
| print_flush( "Waiting for the Search Canary" ) |
| canaryID = SearchCanary.create_canary( |
| ingestion_obj.ingestion_method, |
| "jsonsearchapi-ambassador", |
| timeout=32400, |
| search_opts={ |
| "win_host":domain_info.ie_client_fqdn + ":" + \ |
| domain_info.ambassador_port, |
| "win_user":domain_info.realm_admin, |
| "win_password":domain_info.realm_admin_password}) |
| return canaryID |
| |
| # Merges the documents, if that can be done in the ingestion method |
| # of <ingestion_obj>. Destroys the SearchCanary associated with |
| # <old_canaryID>, and waits until a new canary shows up in a webui |
| # search. Returns the ID of the new canary |
| def merge_docs(domain_info, ingestion_obj, old_canaryID): |
| |
| debug_flush("merge_docs") |
| # If the ingestion object has a merge_docs method, run it |
| if dir(ingestion_obj.__class__).count("merge_docs") == 1: |
| |
| # destroy the Canary, because we'll need a new one to determine |
| # when the merged documents are searchable |
| SearchCanary.destroy_canary( |
| old_canaryID, |
| ingestion_obj.ingestion_method, |
| "jsonsearchapi-ambassador", |
| timeout=32400, |
| search_opts={ |
| "win_host":domain_info.ie_client_fqdn + ":" + \ |
| domain_info.ambassador_port, |
| "win_user":domain_info.realm_admin, |
| "win_password":domain_info.realm_admin_password}) |
| ingestion_obj.merge_docs() |
| print_flush( "Waiting for the Search Canary" ) |
| canaryID = SearchCanary.create_canary( |
| ingestion_obj.ingestion_method, |
| "jsonsearchapi-ambassador", |
| timeout=32400, |
| search_opts={ |
| "win_host":domain_info.ie_client_fqdn + ":" + \ |
| domain_info.ambassador_port, |
| "win_user":domain_info.realm_admin, |
| "win_password":domain_info.realm_admin_password}) |
| return canaryID |
| |
| # Deletes all of the documents from the duck and destroys the search canary |
| def delete_documents(domain_info, users, random_strings, ingestion_obj, |
| canaryID): |
| debug_flush("delete_documents") |
| ingestion_obj.destroy_docs() |
| |
| # wait until the Canary no longer shows up in a web search -- then |
| # we know that all the documents are now off the duck |
| SearchCanary.destroy_canary( |
| canaryID, |
| ingestion_obj.ingestion_method, |
| "jsonsearchapi-ambassador", |
| timeout=32400, |
| search_opts={ |
| "win_host":domain_info.ie_client_fqdn + ":" + \ |
| domain_info.ambassador_port, |
| "win_user":domain_info.realm_admin, |
| "win_password":domain_info.realm_admin_password}) |
| |
| # Make sure that the documents got deleted |
| run_searches(domain_info, users, random_strings, documents_destroyed=1) |
| |
| def create_docs_with_acls(doc_acls, random_strings): |
| debug_flush("create_docs_with_acls") |
| # put together the "documents" in the TestDocs library format |
| documents = [] |
| # make doc_id start at 10000 to avoid collision with 1542 which is the name of a mountain in Iran |
| doc_id = 10000 |
| for acl in doc_acls: |
| # add the SID of each object in each acl |
| # list with the appropriate surrounding tags |
| acl_string = '' |
| for scope in acl.keys(): |
| # the key is the name of the scope |
| acl_string += '<acl scope="%s">' % scope |
| # the value is the list of acl entries |
| for entry in acl[scope]: |
| # deal with blank (i.e. no access) acls |
| if entry == []: |
| acl_string += '<allow></allow>' |
| else: |
| # Put the allow or deny tags around the SID string |
| if entry[ACL_ENTRY_TYPE] == '+': |
| acl_string += '<allow>' + entry[ACL_ENTRY_USER] + \ |
| '</allow>' |
| elif entry[ACL_ENTRY_TYPE] == '-': |
| acl_string += '<deny>' + entry[ACL_ENTRY_USER] + \ |
| '</deny>' |
| else: |
| raise Exception("Could not process doc %d with acl %s" % |
| (doc_id, str(acl))) |
| acl_string += '</acl>' |
| # if there were any acls, put the document designator around them |
| # (a blank acl_string means the document is public) |
| if acl_string: |
| acl_string = '<document-acl>' + acl_string + '</document-acl>' |
| else: |
| acl_string = None |
| # a "document" consists of (contents, id number, acl) |
| documents.append([ |
| # The document consists of a single line -- |
| # "location: randomstring: id" |
| # Locations rotate through those listed above |
| "%s: %s: %d\n" % \ |
| (SQAhelpers.LocationList[doc_id % len(SQAhelpers.LocationList)], |
| random_strings[doc_id % len(random_strings)], |
| doc_id), |
| doc_id, |
| acl_string, |
| # document path will be filled in later |
| '']) |
| doc_id += 1 |
| return documents |
| |
| def ingest_and_search_documents(domains, users, documents, doc_acls, random_strings): |
| # instantiate the ingestion objects. The ingestion objects each |
| # have methods that will ingest/destroy documents using that |
| # particular ingestion type. The CollectionIngestion object also |
| # has a merge method for merging all collections it has created. |
| ingestion_objs = [TestDocsIngestion("HTTP")] |
| errors = 0 |
| |
| for ingestion_obj in ingestion_objs: |
| |
| t1 = time.time() |
| print_flush( "Ingesting %d documents" % len(documents) ) |
| |
| # Create documents just needs some domain to search from. |
| canaryID = create_documents(domains.values()[0], |
| documents, |
| doc_acls, |
| ingestion_obj, |
| debug_info="xMain crawl") |
| |
| t2 = time.time() |
| print_flush( "Ingested %d documents in %0.3f minutes" % (len(documents),(t2-t1)/60.0)) |
| |
| t1 = time.time() |
| print_flush( "Searching %d documents" % len(documents)) |
| |
| dlist = [] |
| i = 0 |
| for domain_info in domains.itervalues(): |
| user_slice = i * len(users)/len(domains) |
| try: |
| run_searches(domain_info, users, random_strings, user_slice=user_slice) |
| except Exception, e: |
| errors = errors + 1 |
| print_flush(traceback.format_exc()) |
| i += 1 |
| |
| t2 = time.time() |
| print_flush( "Searched %d documents in %0.3f minutes" % (len(documents),(t2-t1)/60.0)) |
| |
| t1 = time.time() |
| |
| delete_documents(domain_info, users, random_strings, ingestion_obj, canaryID) |
| |
| t2 = time.time() |
| print_flush( "Deleted %d documents in %0.3f minutes" % (len(documents),(t2-t1)/60.0)) |
| |
| return errors |
| |
| def group_fullname(groupname, groupinfo): |
| """ Returns full domain\name for group. """ |
| |
| if 'domain' in groupinfo: |
| return r'%s\%s' % (groupinfo['domain'], groupname) |
| else: |
| return groupname |
| |
| def create_groups(domains, tag): |
| """ Generate a set of groups for permissions. For d domains, this creates |
| g+u+dl=(d)+(d^2+d)+(d^3+2d^2+d)=d^3+3d^2+3d groups. """ |
| # Kouti, Seitsonen p. 214; There are 5 "levels": |
| # 1. User/computer |
| # 2. Global Groups |
| # 3. Universal Groups |
| # 4. Domain Local Groups |
| |
| # Any upper-level (lower numbered) object can be a member in any |
| # lower-level (higher-numbered) object. Global groups cannot |
| # accept members from other domains. Universal and domain local |
| # groups can, however. Built-in groups are like domain-local |
| # groups with the "domain" "Builtin" (p. 217). |
| |
| # First, create g = d global groups |
| groups = {} |
| i = 0 |
| for domain in domains: |
| groups['G_G_D%d_%s' % (i, tag)] = dict(domain=domain, |
| group_type='glsec', user_members=[], |
| group_members=[]) |
| i += 1 |
| |
| # Next, create u = d*(g+1) = d^2+d universal groups |
| glgroups = dict((k,v) for k,v in groups.iteritems() if v.get('group_type') == 'glsec') |
| |
| for d in range(len(domains)): |
| domain = domains[d] |
| for (i,glg) in zip([i for i in range(len(glgroups))],glgroups.items()): |
| groups['G_U_D%d_%d_%s'% (d, i, tag)] = \ |
| dict(group_type='unsec', |
| group_members=[glg[0]], |
| domain=domain, user_members=[]) |
| groups['G_U_D%d_%d_%s' % (d,len(glgroups),tag)] = \ |
| dict(group_type='unsec', domain=domain, user_members=[], |
| group_members=[]) |
| |
| # Next, create dl = d*(u+g+1) = d*(d^2+d+d+1) = d^3+2d^2+d |
| # domain local groups |
| newgroups = {} |
| i = 0 |
| |
| for d in range(len(domains)): |
| domain = domains[d] |
| count = 0 |
| newgroups['G_DL_D%d_%d_%s'%(i, count, tag)] = \ |
| dict(group_type='dlsec', domain=domain, user_members=[], group_members=[]) |
| count += 1 |
| for (groupname,groupinfo) in groups.iteritems(): |
| count += 1 |
| newgroups['G_DL_D%d_%d_%s'%(i, count, tag)] = \ |
| dict(group_type='dlsec', domain=domain, |
| group_members=[groupname], |
| user_members=[]) |
| i += 1 |
| |
| groups.update(newgroups) |
| return groups |
| |
| def filter_groups(groups, domain): |
| """ Return groups which can be used for permissions for objects in domain. """ |
| debug_flush("filter_groups") |
| return dict((gk, gv) for (gk,gv) in groups.iteritems() \ |
| if gv.get('group_type') != 'dlsec' or gv.get('domain') == domain) |
| |
| def filter_groups_membership(groups, domain): |
| """ Return groups to which a user in domain can belong. """ |
| debug_flush("filter_groups_membership") |
| return dict((gk, gv) for (gk,gv) in groups.iteritems() \ |
| if gv.get('group_type') != 'glsec' or gv.get('domain') == domain) |
| |
| def create_users(domains, allow_groups, deny_groups): |
| """ Create a set of users that have combinations of memberships in |
| allow_groups and deny_groups. Creates 4d^4+16d^3+20d^2+8d+1 users, |
| where d = len(domains). """ |
| debug_flush("create_users") |
| users = {} |
| |
| # Create every combination of accept and deny. Use counting variables |
| # to shorten usernames. |
| d = 0 |
| for domain in domains: |
| i = 0 |
| |
| # the filter_groups_membership will return g+u+(dl/d) = u+g+(u+g+1) groups, |
| # which is 2((d)+(d^2+d))+1 = 2d^2+4d+1. |
| # (2d^2+4d+1)^2 = 2d^2(2d^2+4d+1) + 4d(2d^2+4d+1) + (2d^2+4d+1) |
| # = 4d^4+8d^3+2d^2 + 8d^3+16d^2+4d + 2d^2+4d+1 = 4d^4+16d^3+20d^2+8d+1 users total. |
| domain_allow_groups = filter_groups_membership(dict((k,v) for k,v in \ |
| allow_groups.iteritems() \ |
| if not v['group_members']),domain) |
| for (ik,iv) in domain_allow_groups.iteritems(): |
| j = 0 |
| |
| for (jk,jv) in filter_groups_membership(dict((k,v) for k,v in deny_groups.iteritems() \ |
| if not v['group_members']),domain).iteritems(): |
| username = 'User_%d_%d_D%d' % (i,j,d) |
| users[username] = {'domain': domain, 'groups': [ik, jk]} |
| iv['user_members'].append(username) |
| jv['user_members'].append(username) |
| |
| j = j+1 |
| i = i+1 |
| |
| # Create one user per domain with access to all of the documents. This user |
| # won't be used in the initial searches, but only once documents have been |
| # deleted to verify that these users get no matches. |
| # |
| # We need one in every domain so that they can belong to all global groups. |
| users['SuperUser_D%d' % d] = {'domain': domain, |
| 'groups': domain_allow_groups, |
| 'nosearch': True} |
| for v in domain_allow_groups.itervalues(): |
| v['user_members'].append('SuperUser_D%d' % d) |
| |
| d = d+1 |
| |
| return users |
| |
| def create_doc_acls(allow_groups, deny_groups): |
| """ Create a list of document ACLs to test on documents. """ |
| # Each acl is a dictionary of the form |
| # {scope: [('+'/'-', name), ... ]} |
| # scope is "file", "share", or "dir" |
| # Each name must be in the entities_in_acls dictionary. |
| # An empty dictionary means the document will be public. |
| debug_flush("create_doc_acls") |
| doc_acls = [ |
| {} |
| ] |
| |
| for accept_group, accept_group_info in allow_groups.iteritems(): |
| # Create one document for each element of the cartesian product |
| # of allow_groups and deny_groups minus combinations that have |
| # domain local ACLs from two different domains, since Windows |
| # only allows a document to have domain local groups in ACLs if |
| # the document exists in the domain the group is local to, so |
| # these ACLs can't exist. |
| doc_acls.extend([{'file': [['+', accept_group], ['-', deny_group]]} \ |
| for deny_group, deny_group_info in deny_groups.iteritems() \ |
| if deny_group_info['group_type'] != 'dlsec' or \ |
| accept_group_info['group_type'] != 'dlsec' or |
| accept_group_info['domain'] == deny_group_info['domain']]) |
| return doc_acls |
| |
| def check_group_access(group, groups, acl): |
| """ Returns -1 if explicitly denied, 0 if neither allowed nor denied explicitly |
| (and therefore denied), 1 if allowed. """ |
| # First, check other groups this group is a member of. |
| debug_flush("check_group_access") |
| allowed = 0 |
| for g in groups.iterkeys(): |
| if group in groups[g]['group_members']: |
| ret = check_group_access(g, groups, acl) |
| if ret == -1: |
| return -1 |
| if ret == 1: |
| allowed = 1 |
| |
| # Now check the group we've been explicitly given |
| for acl_entry in acl.get('file',[])+acl.get('share',[]): |
| if acl_entry[0] == '-' and acl_entry[1] == group: |
| return -1 |
| if acl_entry[0] == '+' and acl_entry[1] == group: |
| allowed = 1 |
| return allowed |
| |
| |
| def check_user_access(userinfo, groups, acl): |
| """ See if the user given by userinfo can access the document specified by acl """ |
| |
| debug_flush("check_user_access") |
| allowed = 0 |
| |
| # Empty ACLs mean public. |
| if acl == {}: |
| return True |
| |
| default_groups=["authenticated_users","everyone","networkUsers","users","domainUsers"] |
| user_groups = [g for g in userinfo['groups'] if g in groups] |
| for g in user_groups+default_groups: |
| ret = check_group_access(g, groups, acl) |
| if ret == -1: |
| return False |
| if ret == 1: |
| allowed = 1 |
| if allowed: |
| return True |
| else: |
| return False |
| |
| def resolve_user_doc_access(users,groups,doc_acls): |
| debug_flush("resolve_user_doc_access") |
| for username, userinfo in users.iteritems(): |
| if userinfo.get('nosearch'): |
| continue |
| |
| userinfo['expected_search_results'] = [] |
| for aclnum in range(len(doc_acls)): |
| if check_user_access(userinfo, groups, doc_acls[aclnum]): |
| # bias the acl num by 10000 due to hack to avoid geographic numbers |
| userinfo['expected_search_results'].append(10000+aclnum) |
| |
| def init_users_groups_docacls(domain_id, limit_domains = False): |
| debug_flush("init_users_groups_docacls",domain_id) |
| # Initialize list of domains |
| domain_info = ADTestHelpers.setup_ad_tests(domain_id) |
| |
| # FIXME: generalize to include domain_info or not based upon configuration. |
| # if limit_domains is true only do the first child domain |
| if limit_domains: |
| domains = { domain_info.child_domains[0].ad_domain:domain_info.child_domains[0] } |
| else: |
| domains = dict((d.ad_domain, d) for d in domain_info.child_domains) |
| |
| # dictionary of all groups that we will be creating |
| # group tuple is (sid, exists in AD, type, user members, group members) |
| allow_groups = create_groups(domains.keys(), 'Allow') |
| deny_groups = create_groups(domains.keys(), 'Deny') |
| groups = {} |
| groups.update(allow_groups) |
| groups.update(deny_groups) |
| |
| users = create_users(domains = domains, allow_groups = allow_groups, |
| deny_groups = deny_groups) |
| |
| doc_acls = create_doc_acls(allow_groups, deny_groups) |
| |
| return groups, allow_groups, deny_groups, users, doc_acls, domains, domain_info |
| |
| def update_acl_entries(users,groups,doc_acls): |
| debug_flush("update_acl_entries") |
| # dictionary of all entities that might be found on acls -- |
| # the search users, groups, existing users, a hack for |
| # creating a blank acl, an invalid SID, etc. |
| entities_in_acls = {} |
| entities_in_acls.update(users) |
| entities_in_acls.update(groups) |
| entities_in_acls["everyone"] = {'sid':adtools.EveryoneSID, 'exists_in_ad':1} |
| entities_in_acls["authenticated_users"] = {'sid':adtools.AuthenticatedUsersSID, |
| 'exists_in_ad':1} |
| entities_in_acls["networkUsers"] = {'sid':adtools.NetworkUsersSID, 'exists_in_ad':1} |
| entities_in_acls["users"] = {'sid':adtools.UsersSID, 'exists_in_ad':1} |
| entities_in_acls["invalidSID"] = {'sid':adtools.InvalidSID, 'exists_in_ad':0} |
| entities_in_acls["domainUsers"] = {'sid':adtools.get_sid("Domain Users"), 'exists_in_ad': 1} |
| |
| # replace all of the user/group names with SIDs |
| for acl in doc_acls: |
| for scope in acl.keys(): |
| for entry in acl[scope]: |
| entry[ACL_ENTRY_USER] = entities_in_acls[entry[ACL_ENTRY_USER]]['sid'] |
| |
| def cleanup_users_groups(domain_info, domains, users, groups ): |
| """ clean up users and groups that we may have created """ |
| # We want to try to delete the users from every domain, since if user u1 exists in domain d2, |
| # trying to create d1\u1 will fail. This might happen if the test is reconfigured. |
| debug_flush("cleanup_users_groups") |
| |
| for d in domains.itervalues(): |
| for u in users.iterkeys(): |
| try: |
| adtools.delete_user(u, d.domain_controller_ambassador) |
| except Exception, e: |
| pass |
| |
| # Delete the groups before we try to create them |
| for groupname, groupinfo in groups.iteritems(): |
| if 'domain' in groupinfo: |
| d = domains[groupinfo['domain']] |
| else: |
| d = domain_info |
| try: |
| adtools.delete_group(groupname, d.domain_controller_ambassador, domain=d.ad_domain,verbose=debug) |
| except Exception, e: |
| pass |
| |
| def main(): |
| |
| pp = pprint.PrettyPrinter(indent=2, width=100) |
| |
| # process the command line arguments |
| (options, args) = ADTestHelpers.process_ad_arguments(sys.argv) |
| |
| # if args contain debug then set the debug flag |
| if "debug" in args: |
| global debug |
| debug = True |
| |
| # if args contain nojoin then don't set up and join assume already joined |
| if "nojoin" in args: |
| global nojoin |
| nojoin = True |
| |
| # dictionary of all users that we will be creating |
| # these users will be performing searches |
| # user tuple is (sid, exists in AD, expected search results) |
| |
| (groups, allow_groups, deny_groups, users, doc_acls, domains, domain_info) = \ |
| init_users_groups_docacls(options.domain_id, limit_domains = (options.pre_upgrade or options.post_upgrade)) |
| |
| # Setup DNS to be what we want (primary domain is an A record, everything |
| # else is CNAMEs) |
| duck_name = SQAhelpers.reverse_resolve().split('.')[0] |
| |
| # dns should already be configured for post upgrade test and we want to keep the machine accts |
| if not nojoin and not options.post_upgrade: |
| # make sure there aren't any lingering machine accounts |
| adtools.ad_md_remove_machine_accts( duck_name, domain_info, verbose=debug) |
| # set up DNS |
| adtools.ad_md_setup_dns(duck_name, SQAhelpers.get_host_address(),domain_info,verbose=debug) |
| |
| pp.pprint(users) |
| pp.pprint(zip(range(10000,10000+len(doc_acls)),doc_acls)) |
| |
| if options.cleanup_only: |
| print_flush( "Beginning cleanup. No tests will be run") |
| print_errors = True |
| else: |
| print_flush( "Cleaning up from previous tests") |
| print_errors = False |
| |
| |
| # active_directory_tool join, accept default realm and |
| # Administrator as user and enter Administrator password. |
| # don't join for a post upgrade test, we should already be joined |
| if not nojoin and not options.post_upgrade: |
| print_flush( "Joining the AD domain %s."%domain_info.ad_domain) |
| adtools.ad_md_setup_join(domain_info,verbose=debug) |
| |
| # clean up users and groups |
| cleanup_users_groups(domain_info, domains,users,groups) |
| |
| print_flush( "Force replication of cleanup changes" ) |
| |
| for d in domains.values(): |
| output = d.domain_controller_ambassador.run("repadmin /syncall") |
| print_flush(output) |
| |
| if options.cleanup_only: |
| sys.exit(0) |
| |
| # make sure we can search from all of the domains |
| # New plan, try to wait until the issue with DNS clears up, right now allow up to an hour in 5 minute increments |
| # it may time out the test we'll have to adjust the test timeout once we know when the problem resolves in the domain |
| for d in domains.values(): |
| tries = 24 |
| while tries: |
| try: |
| print "Test search domain %s ie client ambassador %s :"%(d.ad_domain,d.ie_client_ambassador.ambassador) |
| adtools.confirm_user_with_search( "Administrator@"+d.ad_domain, |
| d.realm_admin_password, |
| d.ie_client_ambassador, |
| hostname=duck_name ) |
| print "OK after %d seconds"%((12-tries)*300) |
| break |
| except Exception, e: |
| print "Failed %s"%(str(e)) |
| # This workaround worked manually, but not automatically, there is no evidence that there are cached tickets that |
| # we can detect at this level |
| # print d.ie_client_ambassador.run(r"\\u14e\mcinstall\klist tickets", |
| # user="Administrator@"+d.ad_domain, |
| # password=d.realm_admin_password) |
| # print d.ie_client_ambassador.run(r"\\u14e\mcinstall\Purge.exe", |
| # user="Administrator@"+d.ad_domain, |
| # password=d.realm_admin_password) |
| # print d.ie_client_ambassador.run(r"\\u14e\mcinstall\klist tickets", |
| # user="Administrator@"+d.ad_domain, |
| # password=d.realm_admin_password) |
| tries -= 1 |
| time.sleep(300) |
| else: |
| raise Exception("Failed to get search working!") |
| |
| # create random 8-character strings to use in searches |
| random_strings = [SQAhelpers.create_random_string(16) for i in range(len(groups))] |
| |
| t1 = time.time() |
| |
| print_flush( "Creating %d users" % len(users)) |
| for username,userinfo in users.iteritems(): |
| # It would be nice to use ldif here, but you can't set |
| # the passwords |
| |
| d = domains[userinfo['domain']] |
| |
| try: |
| adtools.create_user(username, makepswd(username), |
| d.domain_controller_ambassador, d.ad_domain, |
| ie_client_ambassador=d.ie_client_ambassador, |
| confirm_user=False) |
| except: |
| print_flush( "Exception creating user %s in domain %s" % (username, d.ad_domain)) |
| raise |
| |
| # mark that it has been created |
| users[username]['exists_in_ad'] = 1 |
| # set SID for use in document acls |
| output = adtools.get_sid(username, userinfo['domain']) |
| users[username]['sid'] = output |
| |
| t2 = time.time() |
| print_flush( "Created %d users in %0.3f minutes" % (len(users),(t2-t1)/60.0)) |
| |
| t1 = time.time() |
| |
| print_flush( "Creating %d groups" % len(groups)) |
| for groupname, groupinfo in groups.iteritems(): |
| # create_group takes the group name and group type string |
| # It would be nice to use ldif here, but we can't because |
| # of circular groups |
| if 'domain' in groupinfo: |
| d = domains[groupinfo['domain']] |
| else: |
| d = domain_info |
| |
| try: |
| adtools.create_group(groupname, groups[groupname]['group_type'], |
| d.ad_domain, d.domain_controller_ambassador) |
| |
| except: |
| print_flush( "Exception creating group %s in domain %s" % (groupname, d.ad_domain)) |
| raise |
| |
| # mark that it has been created |
| groups[groupname]['exists_in_ad'] = 1 |
| # set SID for use in document acls. Because names are unique |
| # across domains, we don't need to specify domain here. |
| groups[groupname]['sid'] = adtools.get_sid(groupname, d.ad_domain) |
| |
| t2 = time.time() |
| print_flush( "Created %d groups in %0.3f minutes" % (len(groups),(t2-t1)/60.0)) |
| |
| t1 = time.time() |
| |
| print_flush( "Populating groups") |
| # this can't be combined with the above loop because there are |
| # circular memberships |
| for groupname, groupinfo in groups.iteritems(): |
| group_domain = groups[groupname]['domain'] |
| #d = domains[groupinfo['domain']] |
| |
| for member in groups[groupname]['user_members']: |
| member_domain = users[member]['domain'] |
| d = domains[member_domain] |
| |
| adtools.add_to_group(member, groupname, d.domain_controller_ambassador, |
| child_domain = member_domain, parent_domain = group_domain, verbose = debug) |
| for member in groups[groupname]['group_members']: |
| member_domain = groups[member]['domain'] |
| d = domains[member_domain] |
| |
| adtools.add_to_group(member, groupname, d.domain_controller_ambassador, |
| child_domain = member_domain, parent_domain = group_domain, verbose = debug) |
| |
| t2 = time.time() |
| print_flush( "Populated %d groups with %d users in %0.3f minutes" % \ |
| (len(groups), len(users),(t2-t1)/60.0)) |
| |
| print_flush( "Resolving Doc Access") |
| resolve_user_doc_access(users,groups,doc_acls) |
| |
| print_flush( "Creating Documents") |
| |
| update_acl_entries(users,groups, doc_acls) |
| |
| documents = create_docs_with_acls(doc_acls, random_strings) |
| |
| print_flush( "Force replication" ) |
| |
| for d in domains.values(): |
| output = d.domain_controller_ambassador.run("repadmin /syncall") |
| print_flush(output) |
| |
| print_flush( "Searching for Documents") |
| # We'll do this for each domain, and test each domain individually. |
| errors = ingest_and_search_documents(domains, users, documents, doc_acls, random_strings) |
| |
| if not errors: |
| print_flush( "All tests have passed. Beginning cleanup.") |
| else: |
| print_flush( "Exiting with errors. Beginning cleanup." ) |
| |
| # clean up users and groups |
| cleanup_users_groups(domain_info, domains,users,groups) |
| |
| # leave the domain |
| if not options.pre_upgrade: |
| adtools.leave_ad( realm_admin_password=domain_info.realm_admin_password, disable_machine_acct=False, delete_machine_acct=True, already_left=0 ) |
| |
| print_flush( "Cleanup complete.") |
| |
| # exit with no error |
| sys.exit(errors) |
| |
| if __name__ == '__main__': |
| main() |
