VCL-1095 - Move unjoining of Windows VMs from Active Directory to earlier in the deprovision process

DataStructure.pm: modified get_domain_credentials: changed name of input parameter and made it optional, if not passed in, will use domain of current image; updated to receive $domain_dns_name as first item in array returned by get_management_node_ad_domain_credentials; added more details to debug notify

Windows.pm:
-modified pre_capture: moved unjoining from domain a little earlier, mainly so setting the Administrator password to the VCL default (from vcld.conf) will not fail if the password doesn't meet domain restrictions, this required adding an extra reboot after unjoining
-modified post_reservation: unjoin computer from domain; this was needed so that reload reservations will be able to unjoin a computer while the previous image is still loaded and it has a way to lookup what credentials are needed to unjoin that image; otherwise, the case exists where a computer needs to be unjoined, but vcld doesn't know which credentials to use for unjoining it
-modified ad_join_ps: cleaned up domain password being written to vcld.log file; added writing addomain_id tag to currentimage.txt file
-modified ad_unjoin: updated to not pass arguments to ad_delete_computer
-modified ad_search: get $domain_username and $domain_password from passed in arguments instead of from calling get_domain_credentials; cleaned up domain password being written to vcld.log file
-modified ad_delete_computer: changed to not accept arguments; get domain_dns_name and credentials from calling get_domain_credentials; if calling that with no arguments returns nothing, try recursively calling self and calling get_domain_credentials with addomain_id from currentimage.txt file; include domain_dns_name and credentials with data passed to ad_search

utils.pm: modified get_management_node_ad_domain_credentials: changed 2nd argument from $domain_dns_name to $domain_id; added $domain_dns_name to beginning of array of returned data; for WHERE clause of query, always use addomain.id since domainDNSName is no longer unique; added domain_dns_name to debug notify

update-vcl.sql:
-changed key on domainDNSName in addomain table from a unique key to just an index; this allows multiple accounts per domain_dns_name
-added DropExistingIndices and AddIndexIfNotExists calls for addomain.domainDNSName

vcl.sql: changed key on domainDNSName in addomain table from a unique key to just an index; this allows multiple accounts per domain_dns_name
diff --git a/managementnode/lib/VCL/DataStructure.pm b/managementnode/lib/VCL/DataStructure.pm
index dd693e3..3d57fdf 100644
--- a/managementnode/lib/VCL/DataStructure.pm
+++ b/managementnode/lib/VCL/DataStructure.pm
@@ -2804,7 +2804,7 @@
 
 =head2 get_domain_credentials
 
- Parameters  : $domain_identifier
+ Parameters  : $imagedomain_id (optional)
  Returns     : array ($username, $domain_password)
  Description : Attempts to determine and decrypt the username and password for
                the domain specified by the argument. 
@@ -2817,23 +2817,25 @@
 		notify($ERRORS{'CRITICAL'}, 0, "subroutine was called as a function, it must be called as a class method");
 		return 0;
 	}
-	
-	my $domain_identifier = shift;
-	if (!defined($domain_identifier)) {
-		notify($ERRORS{'WARNING'}, 0, "domain identifier argument was not supplied");
-		return;
-	}
-	
+
+	my $reservation_id = $self->reservation_id();
 	my $management_node_id = $self->get_management_node_id();
-	
-	my ($username, $secret_id, $encrypted_password) = get_management_node_ad_domain_credentials($management_node_id, $domain_identifier);
-	return unless $username && $secret_id && $encrypted_password;
-	
+
+	my $imagedomain_id = shift || $self->request_data->{reservation}{$reservation_id}{computer}{currentimage}{imagedomain}{id};
+
+	my ($domain_dns_name, $username, $secret_id, $encrypted_password) = get_management_node_ad_domain_credentials($management_node_id, $imagedomain_id);
+	return unless $domain_dns_name && $username && $secret_id && $encrypted_password;
+
 	my $decrypted_password = $self->mn_os->decrypt_cryptsecret($secret_id, $encrypted_password) || return;
 	my $decrypted_password_length = length($decrypted_password);
 	my $decrypted_password_hidden = '*' x $decrypted_password_length;
-	notify($ERRORS{'DEBUG'}, 0, "retrieved credentials for Active Directory domain: '$decrypted_password_hidden' ($decrypted_password_length characters)");
-	return $decrypted_password;
+	notify($ERRORS{'DEBUG'}, 0, "retrieved credentials for Active Directory domain:\n" .
+               "domain ID: $imagedomain_id:\n" .
+               "domain DNS name: $domain_dns_name:\n" .
+               "domain username: $username:\n" .
+               "domain password: $decrypted_password_hidden ($decrypted_password_length characters)"
+        );
+	return ($domain_dns_name, $username, $decrypted_password);
 }
 
 #//////////////////////////////////////////////////////////////////////////////
diff --git a/managementnode/lib/VCL/Module/OS/Windows.pm b/managementnode/lib/VCL/Module/OS/Windows.pm
index f245353..52f7b19 100644
--- a/managementnode/lib/VCL/Module/OS/Windows.pm
+++ b/managementnode/lib/VCL/Module/OS/Windows.pm
@@ -385,6 +385,25 @@
 
 =item *
 
+ If computer is part of Active Directory Domain, unjoin it
+
+=cut
+
+	if ($self->ad_get_current_domain()) {
+		if (!$self->ad_unjoin()) {
+			notify($ERRORS{'WARNING'}, 0, "failed to remove computer from Active Directory domain");
+			return 0;
+		}
+		notify($ERRORS{'DEBUG'}, 0, "computer successfully unjoined from domain, rebooting for change to take effect");
+		# reboot if unjoin successful
+		if (!$self->reboot()) {
+			notify($ERRORS{'WARNING'}, 0, "failed to reboot after unjoining from domain");
+		}
+	}
+
+
+=item *
+
  Set Administrator account password to known value
 
 =cut
@@ -415,20 +434,6 @@
 	if (!$deleted_user_accounts) {
 		notify($ERRORS{'DEBUG'}, 0, "unable to delete user accounts, will try again after reboot");
 	}
-
-=item *
-
- If computer is part of Active Directory Domain, unjoin it
-
-=cut
-
-	if ($self->ad_get_current_domain()) {
-		if (!$self->ad_unjoin()) {
-			notify($ERRORS{'WARNING'}, 0, "failed to remove computer from Active Directory domain");
-			return 0;
-		}
-	}
-
 =item *
 
  Set root as the owner of /home/root
@@ -1087,6 +1092,8 @@
 		return 0;
 	}
 	
+	my $computer_name = $self->data->get_computer_short_name();
+	
 	# Check if custom post_reservation script exists in image
 	my $script_path = '$SYSTEMROOT/vcl_post_reservation.cmd';
 	if ($self->file_exists($script_path)) {
@@ -1097,7 +1104,15 @@
 		notify($ERRORS{'DEBUG'}, 0, "custom post_reservation script does NOT exist in image: $script_path");
 	}
 	
-	return $self->SUPER::post_reservation();
+	$self->SUPER::post_reservation();
+	
+	# Check if the computer is joined to any AD domain
+	my $computer_current_domain_name = $self->ad_get_current_domain();
+	if ($computer_current_domain_name) {
+		$self->ad_delete_computer();
+	}
+	
+	return 1;
 }
 
 #//////////////////////////////////////////////////////////////////////////////
@@ -1123,7 +1138,7 @@
 	# Check if the computer is joined to any AD domain
 	my $computer_current_domain_name = $self->ad_get_current_domain();
 	if ($computer_current_domain_name) {
-		$self->ad_delete_computer($computer_name, $computer_current_domain_name);
+		$self->ad_delete_computer();
 	}
 	
 	return $self->SUPER::pre_reload();
@@ -13785,6 +13800,7 @@
 	my $computer_name	= $self->data->get_computer_short_name();
 	my $image_name	= $self->data->get_image_name();
 	
+	my $image_domain_id = $self->data->get_image_domain_id();
 	my $domain_dns_name = $self->data->get_image_domain_dns_name();
 	my $domain_username = $self->data->get_image_domain_username();
 	my $domain_password = $self->data->get_image_domain_password();
@@ -13817,7 +13833,7 @@
 	notify($ERRORS{'DEBUG'}, 0, "attempting to join $computer_name to AD\n" .
 		"domain DNS name    : $domain_dns_name\n" .
 		"domain user string : $domain_user_string\n" .
-		"domain password    : $domain_password (escaped: $domain_password_escaped)\n" .
+		#"domain password    : $domain_password (escaped: $domain_password_escaped)\n" .
 		"domain computer OU : " . ($computer_ou_dn ? $computer_ou_dn : '<not configured>')
 	);
 	
@@ -13868,12 +13884,17 @@
 \$username = '$domain_user_string'
 \$password = '$domain_password_escaped'
 Write-Host "username (between >*<): `n>\$username<`n"
-Write-Host "password (between >*<): `n>\$password<`n"
 \$ps_credential = New-Object System.Management.Automation.PsCredential(\$username, (ConvertTo-SecureString \$password -AsPlainText -Force))
 Add-Computer -DomainName '$domain_dns_name' -Credential \$ps_credential $domain_computer_command_section -Verbose -ErrorAction Stop
 EOF
+
+# move and uncomment below line to above EOF to include decrypted password in output for debugging
+#Write-Host "password (between >*<): `n>\$password<`n"
+
+	(my $sanitized_password = $domain_password_escaped) =~ s/./*/g;
+	(my $sanitized_script = $ad_powershell_script) =~ s/password = '.*'\n/password = '$sanitized_password'\n/;
 	
-	notify($ERRORS{'DEBUG'}, 0, "attempting to join $computer_name to $domain_dns_name domain using PowerShell script:\n$ad_powershell_script");
+	notify($ERRORS{'DEBUG'}, 0, "attempting to join $computer_name to $domain_dns_name domain using PowerShell script:\n$sanitized_script");
 	my ($exit_status, $output) = $self->run_powershell_as_script($ad_powershell_script, 0, 0); # (path, show output, retain file)
 	if (!defined($output)) {
 		notify($ERRORS{'WARNING'}, 0, "failed to execute PowerShell script to join $computer_name to Active Directory domain");
@@ -13949,6 +13970,8 @@
 		return;
 	}
 	else {
+		$self->set_current_image_tag('addomain_id', $image_domain_id);
+		
 		notify($ERRORS{'DEBUG'}, 0, "successfully joined $computer_name to Active Directory domain: $domain_dns_name, time statistics:\n" .
 			"computer rename reboot : $rename_computer_reboot_duration seconds\n" .
 			"AD join reboot         : $ad_join_reboot_duration seconds\n" .
@@ -14275,7 +14298,7 @@
 	#	
 	#	notify($ERRORS{'OK'}, 0, "removed $computer_name from Active Directory domain, output:\n" . join("\n", @$output));
 	
-	$self->ad_delete_computer($computer_name, $computer_current_domain);
+	$self->ad_delete_computer();
 	return 1;
 }
 
@@ -14393,11 +14416,13 @@
 	my $domain_username;
 	my $domain_password;
 	my $image_domain_dns_name = $self->data->get_image_domain_dns_name(0) || '';
-	if (defined($arguments->{domain_dns_name}) && $arguments->{domain_dns_name} ne $image_domain_dns_name) {
+	if (defined($arguments->{domain_dns_name})) {
 		$domain_dns_name = $arguments->{domain_dns_name};
-		($domain_username, $domain_password) = $self->data->get_domain_credentials($domain_dns_name);
+		$domain_username = $arguments->{domain_username};
+		$domain_password = $arguments->{domain_password};
+		
 		if (!defined($domain_username) || !defined($domain_password)) {
-			notify($ERRORS{'WARNING'}, 0, "unable to search domain: $domain_dns_name, domain DNS name argument was specified but credentials could not be determined from existing 'addomain' table entries");
+			notify($ERRORS{'WARNING'}, 0, "unable to search domain: $domain_dns_name, domain DNS name argument was specified but credentials could not be determined");
 			return;
 		}
 	}
@@ -14463,10 +14488,12 @@
 
 Write-Host "domain: $domain_dns_name"
 Write-Host "domain username (between >*<): >\$domain_username<"
-Write-Host "domain password (between >*<): >\$domain_password<"
-
 EOF
 
+# move and uncomment below line to above EOF to include decrypted password in output for debugging
+#Write-Host "domain password (between >*<): >\$domain_password<"
+
+
 	$powershell_script_contents .= <<'EOF';
 $type = [System.DirectoryServices.ActiveDirectory.DirectoryContextType]"Domain"
 $directory_context = New-Object System.DirectoryServices.ActiveDirectory.DirectoryContext($type, $domain_dns_name, $domain_username, $domain_password)
@@ -14480,7 +14507,7 @@
    else {
       $exception_message = $_.Exception.Message
    }
-   Write-Host "ERROR: failed to connect to $domain_dns_name domain, username: $domain_username, password: $domain_password, error: $exception_message"
+   Write-Host "ERROR: failed to connect to $domain_dns_name domain, username: $domain_username, error: $exception_message"
    exit
 }
 
@@ -14582,16 +14609,11 @@
 
 =head2 ad_delete_computer
 
- Parameters  : $computer_samaccountname (optional), $domain_dns_name (optional)
+ Parameters  : none
  Returns     : boolean
  Description : Deletes a computer object from the active directory domain with a
-               sAMAccountName attribute matching the argument. If no argument is
-               provided, the short name of the reservation computer is used.
-               
-               The sAMAccountName attribute for computers in Active Directory
-               always end with a dollar sign. The trailing dollar sign does not
-               need to be included in the argumenat. One will be added to the
-               LDAP filter used to search for the object to delete.
+					sAMAccountName attribute matching the short name of the
+					reservation computer that is used.
 
 =cut
 
@@ -14601,28 +14623,64 @@
 		notify($ERRORS{'CRITICAL'}, 0, "subroutine was called as a function, it must be called as a class method");
 		return;
 	}
+
+	# static variable to track if function being call recursively
+	CORE::state $recursion = 0;
+
+	my $computer_samaccountname = $self->data->get_computer_short_name();
+
+	my ($domain_dns_name, $username, $decrypted_password);
+	my $return;
+
+	if($recursion == 0) {
+		($domain_dns_name, $username, $decrypted_password) = $self->data->get_domain_credentials();
+	}
+	else {
+		my $addomain_id = $self->get_current_image_tag('addomain_id');
+		$addomain_id =~ s/ \(.*$//;
+		if (!defined($addomain_id)) {
+			return 0;
+		}
+		($domain_dns_name, $username, $decrypted_password) = $self->data->get_domain_credentials($addomain_id);
+	}
+	if (!defined($domain_dns_name) || !defined($username) || !defined($decrypted_password)) {
+		notify($ERRORS{'WARNING'}, 0, "failed to get domain credentials for $computer_samaccountname");
+		if ($recursion == 0) {
+			$recursion = 1;
+			$return = $self->ad_delete_computer();
+			$recursion = 0;
+			return $return;
+		}
+		return;
+	}
 	
-	my ($computer_samaccountname, $domain_dns_name) = @_;
-	
-	$computer_samaccountname = $self->data->get_computer_short_name() unless $computer_samaccountname;
-	
-	# Make sure computer samAccountName does not contain a trailing dollar sign
-	# A dollar sign will be present if retrieved directly from AD
 	$computer_samaccountname =~ s/\$*$/\$/g;
-	
 	my $ad_search_arguments = {
 		'ldap_filter' => {
 			'objectClass' => 'computer',
 			'sAMAccountName' => $computer_samaccountname,
-		}
+		},
+		'domain_dns_name' => $domain_dns_name,
+		'domain_username' => $username,
+		'domain_password' => $decrypted_password,
 	};
 	
-	# If a specific domain was specified, retrieve the username and password for that domain
-	if ($domain_dns_name) {
-		$ad_search_arguments->{domain_dns_name} = $domain_dns_name;
-	}
+	#notify($ERRORS{'DEBUG'}, 0, "attempting to delete Active Directory computer object, arguments:\n" . format_data($ad_search_arguments));
 	
-	return $self->ad_search($ad_search_arguments);
+	$return = $self->ad_search($ad_search_arguments);
+	if ($return == 1) {
+		return 1;
+	}
+	elsif ($recursion == 1) {
+		return 0;
+	}
+	else {
+		notify($ERRORS{'DEBUG'}, 0, "Failed to delete computer from AD using AD domain info from image; trying again with info from currentimage.txt");
+		$recursion = 1;
+		$return = $self->ad_delete_computer();
+		$recursion = 0;
+		return $return;
+	}
 }
 
 #//////////////////////////////////////////////////////////////////////////////
diff --git a/managementnode/lib/VCL/utils.pm b/managementnode/lib/VCL/utils.pm
index c704fe4..fb46a57 100644
--- a/managementnode/lib/VCL/utils.pm
+++ b/managementnode/lib/VCL/utils.pm
@@ -1270,7 +1270,7 @@
 	my ($package, $filename, $line,       $sub)  = caller(0);
 
 	# Mail::Mailer relies on sendmail as written, this causes a "die" on Windows
-	# TODO: Reqork this subroutine to not rely on sendmail
+	# TODO: Rework this subroutine to not rely on sendmail
 	my $osname = lc($^O);
 	if ($osname =~ /win/i) {
 		notify($ERRORS{'OK'}, 0, "sending mail from Windows not yet supported\n-----\nTo: $to\nSubject: $subject\nFrom: $from\n$mailstring\n-----");
@@ -15083,8 +15083,8 @@
 
 =head2 get_management_node_ad_domain_credentials
 
- Parameters  : $management_node_id, $domain_dns_name, $no_cache (optional)
- Returns     : ($username, $secret_id, $encrypted_password)
+ Parameters  : $management_node_id, $domain_id, $no_cache (optional)
+ Returns     : ($domain_dns_name, $username, $secret_id, $encrypted_password)
  Description : Attempts to retrieve the username, encrypted password, and secret
                ID for the domain from the addomain table. This is used if a
                computer needs to be removed from a domain but the reservation
@@ -15094,63 +15094,58 @@
 =cut
 
 sub get_management_node_ad_domain_credentials {
-	my ($management_node_id, $domain_identifier, $no_cache) = @_;
+	my ($management_node_id, $domain_id, $no_cache) = @_;
 	if (!defined($management_node_id)) {
 		notify($ERRORS{'WARNING'}, 0, "management node ID argument was not supplied");
 		return;
 	}
-	elsif (!$domain_identifier) {
+	elsif (!$domain_id) {
 		notify($ERRORS{'WARNING'}, 0, "domain identifier name argument was not specified");
 		return;
 	}
 	
-	if (!$no_cache && defined($ENV{management_node_ad_domain_credentials}{$domain_identifier})) {
-		notify($ERRORS{'DEBUG'}, 0, "returning cached Active Directory credentials for domain: $domain_identifier");
-		return @{$ENV{management_node_ad_domain_credentials}{$domain_identifier}};
+	if (!$no_cache && defined($ENV{management_node_ad_domain_credentials}{$domain_id})) {
+		notify($ERRORS{'DEBUG'}, 0, "returning cached Active Directory credentials for domain: $domain_id");
+		return @{$ENV{management_node_ad_domain_credentials}{$domain_id}};
 	}
 	
 	# Construct the select statement
 	my $select_statement = <<EOF;
 SELECT DISTINCT
+domainDNSName,
 username,
 password,
 secretid
 FROM
 addomain
 WHERE
+addomain.id = $domain_id
 EOF
 	
-	if ($domain_identifier =~ /^\d+$/) {
-		$select_statement .= "addomain.id = $domain_identifier";
-	}
-	else {
-		$select_statement .= "addomain.domainDNSName LIKE '$domain_identifier%'";
-	}
-	
 	# Call the database select subroutine
 	my @selected_rows = database_select($select_statement);
 
 	# Check to make sure 1 row was returned
 	if (scalar @selected_rows == 0) {
-		notify($ERRORS{'DEBUG'}, 0, "Active Directory domain does not exist in the database: $domain_identifier");
+		notify($ERRORS{'DEBUG'}, 0, "Active Directory domain does not exist in the database: $domain_id");
 		return ();
 	}
 
 	# Get the single row returned from the select statement
 	my $row = $selected_rows[0];
+	my $domain_dns_name = $row->{domainDNSName};
 	my $username = $row->{username};
 	my $secret_id = $row->{secretid};
 	my $encrypted_password = $row->{password};
 	
-	
-	
-	notify($ERRORS{'DEBUG'}, 0, "retrieved credentials for domain: $domain_identifier\n" .
+	notify($ERRORS{'DEBUG'}, 0, "retrieved credentials for domain: $domain_id\n" .
+		"domain DNS name    : '$domain_dns_name'\n" .
 		"username           : '$username'\n" .
 		"secret ID          : '$secret_id'\n" .
 		"encrypted password : '$encrypted_password'"
 	);
-	$ENV{management_node_ad_domain_credentials}{$domain_identifier} = [$username, $secret_id, $encrypted_password];
-	return @{$ENV{management_node_ad_domain_credentials}{$domain_identifier}};
+	$ENV{management_node_ad_domain_credentials}{$domain_id} = [$domain_dns_name, $username, $secret_id, $encrypted_password];
+	return @{$ENV{management_node_ad_domain_credentials}{$domain_id}};
 }
 
 #//////////////////////////////////////////////////////////////////////////////
diff --git a/mysql/update-vcl.sql b/mysql/update-vcl.sql
index ce33548..0746771 100644
--- a/mysql/update-vcl.sql
+++ b/mysql/update-vcl.sql
@@ -879,10 +879,13 @@
   `password` varchar(256) NOT NULL default '',
   `secretid` smallint(5) unsigned NOT NULL,
   PRIMARY KEY (`id`),
-  UNIQUE KEY `domainDNSName` (`domainDNSName`),
+  KEY `domainDNSName` (`domainDNSName`),
   KEY `secretid` (`secretid`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 
+CALL DropExistingIndices('addomain', 'domainDNSName');
+CALL AddIndexIfNotExists('addomain', 'domainDNSName');
+
 -- --------------------------------------------------------
 
 --
diff --git a/mysql/vcl.sql b/mysql/vcl.sql
index 95d68ca..58093ca 100644
--- a/mysql/vcl.sql
+++ b/mysql/vcl.sql
@@ -39,7 +39,7 @@
   `password` varchar(256) NOT NULL default '',
   `secretid` smallint(5) unsigned NOT NULL,
   PRIMARY KEY (`id`),
-  UNIQUE KEY `domainDNSName` (`domainDNSName`),
+  KEY `domainDNSName` (`domainDNSName`),
   KEY `secretid` (`secretid`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;