/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.ranger.unixusersync.process;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;
import java.util.HashMap;
import java.util.Date;
import java.util.Arrays;
import java.util.Iterator;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.Table;
import org.apache.log4j.Logger;
import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo;
import org.apache.ranger.ugsyncutil.model.UnixSyncSourceInfo;
import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
import org.apache.ranger.usergroupsync.UserGroupSink;
import org.apache.ranger.usergroupsync.UserGroupSource;

public class UnixUserGroupBuilder implements UserGroupSource {
	
	private static final Logger LOG = Logger.getLogger(UnixUserGroupBuilder.class);
	private final static String OS = System.getProperty("os.name");

	// kept for legacy support
	//public static final String UNIX_USER_PASSWORD_FILE = "/etc/passwd";
	//public static final String UNIX_GROUP_FILE = "/etc/group";

	/** Shell commands to get users and groups */
	static final String LINUX_GET_ALL_USERS_CMD = "getent passwd";
	static final String LINUX_GET_ALL_GROUPS_CMD = "getent group";
	static final String LINUX_GET_GROUP_CMD = "getent group %s";

	// mainly for testing purposes
	// there might be a better way
	static final String MAC_GET_ALL_USERS_CMD = "dscl . -readall /Users UniqueID PrimaryGroupID | " +
			"awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;}" +
			"/RecordName: / {name = $2;i = 0;}/PrimaryGroupID: / {gid = $2;}" +
			"/^ / {if (i == 0) { i++; name = $1;}}" +
			"/UniqueID: / {uid = $2;print name, \"*\", gid, uid;}'";
	static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID | " +
			"awk -v OFS=\":\" '{print $1, \"*\", $2, \"\"}'";
	static final String MAC_GET_GROUP_CMD = "dscl . -read /Groups/%1$s | paste -d, -s - | sed -e 's/:/|/g' | " +
			"awk -v OFS=\":\" -v ORS=\"\\n\" -F, '{print \"%1$s\",\"*\",$6,$4}' | " +
			"sed -e 's/:[^:]*| /:/g' | sed -e 's/ /,/g'";

	static final String BACKEND_PASSWD = "passwd";

	private boolean isUpdateSinkSucc = true;
	private boolean enumerateGroupMembers = false;
	private boolean useNss = false;

	private long lastUpdateTime = 0; // Last time maps were updated
	private long timeout = 0;

	private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
	private Map<String,String> groupId2groupNameMap;
	private Map<String, Map<String, String>> sourceUsers; // Stores username and attr name & value pairs
	private Map<String, Map<String, String>> sourceGroups; // Stores groupname and attr name & value pairs
	private Map<String, Set<String>> sourceGroupUsers;
	private Table<String, String, String> groupUserTable; // groupname, username, group id
	private int minimumUserId  = 0;
	private int minimumGroupId = 0;
	private String unixPasswordFile;
	private String unixGroupFile;

	private long passwordFileModifiedAt = 0;
	private long groupFileModifiedAt = 0;
	private UgsyncAuditInfo ugsyncAuditInfo;
	private UnixSyncSourceInfo unixSyncSourceInfo;
	private boolean isStartupFlag = false;
	Set<String> allGroups = new HashSet<>();


	public static void main(String[] args) throws Throwable {
		UnixUserGroupBuilder ugbuilder = new UnixUserGroupBuilder();
		ugbuilder.init();
		ugbuilder.print();
	}
	
	public UnixUserGroupBuilder() {
		isStartupFlag = true;
		minimumUserId = Integer.parseInt(config.getMinUserId());
		minimumGroupId = Integer.parseInt(config.getMinGroupId());
		unixPasswordFile = config.getUnixPasswordFile();
		unixGroupFile = config.getUnixGroupFile();
		ugsyncAuditInfo = new UgsyncAuditInfo();
		unixSyncSourceInfo = new UnixSyncSourceInfo();
		ugsyncAuditInfo.setSyncSource("Unix");
		ugsyncAuditInfo.setUnixSyncSourceInfo(unixSyncSourceInfo);
		unixSyncSourceInfo.setFileName(unixPasswordFile);
		unixSyncSourceInfo.setMinUserId(config.getMinUserId());
		unixSyncSourceInfo.setMinGroupId(config.getMinGroupId());

		if (LOG.isDebugEnabled()) {
			LOG.debug("Minimum UserId: " + minimumUserId + ", minimum GroupId: " + minimumGroupId);
		}

		timeout = config.getUpdateMillisMin();
		enumerateGroupMembers = config.isGroupEnumerateEnabled();

		if (!config.getUnixBackend().equalsIgnoreCase(BACKEND_PASSWD)) {
			useNss = true;
			unixSyncSourceInfo.setUnixBackend("nss");
		} else {
			LOG.warn("DEPRECATED: Unix backend is configured to use /etc/passwd and /etc/group files directly " +
					"instead of standard system mechanisms.");
			unixSyncSourceInfo.setUnixBackend(BACKEND_PASSWD);
		}

	}

	@Override
	public void init() throws Throwable {
		buildUserGroupInfo();
	}

	@Override
	public boolean isChanged() {
		// If previous update to Ranger admin fails, 
		// we want to retry the sync process even if there are no changes to the sync files
		if (!isUpdateSinkSucc) {
			LOG.info("Previous updateSink failed and hence retry!!");
			return true;
		}
		
		if (useNss)
			return System.currentTimeMillis() - lastUpdateTime > timeout;

		long TempPasswordFileModifiedAt = new File(unixPasswordFile).lastModified();
		if (passwordFileModifiedAt != TempPasswordFileModifiedAt) {
			return true;
		}

		long TempGroupFileModifiedAt = new File(unixGroupFile).lastModified();
		if (groupFileModifiedAt != TempGroupFileModifiedAt) {
			return true;
		}

		return false;
	}


	@Override
	public void updateSink(UserGroupSink sink) throws Throwable {
		DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
		Date lastModifiedTime = new Date(passwordFileModifiedAt);
		Date syncTime = new Date(System.currentTimeMillis());
		unixSyncSourceInfo.setLastModified(formatter.format(lastModifiedTime));
		unixSyncSourceInfo.setSyncTime(formatter.format(syncTime));
		isUpdateSinkSucc = true;
		if (isChanged() || isStartupFlag) {
			buildUserGroupInfo();
			if (LOG.isDebugEnabled()) {
				LOG.debug("Users = " + sourceUsers.keySet());
				LOG.debug("Groups = " + sourceGroups.keySet());
				LOG.debug("GroupUsers = " + sourceGroupUsers.keySet());
			}

			try {
				sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers);
			} catch (Throwable t) {
				LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t);
				isUpdateSinkSucc = false;
			}
		}
		try {
			sink.postUserGroupAuditInfo(ugsyncAuditInfo);
		} catch (Throwable t) {
			LOG.error("sink.postUserGroupAuditInfo failed with exception: ", t);
		}
		isStartupFlag = false;
	}
	
	
	private void buildUserGroupInfo() throws Throwable {
		groupId2groupNameMap = new HashMap<String, String>();
		sourceUsers = new HashMap<>();
		sourceGroups = new HashMap<>();
		sourceGroupUsers = new HashMap<>();
		groupUserTable = HashBasedTable.create();
		allGroups = new HashSet<>();

		if (OS.startsWith("Mac")) {
			buildUnixGroupList(MAC_GET_ALL_GROUPS_CMD, MAC_GET_GROUP_CMD, false);
			buildUnixUserList(MAC_GET_ALL_USERS_CMD);
		} else {
			if (!OS.startsWith("Linux")) {
				LOG.warn("Platform not recognized assuming Linux compatible");
			}
			buildUnixGroupList(LINUX_GET_ALL_GROUPS_CMD, LINUX_GET_GROUP_CMD, true);
			buildUnixUserList(LINUX_GET_ALL_USERS_CMD);
		}

		Iterator<String> groupUserTableIterator = groupUserTable.rowKeySet().iterator();
		while (groupUserTableIterator.hasNext()) {
			String groupName = groupUserTableIterator.next();
			Map<String,String> groupUsersMap =  groupUserTable.row(groupName);
			Set<String> userSet = new HashSet<String>();
			for(String userName : groupUsersMap.keySet()){
				//String transformUserName = userNameTransform(entry.getKey());
				if (sourceUsers.containsKey(userName)) {
					userSet.add(userName);
				}
			}
			sourceGroupUsers.put(groupName, userSet);
		}

		lastUpdateTime = System.currentTimeMillis();

		if (LOG.isDebugEnabled()) {
			print();
		}
	}
	
	private void print() {
		for(String user : sourceUsers.keySet()) {
			if (LOG.isDebugEnabled()) {
				LOG.debug("USER:" + user);
			}
			Set<String> groups = groupUserTable.column(user).keySet();
			if (groups != null) {
				for(String group : groups) {
					if (LOG.isDebugEnabled()) {
						LOG.debug("\tGROUP: " + group);
					}
				}
			}
		}
	}
	
	private void buildUnixUserList(String command) throws Throwable {
		BufferedReader reader = null;
		Map<String, String> userName2uid = new HashMap<String, String>();

		try {
			if (!useNss) {
				File file = new File(unixPasswordFile);
				passwordFileModifiedAt = file.lastModified();
				FileInputStream fis = new FileInputStream(file);
				reader = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8));
			} else {
				Process process = Runtime.getRuntime().exec(
						new String[]{"bash", "-c", command});

				reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8));
			}

			String line = null;

			while ((line = reader.readLine()) != null) {
				if (line.trim().isEmpty())
					continue;

				String[] tokens = line.split(":");

				int len = tokens.length;

				if (len < 4) {
					LOG.warn("Unable to parse: " + line);
					continue;
				}

				String userName = null;
				String userId = null;
				String groupId = null;

				try {
					userName = tokens[0];
					userId = tokens[2];
					groupId = tokens[3];
				}
				catch(ArrayIndexOutOfBoundsException aiobe) {
					LOG.warn("Ignoring line - [" + line + "]: Unable to parse line for getting user information", aiobe);
					continue;
				}

				int numUserId = -1;

				try {
					numUserId = Integer.parseInt(userId);
				} catch (NumberFormatException nfe) {
					LOG.warn("Unix UserId: [" + userId + "]: can not be parsed as valid int. considering as  -1.", nfe);
					numUserId = -1;
				}

				if (numUserId >= minimumUserId) {
					userName2uid.put(userName, userId);
					String groupName = groupId2groupNameMap.get(groupId);
					if (groupName != null) {
						Map<String, String> userAttrMap = new HashMap<>();
						userAttrMap.put("original_name", userName);
						userAttrMap.put("full_name", userName);
						sourceUsers.put(userName, userAttrMap);
						groupUserTable.put(groupName, userName, groupId);
					} else {
						// we are ignoring the possibility that this user was present in /etc/groups.
						LOG.warn("Group Name could not be found for group id: [" + groupId + "]. Skipping adding user [" + userName + "] with id [" + userId + "].");
					}
				} else {
					if (LOG.isDebugEnabled()) {
						LOG.debug("Skipping user [" + userName + "] since its userid [" + userId + "] is less than minuserid limit [" + minimumUserId + "].");
					}
				}
			}
		} finally {
			if (reader != null)
				reader.close();
		}

		if (!useNss)
			return;

		// this does a reverse check as not all users might be listed in getent passwd
		if (enumerateGroupMembers) {
			String line = null;

			if (LOG.isDebugEnabled()) {
				LOG.debug("Start drill down group members");
			}
			for (String userName : groupUserTable.columnKeySet()) {
				// skip users we already now about
				if (sourceUsers.containsKey(userName))
					continue;

				if (LOG.isDebugEnabled()) {
					LOG.debug("Enumerating user " + userName);
				}

				int numUserId = -1;
				try {
					numUserId = Integer.parseInt(userName2uid.get(userName));
				} catch (NumberFormatException nfe) {
					numUserId = -1;
				}

				// if a user comes from an external group we might not have a uid
				if (numUserId < minimumUserId && numUserId != -1)
					continue;


				// "id" is same across Linux / BSD / MacOSX
				// gids are used as id might return groups with spaces, ie "domain users"
				Process process = Runtime.getRuntime().exec(
						new String[]{"bash", "-c", "id -G " + userName});

				try {
					reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
					line = reader.readLine();
				} finally {
					reader.close();
				}

				if (LOG.isDebugEnabled()) {
					LOG.debug("id -G returned " + line);
				}

				if (line == null || line.trim().isEmpty()) {
					LOG.warn("User " + userName + " could not be resolved");
					continue;
				}

				String[] gids = line.split(" ");

				// check if all groups returned by id are visible to ranger
				//ArrayList<String> allowedGroups = new ArrayList<String>();
				for (String gid : gids) {
					int numGroupId = Integer.parseInt(gid);
					if (numGroupId < minimumGroupId)
						continue;

					String groupName = groupId2groupNameMap.get(gid);
					if (groupName != null) {
						groupUserTable.put(groupName, userName, gid);
						//allowedGroups.add(groupName);
					}
				}
				Map<String, String> userAttrMap = new HashMap<>();
				userAttrMap.put("original_name", userName);
				userAttrMap.put("full_name", userName);
				sourceUsers.put(userName, userAttrMap);
			}
			if (LOG.isDebugEnabled()) {
				LOG.debug("End drill down group members");
			}
		}
	}

	private void parseMembers(String line) {
		if (line == null || line.isEmpty())
			return;

		String[] tokens = line.split(":");

		if (tokens.length < 3)
			return;

		String groupName = tokens[0];
		String groupId = tokens[2];
		String groupMembers = null;

		if (tokens.length > 3)
			groupMembers = tokens[3];

		if (groupId2groupNameMap.containsKey(groupId)) {
			groupId2groupNameMap.remove(groupId);
		}

		int numGroupId = Integer.parseInt(groupId);
		if (numGroupId < minimumGroupId)
			return;

		groupId2groupNameMap.put(groupId, groupName);
		Map<String, String> groupAttrMap = new HashMap<>();
		groupAttrMap.put("original_name", groupName);
		groupAttrMap.put("full_name", groupName);
		sourceGroups.put(groupName, groupAttrMap);

		if (groupMembers != null && !groupMembers.trim().isEmpty()) {
			for (String user : groupMembers.split(",")) {
				groupUserTable.put(groupName, user, groupId);
			}
		}
	}

	private void buildUnixGroupList(String allGroupsCmd, String groupCmd, boolean useGid) throws Throwable {
		if (LOG.isDebugEnabled()) {
			LOG.debug("Start enumerating groups");
		}
		BufferedReader reader = null;

		try {
			if (!useNss) {
				File file = new File(unixGroupFile);
				groupFileModifiedAt = file.lastModified();
				FileInputStream fis = new FileInputStream(file);
				reader = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8));
			} else {
				Process process = Runtime.getRuntime().exec(
						new String[]{"bash", "-c", allGroupsCmd});
				reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8));
			}

			String line = null;

			while ((line = reader.readLine()) != null) {
				if (line.trim().isEmpty())
					continue;

				parseMembers(line);
			}
		} finally {
			if (reader != null)
				reader.close();
		}

		if (LOG.isDebugEnabled()) {
			LOG.debug("End enumerating group");
		}

		if (!useNss)
			return;

		if (enumerateGroupMembers) {
			if (LOG.isDebugEnabled()) {
				LOG.debug("Start enumerating group members");
			}
			String line = null;
			Map<String,String> copy = new HashMap<String, String>(groupId2groupNameMap);

			for (Map.Entry<String, String> group : copy.entrySet()) {
				if (LOG.isDebugEnabled()) {
					LOG.debug("Enumerating group: " + group.getValue() + " GID(" + group.getKey() + ")");
				}

				String command;
				if (useGid) {
					command = String.format(groupCmd, group.getKey());
				} else {
					command = String.format(groupCmd, group.getValue());
				}

				String[] cmd = new String[]{"bash", "-c", command + " " + group.getKey()};
				if (LOG.isDebugEnabled()) {
					LOG.debug("Executing: " + Arrays.toString(cmd));
				}

				try {
					Process process = Runtime.getRuntime().exec(cmd);
					reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
					line = reader.readLine();
				} finally {
					if (reader != null)
						reader.close();
				}
				if (LOG.isDebugEnabled()) {
					LOG.debug("bash -c " + command + " for group " + group + " returned " + line);
				}

				parseMembers(line);
			}
			if (LOG.isDebugEnabled()) {
				LOG.debug("End enumerating group members");
			}
		}

		if (config.getEnumerateGroups() != null) {
			String line = null;
			String[] groups = config.getEnumerateGroups().split(",");

			if (LOG.isDebugEnabled()) {
				LOG.debug("Adding extra groups");
			}

			for (String group : groups) {
				String command = String.format(groupCmd, group);
				String[] cmd = new String[]{"bash", "-c", command + " '" + group + "'"};
				if (LOG.isDebugEnabled()) {
					LOG.debug("Executing: " + Arrays.toString(cmd));
				}

				try {
					Process process = Runtime.getRuntime().exec(cmd);
					reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
					line = reader.readLine();
				} finally {
					if (reader != null)
						reader.close();
				}

				if (LOG.isDebugEnabled()) {
					LOG.debug("bash -c " + command + " for group " + group + " returned " + line);
				}

				parseMembers(line);
			}
			if (LOG.isDebugEnabled()) {
				LOG.debug("Done adding extra groups");
			}
		}
	}

	@VisibleForTesting
	Map<String, Set<String>> getGroupUserListMap() {
		return sourceGroupUsers;
	}

	@VisibleForTesting
	Map<String,String> getGroupId2groupNameMap() {
		return groupId2groupNameMap;
	}

	@VisibleForTesting
	Set<String> getUsers() {
		return sourceUsers.keySet();
	}

}