NIFIREG-353 Add ShellUserGroupProvider and relax checks to allow a user to have same identity as a group
This closes #255.
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java
index 916892b..a819e97 100644
--- a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java
+++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java
@@ -611,7 +611,7 @@
@Override
public User addUser(User user) throws AuthorizationAccessException {
- if (tenantExists(baseConfigurableUserGroupProvider, user.getIdentifier(), user.getIdentity())) {
+ if (userExists(baseConfigurableUserGroupProvider, user.getIdentifier(), user.getIdentity())) {
throw new IllegalStateException(String.format("User/user group already exists with the identity '%s'.", user.getIdentity()));
}
return baseConfigurableUserGroupProvider.addUser(user);
@@ -624,7 +624,7 @@
@Override
public User updateUser(User user) throws AuthorizationAccessException {
- if (tenantExists(baseConfigurableUserGroupProvider, user.getIdentifier(), user.getIdentity())) {
+ if (userExists(baseConfigurableUserGroupProvider, user.getIdentifier(), user.getIdentity())) {
throw new IllegalStateException(String.format("User/user group already exists with the identity '%s'.", user.getIdentity()));
}
if (!baseConfigurableUserGroupProvider.isConfigurable(user)) {
@@ -651,7 +651,7 @@
@Override
public Group addGroup(Group group) throws AuthorizationAccessException {
- if (tenantExists(baseConfigurableUserGroupProvider, group.getIdentifier(), group.getName())) {
+ if (groupExists(baseConfigurableUserGroupProvider, group.getIdentifier(), group.getName())) {
throw new IllegalStateException(String.format("User/user group already exists with the identity '%s'.", group.getName()));
}
if (!allGroupUsersExist(baseUserGroupProvider, group)) {
@@ -667,7 +667,7 @@
@Override
public Group updateGroup(Group group) throws AuthorizationAccessException {
- if (tenantExists(baseConfigurableUserGroupProvider, group.getIdentifier(), group.getName())) {
+ if (groupExists(baseConfigurableUserGroupProvider, group.getIdentifier(), group.getName())) {
throw new IllegalStateException(String.format("User/user group already exists with the identity '%s'.", group.getName()));
}
if (!baseConfigurableUserGroupProvider.isConfigurable(group)) {
@@ -796,14 +796,14 @@
// ensure that only one group exists per identity
for (User user : userGroupProvider.getUsers()) {
- if (tenantExists(userGroupProvider, user.getIdentifier(), user.getIdentity())) {
+ if (userExists(userGroupProvider, user.getIdentifier(), user.getIdentity())) {
throw new SecurityProviderCreationException(String.format("Found multiple users/user groups with identity '%s'.", user.getIdentity()));
}
}
// ensure that only one group exists per identity
for (Group group : userGroupProvider.getGroups()) {
- if (tenantExists(userGroupProvider, group.getIdentifier(), group.getName())) {
+ if (groupExists(userGroupProvider, group.getIdentifier(), group.getName())) {
throw new SecurityProviderCreationException(String.format("Found multiple users/user groups with name '%s'.", group.getName()));
}
}
@@ -896,7 +896,7 @@
* @param identity identity of the tenant
* @return true if another user exists with the same identity, false otherwise
*/
- private static boolean tenantExists(final UserGroupProvider userGroupProvider, final String identifier, final String identity) {
+ private static boolean userExists(final UserGroupProvider userGroupProvider, final String identifier, final String identity) {
for (User user : userGroupProvider.getUsers()) {
if (!user.getIdentifier().equals(identifier)
&& user.getIdentity().equals(identity)) {
@@ -904,6 +904,10 @@
}
}
+ return false;
+ }
+
+ private static boolean groupExists(final UserGroupProvider userGroupProvider, final String identifier, final String identity) {
for (Group group : userGroupProvider.getGroups()) {
if (!group.getIdentifier().equals(identifier)
&& group.getName().equals(identity)) {
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/NssShellCommands.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/NssShellCommands.java
new file mode 100644
index 0000000..eef58b0
--- /dev/null
+++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/NssShellCommands.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.security.authorization.shell;
+
+/**
+ * Provides shell commands to read users and groups on NSS-enabled systems.
+ *
+ * See `man 5 nsswitch.conf` for more info.
+ */
+class NssShellCommands implements ShellCommandsProvider {
+ /**
+ * @return Shell command string that will return a list of users.
+ */
+ public String getUsersList() {
+ return "getent passwd | cut -f 1,3,4 -d ':'";
+ }
+
+ /**
+ * @return Shell command string that will return a list of groups.
+ */
+ public String getGroupsList() {
+ return "getent group | cut -f 1,3 -d ':'";
+ }
+
+ /**
+ * @param groupName name of group.
+ * @return Shell command string that will return a list of users for a group.
+ */
+ public String getGroupMembers(String groupName) {
+ return String.format("getent group %s | cut -f 4 -d ':'", groupName);
+ }
+
+ /**
+ * Gets the command for reading a single user by id.
+ *
+ * When executed, this command should output a single line, in the format used by `getUsersList`.
+ *
+ * @param userId name of user.
+ * @return Shell command string that will read a single user.
+ */
+ @Override
+ public String getUserById(String userId) {
+ return String.format("getent passwd %s | cut -f 1,3,4 -d ':'", userId);
+ }
+
+ /**
+ * This method reuses `getUserById` because the getent command is the same for
+ * both uid and username.
+ *
+ * @param userName name of user.
+ * @return Shell command string that will read a single user.
+ */
+ public String getUserByName(String userName) {
+ return getUserById(userName);
+ }
+
+ /**
+ * This method supports gid or group name because getent does.
+ *
+ * @param groupId name of group.
+ * @return Shell command string that will read a single group.
+ */
+ public String getGroupById(String groupId) {
+ return String.format("getent group %s | cut -f 1,3,4 -d ':'", groupId);
+ }
+
+ /**
+ * This gives exit code 0 on all tested distributions.
+ *
+ * @return Shell command string that will exit normally (0) on a suitable system.
+ */
+ public String getSystemCheck() {
+ return "getent --version";
+ }
+}
\ No newline at end of file
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/OsxShellCommands.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/OsxShellCommands.java
new file mode 100644
index 0000000..0591662
--- /dev/null
+++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/OsxShellCommands.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.security.authorization.shell;
+
+/**
+ * Provides shell commands to read users and groups on Mac OSX systems.
+ *
+ * See `man dscl` for more info.
+ */
+class OsxShellCommands implements ShellCommandsProvider {
+ /**
+ * @return Shell command string that will return a list of users.
+ */
+ public String getUsersList() {
+ return "dscl . -readall /Users UniqueID PrimaryGroupID | awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;} /RecordName: / {name = $2;i = 0;}" +
+ "/PrimaryGroupID: / {gid = $2;} /^ / {if (i == 0) { i++; name = $1;}} /UniqueID: / {uid = $2;print name, uid, gid;}' | grep -v ^_";
+ }
+
+ /**
+ * @return Shell command string that will return a list of groups.
+ */
+ public String getGroupsList() {
+ return "dscl . -list /Groups PrimaryGroupID | grep -v '^_' | sed 's/ \\{1,\\}/:/g'";
+ }
+
+ /**
+ *
+ * @param groupName name of group.
+ * @return Shell command string that will return a list of users for a group.
+ */
+ public String getGroupMembers(String groupName) {
+ return String.format("dscl . -read /Groups/%s GroupMembership | cut -f 2- -d ' ' | sed 's/\\ /,/g'", groupName);
+ }
+
+ /**
+ * @param userId name of user.
+ * @return Shell command string that will read a single user.
+ */
+ @Override
+ public String getUserById(String userId) {
+ return String.format("id -P %s | cut -f 1,3,4 -d ':'", userId);
+ }
+
+ /**
+ * @param userName name of user.
+ * @return Shell command string that will read a single user.
+ */
+ public String getUserByName(String userName) {
+ return getUserById(userName); // 'id' command works for both uid/username
+ }
+
+ /**
+ * @param groupId name of group.
+ * @return Shell command string that will read a single group.
+ */
+ public String getGroupById(String groupId) {
+ return String.format(" dscl . -read /Groups/`dscl . -search /Groups gid %s | head -n 1 | cut -f 1` RecordName PrimaryGroupID | awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;} " +
+ "/RecordName: / {name = $2;i = 1;}/PrimaryGroupID: / {gid = $2;}; {if (i==1) {print name,gid,\"\"}}'", groupId);
+ }
+
+ /**
+ * @return Shell command string that will exit normally (0) on a suitable system.
+ */
+ public String getSystemCheck() {
+ return "which dscl";
+ }
+}
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/RemoteShellCommands.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/RemoteShellCommands.java
new file mode 100644
index 0000000..f622409
--- /dev/null
+++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/RemoteShellCommands.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.security.authorization.shell;
+
+class RemoteShellCommands implements ShellCommandsProvider {
+ // Carefully crafted command replacement string:
+ private final static String remoteCommand = "ssh " +
+ "-o 'StrictHostKeyChecking no' " +
+ "-o 'PasswordAuthentication no' " +
+ "-o \"RemoteCommand %s\" " +
+ "-i %s -p %s -l root %s";
+
+ private ShellCommandsProvider innerProvider;
+ private String privateKeyPath;
+ private String remoteHost;
+ private Integer remotePort;
+
+ private RemoteShellCommands() {
+ }
+
+ public static ShellCommandsProvider wrapOtherProvider(ShellCommandsProvider otherProvider, String keyPath, String host, Integer port) {
+ RemoteShellCommands remote = new RemoteShellCommands();
+
+ remote.innerProvider = otherProvider;
+ remote.privateKeyPath = keyPath;
+ remote.remoteHost = host;
+ remote.remotePort = port;
+
+ return remote;
+ }
+
+ public String getUsersList() {
+ return String.format(remoteCommand, innerProvider.getUsersList(), privateKeyPath, remotePort, remoteHost);
+ }
+
+ public String getGroupsList() {
+ return String.format(remoteCommand, innerProvider.getGroupsList(), privateKeyPath, remotePort, remoteHost);
+ }
+
+ public String getGroupMembers(String groupName) {
+ return String.format(remoteCommand, innerProvider.getGroupMembers(groupName), privateKeyPath, remotePort, remoteHost);
+ }
+
+ public String getUserById(String userId) {
+ return String.format(remoteCommand, innerProvider.getUserById(userId), privateKeyPath, remotePort, remoteHost);
+ }
+
+ public String getUserByName(String userName) {
+ return String.format(remoteCommand, innerProvider.getUserByName(userName), privateKeyPath, remotePort, remoteHost);
+ }
+
+ public String getGroupById(String groupId) {
+ return String.format(remoteCommand, innerProvider.getGroupById(groupId), privateKeyPath, remotePort, remoteHost);
+ }
+
+ public String getSystemCheck() {
+ return String.format(remoteCommand, innerProvider.getSystemCheck(), privateKeyPath, remotePort, remoteHost);
+ }
+}
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellCommandsProvider.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellCommandsProvider.java
new file mode 100644
index 0000000..ce3e6a4
--- /dev/null
+++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellCommandsProvider.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.security.authorization.shell;
+
+/**
+ * Common interface for shell command strings to read users and groups.
+ *
+ */
+interface ShellCommandsProvider {
+ /**
+ * Gets the command for listing users.
+ *
+ * When executed, this command should output one record per line in this format:
+ *
+ * `username:user-id:primary-group-id`
+ *
+ * @return Shell command string that will return a list of users.
+ */
+ String getUsersList();
+
+ /**
+ * Gets the command for listing groups.
+ *
+ * When executed, this command should output one record per line in this format:
+ *
+ * `group-name:group-id`
+ *
+ * @return Shell command string that will return a list of groups.
+ */
+ String getGroupsList();
+
+ /**
+ * Gets the command for listing the members of a group.
+ *
+ * When executed, this command should output one line in this format:
+ *
+ * `user-name-1,user-name-2,user-name-n`
+ *
+ * @param groupName name of group.
+ * @return Shell command string that will return a list of users for a group.
+ */
+ String getGroupMembers(String groupName);
+
+ /**
+ * Gets the command for reading a single user by id. Implementations may return null if reading a single
+ * user by id is not supported.
+ *
+ * When executed, this command should output a single line, in the format used by `getUsersList`.
+ *
+ * @param userId name of user.
+ * @return Shell command string that will read a single user.
+ */
+ String getUserById(String userId);
+
+ /**
+ * Gets the command for reading a single user. Implementations may return null if reading a single user by
+ * username is not supported.
+ *
+ * When executed, this command should output a single line, in the format used by `getUsersList`.
+ *
+ * @param userName name of user.
+ * @return Shell command string that will read a single user.
+ */
+ String getUserByName(String userName);
+
+ /**
+ * Gets the command for reading a single group. Implementations may return null if reading a single group
+ * by name is not supported.
+ *
+ * When executed, this command should output a single line, in the format used by `getGroupsList`.
+ *
+ * @param groupId name of group.
+ * @return Shell command string that will read a single group.
+ */
+ String getGroupById(String groupId);
+
+ /**
+ * Gets the command for checking the suitability of the host system.
+ *
+ * The command is expected to exit with status 0 (zero) to indicate success, and any other status
+ * to indicate failure.
+ *
+ * @return Shell command string that will exit normally (0) on a suitable system.
+ */
+ String getSystemCheck();
+}
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellRunner.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellRunner.java
new file mode 100644
index 0000000..ee7ef41
--- /dev/null
+++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellRunner.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.security.authorization.shell;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class ShellRunner {
+ private final static Logger logger = LoggerFactory.getLogger(ShellRunner.class);
+
+ static String SHELL = "sh";
+ static String OPTS = "-c";
+ static Integer TIMEOUT = 60;
+
+ public static List<String> runShell(String command) throws IOException {
+ return runShell(command, "<unknown>");
+ }
+
+ public static List<String> runShell(String command, String description) throws IOException {
+ final ProcessBuilder builder = new ProcessBuilder(SHELL, OPTS, command);
+ final List<String> builderCommand = builder.command();
+
+ logger.debug("Run Command '" + description + "': " + builderCommand);
+ final Process proc = builder.start();
+
+ boolean completed;
+ try {
+ completed = proc.waitFor(TIMEOUT, TimeUnit.SECONDS);
+ } catch (InterruptedException irexc) {
+ throw new IOException(irexc.getMessage(), irexc.getCause());
+ }
+
+ if (!completed) {
+ throw new IllegalStateException("Shell command '" + command + "' did not complete during the allotted time period");
+ }
+
+ if (proc.exitValue() != 0) {
+ try (final Reader stderr = new InputStreamReader(proc.getErrorStream());
+ final BufferedReader reader = new BufferedReader(stderr)) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ logger.warn(line.trim());
+ }
+ }
+ throw new IOException("Command exit non-zero: " + proc.exitValue());
+ }
+
+ final List<String> lines = new ArrayList<>();
+ try (final Reader stdin = new InputStreamReader(proc.getInputStream());
+ final BufferedReader reader = new BufferedReader(stdin)) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ lines.add(line.trim());
+ }
+ }
+
+ return lines;
+ }
+}
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellUserGroupProvider.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellUserGroupProvider.java
new file mode 100644
index 0000000..1d709ac
--- /dev/null
+++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellUserGroupProvider.java
@@ -0,0 +1,678 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.security.authorization.shell;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.registry.security.authorization.AuthorizerConfigurationContext;
+import org.apache.nifi.registry.security.authorization.Group;
+import org.apache.nifi.registry.security.authorization.User;
+import org.apache.nifi.registry.security.authorization.UserAndGroups;
+import org.apache.nifi.registry.security.authorization.UserGroupProvider;
+import org.apache.nifi.registry.security.authorization.UserGroupProviderInitializationContext;
+import org.apache.nifi.registry.security.authorization.exception.AuthorizationAccessException;
+import org.apache.nifi.registry.security.exception.SecurityProviderCreationException;
+import org.apache.nifi.registry.security.exception.SecurityProviderDestructionException;
+import org.apache.nifi.registry.util.FormatUtils;
+import org.apache.nifi.registry.util.PropertyValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/*
+ * ShellUserGroupProvider implements UserGroupProvider by way of shell commands.
+ */
+public class ShellUserGroupProvider implements UserGroupProvider {
+ private final static Logger logger = LoggerFactory.getLogger(ShellUserGroupProvider.class);
+
+ private final static String OS_TYPE_ERROR = "Unsupported operating system.";
+ private final static String SYS_CHECK_ERROR = "System check failed - cannot provide users and groups.";
+ private final static Map<String, User> usersById = new HashMap<>(); // id == identifier
+ private final static Map<String, User> usersByName = new HashMap<>(); // name == identity
+ private final static Map<String, Group> groupsById = new HashMap<>();
+
+ public static final String REFRESH_DELAY_PROPERTY = "Refresh Delay";
+ private static final long MINIMUM_SYNC_INTERVAL_MILLISECONDS = 10_000;
+
+ public static final String EXCLUDE_USER_PROPERTY = "Exclude Users";
+ public static final String EXCLUDE_GROUP_PROPERTY = "Exclude Groups";
+
+ private long fixedDelay;
+ private Pattern excludeUsers;
+ private Pattern excludeGroups;
+
+ // Our scheduler has one thread for users, one for groups:
+ private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2);
+
+ // Our shell timeout, in seconds:
+ @SuppressWarnings("FieldCanBeLocal")
+ private final Integer shellTimeout = 10;
+
+ // Commands selected during initialization:
+ private ShellCommandsProvider selectedShellCommands;
+
+ // Start of the UserGroupProvider implementation. Javadoc strings
+ // copied from the interface definition for reference.
+
+ /**
+ * Retrieves all users. Must be non null
+ *
+ * @return a list of users
+ * @throws AuthorizationAccessException if there was an unexpected error performing the operation
+ */
+ @Override
+ public Set<User> getUsers() throws AuthorizationAccessException {
+ synchronized (usersById) {
+ logger.debug("getUsers has user set of size: " + usersById.size());
+ return new HashSet<>(usersById.values());
+ }
+ }
+
+ /**
+ * Retrieves the user with the given identifier.
+ *
+ * @param identifier the id of the user to retrieve
+ * @return the user with the given id, or null if no matching user was found
+ * @throws AuthorizationAccessException if there was an unexpected error performing the operation
+ */
+ @Override
+ public User getUser(String identifier) throws AuthorizationAccessException {
+ User user;
+
+ synchronized (usersById) {
+ user = usersById.get(identifier);
+ }
+
+ if (user == null) {
+ logger.debug("getUser (by id) user not found: " + identifier);
+ } else {
+ logger.debug("getUser (by id) found user: " + user + " for id: " + identifier);
+ }
+ return user;
+ }
+
+ /**
+ * Retrieves the user with the given identity.
+ *
+ * @param identity the identity of the user to retrieve
+ * @return the user with the given identity, or null if no matching user was found
+ * @throws AuthorizationAccessException if there was an unexpected error performing the operation
+ */
+ @Override
+ public User getUserByIdentity(String identity) throws AuthorizationAccessException {
+ User user;
+
+ synchronized (usersByName) {
+ user = usersByName.get(identity);
+ }
+
+ if (user == null) {
+ refreshOneUser(selectedShellCommands.getUserByName(identity), "Get Single User by Name");
+ user = usersByName.get(identity);
+ }
+
+ if (user == null) {
+ logger.debug("getUser (by name) user not found: " + identity);
+ } else {
+ logger.debug("getUser (by name) found user: " + user.getIdentity() + " for name: " + identity);
+ }
+ return user;
+ }
+
+ /**
+ * Retrieves all groups. Must be non null
+ *
+ * @return a list of groups
+ * @throws AuthorizationAccessException if there was an unexpected error performing the operation
+ */
+ @Override
+ public Set<Group> getGroups() throws AuthorizationAccessException {
+ synchronized (groupsById) {
+ logger.debug("getGroups has group set of size: " + groupsById.size());
+ return new HashSet<>(groupsById.values());
+ }
+ }
+
+ /**
+ * Retrieves a Group by Id.
+ *
+ * @param identifier the identifier of the Group to retrieve
+ * @return the Group with the given identifier, or null if no matching group was found
+ * @throws AuthorizationAccessException if there was an unexpected error performing the operation
+ */
+ @Override
+ public Group getGroup(String identifier) throws AuthorizationAccessException {
+ Group group;
+
+ synchronized (groupsById) {
+ group = groupsById.get(identifier);
+ }
+
+ if (group == null) {
+ refreshOneGroup(selectedShellCommands.getGroupById(identifier), "Get Single Group by Id");
+ group = groupsById.get(identifier);
+ }
+
+ if (group == null) {
+ logger.debug("getGroup (by id) group not found: " + identifier);
+ } else {
+ logger.debug("getGroup (by id) found group: " + group.getName() + " for id: " + identifier);
+ }
+ return group;
+
+ }
+
+ /**
+ * Gets a user and their groups.
+ *
+ * @return the UserAndGroups for the specified identity
+ * @throws AuthorizationAccessException if there was an unexpected error performing the operation
+ */
+ @Override
+ public UserAndGroups getUserAndGroups(String identity) throws AuthorizationAccessException {
+ User user = getUserByIdentity(identity);
+ logger.debug("Retrieved user {} for identity {}", new Object[]{user, identity});
+
+ Set<Group> groups = new HashSet<>();
+ if (user != null) {
+ for (Group g : getGroups()) {
+ if (g.getUsers().contains(user.getIdentifier())) {
+ logger.debug("User {} belongs to group {}", new Object[]{user.getIdentity(), g.getName()});
+ groups.add(g);
+ }
+ }
+ }
+
+ if (groups.isEmpty()) {
+ logger.debug("User {} belongs to no groups", user);
+ }
+
+ return new UserAndGroups() {
+ @Override
+ public User getUser() {
+ return user;
+ }
+
+ @Override
+ public Set<Group> getGroups() {
+ return groups;
+ }
+ };
+ }
+
+ /**
+ * Called immediately after instance creation for implementers to perform additional setup
+ *
+ * @param initializationContext in which to initialize
+ */
+ @Override
+ public void initialize(UserGroupProviderInitializationContext initializationContext) throws SecurityProviderCreationException {
+ }
+
+ /**
+ * Called to configure the Authorizer.
+ *
+ * @param configurationContext at the time of configuration
+ * @throws SecurityProviderCreationException for any issues configuring the provider
+ */
+ @Override
+ public void onConfigured(AuthorizerConfigurationContext configurationContext) throws SecurityProviderCreationException {
+ fixedDelay = getDelayProperty(configurationContext, REFRESH_DELAY_PROPERTY, "5 mins");
+
+ // Our next init step is to select the command set based on the operating system name:
+ ShellCommandsProvider commands = getCommandsProvider();
+
+ if (commands == null) {
+ commands = getCommandsProviderFromName(null);
+ setCommandsProvider(commands);
+ }
+
+ // Our next init step is to run the system check from that command set to determine if the other commands
+ // will work on this host or not.
+ try {
+ ShellRunner.runShell(commands.getSystemCheck());
+ } catch (final Exception e) {
+ logger.error("initialize exception: " + e + " system check command: " + commands.getSystemCheck());
+ throw new SecurityProviderCreationException(SYS_CHECK_ERROR, e);
+ }
+
+ // The next step is to add the user and group exclude regexes:
+ try {
+ excludeGroups = Pattern.compile(getProperty(configurationContext, EXCLUDE_GROUP_PROPERTY, ""));
+ excludeUsers = Pattern.compile(getProperty(configurationContext, EXCLUDE_USER_PROPERTY, ""));
+ } catch (final PatternSyntaxException e) {
+ throw new SecurityProviderCreationException(e);
+ }
+
+ // With our command set selected, and our system check passed, we can pull in the users and groups:
+ refreshUsersAndGroups();
+
+ // And finally, our last init step is to fire off the refresh thread:
+ scheduler.scheduleWithFixedDelay(() -> {
+ try {
+ refreshUsersAndGroups();
+ }catch (final Throwable t) {
+ logger.error("", t);
+ }
+ }, fixedDelay, fixedDelay, TimeUnit.MILLISECONDS);
+
+ }
+
+ private static ShellCommandsProvider getCommandsProviderFromName(String osName) {
+ if (osName == null) {
+ osName = System.getProperty("os.name");
+ }
+
+ ShellCommandsProvider commands;
+ if (osName.startsWith("Linux")) {
+ logger.debug("Selected Linux command set.");
+ commands = new NssShellCommands();
+ } else if (osName.startsWith("Mac OS X")) {
+ logger.debug("Selected OSX command set.");
+ commands = new OsxShellCommands();
+ } else {
+ throw new SecurityProviderCreationException(OS_TYPE_ERROR);
+ }
+ return commands;
+ }
+
+ private String getProperty(AuthorizerConfigurationContext authContext, String propertyName, String defaultValue) {
+ final PropertyValue property = authContext.getProperty(propertyName);
+ final String value;
+
+ if (property != null && property.isSet()) {
+ value = property.getValue();
+ } else {
+ value = defaultValue;
+ }
+ return value;
+
+ }
+
+ private long getDelayProperty(AuthorizerConfigurationContext authContext, String propertyName, String defaultValue) {
+ final PropertyValue intervalProperty = authContext.getProperty(propertyName);
+ final String propertyValue;
+ final long syncInterval;
+
+ if (intervalProperty.isSet()) {
+ propertyValue = intervalProperty.getValue();
+ } else {
+ propertyValue = defaultValue;
+ }
+
+ try {
+ syncInterval = Math.round(FormatUtils.getPreciseTimeDuration(propertyValue, TimeUnit.MILLISECONDS));
+ } catch (final IllegalArgumentException ignored) {
+ throw new SecurityProviderCreationException(String.format("The %s '%s' is not a valid time interval.", propertyName, propertyValue));
+ }
+
+ if (syncInterval < MINIMUM_SYNC_INTERVAL_MILLISECONDS) {
+ throw new SecurityProviderCreationException(String.format("The %s '%s' is below the minimum value of '%d ms'", propertyName, propertyValue, MINIMUM_SYNC_INTERVAL_MILLISECONDS));
+ }
+ return syncInterval;
+ }
+
+ /**
+ * Called immediately before instance destruction for implementers to release resources.
+ *
+ * @throws SecurityProviderDestructionException If pre-destruction fails.
+ */
+ @Override
+ public void preDestruction() throws SecurityProviderDestructionException {
+ try {
+ scheduler.shutdownNow();
+ } catch (final Exception ignored) {
+ }
+ }
+
+ public ShellCommandsProvider getCommandsProvider() {
+ return selectedShellCommands;
+ }
+
+ public void setCommandsProvider(ShellCommandsProvider commandsProvider) {
+ selectedShellCommands = commandsProvider;
+ }
+
+ /**
+ * Refresh a single user.
+ *
+ * @param command Shell command to read a single user. Pre-formatted by caller.
+ * @param description Shell command description.
+ */
+ private void refreshOneUser(String command, String description) {
+ if (command != null) {
+ Map<String, User> idToUser = new HashMap<>();
+ Map<String, User> usernameToUser = new HashMap<>();
+ Map<String, User> gidToUser = new HashMap<>();
+ List<String> userLines;
+
+ try {
+ userLines = ShellRunner.runShell(command, description);
+ rebuildUsers(userLines, idToUser, usernameToUser, gidToUser);
+ } catch (final IOException ioexc) {
+ logger.error("refreshOneUser shell exception: " + ioexc);
+ }
+
+ if (idToUser.size() > 0) {
+ synchronized (usersById) {
+ usersById.putAll(idToUser);
+ }
+ }
+
+ if (usernameToUser.size() > 0) {
+ synchronized (usersByName) {
+ usersByName.putAll(usernameToUser);
+ }
+ }
+ } else {
+ logger.info("Get Single User not supported on this system.");
+ }
+ }
+
+ /**
+ * Refresh a single group.
+ *
+ * @param command Shell command to read a single group. Pre-formatted by caller.
+ * @param description Shell command description.
+ */
+ private void refreshOneGroup(String command, String description) {
+ if (command != null) {
+ Map<String, Group> gidToGroup = new HashMap<>();
+ List<String> groupLines;
+
+ try {
+ groupLines = ShellRunner.runShell(command, description);
+ rebuildGroups(groupLines, gidToGroup);
+ } catch (final IOException ioexc) {
+ logger.error("refreshOneGroup shell exception: " + ioexc);
+ }
+
+ if (gidToGroup.size() > 0) {
+ synchronized (groupsById) {
+ groupsById.putAll(gidToGroup);
+ }
+ }
+ } else {
+ logger.info("Get Single Group not supported on this system.");
+ }
+ }
+
+ /**
+ * This is our entry point for user and group refresh. This method runs the top-level
+ * `getUserList()` and `getGroupsList()` shell commands, then passes those results to the
+ * other methods for record parse, extract, and object construction.
+ */
+ private void refreshUsersAndGroups() {
+ Map<String, User> uidToUser = new HashMap<>();
+ Map<String, User> usernameToUser = new HashMap<>();
+ Map<String, User> gidToUser = new HashMap<>();
+ Map<String, Group> gidToGroup = new HashMap<>();
+
+ List<String> userLines;
+ List<String> groupLines;
+
+ try {
+ userLines = ShellRunner.runShell(selectedShellCommands.getUsersList(), "Get Users List");
+ groupLines = ShellRunner.runShell(selectedShellCommands.getGroupsList(), "Get Groups List");
+ } catch (final IOException ioexc) {
+ logger.error("refreshUsersAndGroups shell exception: " + ioexc);
+ return;
+ }
+
+ rebuildUsers(userLines, uidToUser, usernameToUser, gidToUser);
+ rebuildGroups(groupLines, gidToGroup);
+ reconcilePrimaryGroups(gidToUser, gidToGroup);
+
+ synchronized (usersById) {
+ usersById.clear();
+ usersById.putAll(uidToUser);
+
+ if (logger.isTraceEnabled()) {
+ logger.trace("=== Users by id...");
+ Set<User> sortedUsers = new TreeSet<>(Comparator.comparing(User::getIdentity));
+ sortedUsers.addAll(usersById.values());
+ sortedUsers.forEach(u -> logger.trace("=== " + u.toString()));
+ }
+ }
+
+ synchronized (usersByName) {
+ usersByName.clear();
+ usersByName.putAll(usernameToUser);
+ logger.debug("users now size: " + usersByName.size());
+ }
+
+ synchronized (groupsById) {
+ groupsById.clear();
+ groupsById.putAll(gidToGroup);
+ logger.debug("groups now size: " + groupsById.size());
+
+ if (logger.isTraceEnabled()) {
+ logger.trace("=== Groups by id...");
+ Set<Group> sortedGroups = new TreeSet<>(Comparator.comparing(Group::getName));
+ sortedGroups.addAll(groupsById.values());
+ sortedGroups.forEach(g -> logger.trace("=== " + g.toString()));
+ }
+ }
+ }
+
+ /**
+ * This method parses the output of the `getUsersList()` shell command, where we expect the output
+ * to look like `user-name:user-id:primary-group-id`.
+ * <p>
+ * This method splits each output line on the ":" and attempts to build a User object
+ * from the resulting name, uid, and primary gid. Unusable records are logged.
+ */
+ private void rebuildUsers(List<String> userLines, Map<String, User> idToUser, Map<String, User> usernameToUser, Map<String, User> gidToUser) {
+ userLines.forEach(line -> {
+ logger.trace("Processing user: {}", new Object[]{line});
+
+ String[] record = line.split(":");
+ if (record.length > 2) {
+ String userIdentity = record[0], userIdentifier = record[1], primaryGroupIdentifier = record[2];
+
+ if (!StringUtils.isBlank(userIdentifier) && !StringUtils.isBlank(userIdentity) && !excludeUsers.matcher(userIdentity).matches()) {
+ User user = new User.Builder()
+ .identity(userIdentity)
+ .identifierGenerateFromSeed(getUserIdentifierSeed(userIdentity))
+ .build();
+ idToUser.put(user.getIdentifier(), user);
+ usernameToUser.put(userIdentity, user);
+ logger.debug("Refreshed user {}", new Object[]{user});
+
+ if (!StringUtils.isBlank(primaryGroupIdentifier)) {
+ // create a temporary group to deterministically generate the group id and associate this user
+ Group group = new Group.Builder()
+ .name(primaryGroupIdentifier)
+ .identifierGenerateFromSeed(getGroupIdentifierSeed(primaryGroupIdentifier))
+ .build();
+ gidToUser.put(group.getIdentifier(), user);
+ logger.debug("Associated primary group {} with user {}", new Object[]{group.getIdentifier(), userIdentity});
+ } else {
+ logger.warn("Null or empty primary group id for: " + userIdentity);
+ }
+
+ } else {
+ logger.warn("Null, empty, or skipped user name: " + userIdentity + " or id: " + userIdentifier);
+ }
+ } else {
+ logger.warn("Unexpected record format. Expected 3 or more colon separated values per line.");
+ }
+ });
+ }
+
+ /**
+ * This method parses the output of the `getGroupsList()` shell command, where we expect the output
+ * to look like `group-name:group-id`.
+ * <p>
+ * This method splits each output line on the ":" and attempts to build a Group object
+ * from the resulting name and gid. Unusable records are logged.
+ * <p>
+ * This command also runs the `getGroupMembers(username)` command once per group. The expected output
+ * of that command should look like `group-name-1,group-name-2`.
+ */
+ private void rebuildGroups(List<String> groupLines, Map<String, Group> groupsById) {
+ groupLines.forEach(line -> {
+ logger.trace("Processing group: {}", new Object[]{line});
+
+ String[] record = line.split(":");
+ if (record.length > 1) {
+ Set<String> users = new HashSet<>();
+ String groupName = record[0], groupIdentifier = record[1];
+
+ try {
+ String groupMembersCommand = selectedShellCommands.getGroupMembers(groupName);
+ List<String> memberLines = ShellRunner.runShell(groupMembersCommand);
+ // Use the first line only, and log if the line count isn't exactly one:
+ if (!memberLines.isEmpty()) {
+ String memberLine = memberLines.get(0);
+ if (!StringUtils.isBlank(memberLine)) {
+ String[] members = memberLine.split(",");
+ for (String userIdentity : members) {
+ if (!StringUtils.isBlank(userIdentity)) {
+ User tempUser = new User.Builder()
+ .identity(userIdentity)
+ .identifierGenerateFromSeed(getUserIdentifierSeed(userIdentity))
+ .build();
+ users.add(tempUser.getIdentifier());
+ logger.debug("Added temp user {} for group {}", new Object[]{tempUser, groupName});
+ }
+ }
+ } else {
+ logger.debug("list membership returned no members");
+ }
+ } else {
+ logger.debug("list membership returned zero lines.");
+ }
+ if (memberLines.size() > 1) {
+ logger.error("list membership returned too many lines, only used the first.");
+ }
+
+ } catch (final IOException ioexc) {
+ logger.error("list membership shell exception: " + ioexc);
+ }
+
+ if (!StringUtils.isBlank(groupIdentifier) && !StringUtils.isBlank(groupName) && !excludeGroups.matcher(groupName).matches()) {
+ Group group = new Group.Builder()
+ .name(groupName)
+ .identifierGenerateFromSeed(getGroupIdentifierSeed(groupIdentifier))
+ .addUsers(users)
+ .build();
+ groupsById.put(group.getIdentifier(), group);
+ logger.debug("Refreshed group {}", new Object[] {group});
+ } else {
+ logger.warn("Null, empty, or skipped group name: " + groupName + " or id: " + groupIdentifier);
+ }
+ } else {
+ logger.warn("Unexpected record format. Expected 1 or more comma separated values.");
+ }
+ });
+ }
+
+ /**
+ * This method parses the output of the `getGroupsList()` shell command, where we expect the output
+ * to look like `group-name:group-id`.
+ * <p>
+ * This method splits each output line on the ":" and attempts to build a Group object
+ * from the resulting name and gid.
+ */
+ private void reconcilePrimaryGroups(Map<String, User> uidToUser, Map<String, Group> gidToGroup) {
+ uidToUser.forEach((primaryGid, primaryUser) -> {
+ Group primaryGroup = gidToGroup.get(primaryGid);
+
+ if (primaryGroup == null) {
+ logger.warn("Primary group {} not found for {}", new Object[]{primaryGid, primaryUser.getIdentity()});
+ } else if (!excludeGroups.matcher(primaryGroup.getName()).matches()) {
+ Set<String> groupUsers = primaryGroup.getUsers();
+ if (!groupUsers.contains(primaryUser.getIdentifier())) {
+ Set<String> updatedUserIdentifiers = new HashSet<>(groupUsers);
+ updatedUserIdentifiers.add(primaryUser.getIdentifier());
+
+ Group updatedGroup = new Group.Builder()
+ .identifier(primaryGroup.getIdentifier())
+ .name(primaryGroup.getName())
+ .addUsers(updatedUserIdentifiers)
+ .build();
+ gidToGroup.put(updatedGroup.getIdentifier(), updatedGroup);
+ logger.debug("Added user {} to primary group {}", new Object[]{primaryUser, updatedGroup});
+ } else {
+ logger.debug("Primary group {} already contains user {}", new Object[]{primaryGroup, primaryUser});
+ }
+ } else {
+ logger.debug("Primary group {} excluded from matcher for {}", new Object[]{primaryGroup.getName(), primaryUser.getIdentity()});
+ }
+ });
+ }
+
+ private String getUserIdentifierSeed(final String userIdentifier) {
+ return userIdentifier + "-user";
+ }
+
+ private String getGroupIdentifierSeed(final String groupIdentifier) {
+ return groupIdentifier + "-group";
+ }
+
+
+ /**
+ * @return The fixed refresh delay.
+ */
+ public long getRefreshDelay() {
+ return fixedDelay;
+ }
+
+ /**
+ * Testing concession for clearing the internal caches.
+ */
+ void clearCaches() {
+ synchronized (usersById) {
+ usersById.clear();
+ }
+
+ synchronized (usersByName) {
+ usersByName.clear();
+ }
+
+ synchronized (groupsById) {
+ groupsById.clear();
+ }
+ }
+
+ /**
+ * @return The size of the internal user cache.
+ */
+ public int userCacheSize() {
+ return usersById.size();
+ }
+
+ /**
+ * @return The size of the internal group cache.
+ */
+ public int groupCacheSize() {
+ return groupsById.size();
+ }
+}
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/resources/META-INF/services/org.apache.nifi.registry.security.authorization.UserGroupProvider b/nifi-registry-core/nifi-registry-framework/src/main/resources/META-INF/services/org.apache.nifi.registry.security.authorization.UserGroupProvider
index ee28c07..a4ac129 100644
--- a/nifi-registry-core/nifi-registry-framework/src/main/resources/META-INF/services/org.apache.nifi.registry.security.authorization.UserGroupProvider
+++ b/nifi-registry-core/nifi-registry-framework/src/main/resources/META-INF/services/org.apache.nifi.registry.security.authorization.UserGroupProvider
@@ -15,4 +15,5 @@
org.apache.nifi.registry.security.authorization.CompositeUserGroupProvider
org.apache.nifi.registry.security.authorization.CompositeConfigurableUserGroupProvider
org.apache.nifi.registry.security.authorization.file.FileUserGroupProvider
-org.apache.nifi.registry.security.ldap.tenants.LdapUserGroupProvider
\ No newline at end of file
+org.apache.nifi.registry.security.ldap.tenants.LdapUserGroupProvider
+org.apache.nifi.registry.security.authorization.shell.ShellUserGroupProvider
\ No newline at end of file
diff --git a/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/authorizers.xml b/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/authorizers.xml
index 9f63754..38a6ee8 100644
--- a/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/authorizers.xml
+++ b/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/authorizers.xml
@@ -166,6 +166,25 @@
To enable the ldap-user-group-provider remove 2 lines. This is 2 of 2. -->
<!--
+ The ShellUserGroupProvider provides support for retrieving users and groups by way of shell commands
+ on systems that support `sh`. Implementations available for Linux and Mac OS, and are selected by the
+ provider based on the system property `os.name`.
+
+ 'Refresh Delay' - duration to wait between subsequent refreshes. Default is '5 mins'.
+ 'Exclude Groups' - regular expression used to exclude groups. Default is '', which means no groups are excluded.
+ 'Exclude Users' - regular expression used to exclude users. Default is '', which means no users are excluded.
+ -->
+ <!-- To enable the shell-user-group-provider remove 2 lines. This is 1 of 2.
+ <userGroupProvider>
+ <identifier>shell-user-group-provider</identifier>
+ <class>org.apache.nifi.registry.security.authorization.shell.ShellUserGroupProvider</class>
+ <property name="Refresh Delay">5 mins</property>
+ <property name="Exclude Groups"></property>
+ <property name="Exclude Users"></property>
+ </userGroupProvider>
+ To enable the shell-user-group-provider remove 2 lines. This is 2 of 2. -->
+
+ <!--
The CompositeUserGroupProvider will provide support for retrieving users and groups from multiple sources.
- User Group Provider [unique key] - The identifier of user group providers to load from. The name of
diff --git a/nifi-registry-core/nifi-registry-utils/src/main/java/org/apache/nifi/registry/util/FormatUtils.java b/nifi-registry-core/nifi-registry-utils/src/main/java/org/apache/nifi/registry/util/FormatUtils.java
index c1e353d..aa207c5 100644
--- a/nifi-registry-core/nifi-registry-utils/src/main/java/org/apache/nifi/registry/util/FormatUtils.java
+++ b/nifi-registry-core/nifi-registry-utils/src/main/java/org/apache/nifi/registry/util/FormatUtils.java
@@ -17,12 +17,13 @@
package org.apache.nifi.registry.util;
import java.text.NumberFormat;
+import java.util.Arrays;
+import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class FormatUtils {
-
private static final String UNION = "|";
// for Data Sizes
@@ -41,8 +42,9 @@
private static final String WEEKS = join(UNION, "w", "wk", "wks", "week", "weeks");
private static final String VALID_TIME_UNITS = join(UNION, NANOS, MILLIS, SECS, MINS, HOURS, DAYS, WEEKS);
- public static final String TIME_DURATION_REGEX = "(\\d+)\\s*(" + VALID_TIME_UNITS + ")";
+ public static final String TIME_DURATION_REGEX = "([\\d.]+)\\s*(" + VALID_TIME_UNITS + ")";
public static final Pattern TIME_DURATION_PATTERN = Pattern.compile(TIME_DURATION_REGEX);
+ private static final List<Long> TIME_UNIT_MULTIPLIERS = Arrays.asList(1000L, 1000L, 1000L, 60L, 60L, 24L);
/**
* Formats the specified count by adding commas.
@@ -58,7 +60,7 @@
* Formats the specified duration in 'mm:ss.SSS' format.
*
* @param sourceDuration the duration to format
- * @param sourceUnit the unit to interpret the duration
+ * @param sourceUnit the unit to interpret the duration
* @return representation of the given time data in minutes/seconds
*/
public static String formatMinutesSeconds(final long sourceDuration, final TimeUnit sourceUnit) {
@@ -79,7 +81,7 @@
* Formats the specified duration in 'HH:mm:ss.SSS' format.
*
* @param sourceDuration the duration to format
- * @param sourceUnit the unit to interpret the duration
+ * @param sourceUnit the unit to interpret the duration
* @return representation of the given time data in hours/minutes/seconds
*/
public static String formatHoursMinutesSeconds(final long sourceDuration, final TimeUnit sourceUnit) {
@@ -139,65 +141,230 @@
return format.format(dataSize) + " bytes";
}
+ /**
+ * Returns a time duration in the requested {@link TimeUnit} after parsing the {@code String}
+ * input. If the resulting value is a decimal (i.e.
+ * {@code 25 hours -> TimeUnit.DAYS = 1.04}), the value is rounded.
+ *
+ * @param value the raw String input (i.e. "28 minutes")
+ * @param desiredUnit the requested output {@link TimeUnit}
+ * @return the whole number value of this duration in the requested units
+ * @deprecated As of Apache NiFi 1.9.0, because this method only returns whole numbers, use {@link #getPreciseTimeDuration(String, TimeUnit)} when possible.
+ */
+ @Deprecated
public static long getTimeDuration(final String value, final TimeUnit desiredUnit) {
+ return Math.round(getPreciseTimeDuration(value, desiredUnit));
+ }
+
+ /**
+ * Returns the parsed and converted input in the requested units.
+ * <p>
+ * If the value is {@code 0 <= x < 1} in the provided units, the units will first be converted to a smaller unit to get a value >= 1 (i.e. 0.5 seconds -> 500 milliseconds).
+ * This is because the underlying unit conversion cannot handle decimal values.
+ * <p>
+ * If the value is {@code x >= 1} but x is not a whole number, the units will first be converted to a smaller unit to attempt to get a whole number value (i.e. 1.5 seconds -> 1500 milliseconds).
+ * <p>
+ * If the value is {@code x < 1000} and the units are {@code TimeUnit.NANOSECONDS}, the result will be a whole number of nanoseconds, rounded (i.e. 123.4 ns -> 123 ns).
+ * <p>
+ * This method handles decimal values over {@code 1 ns}, but {@code < 1 ns} will return {@code 0} in any other unit.
+ * <p>
+ * Examples:
+ * <p>
+ * "10 seconds", {@code TimeUnit.MILLISECONDS} -> 10_000.0
+ * "0.010 s", {@code TimeUnit.MILLISECONDS} -> 10.0
+ * "0.010 s", {@code TimeUnit.SECONDS} -> 0.010
+ * "0.010 ns", {@code TimeUnit.NANOSECONDS} -> 1
+ * "0.010 ns", {@code TimeUnit.MICROSECONDS} -> 0
+ *
+ * @param value the {@code String} input
+ * @param desiredUnit the desired output {@link TimeUnit}
+ * @return the parsed and converted amount (without a unit)
+ */
+ public static double getPreciseTimeDuration(final String value, final TimeUnit desiredUnit) {
final Matcher matcher = TIME_DURATION_PATTERN.matcher(value.toLowerCase());
if (!matcher.matches()) {
- throw new IllegalArgumentException("Value '" + value + "' is not a valid Time Duration");
+ throw new IllegalArgumentException("Value '" + value + "' is not a valid time duration");
}
final String duration = matcher.group(1);
final String units = matcher.group(2);
- TimeUnit specifiedTimeUnit = null;
- switch (units.toLowerCase()) {
- case "ns":
- case "nano":
- case "nanos":
- case "nanoseconds":
- specifiedTimeUnit = TimeUnit.NANOSECONDS;
- break;
- case "ms":
- case "milli":
- case "millis":
- case "milliseconds":
- specifiedTimeUnit = TimeUnit.MILLISECONDS;
- break;
- case "s":
- case "sec":
- case "secs":
- case "second":
- case "seconds":
- specifiedTimeUnit = TimeUnit.SECONDS;
- break;
- case "m":
- case "min":
- case "mins":
- case "minute":
- case "minutes":
- specifiedTimeUnit = TimeUnit.MINUTES;
- break;
- case "h":
- case "hr":
- case "hrs":
- case "hour":
- case "hours":
- specifiedTimeUnit = TimeUnit.HOURS;
- break;
- case "d":
- case "day":
- case "days":
- specifiedTimeUnit = TimeUnit.DAYS;
- break;
+
+ double durationVal = Double.parseDouble(duration);
+ TimeUnit specifiedTimeUnit;
+
+ // The TimeUnit enum doesn't have a value for WEEKS, so handle this case independently
+ if (isWeek(units)) {
+ specifiedTimeUnit = TimeUnit.DAYS;
+ durationVal *= 7;
+ } else {
+ specifiedTimeUnit = determineTimeUnit(units);
+ }
+
+ // The units are now guaranteed to be in DAYS or smaller
+ long durationLong;
+ if (durationVal == Math.rint(durationVal)) {
+ durationLong = Math.round(durationVal);
+ } else {
+ // Try reducing the size of the units to make the input a long
+ List wholeResults = makeWholeNumberTime(durationVal, specifiedTimeUnit);
+ durationLong = (long) wholeResults.get(0);
+ specifiedTimeUnit = (TimeUnit) wholeResults.get(1);
+ }
+
+ return desiredUnit.convert(durationLong, specifiedTimeUnit);
+ }
+
+ /**
+ * Converts the provided time duration value to one that can be represented as a whole number.
+ * Returns a {@code List} containing the new value as a {@code long} at index 0 and the
+ * {@link TimeUnit} at index 1. If the incoming value is already whole, it is returned as is.
+ * If the incoming value cannot be made whole, a whole approximation is returned. For values
+ * {@code >= 1 TimeUnit.NANOSECONDS}, the value is rounded (i.e. 123.4 ns -> 123 ns).
+ * For values {@code < 1 TimeUnit.NANOSECONDS}, the constant [1L, {@code TimeUnit.NANOSECONDS}] is returned as the smallest measurable unit of time.
+ * <p>
+ * Examples:
+ * <p>
+ * 1, {@code TimeUnit.SECONDS} -> [1, {@code TimeUnit.SECONDS}]
+ * 1.1, {@code TimeUnit.SECONDS} -> [1100, {@code TimeUnit.MILLISECONDS}]
+ * 0.1, {@code TimeUnit.SECONDS} -> [100, {@code TimeUnit.MILLISECONDS}]
+ * 0.1, {@code TimeUnit.NANOSECONDS} -> [1, {@code TimeUnit.NANOSECONDS}]
+ *
+ * @param decimal the time duration as a decimal
+ * @param timeUnit the current time unit
+ * @return the time duration as a whole number ({@code long}) and the smaller time unit used
+ */
+ protected static List<Object> makeWholeNumberTime(double decimal, TimeUnit timeUnit) {
+ // If the value is already a whole number, return it and the current time unit
+ if (decimal == Math.rint(decimal)) {
+ return Arrays.asList(new Object[]{(long) decimal, timeUnit});
+ } else if (TimeUnit.NANOSECONDS == timeUnit) {
+ // The time unit is as small as possible
+ if (decimal < 1.0) {
+ decimal = 1;
+ } else {
+ decimal = Math.rint(decimal);
+ }
+ return Arrays.asList(new Object[]{(long) decimal, timeUnit});
+ } else {
+ // Determine the next time unit and the respective multiplier
+ TimeUnit smallerTimeUnit = getSmallerTimeUnit(timeUnit);
+ long multiplier = calculateMultiplier(timeUnit, smallerTimeUnit);
+
+ // Recurse with the original number converted to the smaller unit
+ return makeWholeNumberTime(decimal * multiplier, smallerTimeUnit);
+ }
+ }
+
+ /**
+ * Returns the numerical multiplier to convert a value from {@code originalTimeUnit} to
+ * {@code newTimeUnit} (i.e. for {@code TimeUnit.DAYS -> TimeUnit.MINUTES} would return
+ * 24 * 60 = 1440). If the original and new units are the same, returns 1. If the new unit
+ * is larger than the original (i.e. the result would be less than 1), throws an
+ * {@link IllegalArgumentException}.
+ *
+ * @param originalTimeUnit the source time unit
+ * @param newTimeUnit the destination time unit
+ * @return the numerical multiplier between the units
+ */
+ protected static long calculateMultiplier(TimeUnit originalTimeUnit, TimeUnit newTimeUnit) {
+ if (originalTimeUnit == newTimeUnit) {
+ return 1;
+ } else if (originalTimeUnit.ordinal() < newTimeUnit.ordinal()) {
+ throw new IllegalArgumentException("The original time unit '" + originalTimeUnit + "' must be larger than the new time unit '" + newTimeUnit + "'");
+ } else {
+ int originalOrd = originalTimeUnit.ordinal();
+ int newOrd = newTimeUnit.ordinal();
+
+ List<Long> unitMultipliers = TIME_UNIT_MULTIPLIERS.subList(newOrd, originalOrd);
+ return unitMultipliers.stream().reduce(1L, (a, b) -> (long) a * b);
+ }
+ }
+
+ /**
+ * Returns the next smallest {@link TimeUnit} (i.e. {@code TimeUnit.DAYS -> TimeUnit.HOURS}).
+ * If the parameter is {@code null} or {@code TimeUnit.NANOSECONDS}, an
+ * {@link IllegalArgumentException} is thrown because there is no valid smaller TimeUnit.
+ *
+ * @param originalUnit the TimeUnit
+ * @return the next smaller TimeUnit
+ */
+ protected static TimeUnit getSmallerTimeUnit(TimeUnit originalUnit) {
+ if (originalUnit == null || TimeUnit.NANOSECONDS == originalUnit) {
+ throw new IllegalArgumentException("Cannot determine a smaller time unit than '" + originalUnit + "'");
+ } else {
+ return TimeUnit.values()[originalUnit.ordinal() - 1];
+ }
+ }
+
+ /**
+ * Returns {@code true} if this raw unit {@code String} is parsed as representing "weeks", which does not have a value in the {@link TimeUnit} enum.
+ *
+ * @param rawUnit the String containing the desired unit
+ * @return true if the unit is "weeks"; false otherwise
+ */
+ protected static boolean isWeek(final String rawUnit) {
+ switch (rawUnit) {
case "w":
case "wk":
case "wks":
case "week":
case "weeks":
- final long durationVal = Long.parseLong(duration);
- return desiredUnit.convert(durationVal, TimeUnit.DAYS)*7;
+ return true;
+ default:
+ return false;
}
+ }
- final long durationVal = Long.parseLong(duration);
- return desiredUnit.convert(durationVal, specifiedTimeUnit);
+ /**
+ * Returns the {@link TimeUnit} enum that maps to the provided raw {@code String} input. The
+ * highest time unit is {@code TimeUnit.DAYS}. Any input that cannot be parsed will result in
+ * an {@link IllegalArgumentException}.
+ *
+ * @param rawUnit the String to parse
+ * @return the TimeUnit
+ */
+ protected static TimeUnit determineTimeUnit(String rawUnit) {
+ switch (rawUnit.toLowerCase()) {
+ case "ns":
+ case "nano":
+ case "nanos":
+ case "nanoseconds":
+ return TimeUnit.NANOSECONDS;
+ case "µs":
+ case "micro":
+ case "micros":
+ case "microseconds":
+ return TimeUnit.MICROSECONDS;
+ case "ms":
+ case "milli":
+ case "millis":
+ case "milliseconds":
+ return TimeUnit.MILLISECONDS;
+ case "s":
+ case "sec":
+ case "secs":
+ case "second":
+ case "seconds":
+ return TimeUnit.SECONDS;
+ case "m":
+ case "min":
+ case "mins":
+ case "minute":
+ case "minutes":
+ return TimeUnit.MINUTES;
+ case "h":
+ case "hr":
+ case "hrs":
+ case "hour":
+ case "hours":
+ return TimeUnit.HOURS;
+ case "d":
+ case "day":
+ case "days":
+ return TimeUnit.DAYS;
+ default:
+ throw new IllegalArgumentException("Could not parse '" + rawUnit + "' to TimeUnit");
+ }
}
public static String formatUtilization(final double utilization) {
@@ -225,15 +392,15 @@
* 3 seconds, 8 millis, 3 nanos - if includeTotalNanos = false,
* 3 seconds, 8 millis, 3 nanos (3008000003 nanos) - if includeTotalNanos = true
*
- * @param nanos the number of nanoseconds to format
+ * @param nanos the number of nanoseconds to format
* @param includeTotalNanos whether or not to include the total number of nanoseconds in parentheses in the returned value
* @return a human-readable String that is a formatted representation of the given number of nanoseconds.
*/
public static String formatNanos(final long nanos, final boolean includeTotalNanos) {
final StringBuilder sb = new StringBuilder();
- final long seconds = nanos > 1000000000L ? nanos / 1000000000L : 0L;
- long millis = nanos > 1000000L ? nanos / 1000000L : 0L;
+ final long seconds = nanos >= 1000000000L ? nanos / 1000000000L : 0L;
+ long millis = nanos >= 1000000L ? nanos / 1000000L : 0L;
final long nanosLeft = nanos % 1000000L;
if (seconds > 0) {
@@ -258,4 +425,5 @@
return sb.toString();
}
+
}