NIFIREG-227 GitFlowPersistenceProvider option to clone repo on startup
- Remove the extra file creation in clone repo
- Fix checkstyle issues
- Added documentation and made changes as per suggestions made by HorizonNet
- Javadoc and documentation improvements
This closes #248.
diff --git a/nifi-registry-core/nifi-registry-docs/src/main/asciidoc/administration-guide.adoc b/nifi-registry-core/nifi-registry-docs/src/main/asciidoc/administration-guide.adoc
index 90b80d5..a9470b4 100644
--- a/nifi-registry-core/nifi-registry-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-registry-core/nifi-registry-docs/src/main/asciidoc/administration-guide.adoc
@@ -1315,11 +1315,13 @@
link:https://git-scm.com/book/en/v2/Git-Internals-The-Refspec[https://git-scm.com/book/en/v2/Git-Internals-The-Refspec^]).
|`Remote Access User`|This username is used to make push requests to the remote repository when `Remote To Push` is enabled, and the remote repository is accessed by HTTP protocol. If SSH is used, user authentication is done with SSH keys.
|`Remote Access Password`|The password for the `Remote Access User`.
+|`Remote Clone Repository`|Remote repository URI to use to clone into `Flow Storage Directory`, if local repository is not present in `Flow Storage Directory`. If left empty the git directory needs to be configured as per <<Initialize Git directory>>. If URI is provided then `Remote Access User` and `Remote Access Password` also should be present.
+Currently, default branch of remote will be cloned.
|====
===== Initialize Git directory
-In order to use `GitFlowPersistenceRepository`, you need to prepare a Git directory on the local file system. You can do so by initializing a directory with `git init` command, or clone an existing Git project from a remote Git repository by `git clone` command.
+In order to use `GitFlowPersistenceRepository`, you need to prepare a Git directory on the local file system. You can do so by initializing a directory with `git init` command, or clone an existing Git project from a remote Git repository by `git clone` command. If you want to clone the default branch of remote repository automatically, set the `Remote Clone Repository` as described above.
- `git init` command
link:https://git-scm.com/docs/git-init[https://git-scm.com/docs/git-init^]
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/flow/git/GitFlowMetaData.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/flow/git/GitFlowMetaData.java
index c5ee277..6a972af 100644
--- a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/flow/git/GitFlowMetaData.java
+++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/flow/git/GitFlowMetaData.java
@@ -18,12 +18,15 @@
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.LsRemoteCommand;
import org.eclipse.jgit.api.PushCommand;
import org.eclipse.jgit.api.Status;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.NoHeadException;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.RepositoryCache;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
@@ -32,6 +35,7 @@
import org.eclipse.jgit.transport.RemoteConfig;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.treewalk.TreeWalk;
+import org.eclipse.jgit.util.FS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@@ -131,6 +135,78 @@
return builder.build();
}
+ private static boolean hasAtLeastOneReference(Repository repo) {
+ logger.info("Checking references for repository {}", repo.toString());
+ for (Ref ref : repo.getAllRefs().values()) {
+ if (ref.getObjectId() == null) {
+ continue;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Check if the provided local repository exists or not, provided by the 'Flow Storage Directory'
+ * configuration in the providers.xml.
+ * @param localRepo {@link File} object of the 'Flow Storage Directory' configuration
+ * @return true if the local repository exists, false otherwise
+ * @throws IOException if the .git directory of the local repository cannot be opened
+ */
+ public boolean localRepoExists(File localRepo) throws IOException {
+ if (!localRepo.isDirectory()) {
+ logger.info("{} is not a directory or does not exist.", localRepo.getPath());
+ return false;
+ }
+
+ if (RepositoryCache.FileKey.isGitRepository(new File(localRepo.getPath()+"/.git"), FS.DETECTED)) {
+ final Git git = Git.open(new File(localRepo.getPath() + "/.git"));
+ final Repository repository = git.getRepository();
+ logger.info("Checking for git references in {}", localRepo.getPath());
+ final boolean referenceExists = hasAtLeastOneReference(repository);
+ if (referenceExists) {
+ logger.info("{} local repository exists with references so no need to clone remote", localRepo.getPath());
+ }
+ // Can be an empty repository if no references are present should we pull from remote?
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Validate if the provided 'Remote Clone Repository' configuration in the providers.xml exists or not.
+ * If the remote repository does not exist, an {@link IllegalArgumentException} will be thrown.
+ * @param remoteRepository the URI value of the 'Remote Clone Repository' configuration
+ * @throws IOException if creating the repository fails
+ */
+ public void remoteRepoExists(String remoteRepository) throws IOException {
+ final Git git = new Git(FileRepositoryBuilder.create(new File(remoteRepository)));
+ final LsRemoteCommand lsCmd = git.lsRemote();
+ try {
+ lsCmd.setRemote(remoteRepository);
+ lsCmd.setCredentialsProvider(this.credentialsProvider);
+ lsCmd.call();
+ } catch (Exception e){
+ throw new IllegalArgumentException("InvalidRemoteRepository : Given remote repository is not valid");
+ }
+ }
+
+ /**
+ * If validation of remote clone repository throws no exception then clone the repository given
+ * in the 'Remote Clone Repository' configuration. Currently the default branch of remote will be cloned.
+ * @param localRepo {@link File} object of the 'Flow Storage Directory' configuration
+ * @param remoteRepository the URI value of the 'Remote Clone Repository' configuration
+ * @throws GitAPIException if unable to call the remote repository
+ */
+ public void cloneRepository(File localRepo, String remoteRepository) throws GitAPIException {
+ logger.info("Cloning the repository {} in {}", remoteRepository, localRepo.getPath());
+ Git.cloneRepository()
+ .setURI(remoteRepository)
+ .setCredentialsProvider(this.credentialsProvider)
+ .setDirectory(localRepo)
+ .call();
+ }
+
@SuppressWarnings("unchecked")
public void loadGitRepository(File gitProjectRootDir) throws IOException, GitAPIException {
gitRepo = openRepository(gitProjectRootDir);
diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/flow/git/GitFlowPersistenceProvider.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/flow/git/GitFlowPersistenceProvider.java
index 08fa467..d511f87 100644
--- a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/flow/git/GitFlowPersistenceProvider.java
+++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/provider/flow/git/GitFlowPersistenceProvider.java
@@ -50,6 +50,7 @@
private static final String REMOTE_TO_PUSH = "Remote To Push";
private static final String REMOTE_ACCESS_USER = "Remote Access User";
private static final String REMOTE_ACCESS_PASSWORD = "Remote Access Password";
+ private static final String REMOTE_CLONE_REPOSITORY = "Remote Clone Repository";
static final String SNAPSHOT_EXTENSION = ".snapshot";
private File flowStorageDir;
@@ -73,6 +74,13 @@
final String remoteUser = props.get(REMOTE_ACCESS_USER);
final String remotePassword = props.get(REMOTE_ACCESS_PASSWORD);
+ final String remoteRepo = props.get(REMOTE_CLONE_REPOSITORY);
+ if (!isEmpty(remoteRepo)) {
+ if (isEmpty(remoteUser) || isEmpty(remotePassword)) {
+ throw new ProviderCreationException(format("The property %s needs remote username and remote password",
+ REMOTE_CLONE_REPOSITORY));
+ }
+ }
if (!isEmpty(remoteUser) && isEmpty(remotePassword)) {
throw new ProviderCreationException(format("The property %s is specified but %s is not." +
" %s is required for username password authentication.",
@@ -84,6 +92,11 @@
try {
flowStorageDir = new File(flowStorageDirValue);
+ final boolean localRepoExists = flowMetaData.localRepoExists(flowStorageDir);
+ if (remoteRepo != null && !remoteRepo.isEmpty() && !localRepoExists){
+ flowMetaData.remoteRepoExists(remoteRepo);
+ flowMetaData.cloneRepository(flowStorageDir, remoteRepo);
+ }
flowMetaData.loadGitRepository(flowStorageDir);
flowMetaData.startPushThread();
logger.info("Configured GitFlowPersistenceProvider with Flow Storage Directory {}",
diff --git a/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/providers.xml b/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/providers.xml
index 3eb138f..70169b5 100644
--- a/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/providers.xml
+++ b/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/providers.xml
@@ -33,6 +33,7 @@
<property name="Remote To Push"></property>
<property name="Remote Access User"></property>
<property name="Remote Access Password"></property>
+ <property name="Remote Clone Repository"></property>
</flowPersistenceProvider>
-->
