Fetching directory metadata directly through agents using a custom sync - rpc protocol with consul backend
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java b/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
index a915348..e4d5deb 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java
@@ -20,6 +20,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.airavata.mft.admin.models.rpc.SyncRPCRequest;
import org.apache.airavata.mft.admin.models.rpc.SyncRPCResponse;
+import org.apache.airavata.mft.core.DirectoryResourceMetadata;
import org.apache.airavata.mft.core.FileResourceMetadata;
import org.apache.airavata.mft.core.MetadataCollectorResolver;
import org.apache.airavata.mft.core.api.MetadataCollector;
@@ -63,6 +64,20 @@
return mapper.writeValueAsString(fileResourceMetadata);
}
break;
+ case "getDirectoryResourceMetadata":
+ resourceId = request.getParameters().get("resourceId");
+ resourceType = request.getParameters().get("resourceType");
+ resourceToken = request.getParameters().get("resourceToken");
+ mftAuthorizationToken = request.getParameters().get("mftAuthorizationToken");
+
+ metadataCollectorOp = MetadataCollectorResolver.resolveMetadataCollector(resourceType);
+ if (metadataCollectorOp.isPresent()) {
+ MetadataCollector metadataCollector = metadataCollectorOp.get();
+ metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort);
+ DirectoryResourceMetadata dirResourceMetadata = metadataCollector.getDirectoryResourceMetadata(resourceId, resourceToken);
+ return mapper.writeValueAsString(dirResourceMetadata);
+ }
+ break;
}
logger.error("Unknown method type specified {}", request.getMethod());
throw new Exception("Unknown method " + request.getMethod());
diff --git a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
index a2c7511..21f85bb 100644
--- a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
+++ b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
@@ -26,6 +26,7 @@
import org.apache.airavata.mft.admin.models.rpc.SyncRPCRequest;
import org.apache.airavata.mft.admin.models.rpc.SyncRPCResponse;
import org.apache.airavata.mft.api.service.*;
+import org.apache.airavata.mft.core.DirectoryResourceMetadata;
import org.apache.airavata.mft.core.FileResourceMetadata;
import org.apache.airavata.mft.core.MetadataCollectorResolver;
import org.apache.airavata.mft.core.api.MetadataCollector;
@@ -172,13 +173,55 @@
rpcResponse.getErrorAsStr()));
}
} catch (Exception e) {
- logger.error("Error while fetching resource metadata for resource " + request.getResourceId(), e);
- responseObserver.onError(new Exception("Failed to fetch resource metadata", e));
+ logger.error("Error while fetching resource metadata for file resource " + request.getResourceId(), e);
+ responseObserver.onError(new Exception("Failed to fetch file resource metadata", e));
}
}
@Override
public void getDirectoryResourceMetadata(FetchResourceMetadataRequest request, StreamObserver<DirectoryMetadataResponse> responseObserver) {
- super.getDirectoryResourceMetadata(request, responseObserver);
+ try {
+ SyncRPCResponse rpcResponse = agentRPCClient.sendSyncRequest(SyncRPCRequest.SyncRPCRequestBuilder.builder()
+ .withAgentId(request.getTargetAgentId())
+ .withMessageId(UUID.randomUUID().toString())
+ .withMethod("getDirectoryResourceMetadata")
+ .withParameter("resourceId", request.getResourceId())
+ .withParameter("resourceType", request.getResourceType())
+ .withParameter("resourceToken", request.getResourceToken())
+ .withParameter("mftAuthorizationToken", request.getMftAuthorizationToken())
+ .build());
+
+ switch (rpcResponse.getResponseStatus()) {
+ case SUCCESS:
+ DirectoryResourceMetadata dirResourceMetadata = jsonMapper.readValue(rpcResponse.getResponseAsStr(), DirectoryResourceMetadata.class);
+ DirectoryMetadataResponse.Builder responseBuilder = DirectoryMetadataResponse.newBuilder();
+ dozerBeanMapper.map(dirResourceMetadata, responseBuilder);
+
+ // As dozer mapper can't map collections in protobuf, do it manually for directories and files
+ for (DirectoryResourceMetadata dm : dirResourceMetadata.getDirectories()) {
+ DirectoryMetadataResponse.Builder db = DirectoryMetadataResponse.newBuilder();
+ dozerBeanMapper.map(dm, db);
+ responseBuilder.addDirectories(db);
+ }
+
+ for (FileResourceMetadata fm : dirResourceMetadata.getFiles()) {
+ FileMetadataResponse.Builder fb = FileMetadataResponse.newBuilder();
+ dozerBeanMapper.map(fm, fb);
+ responseBuilder.addFiles(fb);
+ }
+
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
+ return;
+ case FAIL:
+ logger.error("Errored while processing the fetch directory metadata response for resource id {}. Error msg : {}",
+ request.getResourceId(), rpcResponse.getErrorAsStr());
+ responseObserver.onError(new Exception("Errored while processing the the fetch directory metadata response. Error msg : " +
+ rpcResponse.getErrorAsStr()));
+ }
+ } catch (Exception e) {
+ logger.error("Error while fetching directory resource metadata for resource " + request.getResourceId(), e);
+ responseObserver.onError(new Exception("Failed to fetch directory resource metadata", e));
+ }
}
}
diff --git a/examples/src/main/java/org/apache/airavata/mft/examples/metadata/SCPExample.java b/examples/src/main/java/org/apache/airavata/mft/examples/metadata/SCPExample.java
index ed378b8..4d4e0a7 100644
--- a/examples/src/main/java/org/apache/airavata/mft/examples/metadata/SCPExample.java
+++ b/examples/src/main/java/org/apache/airavata/mft/examples/metadata/SCPExample.java
@@ -1,6 +1,7 @@
package org.apache.airavata.mft.examples.metadata;
import org.apache.airavata.mft.api.client.MFTApiClient;
+import org.apache.airavata.mft.api.service.DirectoryMetadataResponse;
import org.apache.airavata.mft.api.service.FetchResourceMetadataRequest;
import org.apache.airavata.mft.api.service.FileMetadataResponse;
import org.apache.airavata.mft.api.service.MFTApiServiceGrpc;
@@ -20,5 +21,18 @@
System.out.println("File metadata response ");
System.out.println(fileResourceMetadata);
System.out.println("Time for processing : " + (endTime - startTime) + " ms");
+
+ startTime = System.currentTimeMillis();
+ DirectoryMetadataResponse directoryMetadataResponse = client.getDirectoryResourceMetadata(FetchResourceMetadataRequest.newBuilder()
+ .setResourceId("remote-ssh-dir-resource")
+ .setResourceType("SCP")
+ .setResourceToken("local-ssh-cred")
+ .setTargetAgentId("agent0")
+ .build());
+ endTime = System.currentTimeMillis();
+
+ System.out.println("Directory metadata response ");
+ System.out.println(directoryMetadataResponse);
+ System.out.println("Time for processing : " + (endTime - startTime) + " ms");
}
}
