| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.oodt.cas.crawl.action; |
| |
| //JDK imports |
| import java.io.File; |
| import java.net.URL; |
| import java.util.List; |
| import java.util.Map; |
| |
| //OODT imports |
| import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException; |
| import org.apache.oodt.cas.filemgr.ingest.Ingester; |
| import org.apache.oodt.cas.filemgr.ingest.StdIngester; |
| import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys; |
| import org.apache.oodt.cas.metadata.Metadata; |
| import org.apache.oodt.cas.metadata.util.PathUtils; |
| |
| //Spring imports |
| import org.springframework.beans.factory.annotation.Required; |
| |
| /** |
| * This action allows the crawler to ingest an ancillary file while crawling |
| * other files. For example, if the crawler is configured to pick up images then |
| * this action could be used to pick up the thumbnails. The ancillary file will |
| * be submitted to the file manager given the product type specified and an |
| * identifier placed in the metadata of the file being ingested. One will have |
| * to set up the product type in the file manager policy to support this action. |
| * This action does not perform metadata extraction and if this is required one |
| * should consider using a server side metadata extractor configured with the |
| * file manager. |
| * |
| * @author pramirez (Paul Ramirez) |
| * @author mattmann (Chris Mattmann) |
| */ |
| public class IngestAncillary extends FileBasedAction { |
| private String fileManagerUrl; |
| private String dataTransferService; |
| private String relatedKey; |
| private String productType; |
| private Map<String, List<String>> ancillaryMetadata; |
| private List<String> copyKeys; |
| private List<String> replaceDynamicKeys; |
| private List<String> writeBackKeys; |
| private String writeBackKeyPrefix; |
| |
| public IngestAncillary() { |
| super(); |
| } |
| |
| @Override |
| public boolean performFileAction(File actionFile, Metadata metadata) |
| throws CrawlerActionException { |
| Metadata ingestMetadata = new Metadata(); |
| if (ancillaryMetadata != null) { |
| for (String key : this.ancillaryMetadata.keySet()) { |
| for (String value : this.ancillaryMetadata.get(key)) { |
| ingestMetadata.addMetadata(key, PathUtils.replaceEnvVariables(value)); |
| } |
| } |
| } |
| |
| if (copyKeys != null) { |
| for (String copyKey : copyKeys) { |
| if (metadata.containsKey(copyKey)) { |
| ingestMetadata.addMetadata(copyKey, metadata.getAllMetadata(copyKey)); |
| } |
| } |
| } |
| |
| if (ingestMetadata.getMetadata(CoreMetKeys.FILE_LOCATION) == null) { |
| ingestMetadata.addMetadata(CoreMetKeys.FILE_LOCATION, |
| actionFile.getParent()); |
| } |
| if (ingestMetadata.getMetadata(CoreMetKeys.FILENAME) == null) { |
| ingestMetadata.addMetadata(CoreMetKeys.FILENAME, actionFile.getName()); |
| } |
| if (ingestMetadata.getMetadata(CoreMetKeys.PRODUCT_NAME) == null) { |
| ingestMetadata.addMetadata(CoreMetKeys.PRODUCT_NAME, actionFile.getName() |
| .substring(0, actionFile.getName().indexOf("."))); |
| } |
| if (ingestMetadata.getMetadata(CoreMetKeys.PRODUCT_TYPE) == null) { |
| ingestMetadata.addMetadata(CoreMetKeys.PRODUCT_TYPE, productType); |
| } |
| |
| try { |
| Ingester ingester = new StdIngester(dataTransferService); |
| String identifier = ingester.ingest(new URL(this.fileManagerUrl), |
| actionFile, ingestMetadata); |
| if (identifier != null) { |
| LOG.info("Succesfully ingested ancillary file " |
| + actionFile.getAbsolutePath() + " with identifier " + identifier); |
| } else { |
| LOG.severe("Failed to ingest ancillary file " |
| + actionFile.getAbsolutePath() |
| + " identifer was not returned from file manager"); |
| } |
| |
| if (relatedKey != null) { |
| metadata.addMetadata(relatedKey, identifier); |
| } |
| |
| if (writeBackKeys != null) { |
| if (writeBackKeyPrefix == null) { |
| writeBackKeyPrefix = ""; |
| } |
| ingestMetadata.addMetadata(writeBackKeyPrefix + CoreMetKeys.PRODUCT_ID, |
| identifier); |
| for (String writeBackKey : writeBackKeys) { |
| metadata.addMetadata(writeBackKeyPrefix + writeBackKey, |
| ingestMetadata.getAllMetadata(writeBackKey)); |
| } |
| } |
| |
| if (replaceDynamicKeys != null) { |
| for (String replaceDynamicKey : replaceDynamicKeys) { |
| String value = PathUtils.replaceEnvVariables( |
| metadata.getMetadata(replaceDynamicKey), metadata); |
| metadata.removeMetadata(replaceDynamicKey); |
| metadata.addMetadata(replaceDynamicKey, value); |
| } |
| } |
| |
| } catch (Exception ex) { |
| LOG.severe("Failed to ingest ancillary file " |
| + actionFile.getAbsolutePath()); |
| LOG.severe(ex.getMessage()); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| public void setReplaceDynamicKeys(List<String> replaceDynamicKeys) { |
| this.replaceDynamicKeys = replaceDynamicKeys; |
| } |
| |
| @Required |
| public void setFileManagerUrl(String fileManagerUrl) { |
| this.fileManagerUrl = fileManagerUrl; |
| } |
| |
| @Required |
| public void setDataTransferService(String dataTransferService) { |
| this.dataTransferService = dataTransferService; |
| } |
| |
| public void setRelatedKey(String relatedKey) { |
| this.relatedKey = relatedKey; |
| } |
| |
| public void setAncillaryMetadata(Map<String, List<String>> ancillaryMetadata) { |
| this.ancillaryMetadata = ancillaryMetadata; |
| } |
| |
| @Required |
| public void setProductType(String productType) { |
| this.productType = productType; |
| } |
| |
| public void setCopyKeys(List<String> copyKeys) { |
| this.copyKeys = copyKeys; |
| } |
| |
| public void setWriteBackKeys(List<String> writeBackKeys) { |
| this.writeBackKeys = writeBackKeys; |
| } |
| |
| public void setWriteBackKeyPrefix(String writeBackKeyPrefix) { |
| this.writeBackKeyPrefix = writeBackKeyPrefix; |
| } |
| |
| public static void main(String[] args) throws Exception { |
| if (args.length != 6) { |
| System.out |
| .println("Usage: java " |
| + IngestAncillary.class.getName() |
| + " <serviceUrl> <transferServiceClass> <productFile> <fileSuffix> <relatedKey> <productType"); |
| System.exit(-1); |
| } |
| IngestAncillary ingest = new IngestAncillary(); |
| ingest.setFileManagerUrl(args[0]); |
| ingest.setDataTransferService(args[1]); |
| File product = new File(args[2]); |
| ingest.setFileSuffix(args[3]); |
| ingest.setRelatedKey(args[4]); |
| ingest.setProductType(args[5]); |
| Metadata metadata = new Metadata(); |
| ingest.performAction(product, metadata); |
| System.out.println("Ingested " + args[5] + ": " |
| + metadata.getMetadata(args[4])); |
| } |
| } |