| /** |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.oodt.opendapps.extractors; |
| |
| //JDK imports |
| import java.net.MalformedURLException; |
| import java.net.URL; |
| import java.util.UUID; |
| import java.util.logging.Level; |
| import java.util.logging.Logger; |
| |
| //OODT imports |
| import org.apache.oodt.cas.metadata.Metadata; |
| import org.apache.oodt.opendapps.config.OpendapConfig; |
| import org.apache.oodt.opendapps.util.ProfileUtils; |
| |
| |
| //Spring imports |
| import org.springframework.util.StringUtils; |
| |
| //THREDDS imports |
| import thredds.catalog.InvAccess; |
| import thredds.catalog.InvDataset; |
| import thredds.catalog.InvDocumentation; |
| import thredds.catalog.InvProperty; |
| import thredds.catalog.ServiceType; |
| import thredds.catalog.ThreddsMetadata.Contributor; |
| import thredds.catalog.ThreddsMetadata.GeospatialCoverage; |
| import thredds.catalog.ThreddsMetadata.Range; |
| import thredds.catalog.ThreddsMetadata.Source; |
| import thredds.catalog.ThreddsMetadata.Vocab; |
| import ucar.nc2.units.DateType; |
| import ucar.unidata.geoloc.LatLonRect; |
| |
| /** |
| * Implementation of {@link MetadataExtractor} that extracts metadata from a |
| * Thredds dataset. |
| * |
| * @author Luca Cinquini |
| * |
| */ |
| public class ThreddsMetadataExtractor implements MetadataExtractor { |
| |
| private static Logger LOG = Logger.getLogger(ThreddsMetadataExtractor.class |
| .getName()); |
| |
| // constant missing for 4.2 version of NetCDF library |
| public final static String SERVICE_TYPE_NCML = "NCML"; |
| |
| /** |
| * The source of metadata to be extracted. |
| */ |
| private final InvDataset dataset; |
| |
| public ThreddsMetadataExtractor(final InvDataset dataset) { |
| this.dataset = dataset; |
| } |
| |
| public void extract(Metadata met, OpendapConfig config) { |
| |
| LOG.log(Level.INFO, "Crawling catalog URL=" + dataset.getCatalogUrl() |
| + " dataset ID=" + dataset.getID()); |
| |
| ProfileUtils.addIfNotNull(met, "Authority", dataset.getAuthority()); |
| ProfileUtils.addIfNotNull(met, "CatalogUrl", dataset.getCatalogUrl()); |
| try { |
| ProfileUtils.addIfNotNull(met, "Host", (new URL(dataset.getCatalogUrl())).getHost() ); |
| } catch(MalformedURLException e) { |
| LOG.log(Level.WARNING, e.getMessage()); |
| } |
| ProfileUtils.addIfNotNull(met, "DatasetFullName", dataset.getFullName()); |
| if (dataset.getContributors() != null) { |
| for (Contributor contributor : dataset.getContributors()) { |
| ProfileUtils.addIfNotNull(met, "Contributor", contributor.getName()); |
| } |
| } |
| |
| if (dataset.getCreators() != null) { |
| for (Source source : dataset.getCreators()) { |
| ProfileUtils.addIfNotNull(met, "Creator", source.getName()); |
| } |
| } |
| |
| if (dataset.getDataFormatType() != null) { |
| ProfileUtils.addIfNotNull(met, "DataFormatType", dataset |
| .getDataFormatType().toString()); |
| } |
| |
| if (dataset.getDataType() != null) { |
| ProfileUtils.addIfNotNull(met, "DataType", dataset.getDataType() |
| .toString()); |
| } |
| |
| if (dataset.getDates() != null) { |
| for (DateType dateType : dataset.getDates()) { |
| String dateString = null; |
| try { |
| dateString = ProfileUtils.toISO8601(dateType.getDate()); |
| } catch (Exception e) { |
| LOG.log(Level.WARNING, |
| "Error converting date: [" + dateType.getDate() + "]: Message: " |
| + e.getMessage()); |
| } |
| ProfileUtils.addIfNotNull(met, "Dates", dateString); |
| } |
| } |
| |
| if (dataset.getDocumentation() != null) { |
| for (InvDocumentation doc : dataset.getDocumentation()) { |
| // textual documentation |
| if (StringUtils.hasText(doc.getInlineContent())) { |
| if (StringUtils.hasText(doc.getType())) { |
| // use specific documentation type, when available |
| ProfileUtils.addIfNotNull(met, doc.getType(), doc.getInlineContent()); |
| } else { |
| // otherwise use generic "Documentation" tag |
| ProfileUtils.addIfNotNull(met, "Documentation", doc.getInlineContent()); |
| } |
| } |
| // hyperlinked documentation |
| if (StringUtils.hasText(doc.getXlinkHref())) { |
| String tuple = this.encodeXlinkTuple(doc.getXlinkHref(), doc.getXlinkTitle(), doc.getType()); |
| ProfileUtils.addIfNotNull(met, "Xlink", tuple); |
| } |
| |
| } |
| } |
| |
| ProfileUtils.addIfNotNull(met, "FullName", dataset.getFullName()); |
| GeospatialCoverage geoCoverage = dataset.getGeospatialCoverage(); |
| |
| if (geoCoverage != null) { |
| |
| LatLonRect bbox = geoCoverage.getBoundingBox(); |
| if (bbox != null) { |
| ProfileUtils.addIfNotNull(met, "SouthwestBC", bbox.getLowerLeftPoint() |
| .toString()); |
| ProfileUtils.addIfNotNull(met, "NorthwestBC", bbox.getUpperLeftPoint() |
| .toString()); |
| ProfileUtils.addIfNotNull(met, "NortheastBC", bbox.getUpperRightPoint() |
| .toString()); |
| ProfileUtils.addIfNotNull(met, "SoutheastBC", bbox.getLowerRightPoint() |
| .toString()); |
| } |
| |
| // try north south, east west |
| if (geoCoverage.getNorthSouthRange() != null) { |
| Range nsRange = geoCoverage.getNorthSouthRange(); |
| ProfileUtils.addIfNotNull(met, "NorthSouthRangeStart", |
| String.valueOf(nsRange.getStart())); |
| ProfileUtils.addIfNotNull(met, "NorthSouthRangeResolution", |
| String.valueOf(nsRange.getResolution())); |
| ProfileUtils.addIfNotNull(met, "NorthSouthRangeSize", |
| String.valueOf(nsRange.getSize())); |
| ProfileUtils.addIfNotNull(met, "NorthSouthRangeUnits", |
| nsRange.getUnits()); |
| ProfileUtils.addIfNotNull(met, "NorthSouthRangeStop", |
| String.valueOf(nsRange.getStart()+nsRange.getSize())); |
| } |
| |
| if (geoCoverage.getEastWestRange() != null) { |
| Range nsRange = geoCoverage.getEastWestRange(); |
| ProfileUtils.addIfNotNull(met, "EastWestRangeStart", |
| String.valueOf(nsRange.getStart())); |
| ProfileUtils.addIfNotNull(met, "EastWestRangeResolution", |
| String.valueOf(nsRange.getResolution())); |
| ProfileUtils.addIfNotNull(met, "EastWestRangeSize", |
| String.valueOf(nsRange.getSize())); |
| ProfileUtils.addIfNotNull(met, "EastWestRangeUnits", |
| nsRange.getUnits()); |
| ProfileUtils.addIfNotNull(met, "EastWestRangeStop", |
| String.valueOf(nsRange.getStart()+nsRange.getSize())); |
| } |
| |
| ProfileUtils.addIfNotNull(met, "GeospatialCoverageLatitudeResolution", |
| String.valueOf(dataset.getGeospatialCoverage().getLatResolution())); |
| ProfileUtils.addIfNotNull(met, "GeospatialCoverageLongitudeResolution", |
| String.valueOf(dataset.getGeospatialCoverage().getLonResolution())); |
| |
| // add geo-spatial coverage alternative form |
| ProfileUtils.addIfNotNull(met, "GeospatialCoverageLatSouth", String.valueOf(dataset.getGeospatialCoverage().getLatSouth())); |
| ProfileUtils.addIfNotNull(met, "GeospatialCoverageLatNorth", String.valueOf(dataset.getGeospatialCoverage().getLatNorth())); |
| ProfileUtils.addIfNotNull(met, "GeospatialCoverageLonWest", String.valueOf(dataset.getGeospatialCoverage().getLonWest())); |
| ProfileUtils.addIfNotNull(met, "GeospatialCoverageLonEast", String.valueOf(dataset.getGeospatialCoverage().getLonEast())); |
| |
| if (dataset.getGeospatialCoverage().getNames() != null) { |
| for (Vocab gName : dataset.getGeospatialCoverage().getNames()) { |
| ProfileUtils.addIfNotNull(met, "GeospatialCoverage", gName.getText()); |
| } |
| } |
| |
| } |
| |
| ProfileUtils.addIfNotNull(met, "History", dataset.getHistory()); |
| if (dataset.getKeywords() != null) { |
| for (Vocab vocab : dataset.getKeywords()) { |
| ProfileUtils.addIfNotNull(met, "Keywords", vocab.getText()); |
| } |
| } |
| ProfileUtils.addIfNotNull(met, "Name", dataset.getName()); |
| ProfileUtils.addIfNotNull(met, "Processing", dataset.getProcessing()); |
| if (dataset.getProjects() != null) { |
| for (Vocab vocab : dataset.getProjects()) { |
| ProfileUtils.addIfNotNull(met, "Projects", vocab.getText()); |
| } |
| } |
| |
| if (dataset.getProperties() != null) { |
| for (InvProperty prop : dataset.getProperties()) { |
| ProfileUtils.addIfNotNull(met, prop.getName(), prop.getValue()); |
| } |
| } |
| |
| if (dataset.getPublishers() != null) { |
| for (Source source : dataset.getPublishers()) { |
| // Note: use "Publisher" (singular) as from the OODT profile specification |
| ProfileUtils.addIfNotNull(met, "Publisher", source.getName()); |
| } |
| } |
| |
| ProfileUtils.addIfNotNull(met, "RestrictAccess", |
| dataset.getRestrictAccess()); |
| if (dataset.getTimeCoverage() != null) { |
| String startDateTimeStr = null, endDateTimeStr = null; |
| try { |
| startDateTimeStr = ProfileUtils.toISO8601(dataset.getTimeCoverage() |
| .getStart().getDate()); |
| endDateTimeStr = ProfileUtils.toISO8601(dataset.getTimeCoverage() |
| .getEnd().getDate()); |
| } catch (Exception e) { |
| LOG.log( |
| Level.WARNING, |
| "Error converting start/end date time strings: Message: " |
| + e.getMessage()); |
| } |
| |
| ProfileUtils.addIfNotNull(met, "StartDateTime", startDateTimeStr); |
| ProfileUtils.addIfNotNull(met, "EndDateTime", endDateTimeStr); |
| } |
| |
| if (dataset.getTimeCoverage() != null |
| && dataset.getTimeCoverage().getResolution() != null) { |
| ProfileUtils.addIfNotNull(met, "TimeCoverageResolution", dataset |
| .getTimeCoverage().getResolution().getText()); |
| } |
| // dataset unique ID |
| ProfileUtils.addIfNotNull(met, "UniqueID", dataset.getUniqueID()); |
| |
| // dataset ID is typically not null |
| ProfileUtils.addIfNotNull(met, "ID", dataset.getID()); |
| |
| // generate a UUID for each dataset, to be used as profile ID |
| ProfileUtils.addIfNotNull(met, "UUID", UUID.randomUUID().toString()); |
| |
| // store access services - only the OpenDAP endpoint for now |
| for (InvAccess access : dataset.getAccess()) { |
| String url = access.getStandardUri().toString(); |
| String type = access.getService().getServiceType().toString(); |
| String name = access.getService().getName(); |
| |
| // add opendap access URL |
| if (type.equalsIgnoreCase(ServiceType.OPENDAP.toString())) { |
| // store opendap URL |
| ProfileUtils.addIfNotNull(met,"OpendapUrl",url); |
| // note: special processing of opendap endpoints since URL in thredds catalog is unusable without a suffix |
| ProfileUtils.addIfNotNull(met,"Access", this.encodeAccessTuple(url+".html", ProfileUtils.MIME_TYPE_OPENDAP_HTML, type)); |
| } |
| } |
| // add TREDDS XML catalog URL |
| String url = dataset.getCatalogUrl(); // catalog_url.xml#dataset_id |
| ProfileUtils.addIfNotNull(met,"Access", this.encodeAccessTuple(url, ProfileUtils.MIME_TYPE_THREDDS, "Catalog/XML")); |
| ProfileUtils.addIfNotNull(met,"Access", this.encodeAccessTuple(url.replaceAll("\\.xml", ".html"), ProfileUtils.MIME_TYPE_HTML, "Catalog/HTML")); |
| |
| } |
| |
| /** |
| * Utility method that joins the parts of an xlink tuple (href, title, type) with a delimiting character. |
| * @param href : the xlink URL, must be not null |
| * @param title : the xlink title, may be null |
| * @param type : the xlink type, may be null |
| * @return |
| */ |
| private String encodeXlinkTuple(final String href, final String title, final String type) { |
| final StringBuilder tuple = new StringBuilder(); |
| tuple.append(href) |
| .append(ProfileUtils.CHAR) |
| .append(StringUtils.hasText(title) ? title : "Reference" ) |
| .append(ProfileUtils.CHAR) |
| .append(StringUtils.hasText(type) ? type : "HTML" ); |
| return tuple.toString(); |
| } |
| |
| /** |
| * Utility method that joins the part of A THREDDS access point (url, service type, service name) with a delimiting character |
| * @param url : the access URL |
| * @param type : the service type, mapped to a mime type |
| * @param name : the service name |
| * @return |
| */ |
| private String encodeAccessTuple(final String url, final String type, final String name) { |
| final StringBuilder tuple = new StringBuilder(); |
| tuple.append(url) |
| .append(ProfileUtils.CHAR) |
| .append(StringUtils.hasText(type) ? type : "") |
| .append(ProfileUtils.CHAR) |
| .append(StringUtils.hasText(name) ? name : ""); |
| return tuple.toString(); |
| } |
| |
| } |
