| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.stanbol.enhancer.engines.tika; |
| |
| import static java.util.Collections.singleton; |
| import static org.apache.commons.io.IOUtils.closeQuietly; |
| import static org.apache.stanbol.enhancer.engines.tika.TikaEngine.XHTML; |
| import static org.apache.stanbol.enhancer.servicesapi.EnhancementEngine.CANNOT_ENHANCE; |
| import static org.apache.tika.mime.MediaType.OCTET_STREAM; |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertFalse; |
| import static org.junit.Assert.assertNotNull; |
| import static org.junit.Assert.assertNull; |
| import static org.junit.Assert.assertTrue; |
| import static org.junit.Assert.fail; |
| |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.InputStreamReader; |
| import java.io.Reader; |
| import java.nio.charset.Charset; |
| import java.text.DateFormatSymbols; |
| import java.text.ParseException; |
| import java.text.SimpleDateFormat; |
| import java.util.Arrays; |
| import java.util.Date; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.Locale; |
| import java.util.Map.Entry; |
| import java.util.Set; |
| import java.util.regex.Pattern; |
| |
| import org.apache.clerezza.rdf.core.Literal; |
| import org.apache.clerezza.rdf.core.LiteralFactory; |
| import org.apache.clerezza.rdf.core.NonLiteral; |
| import org.apache.clerezza.rdf.core.PlainLiteral; |
| import org.apache.clerezza.rdf.core.Resource; |
| import org.apache.clerezza.rdf.core.Triple; |
| import org.apache.clerezza.rdf.core.TypedLiteral; |
| import org.apache.clerezza.rdf.core.UriRef; |
| import org.apache.clerezza.rdf.ontologies.RDF; |
| import org.apache.clerezza.rdf.ontologies.XSD; |
| import org.apache.commons.io.IOUtils; |
| import org.apache.commons.io.LineIterator; |
| import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory; |
| import org.apache.stanbol.enhancer.servicesapi.Blob; |
| import org.apache.stanbol.enhancer.servicesapi.ContentItem; |
| import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory; |
| import org.apache.stanbol.enhancer.servicesapi.EngineException; |
| import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper; |
| import org.apache.stanbol.enhancer.servicesapi.impl.StreamSource; |
| import org.apache.stanbol.enhancer.servicesapi.impl.StringSource; |
| import org.apache.stanbol.enhancer.servicesapi.rdf.NamespaceEnum; |
| import org.junit.After; |
| import org.junit.AfterClass; |
| import org.junit.Assert; |
| import org.junit.Before; |
| import org.junit.BeforeClass; |
| import org.junit.Test; |
| import org.osgi.service.cm.ConfigurationException; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| public class TikaEngineTest { |
| |
| private static final Logger log = LoggerFactory.getLogger(TikaEngineTest.class); |
| private static final ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance(); |
| private static TikaEngine engine; |
| private static MockComponentContext context; |
| private static LiteralFactory lf = LiteralFactory.getInstance(); |
| /** |
| * Required to make this test independent of the timeZone of the local host. |
| */ |
| private static SimpleDateFormat dateDefaultTimezone = |
| new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", new DateFormatSymbols(Locale.US)); |
| |
| @BeforeClass |
| public static void setUpServices() throws IOException { |
| context = new MockComponentContext(); |
| context.properties.put(TikaEngine.PROPERTY_NAME, "tika"); |
| } |
| |
| @Before |
| public void bindServices() throws ConfigurationException { |
| if(engine == null){ |
| engine = new TikaEngine(ciFactory); |
| engine.activate(context); |
| } |
| } |
| |
| @Test |
| public void testHtml() throws EngineException, IOException { |
| log.info(">>> testHtml <<<"); |
| ContentItem ci = createContentItem("test.html", "text/html; charset=UTF-8"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities such as Paris and people such as Bob Marley."); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| Blob xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| assertContentRegexp(xhtmlBlob, |
| "<html xmlns=\"http://www.w3.org/1999/xhtml\">", |
| "<head>", |
| "<meta name=", |
| "<title>The Apache Stanbol Enhancer</title>", |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities", |
| "</body></html>"); |
| } |
| @Test |
| public void testPdf() throws EngineException, IOException { |
| log.info(">>> testPdf <<<"); |
| //PDF created by Apple Pages |
| ContentItem ci = createContentItem("test.pdf", "application/pdf"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities "); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| Blob xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| assertContentRegexp(xhtmlBlob, |
| "<html xmlns=\"http://www.w3.org/1999/xhtml\">", |
| "<head>", |
| "<meta name=", |
| "<div class=\"page\">", |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities", |
| "</body></html>"); |
| |
| //PDF created by OpenOffice |
| ci = createContentItem("test2.pdf", "application/pdf"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| //validate plain text results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities"); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| assertContentRegexp(xhtmlBlob, |
| "<html xmlns=\"http://www.w3.org/1999/xhtml\">", |
| "<head>", |
| "<meta name=", |
| "<div class=\"page\">", |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities", |
| "</body></html>"); |
| } |
| @Test |
| public void testMsWord() throws EngineException, IOException { |
| log.info(">>> testMsWord <<<"); |
| ContentItem ci = createContentItem("test.doc", "application/msword"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities such as Paris and people such as Bob Marley."); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| Blob xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| assertContentRegexp(xhtmlBlob, |
| "<html xmlns=\"http://www.w3.org/1999/xhtml\">", |
| "<head>", |
| "<meta name=", |
| "<title>", |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities", |
| "</body></html>"); } |
| @Test |
| public void testRtf() throws EngineException, IOException { |
| log.info(">>> testRtf <<<"); |
| ContentItem ci = createContentItem("test.rtf", "application/rtf"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities such as Paris and people such as Bob Marley."); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| Blob xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| assertContentRegexp(xhtmlBlob, |
| "<html xmlns=\"http://www.w3.org/1999/xhtml\">", |
| "<head>", |
| "<meta name=", |
| "<title>", |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities", |
| "</body></html>"); |
| } |
| @Test |
| public void testOdt() throws EngineException, IOException { |
| log.info(">>> testOdt <<<"); |
| ContentItem ci = createContentItem("test.odt", "application/vnd.oasis.opendocument.text"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities such as Paris and people such as Bob Marley."); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| Blob xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| assertContentRegexp(xhtmlBlob, |
| "<html xmlns=\"http://www.w3.org/1999/xhtml\">", |
| "<head>", |
| "<meta name=", |
| "<title>", |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities", |
| "</body></html>"); |
| } |
| @Test |
| public void testEMail() throws EngineException, IOException, ParseException { |
| log.info(">>> testEMail <<<"); |
| ContentItem ci = createContentItem("test.email.txt", "message/rfc822"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "Julien Nioche commented on TIKA-461:", |
| "I'll have a look at mime4j and try to use it in Tika", |
| "> RFC822 messages not parsed", |
| "Key: TIKA-461", |
| "URL: https://issues.apache.org/jira/browse/TIKA-461"); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| Blob xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| assertContentRegexp(xhtmlBlob, |
| "<html xmlns=\"http://www.w3.org/1999/xhtml\">", |
| "<title>\\[jira\\] Commented: \\(TIKA-461\\) RFC822 messages not parsed</title>", |
| "<body><p>", |
| "Julien Nioche commented on TIKA-461:", |
| "I'll have a look at mime4j and try to use it in Tika", |
| "> RFC822 messages not parsed", |
| "Key: TIKA-461", |
| "URL: https://issues.apache.org/jira/browse/TIKA-461"); |
| //no check the extracted metadata! |
| //DC |
| //STANBOL-757: dc:date no longer added by Tika 1.2 (dc:created is still present) |
| //verifyValue(ci, new UriRef(NamespaceEnum.dc+"date"), XSD.dateTime,"2010-09-06T09:25:34Z"); |
| verifyValue(ci, new UriRef(NamespaceEnum.dc+"format"), null,"message/rfc822"); |
| //STANBOL-757: dc:subject no longer added by Tika1.2 (dc:title is used instead) |
| //verifyValue(ci, new UriRef(NamespaceEnum.dc+"subject"), null,"[jira] Commented: (TIKA-461) RFC822 messages not parsed"); |
| verifyValue(ci, new UriRef(NamespaceEnum.dc+"title"), null,"[jira] Commented: (TIKA-461) RFC822 messages not parsed"); |
| verifyValue(ci, new UriRef(NamespaceEnum.dc+"creator"), null,"Julien Nioche (JIRA) <jira@apache.org>"); |
| verifyValue(ci, new UriRef(NamespaceEnum.dc+"created"), XSD.dateTime,"2010-09-06T09:25:34Z"); |
| |
| //Media Ontology |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"creationDate"),XSD.dateTime,"2010-09-06T09:25:34Z"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"hasFormat"),null,"message/rfc822"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"hasCreator"),null,"Julien Nioche (JIRA) <jira@apache.org>"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"hasContributor"),null,"Julien Nioche (JIRA) <jira@apache.org>"); |
| //STANBOL-757: This was present with Tika 1.1 because its mapping from dc:subject |
| // verifyValue(ci, new UriRef(NamespaceEnum.media+"hasKeyword"),null,"[jira] Commented: (TIKA-461) RFC822 messages not parsed"); |
| |
| |
| //Nepomuk Message |
| String message = "http://www.semanticdesktop.org/ontologies/2007/03/22/nmo#"; |
| verifyValue(ci, new UriRef(message+"from"),null,"Julien Nioche (JIRA) <jira@apache.org>"); |
| verifyValue(ci, new UriRef(message+"to"),null,"dev@tika.apache.org"); |
| |
| } |
| @Test |
| public void testMp3() throws EngineException, IOException, ParseException { |
| log.info(">>> testMp3 <<<"); |
| ContentItem ci = createContentItem("testMP3id3v24.mp3", "audio/mpeg"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "Test Title", |
| "Test Artist", |
| "Test Album"); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| Blob xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| //Test AudioTrack metadata |
| NonLiteral audioTrack = verifyNonLiteral(ci, new UriRef(NamespaceEnum.media+"hasTrack")); |
| //types |
| verifyValues(ci, audioTrack, RDF.type, |
| new UriRef(NamespaceEnum.media+"MediaFragment"), |
| new UriRef(NamespaceEnum.media+"Track"), |
| new UriRef(NamespaceEnum.media+"AudioTrack")); |
| //properties |
| verifyValue(ci, audioTrack, new UriRef(NamespaceEnum.media+"hasFormat"), XSD.string, "Stereo"); |
| verifyValue(ci, audioTrack, new UriRef(NamespaceEnum.media+"samplingRate"), XSD.int_, "44100"); |
| verifyValue(ci, audioTrack, new UriRef(NamespaceEnum.media+"hasCompression"), XSD.string, "MP3"); |
| } |
| /** |
| * Tests mappings for the Mp4 metadata extraction capabilities added to |
| * Tika 1.1 (STANBOL-627) |
| * @throws EngineException |
| * @throws IOException |
| * @throws ParseException |
| */ |
| @Test |
| public void testMp4() throws EngineException, IOException, ParseException { |
| log.info(">>> testMp4 <<<"); |
| ContentItem ci = createContentItem("testMP4.m4a", "audio/mp4"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "Test Title", |
| "Test Artist", |
| "Test Album"); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| Blob xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| //Test AudioTrack metadata |
| NonLiteral audioTrack = verifyNonLiteral(ci, new UriRef(NamespaceEnum.media+"hasTrack")); |
| //types |
| verifyValues(ci, audioTrack, RDF.type, |
| new UriRef(NamespaceEnum.media+"MediaFragment"), |
| new UriRef(NamespaceEnum.media+"Track"), |
| new UriRef(NamespaceEnum.media+"AudioTrack")); |
| //properties |
| verifyValue(ci, audioTrack, new UriRef(NamespaceEnum.media+"hasFormat"), XSD.string, "Stereo"); |
| verifyValue(ci, audioTrack, new UriRef(NamespaceEnum.media+"samplingRate"), XSD.int_, "44100"); |
| verifyValue(ci, audioTrack, new UriRef(NamespaceEnum.media+"hasCompression"), XSD.string, "M4A"); |
| } |
| @Test |
| public void testGEOMetadata() throws EngineException, IOException, ParseException{ |
| log.info(">>> testGEOMetadata <<<"); |
| //first validate Media Resource Ontology |
| UriRef hasLocation = new UriRef(NamespaceEnum.media+"hasLocation"); |
| UriRef locationLatitude = new UriRef(NamespaceEnum.media+"locationLatitude"); |
| UriRef locationLongitude = new UriRef(NamespaceEnum.media+"locationLongitude"); |
| //UriRef locationAltitude = new UriRef(NamespaceEnum.media+"locationAltitude"); |
| ContentItem ci = createContentItem("testJPEG_GEO.jpg", OCTET_STREAM.toString());//"video/x-ms-asf"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Iterator<Triple> it = ci.getMetadata().filter(ci.getUri(),hasLocation, null); |
| assertTrue(it.hasNext()); |
| Resource r = it.next().getObject(); |
| assertFalse(it.hasNext()); |
| assertTrue(r instanceof NonLiteral); |
| NonLiteral location = verifyNonLiteral(ci, hasLocation); |
| //lat |
| verifyValue(ci, location, locationLatitude, XSD.double_, "12.54321"); |
| //long |
| verifyValue(ci, location, locationLongitude, XSD.double_, "-54.1234"); |
| |
| //second the GEO ont |
| UriRef lat = new UriRef(NamespaceEnum.geo+"lat"); |
| UriRef lon = new UriRef(NamespaceEnum.geo+"long"); |
| //lat |
| verifyValue(ci, lat, XSD.double_, "12.54321"); |
| //long |
| verifyValue(ci, lon, XSD.double_, "-54.1234"); |
| } |
| |
| |
| |
| public void testMetadata() throws EngineException, ParseException, IOException{ |
| log.info(">>> testMetadata <<<"); |
| ContentItem ci = createContentItem("testMP3id3v24.mp3", "audio/mpeg"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| verifyValue(ci,new UriRef(NamespaceEnum.dc+"creator"),null,"Test Artist"); |
| verifyValue(ci, new UriRef(NamespaceEnum.dc+"title"),null,"Test Album"); |
| verifyValue(ci, new UriRef(NamespaceEnum.dc+"format"),null,"audio/mpeg"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"hasFormat"),null,"audio/mpeg"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"mainOriginalTitle"),null,"Test Album"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"hasContributor"),null,"Test Artist"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"releaseDate"),XSD.string,"2008"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"hasGenre"),null,"Rock"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"hasCreator"),null,"Test Artist"); |
| } |
| @Test |
| public void testExifMetadata() throws EngineException, ParseException, IOException { |
| log.info(">>> testExifMetadata <<<"); |
| String exif = "http://www.semanticdesktop.org/ontologies/2007/05/10/nexif#"; |
| ContentItem ci = createContentItem("testJPEG_EXIF.jpg", "image/jpeg"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| verifyValue(ci, new UriRef(exif+"make"),null,"Canon"); |
| verifyValue(ci, new UriRef(exif+"software"),null,"Adobe Photoshop CS3 Macintosh"); |
| verifyValue(ci, new UriRef(exif+"dateTimeOriginal"),XSD.dateTime,"2009-08-11T09:09:45"); |
| verifyValue(ci, new UriRef(exif+"relatedImageWidth"),XSD.int_,"100"); |
| verifyValue(ci, new UriRef(exif+"fNumber"),XSD.double_,"5.6"); |
| verifyValue(ci, new UriRef(exif+"model"),null,"Canon EOS 40D"); |
| verifyValue(ci, new UriRef(exif+"isoSpeedRatings"),XSD.int_,"400"); |
| verifyValue(ci, new UriRef(exif+"xResolution"),XSD.double_,"240.0"); |
| verifyValue(ci, new UriRef(exif+"flash"),XSD.boolean_,"false"); |
| verifyValue(ci, new UriRef(exif+"exposureTime"),XSD.double_,"6.25E-4"); |
| verifyValue(ci, new UriRef(exif+"yResolution"),XSD.double_,"240.0"); |
| verifyValue(ci, new UriRef(exif+"resolutionUnit"),XSD.string,"Inch"); |
| verifyValue(ci, new UriRef(exif+"focalLength"),XSD.double_,"194.0"); |
| verifyValue(ci, new UriRef(exif+"relatedImageLength"),XSD.int_,"68"); |
| verifyValue(ci, new UriRef(exif+"bitsPerSample"),XSD.int_,"8"); |
| //also Media Ontology mappings for Exif |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"frameHeight"),XSD.int_,"68"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"frameWidth"),XSD.int_,"100"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"hasFormat"),null,"image/jpeg"); |
| verifyValue(ci, new UriRef(NamespaceEnum.media+"creationDate"),XSD.dateTime,"2009-08-11T09:09:45"); |
| verifyValues(ci, new UriRef(NamespaceEnum.media+"hasKeyword"),null,"serbor","moscow-birds","canon-55-250"); |
| //and finally the mapped DC properties |
| verifyValue(ci, new UriRef(NamespaceEnum.dc+"format"),null,"image/jpeg"); |
| verifyValue(ci, new UriRef(NamespaceEnum.dc+"created"),XSD.dateTime,"2009-08-11T09:09:45"); |
| verifyValue(ci, new UriRef(NamespaceEnum.dc+"modified"),XSD.dateTime,"2009-10-02T23:02:49"); |
| verifyValues(ci, new UriRef(NamespaceEnum.dc+"subject"), null, "serbor","moscow-birds","canon-55-250"); |
| } |
| |
| @Test |
| public void testContentTypeDetection() throws EngineException, IOException { |
| log.info(">>> testContentTypeDetection <<<"); |
| ContentItem ci = createContentItem("test.pdf", OCTET_STREAM.toString()); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities"); |
| //validate XHTML results |
| contentPart = ContentItemHelper.getBlob(ci, |
| singleton("application/xhtml+xml")); |
| assertNotNull(contentPart); |
| Blob xhtmlBlob = contentPart.getValue(); |
| assertNotNull(xhtmlBlob); |
| assertContentRegexp(xhtmlBlob, |
| "<html xmlns=\"http://www.w3.org/1999/xhtml\">", |
| "<head>", |
| "<meta name=", |
| "<div class=\"page\">", |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities", |
| "</body></html>"); |
| } |
| /** |
| * Tests that text is not processed |
| */ |
| @Test |
| public void testText() throws EngineException, IOException { |
| log.info(">>> testText <<<"); |
| String text = "The Stanbol enhancer can detect famous cities such as " + |
| "Paris and people such as Bob Marley."; |
| ContentItem ci = ciFactory.createContentItem(new StringSource(text)); |
| Assert.assertEquals(1, ContentItemHelper.getContentParts(ci, Blob.class).size()); |
| } |
| @Test |
| public void testUnsupported() throws EngineException, IOException { |
| log.info(">>> testUnsupported <<<"); |
| ContentItem ci = createContentItem("test.pages", "application/x-iwork-pages-sffpages"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| //it MUST NOT give an error but also not add a content part |
| assertNull(contentPart); |
| //only the original content |
| assertEquals(1, ContentItemHelper.getContentParts(ci, Blob.class).size()); |
| } |
| @Test |
| public void testXhtml() throws EngineException, IOException { |
| log.info(">>> testXhtml <<<"); |
| ContentItem ci = createContentItem("test.xhtml", XHTML.toString()+"; charset=UTF-8"); |
| assertFalse(engine.canEnhance(ci) == CANNOT_ENHANCE); |
| engine.computeEnhancements(ci); |
| Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, |
| singleton("text/plain")); |
| assertNotNull(contentPart); |
| Blob plainTextBlob = contentPart.getValue(); |
| assertNotNull(plainTextBlob); |
| assertContentRegexp(plainTextBlob, |
| "The Apache Stanbol Enhancer", |
| "The Stanbol enhancer can detect famous cities"); |
| //only the original and the plain text |
| // this asserts that no xhtml is parsed from the parsed xhtml content |
| assertEquals(2, ContentItemHelper.getContentParts(ci, Blob.class).size()); |
| } |
| |
| private ContentItem createContentItem(String resourceName, String contentType) throws IOException { |
| InputStream in = TikaEngineTest.class.getClassLoader().getResourceAsStream(resourceName); |
| assertNotNull(in); |
| return ciFactory.createContentItem(new StreamSource(in,contentType)); |
| } |
| /** |
| * Tests if the parsed regex pattern are contained in any line of the parsed |
| * test |
| * @throws IOException |
| */ |
| public void assertContentRegexp(Blob blob, String... regexp) throws IOException { |
| Charset charset; |
| if(blob.getParameter().containsKey("charset")){ |
| charset = Charset.forName(blob.getParameter().get("charset")); |
| } else { |
| charset = Charset.defaultCharset(); |
| } |
| Reader reader = null; |
| nextPattern: |
| for (String expr : regexp) { |
| if(reader != null){ |
| closeQuietly(reader); |
| } |
| final Pattern p = Pattern.compile(".*" + expr + ".*"); |
| reader = new InputStreamReader(blob.getStream(), charset); |
| final LineIterator it = new LineIterator(reader); |
| while (it.hasNext()) { |
| final String line = it.nextLine(); |
| if (p.matcher(line).matches()) { |
| continue nextPattern; |
| } |
| } |
| fail(this + ": no match for regexp '" + expr + "', content=\n" + |
| IOUtils.toString(blob.getStream(), charset.toString())); |
| } |
| } |
| @After |
| public void unbindServices() {/*nothing to do */} |
| |
| @AfterClass |
| public static void shutdownServices() { |
| engine.deactivate(context); |
| engine = null; |
| } |
| |
| /* |
| * Internal helper methods |
| */ |
| private NonLiteral verifyNonLiteral(ContentItem ci, UriRef property){ |
| return verifyNonLiteral(ci, ci.getUri(), property); |
| } |
| private static NonLiteral verifyNonLiteral(ContentItem ci, UriRef subject, UriRef property){ |
| Iterator<Triple> it = ci.getMetadata().filter(subject,property, null); |
| assertTrue(it.hasNext()); |
| Resource r = it.next().getObject(); |
| assertFalse(it.hasNext()); |
| assertTrue(r instanceof NonLiteral); |
| return (NonLiteral)r; |
| } |
| private static UriRef verifyValue(ContentItem ci, UriRef property, UriRef value){ |
| return verifyValue(ci, ci.getUri(), property, value); |
| } |
| private static UriRef verifyValue(ContentItem ci, NonLiteral subject, UriRef property, UriRef value){ |
| Iterator<Triple> it = ci.getMetadata().filter(subject,property, null); |
| assertTrue(it.hasNext()); |
| Resource r = it.next().getObject(); |
| assertFalse(it.hasNext()); |
| assertTrue(r instanceof UriRef); |
| assertEquals(value,r); |
| return (UriRef)r; |
| } |
| private static Literal verifyValue(ContentItem ci, UriRef property, UriRef dataType, String lexValue) throws ParseException{ |
| return verifyValue(ci, ci.getUri(), property, dataType, lexValue); |
| } |
| private static Literal verifyValue(ContentItem ci, NonLiteral subject, UriRef property, UriRef dataType, String lexValue) throws ParseException{ |
| Iterator<Triple> it = ci.getMetadata().filter(subject,property, null); |
| assertTrue(it.hasNext()); |
| Resource r = it.next().getObject(); |
| assertFalse(it.hasNext()); |
| if(dataType == null){ |
| assertTrue(r instanceof PlainLiteral); |
| } else { |
| assertTrue(r instanceof TypedLiteral); |
| assertEquals(dataType, ((TypedLiteral)r).getDataType()); |
| } |
| //if we check dates and the lexical value is not UTC than we need to |
| //consider the time zone of the host running this test |
| if(XSD.dateTime.equals(dataType) && lexValue.charAt(lexValue.length()-1) != 'Z'){ |
| Date expectedDate = dateDefaultTimezone.parse(lexValue); |
| assertEquals(expectedDate, lf.createObject(Date.class, ((TypedLiteral)r))); |
| } else { |
| assertEquals(lexValue,((Literal)r).getLexicalForm()); |
| } |
| return (Literal)r; |
| } |
| private static Set<Literal> verifyValues(ContentItem ci, UriRef property, UriRef dataType, String...lexValues){ |
| return verifyValues(ci, ci.getUri(), property, dataType, lexValues); |
| } |
| private static Set<Literal> verifyValues(ContentItem ci, NonLiteral subject, UriRef property, UriRef dataType, String...lexValues){ |
| Iterator<Triple> it = ci.getMetadata().filter(subject,property, null); |
| assertTrue(it.hasNext()); |
| Set<String> expected = new HashSet<String>(Arrays.asList(lexValues)); |
| Set<Literal> found = new HashSet<Literal>(expected.size()); |
| while(it.hasNext()){ |
| Resource r = it.next().getObject(); |
| if(dataType == null){ |
| assertTrue(r instanceof PlainLiteral); |
| } else { |
| assertTrue(r instanceof TypedLiteral); |
| assertEquals(dataType, ((TypedLiteral)r).getDataType()); |
| } |
| assertTrue(expected.remove(((Literal)r).getLexicalForm())); |
| found.add((Literal)r); |
| } |
| return found; |
| } |
| private static Set<NonLiteral> verifyValues(ContentItem ci, NonLiteral subject, UriRef property, NonLiteral...references){ |
| Iterator<Triple> it = ci.getMetadata().filter(subject,property, null); |
| assertTrue(it.hasNext()); |
| Set<NonLiteral> expected = new HashSet<NonLiteral>(Arrays.asList(references)); |
| Set<NonLiteral> found = new HashSet<NonLiteral>(expected.size()); |
| while(it.hasNext()){ |
| Resource r = it.next().getObject(); |
| assertTrue(r instanceof NonLiteral); |
| assertTrue(expected.remove(r)); |
| found.add((NonLiteral)r); |
| } |
| return found; |
| } |
| |
| } |
