enhancer/generic/nlp-json/src/main/java/org/apache/stanbol/enhancer/nlp/json/AnalyzedTextParser.java

/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements.  See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.stanbol.enhancer.nlp.json;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;

import javax.swing.JPanel;

import org.apache.commons.io.IOUtils;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.ConfigurationPolicy;
import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.Service;
import org.apache.stanbol.enhancer.nlp.json.valuetype.ValueTypeParser;
import org.apache.stanbol.enhancer.nlp.json.valuetype.ValueTypeParserRegistry;
import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
import org.apache.stanbol.enhancer.nlp.model.AnalysedTextFactory;
import org.apache.stanbol.enhancer.nlp.model.Span;
import org.apache.stanbol.enhancer.nlp.model.Span.SpanTypeEnum;
import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
import org.apache.stanbol.enhancer.servicesapi.Blob;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonToken;
import org.codehaus.jackson.io.SerializedString;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.ArrayNode;
import org.codehaus.jackson.node.ObjectNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Component(immediate=true,policy=ConfigurationPolicy.IGNORE)
@Service(value=AnalyzedTextParser.class)
public class AnalyzedTextParser {
    
    private final Logger log = LoggerFactory.getLogger(AnalyzedTextParser.class);
    
    private final static Charset UTF8 = Charset.forName("UTF-8");
    
    private static AnalyzedTextParser defaultInstance;
    
    protected ObjectMapper mapper = new ObjectMapper();    
    /**
     * Can be used when running outside of OSGI to obtain the default (singleton)
     * instance.
     * @return
     */
    public static final AnalyzedTextParser getDefaultInstance(){
        if(defaultInstance == null){
            defaultInstance = new AnalyzedTextParser(
                ValueTypeParserRegistry.getInstance());
        }
        return defaultInstance;
    }
    
    /**
     * Default constructor used by OSGI
     */
    public AnalyzedTextParser() {}
    
    /**
     * Constructs a new Parser instance for the parsed {@link ValueTypeParserRegistry}
     * instance. Typically this constructor should not be used as usages within
     * an OSGI environment MUST lookup the service via the service registry.
     * Usages outside an OSGI environment should prefer to use the
     * {@link #getDefaultInstance()} instance to obtain the singleton instance.
     * @param vtsr
     */
    public AnalyzedTextParser(ValueTypeParserRegistry vtpr){
        if(vtpr == null){
            throw new IllegalArgumentException("The parsed ValueTypeParserRegistry MUST NOT be NULL!");
        }
        this.valueTypeParserRegistry = vtpr;
    }
    
    @Reference
    protected ValueTypeParserRegistry valueTypeParserRegistry;
    
    /**
     * Parses {@link AnalysedText} {@link Span}s including annotations from the 
     * {@link InputStream}. The {@link AnalysedText} instance that is going to
     * be enrichted with the parsed data needs to be parsed. In the simplest case
     * the caller can create an empty instance by using a 
     * {@link AnalysedTextFactory}.
     * @param in The stream to read the data from
     * @param charset the {@link Charset} used by the stream
     * @param at The {@link AnalysedText} instance used to add the data to
     * @return the parsed {@link AnalysedText} instance enrichted with the
     * information parsed from the Stream
     * @throws IOException on any Error while reading or parsing the data
     * from the Stream
     */
    public AnalysedText parse(InputStream in, Charset charset, final AnalysedText at) throws IOException {
        if(in == null){
            throw new IllegalArgumentException("The parsed InputStream MUST NOT be NULL!");
        }
        if(charset == null){
            charset = UTF8;
        }
        JsonParser parser = mapper.getJsonFactory().createJsonParser(new InputStreamReader(in, charset));
        if(parser.nextToken() != JsonToken.START_OBJECT) { //start object
            throw new IOException("JSON serialized AnalyzedTexts MUST use a JSON Object as Root!");
        }
        if(!parser.nextFieldName(new SerializedString("spans"))){
            throw new IOException("JSON serialized AnalyzedText MUST define the 'spans' field as first entry "
                + "in the root JSON object!");
        }
        if(parser.nextValue() != JsonToken.START_ARRAY){
            throw new IOException("The value of the 'span' field MUST BE an Json Array!");
        }
        boolean first = true;
        while(parser.nextValue() == JsonToken.START_OBJECT){
            if(first){
                parseAnalyzedTextSpan(parser.readValueAsTree(), at);
                first = false;
            } else {
                parseSpan(at, parser.readValueAsTree());
            }
        }
        return at;
    }

    private void parseAnalyzedTextSpan(JsonNode node, AnalysedText at) throws IOException {
        if(node.isObject()){
            ObjectNode jSpan = (ObjectNode)node;
            int[] spanPos = new int[]{-1,-1}; 
            Collection<Entry<String,JsonNode>> jAnnotations = new ArrayList<Entry<String,JsonNode>>(4);
            SpanTypeEnum spanType = parseSpanData(jSpan, spanPos, jAnnotations);
            if(spanType != SpanTypeEnum.Text || spanPos[0] != 0 || spanPos[1] < 0){
                throw new IOException("The AnalyzedText span MUST have the SpanType 'text', a "
                        + "start position of '0' and an end position (ignored, json: "+jSpan);
            }
            if(at.getEnd() != spanPos[1]){
                throw new IOException("The size of the local text '"+at.getEnd()+"' does not "
                    + "match the span of the parsed AnalyzedText ["+spanPos[0]+","+spanPos[1]+"]!");
            }
            parseAnnotations(at, jAnnotations);
        } else {
            throw new IOException("Unable to parse AnalyzedText span form JsonNode "+node+" (expected JSON object)!");
        }
        
    }
    
    private void parseSpan(AnalysedText at, JsonNode node) throws IOException {
        if(node.isObject()){
            ObjectNode jSpan = (ObjectNode)node;
            int[] spanPos = new int[]{-1,-1}; 
            Collection<Entry<String,JsonNode>> jAnnotations = new ArrayList<Entry<String,JsonNode>>(4);
            SpanTypeEnum spanType = parseSpanData(jSpan, spanPos, jAnnotations);
            if(spanType == null || spanPos[0] < 0 || spanPos[1] < 0){
                log.warn("Illegal or missing span type, start and/or end position (ignored, json: "+jSpan);
                return;
            }
            //now create the Span
            Span span;
            switch (spanType) {
                case Text:
                    log.warn("Encounterd 'Text' span that is not the first span in the "
                        + "'spans' array (ignored, json: "+node+")");
                    return;
                case TextSection:
                    log.warn("Encountered 'TextSection' span. This SpanTypeEnum entry "
                        + "is currently unused. If this is no longer the case please "
                        + "update this implementation (ignored, json: "+node+")"); 
                    return;
                case Sentence:
                    span = at.addSentence(spanPos[0], spanPos[1]);
                    break;
                case Chunk:
                    span = at.addChunk(spanPos[0], spanPos[1]);
                    break;
                case Token:
                    span = at.addToken(spanPos[0], spanPos[1]);
                    break;
                default:
                    log.warn("Unsupported SpanTypeEnum  '"+spanType+"'!. Please "
                            + "update this implementation (ignored, json: "+node+")"); 
                    return;
            }
            if(!jAnnotations.isEmpty()){
                parseAnnotations(span,jAnnotations);
            }
        } else {
            log.warn("Unable to parse Span form JsonNode "+node+" (expected JSON object)!");
        }
    }

    /**
     * @param jSpan
     * @param spanPos
     * @param jAnnotations
     * @return the type of the parsed span
     */
    private SpanTypeEnum parseSpanData(ObjectNode jSpan, int[] spanPos,
            Collection<Entry<String,JsonNode>> jAnnotations) {
        SpanTypeEnum spanType = null;
        for(Iterator<Entry<String,JsonNode>> fields = jSpan.getFields(); fields.hasNext();){
            Entry<String,JsonNode> field = fields.next();
            if("type".equals(field.getKey())){
                if(field.getValue().isTextual()){
                    spanType = SpanTypeEnum.valueOf(field.getValue().getTextValue());
                } else if(field.getValue().isInt()){
                    spanType = SpanTypeEnum.values()[field.getValue().getIntValue()];
                } else {
                    log.warn("Unable to parse SpanType form JSON field "+field +" (ignored, json: "+jSpan+")");
                    return null;
                }
            } else if("start".equals(field.getKey())){
                if(field.getValue().isInt()){
                    spanPos[0] = field.getValue().getIntValue();
                } else {
                    log.warn("Unable to parse span start position form JSON field "
                            +field +" (ignored, json: "+jSpan+")");
                    return null;
                }
            } else if("end".equals(field.getKey())){
                if(field.getValue().isInt()){
                    spanPos[1] = field.getValue().getIntValue();
                } else {
                    log.warn("Unable to parse span end position form JSON field "
                            +field +" (ignored, json: "+jSpan+")");
                    return null;
                }
            } else {
                jAnnotations.add(field);
            }
        }
        if(spanType == null){
            log.warn("Missing required field 'type' defining the type of the Span!");
        }
        return spanType;
    }


    private void parseAnnotations(Span span, Collection<Entry<String,JsonNode>> jAnnotations) throws IOException {
        for(Entry<String,JsonNode> jAnnotation : jAnnotations){
            if(jAnnotation.getValue().isObject()){
                parseAnnotation(span, jAnnotation.getKey(), (ObjectNode)jAnnotation.getValue());
            } else if(jAnnotation.getValue().isArray()){
                ArrayNode jValues = (ArrayNode)jAnnotation.getValue();
                for(int i=0;i< jValues.size();i++){
                    JsonNode jValue = jValues.get(i);
                    if(jValue.isObject()){
                        parseAnnotation(span, jAnnotation.getKey(), (ObjectNode)jValue);
                    } else {
                        log.warn("unable to parse the {} value of the annotation {} "
                            + "because value is no JSON object (ignored, json: {}",
                            new Object[]{i,jAnnotation.getKey(),jAnnotation.getValue()});
                    }
                }
            } else {
                log.warn("unable to parse Annotation {} because value is no JSON object (ignored, json: {}",
                    jAnnotation.getKey(),jAnnotation.getValue());
            }
        }
    
    }

    private void parseAnnotation(Span span, String key, ObjectNode jValue) throws IOException {
        JsonNode jClass = jValue.path("class");
        if(!jClass.isTextual()){
            log.warn("unable to parse Annotation {} because 'class' field "
                + "is not set or not a stringis no JSON object (ignored, json: {}",
                key,jValue);
            return;
        }
        Class<?> clazz;
        try {
            clazz = AnalyzedTextParser.class.getClassLoader().loadClass(jClass.getTextValue());
        } catch (ClassNotFoundException e) {
            log.warn("Unable to parse Annotation "+key 
                + " because the 'class' "+jClass.getTextValue()+" of the "
                + "the value can not be resolved (ignored, json: "+jValue+")",e);
            return;
        }
        ValueTypeParser<?> parser = this.valueTypeParserRegistry.getParser(clazz);
        Object value;
        if(parser != null){
            value = parser.parse(jValue);
        } else {
            JsonNode valueNode = jValue.path("value");
            if(valueNode.isMissingNode()){
                log.warn("unable to parse value for annotation {} because the "
                    + "field 'value' is not present (ignored, json: {}",
                    key,jValue);
                return;
            } else {
                try {
                    value = mapper.treeToValue(valueNode, clazz);
                } catch (JsonParseException e) {
                    log.warn("unable to parse value for annotation "
                            + key+ "because the value can"
                            + "not be converted to the class "+ clazz.getName()
                            + "(ignored, json: "+jValue+")",e);
                    return;
                } catch (JsonMappingException e) {
                    log.warn("unable to parse value for annotation "
                            + key+ "because the value can"
                            + "not be converted to the class "+ clazz.getName()
                            + "(ignored, json: "+jValue+")",e);
                    return;
                }
            }
        }
        JsonNode jProb = jValue.path("prob");
        if(!jProb.isDouble()){
            span.addValue(key, Value.value(value));
        } else {
            span.addValue(key, Value.value(value,jProb.getDoubleValue()));
        }        
    }


    /**
     * Parses the SpanType for the parsed {@link ObjectNode} representing a {@link Span}
     * @param jSpan the JSON root node of the span
     * @return the type or <code>null</code> if the information is missing
     */
    private SpanTypeEnum parseSpanType(ObjectNode jSpan) {
        EnumSet<SpanTypeEnum> spanTypes = JsonUtils.parseEnum(jSpan, "type", SpanTypeEnum.class);
        if(spanTypes.isEmpty()){
            log.warn("Unable to parse Span with missing 'type' (json: "+jSpan+")!");
            return null;
        }
        if(spanTypes.size() > 1){
            log.warn("Found Span with multiple 'types' (Json:"+jSpan+")!");
        }
        return spanTypes.iterator().next();
    }
    
}