/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.schema;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.Reader;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.document.Field;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.AttributeSource.State;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.schema.PreAnalyzedField.ParseResult;
import org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser;

/**
 * Simple plain text format parser for {@link PreAnalyzedField}.
 * <h2>Serialization format</h2>
 * <p>The format of the serialization is as follows:
 * <pre>
 * content ::= version (stored)? tokens
 * version ::= digit+ " "
 * ; stored field value - any "=" inside must be escaped!
 * stored ::= "=" text "="
 * tokens ::= (token ((" ") + token)*)*
 * token ::= text ("," attrib)*
 * attrib ::= name '=' value
 * name ::= text
 * value ::= text
 * </pre>
 * <p>Special characters in "text" values can be escaped
 * using the escape character \ . The following escape sequences are recognized:
 * <pre>
 * "\ " - literal space character
 * "\," - literal , character
 * "\=" - literal = character
 * "\\" - literal \ character
 * "\n" - newline
 * "\r" - carriage return
 * "\t" - horizontal tab
 * </pre>
 * Please note that Unicode sequences (e.g. &#92;u0001) are not supported.
 * <h2>Supported attribute names</h2>
 * The following token attributes are supported, and identified with short
 * symbolic names:
 * <pre>
 * i - position increment (integer)
 * s - token offset, start position (integer)
 * e - token offset, end position (integer)
 * t - token type (string)
 * f - token flags (hexadecimal integer)
 * p - payload (bytes in hexadecimal format; whitespace is ignored)
 * </pre>
 * Token offsets are tracked and implicitly added to the token stream -
 * the start and end offsets consider only the term text and whitespace,
 * and exclude the space taken by token attributes.
 * <h2>Example token streams</h2>
 * <pre>
 * 1 one two three
  - version 1
  - stored: 'null'
  - tok: '(term=one,startOffset=0,endOffset=3)'
  - tok: '(term=two,startOffset=4,endOffset=7)'
  - tok: '(term=three,startOffset=8,endOffset=13)'
 1 one  two   three 
  - version 1
  - stored: 'null'
  - tok: '(term=one,startOffset=0,endOffset=3)'
  - tok: '(term=two,startOffset=5,endOffset=8)'
  - tok: '(term=three,startOffset=11,endOffset=16)'
1 one,s=123,e=128,i=22  two three,s=20,e=22
  - version 1
  - stored: 'null'
  - tok: '(term=one,positionIncrement=22,startOffset=123,endOffset=128)'
  - tok: '(term=two,positionIncrement=1,startOffset=5,endOffset=8)'
  - tok: '(term=three,positionIncrement=1,startOffset=20,endOffset=22)'
1 \ one\ \,,i=22,a=\, two\=

  \n,\ =\   \
  - version 1
  - stored: 'null'
  - tok: '(term= one ,,positionIncrement=22,startOffset=0,endOffset=6)'
  - tok: '(term=two=

  
 ,positionIncrement=1,startOffset=7,endOffset=15)'
  - tok: '(term=\,positionIncrement=1,startOffset=17,endOffset=18)'
1 ,i=22 ,i=33,s=2,e=20 , 
  - version 1
  - stored: 'null'
  - tok: '(term=,positionIncrement=22,startOffset=0,endOffset=0)'
  - tok: '(term=,positionIncrement=33,startOffset=2,endOffset=20)'
  - tok: '(term=,positionIncrement=1,startOffset=2,endOffset=2)'
1 =This is the stored part with \= 
 \n    \t escapes.=one two three 
  - version 1
  - stored: 'This is the stored part with = 
 \n    \t escapes.'
  - tok: '(term=one,startOffset=0,endOffset=3)'
  - tok: '(term=two,startOffset=4,endOffset=7)'
  - tok: '(term=three,startOffset=8,endOffset=13)'
1 ==
  - version 1
  - stored: ''
  - (no tokens)
1 =this is a test.=
  - version 1
  - stored: 'this is a test.'
  - (no tokens)
 * </pre> 
 */
public final class SimplePreAnalyzedParser implements PreAnalyzedParser {
  static final String VERSION = "1";
  
  private static class Tok {
    StringBuilder token = new StringBuilder();
    Map<String, String> attr = new HashMap<>();
    
    public boolean isEmpty() {
      return token.length() == 0 && attr.size() == 0;
    }
    
    public void reset() {
      token.setLength(0);
      attr.clear();
    }
    
    @Override
    public String toString() {
      return "tok='" + token + "',attr=" + attr;
    }
  }
  
  // parser state
  private static enum S {TOKEN, NAME, VALUE, UNDEF};
  
  private static final byte[] EMPTY_BYTES = new byte[0];
  
  /** Utility method to convert a hex string to a byte array. */
  static byte[] hexToBytes(String hex) {
    if (hex == null) {
      return EMPTY_BYTES;
    }
    hex = hex.replaceAll("\\s+", "");
    if (hex.length() == 0) {
      return EMPTY_BYTES;
    }
    ByteArrayOutputStream baos = new ByteArrayOutputStream(hex.length() / 2);
    byte b;
    for (int i = 0; i < hex.length(); i++) {
      int high = charToNibble(hex.charAt(i));
      int low = 0;
      if (i < hex.length() - 1) {
        i++;
        low = charToNibble(hex.charAt(i));
      }
      b = (byte)(high << 4 | low);
      baos.write(b);
    }
    return baos.toByteArray();
  }

  static final int charToNibble(char c) {
    if (c >= '0' && c <= '9') {
      return c - '0';
    } else if (c >= 'a' && c <= 'f') {
      return 0xa + (c - 'a');
    } else if (c >= 'A' && c <= 'F') {
      return 0xA + (c - 'A');
    } else {
      throw new RuntimeException("Not a hex character: '" + c + "'");
    }
  }
  
  static String bytesToHex(byte bytes[], int offset, int length) {
    StringBuilder sb = new StringBuilder();
    for (int i = offset; i < offset + length; ++i) {
      sb.append(Integer.toHexString(0x0100 + (bytes[i] & 0x00FF))
                       .substring(1));
    }
    return sb.toString();
  }
  
  public SimplePreAnalyzedParser() {
    
  }

  @Override
  public ParseResult parse(Reader reader, AttributeSource parent) throws IOException {
    ParseResult res = new ParseResult();
    StringBuilder sb = new StringBuilder();
    char[] buf = new char[128];
    int cnt;
    while ((cnt = reader.read(buf)) > 0) {
      sb.append(buf, 0, cnt);
    }
    String val = sb.toString();
    // empty string - accept even without version number
    if (val.length() == 0) {
      return res;
    }
    // first consume the version
    int idx = val.indexOf(' ');
    if (idx == -1) {
      throw new IOException("Missing VERSION token");
    }
    String version = val.substring(0, idx);
    if (!VERSION.equals(version)) {
      throw new IOException("Unknown VERSION " + version);
    }
    val = val.substring(idx + 1);
    // then consume the optional stored part
    int tsStart = 0;
    boolean hasStored = false;
    StringBuilder storedBuf = new StringBuilder();
    if (val.charAt(0) == '=') {
      hasStored = true;
      if (val.length() > 1) {
        for (int i = 1; i < val.length(); i++) {
          char c = val.charAt(i);
          if (c == '\\') {
            if (i < val.length() - 1) {
              c = val.charAt(++i);
              if (c == '=') { // we recognize only \= escape in the stored part
                storedBuf.append('=');
              } else {
                storedBuf.append('\\');
                storedBuf.append(c);
                continue;
              }
            } else {
              storedBuf.append(c);
              continue;
            }
          } else if (c == '=') {
            // end of stored text
            tsStart = i + 1;
            break;
          } else {
            storedBuf.append(c);
          }
        }
        if (tsStart == 0) { // missing end-of-stored marker
          throw new IOException("Missing end marker of stored part");
        }
      } else {
        throw new IOException("Unexpected end of stored field");
      }
    }
    if (hasStored) {
      res.str = storedBuf.toString();
    }
    Tok tok = new Tok();
    StringBuilder attName = new StringBuilder();
    StringBuilder attVal = new StringBuilder();
    // parser state
    S s = S.UNDEF;
    int lastPos = 0;
    for (int i = tsStart; i < val.length(); i++) {
      char c = val.charAt(i);
      if (c == ' ') {
        // collect leftovers
        switch (s) {
        case VALUE :
          if (attVal.length() == 0) {
            throw new IOException("Unexpected character '" + c + "' at position " + i + " - empty value of attribute.");
          }
          if (attName.length() > 0) {
            tok.attr.put(attName.toString(), attVal.toString());
          }
          break;
        case NAME: // attr name without a value ?
          if (attName.length() > 0) {
            throw new IOException("Unexpected character '" + c + "' at position " + i + " - missing attribute value.");
          } else {
            // accept missing att name and value
          }
          break;
        case TOKEN:
        case UNDEF:
          // do nothing, advance to next token
        }
        attName.setLength(0);
        attVal.setLength(0);
        if (!tok.isEmpty() || s == S.NAME) {
          AttributeSource.State state = createState(parent, tok, lastPos);
          if (state != null) res.states.add(state.clone());
        }
        // reset tok
        s = S.UNDEF;
        tok.reset();
        // skip
        lastPos++;
        continue;
      }
      StringBuilder tgt = null;
      switch (s) {
      case TOKEN:
        tgt = tok.token;
        break;
      case NAME:
        tgt = attName;
        break;
      case VALUE:
        tgt = attVal;
        break;
      case UNDEF:
        tgt = tok.token;
        s = S.TOKEN;
      }
      if (c == '\\') {
        if (s == S.TOKEN) lastPos++;
        if (i >= val.length() - 1) { // end
          
          tgt.append(c);
          continue;
        } else {
          c = val.charAt(++i);
          switch (c) {
          case '\\' :
          case '=' :
          case ',' :
          case ' ' :
            tgt.append(c);
            break;
          case 'n':
            tgt.append('\n');
            break;
          case 'r':
            tgt.append('\r');
            break;
          case 't':
            tgt.append('\t');
            break;
          default:
            tgt.append('\\');
            tgt.append(c);
            lastPos++;
          }
        }
      } else {
        // state switch
        if (c == ',') {
          if (s == S.TOKEN) {
            s = S.NAME;
          } else if (s == S.VALUE) { // end of value, start of next attr
            if (attVal.length() == 0) {
              throw new IOException("Unexpected character '" + c + "' at position " + i + " - empty value of attribute.");
            }
            if (attName.length() > 0 && attVal.length() > 0) {
              tok.attr.put(attName.toString(), attVal.toString());
            }
            // reset
            attName.setLength(0);
            attVal.setLength(0);
            s = S.NAME;
          } else {
            throw new IOException("Unexpected character '" + c + "' at position " + i + " - missing attribute value.");
          }
        } else if (c == '=') {
          if (s == S.NAME) {
            s = S.VALUE;
          } else {
            throw new IOException("Unexpected character '" + c + "' at position " + i + " - empty value of attribute.");
          }
        } else {
          tgt.append(c);
          if (s == S.TOKEN) lastPos++;
        }
      }
    }
    // collect leftovers
    if (!tok.isEmpty() || s == S.NAME || s == S.VALUE) {
      // remaining attrib?
      if (s == S.VALUE) {
        if (attName.length() > 0 && attVal.length() > 0) {
          tok.attr.put(attName.toString(), attVal.toString());
        }        
      }
      AttributeSource.State state = createState(parent, tok, lastPos);
      if (state != null) res.states.add(state.clone());
    }
    return res;
  }
  
  private static AttributeSource.State createState(AttributeSource a, Tok state, int tokenEnd) {
    a.clearAttributes();
    CharTermAttribute termAtt = a.addAttribute(CharTermAttribute.class);
    char[] tokChars = state.token.toString().toCharArray();
    termAtt.copyBuffer(tokChars, 0, tokChars.length);
    int tokenStart = tokenEnd - state.token.length();
    for (Entry<String, String> e : state.attr.entrySet()) {
      String k = e.getKey();
      if (k.equals("i")) {
        // position increment
        int incr = Integer.parseInt(e.getValue());
        PositionIncrementAttribute posIncr = a.addAttribute(PositionIncrementAttribute.class);
        posIncr.setPositionIncrement(incr);
      } else if (k.equals("s")) {
        tokenStart = Integer.parseInt(e.getValue());
      } else if (k.equals("e")) {
        tokenEnd = Integer.parseInt(e.getValue());
      } else if (k.equals("y")) {
        TypeAttribute type = a.addAttribute(TypeAttribute.class);
        type.setType(e.getValue());
      } else if (k.equals("f")) {
        FlagsAttribute flags = a.addAttribute(FlagsAttribute.class);
        int f = Integer.parseInt(e.getValue(), 16);
        flags.setFlags(f);
      } else if (k.equals("p")) {
        PayloadAttribute p = a.addAttribute(PayloadAttribute.class);
        byte[] data = hexToBytes(e.getValue());
        if (data != null && data.length > 0) {
          p.setPayload(new BytesRef(data));
        }
      } else {
        // unknown attribute
      }
    }
    // handle offset attr
    OffsetAttribute offset = a.addAttribute(OffsetAttribute.class);
    offset.setOffset(tokenStart, tokenEnd);
    State resState = a.captureState();
    a.clearAttributes();
    return resState;
  }

  @Override
  public String toFormattedString(Field f) throws IOException {
    StringBuilder sb = new StringBuilder();
    sb.append(VERSION + " ");
    if (f.fieldType().stored()) {
      String s = f.stringValue();
      if (s != null) {
        // encode the equals sign
        s = s.replaceAll("=", "\\=");
        sb.append('=');
        sb.append(s);
        sb.append('=');
      }
    }
    TokenStream ts = f.tokenStreamValue();
    if (ts != null) {
      StringBuilder tok = new StringBuilder();
      boolean next = false;
      while (ts.incrementToken()) {
        if (next) {
          sb.append(' ');
        } else {
          next = true;
        }
        tok.setLength(0);
        Iterator<Class<? extends Attribute>> it = ts.getAttributeClassesIterator();
        String cTerm = null;
        String tTerm = null;
        while (it.hasNext()) {
          Class<? extends Attribute> cl = it.next();
          Attribute att = ts.getAttribute(cl);
          if (att == null) {
            continue;
          }
          if (cl.isAssignableFrom(CharTermAttribute.class)) {
            CharTermAttribute catt = (CharTermAttribute)att;
            cTerm = escape(catt.buffer(), catt.length());
          } else if (cl.isAssignableFrom(TermToBytesRefAttribute.class)) {
            TermToBytesRefAttribute tatt = (TermToBytesRefAttribute)att;
            char[] tTermChars = tatt.getBytesRef().utf8ToString().toCharArray();
            tTerm = escape(tTermChars, tTermChars.length);
          } else {
            if (tok.length() > 0) tok.append(',');
            if (cl.isAssignableFrom(FlagsAttribute.class)) {
              tok.append("f=").append(Integer.toHexString(((FlagsAttribute) att).getFlags()));
            } else if (cl.isAssignableFrom(OffsetAttribute.class)) {
              tok.append("s=").append(((OffsetAttribute) att).startOffset()).append(",e=").append(((OffsetAttribute) att).endOffset());
            } else if (cl.isAssignableFrom(PayloadAttribute.class)) {
              BytesRef p = ((PayloadAttribute)att).getPayload();
              if (p != null && p.length > 0) {
                tok.append("p=").append(bytesToHex(p.bytes, p.offset, p.length));
              } else if (tok.length() > 0) {
                tok.setLength(tok.length() - 1); // remove the last comma
              }
            } else if (cl.isAssignableFrom(PositionIncrementAttribute.class)) {
              tok.append("i=").append(((PositionIncrementAttribute) att).getPositionIncrement());
            } else if (cl.isAssignableFrom(TypeAttribute.class)) {
              tok.append("y=").append(escape(((TypeAttribute) att).type()));
            } else {
              
              tok.append(cl.getName()).append('=').append(escape(att.toString()));
            }
          }
        }
        String term = null;
        if (cTerm != null) {
          term = cTerm;
        } else {
          term = tTerm;
        }
        if (term != null && term.length() > 0) {
          if (tok.length() > 0) {
            tok.insert(0, term + ",");
          } else {
            tok.insert(0, term);
          }
        }
        sb.append(tok);
      }
    }
    return sb.toString();
  }
    
  String escape(String val) {
    return escape(val.toCharArray(), val.length());
  }
  
  String escape(char[] val, int len) {
    if (val == null || len == 0) {
      return "";
    }
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < len; i++) {
      switch (val[i]) {
      case '\\' :
      case '=' :
      case ',' :
      case ' ' :
        sb.append('\\');
        sb.append(val[i]);
        break;
      case '\n' :
        sb.append('\\');
        sb.append('n');
        break;
      case '\r' :
        sb.append('\\');
        sb.append('r');
        break;
      case '\t' :
        sb.append('\\');
        sb.append('t');
        break;
      default:
        sb.append(val[i]);
      }
    }
    return sb.toString();
  }
  
}