tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/PreScanner.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.tika.parser.html.charsetdetector;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.AbstractMap;
import java.util.BitSet;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * A scanner meant to detect charset meta tags in a byte stream
 * See: https://html.spec.whatwg.org/multipage/parsing.html#prescan-a-byte-stream-to-determine-its-encoding
 */
class PreScanner {

    private static final Pattern CHARSET_PATTERN =
            Pattern.compile("charset\\s*=\\s*([\"']?)([^\"'\\s;]+)\\1");
    private static final byte[] COMMENT_START = {(byte) '<', (byte) '!', (byte) '-', (byte) '-'};
    private static final byte[] COMMENT_END = {(byte) '-', (byte) '-', (byte) '>'};
    private static final byte[] META_TAG_START =
            {(byte) '<', (byte) 'm', (byte) 'e', (byte) 't', (byte) 'a'};
    private static final byte SLASH = (byte) '/';
    private static final byte EQUAL = (byte) '=';
    private static final byte TAG_START = (byte) '<';
    private static final byte TAG_END = (byte) '>';
    private static final BitSet QUOTE = bitSet('"', '\'');

    private static final BitSet WHITESPACE = bitSet(0x09, 0x0A, 0x0C, 0x0D, 0x0D, 0x20);
    private static final BitSet SPACE_OR_TAG_END = bitSet(WHITESPACE, TAG_END);
    private static final BitSet SPACE_OR_SLASH = bitSet(WHITESPACE, SLASH);
    private static final BitSet SPECIAL_TAGS = bitSet('!', '/', '?');

    private static final byte[] UTF8_BOM = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF};
    private static final byte[] UTF16_BE_BOM = {(byte) 0xFE, (byte) 0xFF};
    private static final byte[] UTF16_LE_BOM = {(byte) 0xFF, (byte) 0xFE};
    private static final byte LOWER_A = (byte) 'a';
    private static final byte LOWER_Z = (byte) 'z';
    private static final byte UPPER_A = (byte) 'A';
    private static final byte UPPER_Z = (byte) 'Z';
    private BufferedInputStream stream;
    private CharsetDetectionResult detectedCharset = CharsetDetectionResult.notFound();

    PreScanner(InputStream inputStream) {
        this.stream = new BufferedInputStream(inputStream);
    }

    private static BitSet bitSet(int... bs) {
        BitSet bitSet = new BitSet(0xFF);
        for (int b : bs) bitSet.set(b);
        return bitSet;
    }

    private static BitSet bitSet(BitSet base, int... bs) {
        BitSet bitSet = (BitSet) base.clone();
        for (int b : bs) bitSet.set(b);
        return bitSet;
    }

    static String getEncodingFromMeta(String attributeValue) {
        Matcher matcher = CHARSET_PATTERN.matcher(attributeValue);
        if (!matcher.find()) {
            return null;
        }
        return matcher.group(2);
    }

    private static boolean contains(BitSet bitSet, byte b) {
        return bitSet.get(b & 0xFF);
    }

    Charset scan() {
        while (processAtLeastOneByte()) {
            if (detectedCharset.isFound()) {
                return detectedCharset.getCharset();
            }
        }
        return null;
    }

    Charset detectBOM() {
        try {
            if (expect(UTF8_BOM)) {
                return StandardCharsets.UTF_8;
            } else if (expect(UTF16_BE_BOM)) {
                return StandardCharsets.UTF_16BE;
            } else if (expect(UTF16_LE_BOM)) {
                return StandardCharsets.UTF_16LE;
            }
        } catch (IOException e) { /* stream could not be read, also return null */ }
        return null;
    }

    private boolean processAtLeastOneByte() {
        try {
            return processComment() || processMeta() || processTag() || processSpecialTag() ||
                    processAny();
        } catch (IOException e) {
            return false;
        }
    }

    private boolean processAny() throws IOException {
        int read = stream.read();
        return read != -1;
    }

    private boolean processTag() throws IOException {
        stream.mark(3);
        if (read() == TAG_START) {
            int read = stream.read();
            if (read == SLASH) {
                read = stream.read();
            }
            if ((LOWER_A <= read && read <= LOWER_Z) || (UPPER_A <= read && read <= UPPER_Z)) {
                do {
                    stream.mark(1);
                } while (!contains(SPACE_OR_TAG_END, read()));
                stream.reset();
                while (getAttribute() != null) {/* ignore the attribute*/}
                return true;
            }
        }
        stream.reset();
        return false;
    }

    private boolean processSpecialTag() throws IOException {
        stream.mark(2);
        if (read() == TAG_START && contains(SPECIAL_TAGS, read())) {
            skipUntil(TAG_END);
            return true;
        }
        stream.reset();
        return false;
    }

    private boolean processMeta() throws IOException {
        stream.mark(6); // len("<meta ") == 6
        if (readCaseInsensitive(META_TAG_START) && contains(SPACE_OR_SLASH, read())) {
            MetaProcessor metaProcessor = new MetaProcessor();
            for (Map.Entry<String, String> attribute = getAttribute(); attribute != null;
                    attribute = getAttribute()) {
                metaProcessor.processAttribute(attribute);
            }
            metaProcessor.updateDetectedCharset(detectedCharset);
            return true;
        }
        stream.reset();
        return false;
    }

    /**
     * Read an attribute from the stream
     *
     * @return the attribute as a Map.Entry, where the key is the attribute's name and
     * the value is the attribute's value. If there is no attribute, return null
     */
    private Map.Entry<String, String> getAttribute() throws IOException {
        String name = getAttributeName();
        if (name == null) {
            return null;
        }

        if (!expect(EQUAL)) {
            return new AbstractMap.SimpleEntry<>(name, "");
        }
        skipAll(WHITESPACE);

        String value = getAttributeValue();
        return new AbstractMap.SimpleEntry<>(name, value);
    }

    private String getAttributeName() throws IOException {
        skipAll(SPACE_OR_SLASH);
        if (expect(TAG_END)) {
            return null;
        }
        StringBuilder name = new StringBuilder();
        while (!(peek() == EQUAL && name.length() > 0) && !(peek() == TAG_END || peek() == SLASH) &&
                !skipAll(WHITESPACE)) {
            name.append((char) getLowerCaseChar());
        }
        return name.toString();
    }

    private String getAttributeValue() throws IOException {
        StringBuilder value = new StringBuilder();
        stream.mark(1);
        byte quote = read();
        if (contains(QUOTE, quote)) {
            for (byte b = getLowerCaseChar(); b != quote; b = getLowerCaseChar()) {
                value.append((char) b);
            }
        } else {
            stream.reset();
            for (byte b = getLowerCaseChar(); !contains(SPACE_OR_TAG_END, b);
                    b = getLowerCaseChar()) {
                value.append((char) b);
                stream.mark(1);
            }
            stream.reset(); // unread the space or tag end
        }
        return value.toString();
    }

    private boolean skipAll(BitSet bitSet) throws IOException {
        boolean skipped = false;
        stream.mark(1);
        for (byte read = read(); contains(bitSet, read); read = read()) {
            skipped = true;
            stream.mark(1);
        }
        stream.reset();
        return skipped;
    }

    private byte getLowerCaseChar() throws IOException {
        byte nextPoint = read();
        if (nextPoint >= 'A' && nextPoint <= 'Z') {
            nextPoint += 0x20; // lowercase
        }
        return nextPoint;
    }

    private boolean processComment() throws IOException {
        if (!expect(COMMENT_START)) {
            return false;
        }
        if (!expect(TAG_END)) {
            skipUntil(COMMENT_END);
        }
        return true;
    }

    private boolean expect(byte... expected) throws IOException {
        stream.mark(expected.length);
        for (byte b : expected) {
            byte read = read();
            if (read != b) {
                stream.reset();
                return false;
            }
        }
        return true;
    }

    private void skipUntil(byte... expected) throws IOException {
        while (!expect(expected)) {
            if (stream.read() == -1) {
                return;
            }
        }
    }

    private boolean readCaseInsensitive(byte... bs) throws IOException {
        for (byte b : bs)
            if (getLowerCaseChar() != b) {
                return false;
            }
        return true;
    }

    private byte read() throws IOException {
        int r = stream.read();
        if (r == -1) {
            throw new IOException();
        }
        return (byte) r;
    }

    private byte peek() throws IOException {
        stream.mark(1);
        byte b = read();
        stream.reset();
        return b;
    }
}