blob: 54f39c4fc2e9dcdf442fb5da092cd1e83e4bfca5 [file]
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HexFormat;
import java.util.List;
/**
* Generates the bundled {@code EmojiSequences.txt} from the Unicode Emoji {@code emoji-test.txt}
* and {@code emoji-data.txt} of one release. The release, the SHA-256 of both source files and
* the expected number of qualified sequences are pinned in {@link #RELEASE}, so a version
* bump is a change of that one table.
*
* <p>Usage: {@code UnicodeEmojiSequenceGenerator emoji-test.txt emoji-data.txt output.txt}.</p>
*/
public final class UnicodeEmojiSequenceGenerator {
/** Unicode Emoji 17.0, the version of the other bundled Unicode data files. */
private static final Release RELEASE = new Release("17.0", "2025",
"1d8a944f88d7952f7ef7c5167fef3c67995bcae24543949710231b03a201acda",
"2cb2bb9455cda83e8481541ecf5b6dfda66a3bb89efa3fa7c5297eccf607b72b",
4973);
private static final String OUTPUT_NAME = "EmojiSequences.txt";
private static final String VERSION_LINE = "# Version: ";
private static final String FULLY_QUALIFIED = "fully-qualified";
private static final String MINIMALLY_QUALIFIED = "minimally-qualified";
private static final String EMOJI_COMPONENT = "Emoji_Component";
private static final String SEQUENCE_RECORD = "S;";
private static final String COMPONENT_RECORD = "C;";
private static final String RANGE_SEPARATOR = "..";
private static final char CODE_POINT_SEPARATOR = ' ';
private static final char FIELD_SEPARATOR = ';';
private static final char COMMENT = '#';
private UnicodeEmojiSequenceGenerator() {
}
/**
* Verifies the two source files against {@link #RELEASE} and writes the inventory.
*
* @param args The paths of {@code emoji-test.txt}, {@code emoji-data.txt} and the output file.
* @throws IOException Thrown if a source file cannot be read, has the wrong checksum or
* version, or has unexpected content.
*/
public static void main(String[] args) throws IOException {
if (args.length != 3) {
throw new IllegalArgumentException(
"usage: UnicodeEmojiSequenceGenerator emoji-test.txt emoji-data.txt output.txt");
}
new UnicodeEmojiSequenceGenerator().generate(Path.of(args[0]), Path.of(args[1]),
Path.of(args[2]));
}
/**
* Verifies the two source files against {@link #RELEASE} and writes the inventory.
*
* @param emojiTestFile The {@code emoji-test.txt} file.
* @param emojiDataFile The {@code emoji-data.txt} file.
* @param target The output file.
* @throws IOException Thrown if a source file cannot be read, has the wrong checksum or
* version, or has unexpected content, or the output cannot be written.
*/
private void generate(Path emojiTestFile, Path emojiDataFile, Path target) throws IOException {
List<String> emojiTest = readSource(emojiTestFile, RELEASE.emojiTestSha256());
List<String> emojiData = readSource(emojiDataFile, RELEASE.emojiDataSha256());
List<String> sequences = readQualifiedSequences(emojiTest);
List<String> components = readComponentRanges(emojiData);
List<String> output = new ArrayList<>(sequences.size() + components.size() + 11);
output.add("# " + OUTPUT_NAME);
output.add("# Copyright (c) 1991-" + RELEASE.copyrightYear() + " Unicode, Inc.");
output.add("# For terms of use and license, see https://www.unicode.org/license.txt");
output.add("#");
output.add("# Unicode Emoji fully-qualified and minimally-qualified sequences, derived from");
output.add("# emoji-test.txt and emoji-data.txt under Unicode License V3.");
output.add(VERSION_LINE + RELEASE.version());
output.add("# emoji-test.txt SHA-256: " + RELEASE.emojiTestSha256());
output.add("# emoji-data.txt SHA-256: " + RELEASE.emojiDataSha256());
output.add("# S records are exact sequences; C records are Emoji_Component ranges.");
output.add("# Generated by dev/UnicodeEmojiSequenceGenerator.java; do not edit.");
for (String sequence : sequences) {
output.add(SEQUENCE_RECORD + sequence);
}
for (String component : components) {
output.add(COMPONENT_RECORD + component);
}
if (target.getParent() != null) {
Files.createDirectories(target.getParent());
}
Files.write(target, output, StandardCharsets.US_ASCII);
System.out.printf("wrote %d sequences and %d component ranges%n",
sequences.size(), components.size());
}
/**
* Reads the code point sequences that {@code emoji-test.txt} marks {@code fully-qualified} or
* {@code minimally-qualified}, in file order. Unqualified sequences, whose first code point
* lacks a needed U+FE0F, are left out: they start with a symbol in text presentation.
*
* @param lines The lines of {@code emoji-test.txt}.
* @return The sequences, each as upper case hex code points separated by one space.
* @throws IOException Thrown if a line holds a malformed code point, or the file does not
* hold exactly the number of sequences pinned in {@link #RELEASE}.
*/
private List<String> readQualifiedSequences(List<String> lines) throws IOException {
List<String> result = new ArrayList<>();
for (String line : lines) {
int semicolon = line.indexOf(FIELD_SEPARATOR);
int comment = line.indexOf(COMMENT, semicolon + 1);
if (semicolon < 0 || comment < 0) {
continue;
}
String status = line.substring(semicolon + 1, comment).trim();
if (status.equals(FULLY_QUALIFIED) || status.equals(MINIMALLY_QUALIFIED)) {
result.add(normalizeCodePoints(line.substring(0, semicolon)));
}
}
if (result.size() != RELEASE.sequenceCount()) {
throw new IOException("expected " + RELEASE.sequenceCount()
+ " qualified sequences, found " + result.size());
}
return result;
}
/**
* Reads the code point ranges that {@code emoji-data.txt} assigns the {@code Emoji_Component}
* property, in file order.
*
* @param lines The lines of {@code emoji-data.txt}.
* @return The ranges, each as {@code XXXX..YYYY} or a single code point.
* @throws IOException Thrown if a line holds a malformed code point, or the file has no
* {@code Emoji_Component} line.
*/
private List<String> readComponentRanges(List<String> lines) throws IOException {
List<String> result = new ArrayList<>();
for (String line : lines) {
int semicolon = line.indexOf(FIELD_SEPARATOR);
int comment = line.indexOf(COMMENT, semicolon + 1);
if (semicolon < 0 || comment < 0) {
continue;
}
if (line.substring(semicolon + 1, comment).trim().equals(EMOJI_COMPONENT)) {
result.add(normalizeRange(line.substring(0, semicolon)));
}
}
if (result.isEmpty()) {
throw new IOException("emoji-data.txt has no Emoji_Component ranges");
}
return result;
}
/**
* Rewrites a whitespace separated list of hex code points as four or more upper case hex
* digits each, separated by one space.
*
* @param value The code point column of an {@code emoji-test.txt} line.
* @return The normalized sequence.
* @throws IOException Thrown if the column is empty or holds a malformed code point.
*/
private String normalizeCodePoints(String value) throws IOException {
StringBuilder normalized = new StringBuilder();
int start = -1;
for (int i = 0; i <= value.length(); i++) {
if (i == value.length() || Character.isWhitespace(value.charAt(i))) {
if (start >= 0) {
if (normalized.length() > 0) {
normalized.append(CODE_POINT_SEPARATOR);
}
normalized.append(formatCodePoint(parseHex(value.substring(start, i))));
start = -1;
}
} else if (start < 0) {
start = i;
}
}
if (normalized.length() == 0) {
throw new IOException("empty code point sequence");
}
return normalized.toString();
}
/**
* Rewrites a code point or a {@code XXXX..YYYY} range with four or more upper case hex digits
* per code point.
*
* @param value The code point column of an {@code emoji-data.txt} line.
* @return The normalized range.
* @throws IOException Thrown if a code point is malformed.
*/
private String normalizeRange(String value) throws IOException {
String trimmed = value.trim();
int dots = trimmed.indexOf(RANGE_SEPARATOR);
if (dots < 0) {
return formatCodePoint(parseHex(trimmed));
}
return formatCodePoint(parseHex(trimmed.substring(0, dots))) + RANGE_SEPARATOR
+ formatCodePoint(parseHex(trimmed.substring(dots + RANGE_SEPARATOR.length())));
}
/**
* Parses one hex code point.
*
* @param value The hex digits.
* @return The code point.
* @throws IOException Thrown if the digits are not hex or not a Unicode code point.
*/
private int parseHex(String value) throws IOException {
try {
int codePoint = Integer.parseInt(value, 16);
if (!Character.isValidCodePoint(codePoint)) {
throw new IOException("invalid code point: " + value);
}
return codePoint;
} catch (NumberFormatException e) {
throw new IOException("invalid hexadecimal code point: " + value, e);
}
}
/**
* Formats a code point as at least four upper case hex digits, the way the Unicode data files
* do.
*
* @param codePoint The code point.
* @return The hex digits.
*/
private String formatCodePoint(int codePoint) {
return String.format("%04X", codePoint);
}
/**
* Reads a source file once and checks its SHA-256 and its {@code # Version:} header line
* against {@link #RELEASE}. Both emoji files use the two part form, for example
* {@code # Version: 17.0}.
*
* @param source The file to read.
* @param expectedSha256 The expected SHA-256 as lower case hex.
* @return The lines of the file.
* @throws IOException Thrown if the file cannot be read, its checksum differs, or it is of
* another version.
*/
private List<String> readSource(Path source, String expectedSha256) throws IOException {
byte[] bytes = Files.readAllBytes(source);
String actual;
try {
actual = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-256 is unavailable", e);
}
if (!actual.equals(expectedSha256)) {
throw new IOException(source + " SHA-256 mismatch: expected " + expectedSha256
+ ", found " + actual);
}
List<String> lines = new String(bytes, StandardCharsets.UTF_8).lines().toList();
if (!lines.contains(VERSION_LINE + RELEASE.version())) {
throw new IOException(source + " is not Unicode Emoji version " + RELEASE.version());
}
return lines;
}
/**
* The pinned Unicode Emoji release.
*
* @param version The version as it appears in the {@code # Version:} header of both files.
* @param copyrightYear The last year of the Unicode copyright line of that release.
* @param emojiTestSha256 The SHA-256 of {@code emoji-test.txt}.
* @param emojiDataSha256 The SHA-256 of {@code emoji-data.txt}.
* @param sequenceCount The number of fully-qualified and minimally-qualified sequences in
* {@code emoji-test.txt}.
*/
private record Release(String version, String copyrightYear, String emojiTestSha256,
String emojiDataSha256, int sequenceCount) {
}
}