STANBOL-1279 : Named Entity co-reference resolution engine based on yago/dbpedia contextual information
Added Entity Coreference Engine and the entity-coref-dbpedia data bundle.
git-svn-id: https://svn.apache.org/repos/asf/stanbol/trunk@1692320 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/data/pom.xml b/data/pom.xml
index 64cccdd..f169691 100644
--- a/data/pom.xml
+++ b/data/pom.xml
@@ -61,6 +61,7 @@
<module>sites/dbpedia</module>
<module>sites/dbpediacached</module>
+ <!--module>sites/entity-coref-dbpedia</module-->
<module>registries/default</module>
diff --git a/data/sites/entity-coref-dbpedia/README.md b/data/sites/entity-coref-dbpedia/README.md
new file mode 100644
index 0000000..c7f98a4
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/README.md
@@ -0,0 +1,67 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+## Building the DbPedia 3.9 index with yago types
+
+This index will contain the yago rdf:types and several spatial/org membership properties from the DBpedia index.
+NOTE: At the moment the index is available only for english.
+
+### (1) Follow the instructions at entityhub/indexing/dbpedia/README.md and build the dbpedia index with the following configuration:
+
+#### (1) Use the RDF dumps (in N-Triple format) from :
+ http://downloads.dbpedia.org/3.9/dbpedia_3.9.owl
+ http://downloads.dbpedia.org/3.9/en/labels_en.nt.bz2
+ http://downloads.dbpedia.org/3.9/en/instance_types_en.nt.bz2
+ http://downloads.dbpedia.org/3.9/en/mappingbased_properties_en.nt.bz2
+ http://downloads.dbpedia.org/3.9/links/yago_types.nt.bz2
+
+#### (2) The mappings.txt file must contain the following entries:
+
+ rdfs:label | d=entityhub:text
+ rdf:type | d=entityhub:ref
+ dbp-ont:birthPlace | d=entityhub:ref
+ dbp-ont:region | d=entityhub:ref
+ dbp-ont:foundationPlace | d=entityhub:ref
+ dbp-ont:locationCity | d=entityhub:ref
+ dbp-ont:location | d=entityhub:ref
+ dbp-ont:hometown | d=entityhub:ref
+ dbp-ont:country | d=entityhub:ref
+ dbp-ont:occupation | d=entityhub:ref
+ dbp-ont:associatedBand | d=entityhub:ref
+ dbp-ont:employer | d=entityhub:ref
+
+#### (3) Change the indexing/config/indexing.properties file to include the following attributes:
+ name=entity-coref-dbpedia
+ description=DBpedia.org
+
+### (2) Run the script /dbpedia_yag_classes/build_yago_dbpedia_labels.sh which will create the dbpedia_yago_classes_labels.nt.bz2 archive
+which contains the labels of the yago types.
+
+### (3) Follow the instructions at entityhub/indexing/genericrdf/README.md and rebuild the dbpedia index in order to include the
+aforementioned yago types labels. After you init the indexer but before you run it go through the following steps:
+
+#### (1) Copy the dbpedia_yago_classes_labels.nt.bz2 to the indexing\resources\rdfdata folder.
+
+#### (2) Change the indexing/config/indexing.properties to include the following attributes:
+
+ name=entity-coref-dbpedia
+ description=DBpedia.org
+
+#### (3) The indexing/config/mappings.txt file must only contain the rdfs:label attribute
+
+#### (4) Copy the contents of the indexing/destination folder from the results of point ### (1) to the /indexing/destination folder
+of the generic rdf indexing at point ### (3).
diff --git a/data/sites/entity-coref-dbpedia/dbpedia_yago_classes/build_yago_dbpedia_labels.sh b/data/sites/entity-coref-dbpedia/dbpedia_yago_classes/build_yago_dbpedia_labels.sh
new file mode 100644
index 0000000..0ea3cf3
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/dbpedia_yago_classes/build_yago_dbpedia_labels.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+YAGO=http://resources.mpi-inf.mpg.de/yago-naga/yago/download/yago/
+
+files=(yagoLabels.ttl.7z \
+ yagoDBpediaClasses.ttl.7z
+ )
+YAGO_LABELS=yagoLabels.ttl
+YAGO_DBPEDIA_CLASSES=yagoDBpediaClasses.ttl
+YAGO_WORDNET_LABELS=yago_wordnet_labels
+YAGO_WORDNET_DBPEDIA_CLASSES=yago_wordnet_dbpedia_classes
+DBPEDIA_YAGO_CLASS_LABELS_NT=dbpedia_yago_classes_labels.nt
+
+# First, download and decompress the necessary yago files.
+for i in "${files[@]}"
+do
+ :
+ if [ ! -f ${i} ];
+ then
+ url=${YAGO}/${i}
+ wget -c ${url}
+ fi
+
+ echo "Unzipping ${i}"
+ 7za e ${i}
+done
+
+# Second, create a file with <wordnet_class> rdfs:label "label" format.
+grep '^<wordnet_' ${YAGO_LABELS} | grep 'rdfs:label' > ${YAGO_WORDNET_LABELS}
+
+# Third, create a file with wordnet to dbpedia yago class mappings.
+grep '^<wordnet_' ${YAGO_DBPEDIA_CLASSES} > ${YAGO_WORDNET_DBPEDIA_CLASSES}
+
+# Last, create the nt file which will contain the dbpedia yago class and its labels.
+touch ${DBPEDIA_YAGO_CLASS_LABELS_NT};
+
+YAGO_LABELS_NUM_LINES=$(wc -l < ${YAGO_WORDNET_LABELS})
+PROCESSED_LINES=0
+NOT_FOUND_LINES=0
+
+echo -e "\n\nStarting to process $YAGO_LABELS_NUM_LINES lines"
+
+while read line
+do
+ wordnet_class=`echo $line | awk '{print $1}'`;
+ dbpedia_class=`grep $wordnet_class $YAGO_WORDNET_DBPEDIA_CLASSES | awk '{split($0,a," "); print a[3]}'`;
+
+ if [ -z "$dbpedia_class" ]
+ then
+ ((NOT_FOUND_LINES++));
+ continue;
+ fi
+
+ mapped_line=${line/$wordnet_class/$dbpedia_class};
+ mapped_line_with_label=${mapped_line/rdfs:label/<http://www.w3.org/2000/01/rdf-schema#label>};
+ mapped_line_with_label_lang=${mapped_line_with_label/@eng/@en};
+
+ echo $mapped_line_with_label_lang >> ${DBPEDIA_YAGO_CLASS_LABELS_NT};
+
+ ((PROCESSED_LINES++));
+ echo -ne "$PROCESSED_LINES/$YAGO_LABELS_NUM_LINES processed\r";
+done < ${YAGO_WORDNET_LABELS}
+
+echo -e "\nDone processing lines. Skipped $NOT_FOUND_LINES not found dbpedia classes. Creating .nt archive."
+bzip2 ${DBPEDIA_YAGO_CLASS_LABELS_NT}
+
+# Cleanup
+rm ${YAGO_LABELS}
+rm ${YAGO_DBPEDIA_CLASSES}
+rm ${YAGO_WORDNET_LABELS}
+rm ${YAGO_WORDNET_DBPEDIA_CLASSES}
\ No newline at end of file
diff --git a/data/sites/entity-coref-dbpedia/download_index.xml b/data/sites/entity-coref-dbpedia/download_index.xml
new file mode 100644
index 0000000..a67e6d8
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/download_index.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="Index Download Helper" default="download" basedir=".">
+ <description>
+ Contains only a single target that is used by the Maven Ant
+ Plugin to download the Index parsed via 'index.url'
+ to 'target.directory'
+ </description>
+
+ <target name="download">
+ <copy todir="${target.directory}" flatten="true">
+ <resources>
+ <url url="${index.url}"/>
+ </resources>
+ </copy>
+ </target>
+</project>
diff --git a/data/sites/entity-coref-dbpedia/pom.xml b/data/sites/entity-coref-dbpedia/pom.xml
new file mode 100644
index 0000000..12f6b53
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/pom.xml
@@ -0,0 +1,163 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.data.parent</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ <relativePath>../../parent</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.data.sites.entity-coref-dbpedia</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ <packaging>bundle</packaging>
+
+ <name>Apache Stanbol Data: DBpedia.org and Yago.org</name>
+ <description>
+ This bundle allows to use DBpedia and Yago as Referenced Site for the
+ Entity Co-reference Engine. The build also downloads the dbpedia index.
+ </description>
+
+ <inceptionYear>2015</inceptionYear>
+ <licenses>
+ <license>
+ <name>Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ <comments>A business-friendly OSS license</comments>
+ </license>
+ </licenses>
+
+ <scm>
+ <connection>scm:svn:http://svn.apache.org/repos/asf/stanbol/trunk/data/sites/entity-coref-dbpedia</connection>
+ <developerConnection>scm:svn:https://svn.apache.org/repos/asf/stanbol/trunk/data/sites/entity-coref-dbpedia</developerConnection>
+ <url>http://stanbol.apache.org/</url>
+ </scm>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <dbpedia.default.index.url>https://github.com/cpetroaca/stanbol-indices/releases/download/trunk/entity-coref-dbpedia.solrindex.zip</dbpedia.default.index.url>
+ <dbpedia.default.path>org/apache/stanbol/data/site/entity-coref-dbpedia/default</dbpedia.default.path>
+ <dbpedia.default.index.path>${dbpedia.default.path}/index</dbpedia.default.index.path>
+ <dbpedia.default.config.path>${dbpedia.default.path}/config</dbpedia.default.config.path>
+ <downloadWarning>
+*
+* WARNING - this build downloads a DPBedia index
+* that is *not* licensed under the Apache License, but under the
+* Creative Commons Attribution-ShareAlike 3.0 Unported License.
+*
+ </downloadWarning>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <inherited>true</inherited>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <!--
+ Extension used to provide files in that directory to the
+ DataFileProvider
+ -->
+ <Data-Files>${dbpedia.default.index.path}</Data-Files>
+ <!--
+ Use a priority lower than 0 to allow providers without a
+ defined ranking to override this default data.
+ -->
+ <Data-Files-Priority>-100</Data-Files-Priority>
+ <!--
+ Extension used by the Bundle-Installer to load OSGI
+ component configuration
+ -->
+ <Install-Path>${dbpedia.default.config.path}</Install-Path>
+ <_versionpolicy>$${version;===;${@}}</_versionpolicy>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>download</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <!--
+ TODO: I would like to add an "unless" constraint to the
+ target that prevents execution if Maven operates in offline
+ mode. However I was not able to find out how to obtain this
+ information. ${settings.offline} (as noted by several
+ resources) does not work.
+ Until fixed builds will fail if no internetconnection is
+ available!
+ -->
+ <target>
+ <property name="target.directory" value="${project.basedir}/downloads/resources/${dbpedia.default.index.path}" />
+ <property name="index.url" value="${dbpedia.default.index.url}" />
+
+ <echo message="copy Solr Index " />
+ <echo message=" FROM ${index.url} " />
+ <echo message=" TO ${target.directory}" />
+
+ <ant antfile="${basedir}/download_index.xml">
+ <target name="download" />
+ </ant>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <!-- AL20 licensed. See src/main/resources/README -->
+ <exclude>src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/entity-coref-dbpedia.solrindex.ref</exclude>
+ <exclude>src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.enhancer.engines.entitytagging.impl.NamedEntityTaggingEngine-entity-coref-dbpedia.config</exclude>
+ <exclude>src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.core.site.CacheImpl-entity-coref-dbpedia.config</exclude>
+ <exclude>src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.site.referencedSite-entity-coref-dbpedia.config</exclude>
+ <exclude>src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-entity-coref-dbpedia.config</exclude>
+
+ <!-- Licensed under Creative Commons Attribution-ShareAlike 3.0 Unported License -->
+ <exclude>src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/index/entity-coref-dbpedia.solrindex.zip</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <!-- DBPedia dumps can be deployed -->
+ <skip>false</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/data/sites/entity-coref-dbpedia/src/main/appended-resources/META-INF/LICENSE b/data/sites/entity-coref-dbpedia/src/main/appended-resources/META-INF/LICENSE
new file mode 100644
index 0000000..eb675bd
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/src/main/appended-resources/META-INF/LICENSE
@@ -0,0 +1,362 @@
+
+-----------------------------------------------------------------------------
+
+Creative Commons Legal Code
+
+Attribution-ShareAlike 3.0 Unported
+
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+ LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN
+ ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+ INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+ REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR
+ DAMAGES RESULTING FROM ITS USE.
+
+License
+
+THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE
+COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY
+COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS
+AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
+
+BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE
+TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY
+BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS
+CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND
+CONDITIONS.
+
+1. Definitions
+
+ a. "Adaptation" means a work based upon the Work, or upon the Work and
+ other pre-existing works, such as a translation, adaptation,
+ derivative work, arrangement of music or other alterations of a
+ literary or artistic work, or phonogram or performance and includes
+ cinematographic adaptations or any other form in which the Work may be
+ recast, transformed, or adapted including in any form recognizably
+ derived from the original, except that a work that constitutes a
+ Collection will not be considered an Adaptation for the purpose of
+ this License. For the avoidance of doubt, where the Work is a musical
+ work, performance or phonogram, the synchronization of the Work in
+ timed-relation with a moving image ("synching") will be considered an
+ Adaptation for the purpose of this License.
+ b. "Collection" means a collection of literary or artistic works, such as
+ encyclopedias and anthologies, or performances, phonograms or
+ broadcasts, or other works or subject matter other than works listed
+ in Section 1(f) below, which, by reason of the selection and
+ arrangement of their contents, constitute intellectual creations, in
+ which the Work is included in its entirety in unmodified form along
+ with one or more other contributions, each constituting separate and
+ independent works in themselves, which together are assembled into a
+ collective whole. A work that constitutes a Collection will not be
+ considered an Adaptation (as defined below) for the purposes of this
+ License.
+ c. "Creative Commons Compatible License" means a license that is listed
+ at http://creativecommons.org/compatiblelicenses that has been
+ approved by Creative Commons as being essentially equivalent to this
+ License, including, at a minimum, because that license: (i) contains
+ terms that have the same purpose, meaning and effect as the License
+ Elements of this License; and, (ii) explicitly permits the relicensing
+ of adaptations of works made available under that license under this
+ License or a Creative Commons jurisdiction license with the same
+ License Elements as this License.
+ d. "Distribute" means to make available to the public the original and
+ copies of the Work or Adaptation, as appropriate, through sale or
+ other transfer of ownership.
+ e. "License Elements" means the following high-level license attributes
+ as selected by Licensor and indicated in the title of this License:
+ Attribution, ShareAlike.
+ f. "Licensor" means the individual, individuals, entity or entities that
+ offer(s) the Work under the terms of this License.
+ g. "Original Author" means, in the case of a literary or artistic work,
+ the individual, individuals, entity or entities who created the Work
+ or if no individual or entity can be identified, the publisher; and in
+ addition (i) in the case of a performance the actors, singers,
+ musicians, dancers, and other persons who act, sing, deliver, declaim,
+ play in, interpret or otherwise perform literary or artistic works or
+ expressions of folklore; (ii) in the case of a phonogram the producer
+ being the person or legal entity who first fixes the sounds of a
+ performance or other sounds; and, (iii) in the case of broadcasts, the
+ organization that transmits the broadcast.
+ h. "Work" means the literary and/or artistic work offered under the terms
+ of this License including without limitation any production in the
+ literary, scientific and artistic domain, whatever may be the mode or
+ form of its expression including digital form, such as a book,
+ pamphlet and other writing; a lecture, address, sermon or other work
+ of the same nature; a dramatic or dramatico-musical work; a
+ choreographic work or entertainment in dumb show; a musical
+ composition with or without words; a cinematographic work to which are
+ assimilated works expressed by a process analogous to cinematography;
+ a work of drawing, painting, architecture, sculpture, engraving or
+ lithography; a photographic work to which are assimilated works
+ expressed by a process analogous to photography; a work of applied
+ art; an illustration, map, plan, sketch or three-dimensional work
+ relative to geography, topography, architecture or science; a
+ performance; a broadcast; a phonogram; a compilation of data to the
+ extent it is protected as a copyrightable work; or a work performed by
+ a variety or circus performer to the extent it is not otherwise
+ considered a literary or artistic work.
+ i. "You" means an individual or entity exercising rights under this
+ License who has not previously violated the terms of this License with
+ respect to the Work, or who has received express permission from the
+ Licensor to exercise rights under this License despite a previous
+ violation.
+ j. "Publicly Perform" means to perform public recitations of the Work and
+ to communicate to the public those public recitations, by any means or
+ process, including by wire or wireless means or public digital
+ performances; to make available to the public Works in such a way that
+ members of the public may access these Works from a place and at a
+ place individually chosen by them; to perform the Work to the public
+ by any means or process and the communication to the public of the
+ performances of the Work, including by public digital performance; to
+ broadcast and rebroadcast the Work by any means including signs,
+ sounds or images.
+ k. "Reproduce" means to make copies of the Work by any means including
+ without limitation by sound or visual recordings and the right of
+ fixation and reproducing fixations of the Work, including storage of a
+ protected performance or phonogram in digital form or other electronic
+ medium.
+
+2. Fair Dealing Rights. Nothing in this License is intended to reduce,
+limit, or restrict any uses free from copyright or rights arising from
+limitations or exceptions that are provided for in connection with the
+copyright protection under copyright law or other applicable laws.
+
+3. License Grant. Subject to the terms and conditions of this License,
+Licensor hereby grants You a worldwide, royalty-free, non-exclusive,
+perpetual (for the duration of the applicable copyright) license to
+exercise the rights in the Work as stated below:
+
+ a. to Reproduce the Work, to incorporate the Work into one or more
+ Collections, and to Reproduce the Work as incorporated in the
+ Collections;
+ b. to create and Reproduce Adaptations provided that any such Adaptation,
+ including any translation in any medium, takes reasonable steps to
+ clearly label, demarcate or otherwise identify that changes were made
+ to the original Work. For example, a translation could be marked "The
+ original work was translated from English to Spanish," or a
+ modification could indicate "The original work has been modified.";
+ c. to Distribute and Publicly Perform the Work including as incorporated
+ in Collections; and,
+ d. to Distribute and Publicly Perform Adaptations.
+ e. For the avoidance of doubt:
+
+ i. Non-waivable Compulsory License Schemes. In those jurisdictions in
+ which the right to collect royalties through any statutory or
+ compulsory licensing scheme cannot be waived, the Licensor
+ reserves the exclusive right to collect such royalties for any
+ exercise by You of the rights granted under this License;
+ ii. Waivable Compulsory License Schemes. In those jurisdictions in
+ which the right to collect royalties through any statutory or
+ compulsory licensing scheme can be waived, the Licensor waives the
+ exclusive right to collect such royalties for any exercise by You
+ of the rights granted under this License; and,
+ iii. Voluntary License Schemes. The Licensor waives the right to
+ collect royalties, whether individually or, in the event that the
+ Licensor is a member of a collecting society that administers
+ voluntary licensing schemes, via that society, from any exercise
+ by You of the rights granted under this License.
+
+The above rights may be exercised in all media and formats whether now
+known or hereafter devised. The above rights include the right to make
+such modifications as are technically necessary to exercise the rights in
+other media and formats. Subject to Section 8(f), all rights not expressly
+granted by Licensor are hereby reserved.
+
+4. Restrictions. The license granted in Section 3 above is expressly made
+subject to and limited by the following restrictions:
+
+ a. You may Distribute or Publicly Perform the Work only under the terms
+ of this License. You must include a copy of, or the Uniform Resource
+ Identifier (URI) for, this License with every copy of the Work You
+ Distribute or Publicly Perform. You may not offer or impose any terms
+ on the Work that restrict the terms of this License or the ability of
+ the recipient of the Work to exercise the rights granted to that
+ recipient under the terms of the License. You may not sublicense the
+ Work. You must keep intact all notices that refer to this License and
+ to the disclaimer of warranties with every copy of the Work You
+ Distribute or Publicly Perform. When You Distribute or Publicly
+ Perform the Work, You may not impose any effective technological
+ measures on the Work that restrict the ability of a recipient of the
+ Work from You to exercise the rights granted to that recipient under
+ the terms of the License. This Section 4(a) applies to the Work as
+ incorporated in a Collection, but this does not require the Collection
+ apart from the Work itself to be made subject to the terms of this
+ License. If You create a Collection, upon notice from any Licensor You
+ must, to the extent practicable, remove from the Collection any credit
+ as required by Section 4(c), as requested. If You create an
+ Adaptation, upon notice from any Licensor You must, to the extent
+ practicable, remove from the Adaptation any credit as required by
+ Section 4(c), as requested.
+ b. You may Distribute or Publicly Perform an Adaptation only under the
+ terms of: (i) this License; (ii) a later version of this License with
+ the same License Elements as this License; (iii) a Creative Commons
+ jurisdiction license (either this or a later license version) that
+ contains the same License Elements as this License (e.g.,
+ Attribution-ShareAlike 3.0 US)); (iv) a Creative Commons Compatible
+ License. If you license the Adaptation under one of the licenses
+ mentioned in (iv), you must comply with the terms of that license. If
+ you license the Adaptation under the terms of any of the licenses
+ mentioned in (i), (ii) or (iii) (the "Applicable License"), you must
+ comply with the terms of the Applicable License generally and the
+ following provisions: (I) You must include a copy of, or the URI for,
+ the Applicable License with every copy of each Adaptation You
+ Distribute or Publicly Perform; (II) You may not offer or impose any
+ terms on the Adaptation that restrict the terms of the Applicable
+ License or the ability of the recipient of the Adaptation to exercise
+ the rights granted to that recipient under the terms of the Applicable
+ License; (III) You must keep intact all notices that refer to the
+ Applicable License and to the disclaimer of warranties with every copy
+ of the Work as included in the Adaptation You Distribute or Publicly
+ Perform; (IV) when You Distribute or Publicly Perform the Adaptation,
+ You may not impose any effective technological measures on the
+ Adaptation that restrict the ability of a recipient of the Adaptation
+ from You to exercise the rights granted to that recipient under the
+ terms of the Applicable License. This Section 4(b) applies to the
+ Adaptation as incorporated in a Collection, but this does not require
+ the Collection apart from the Adaptation itself to be made subject to
+ the terms of the Applicable License.
+ c. If You Distribute, or Publicly Perform the Work or any Adaptations or
+ Collections, You must, unless a request has been made pursuant to
+ Section 4(a), keep intact all copyright notices for the Work and
+ provide, reasonable to the medium or means You are utilizing: (i) the
+ name of the Original Author (or pseudonym, if applicable) if supplied,
+ and/or if the Original Author and/or Licensor designate another party
+ or parties (e.g., a sponsor institute, publishing entity, journal) for
+ attribution ("Attribution Parties") in Licensor's copyright notice,
+ terms of service or by other reasonable means, the name of such party
+ or parties; (ii) the title of the Work if supplied; (iii) to the
+ extent reasonably practicable, the URI, if any, that Licensor
+ specifies to be associated with the Work, unless such URI does not
+ refer to the copyright notice or licensing information for the Work;
+ and (iv) , consistent with Ssection 3(b), in the case of an
+ Adaptation, a credit identifying the use of the Work in the Adaptation
+ (e.g., "French translation of the Work by Original Author," or
+ "Screenplay based on original Work by Original Author"). The credit
+ required by this Section 4(c) may be implemented in any reasonable
+ manner; provided, however, that in the case of a Adaptation or
+ Collection, at a minimum such credit will appear, if a credit for all
+ contributing authors of the Adaptation or Collection appears, then as
+ part of these credits and in a manner at least as prominent as the
+ credits for the other contributing authors. For the avoidance of
+ doubt, You may only use the credit required by this Section for the
+ purpose of attribution in the manner set out above and, by exercising
+ Your rights under this License, You may not implicitly or explicitly
+ assert or imply any connection with, sponsorship or endorsement by the
+ Original Author, Licensor and/or Attribution Parties, as appropriate,
+ of You or Your use of the Work, without the separate, express prior
+ written permission of the Original Author, Licensor and/or Attribution
+ Parties.
+ d. Except as otherwise agreed in writing by the Licensor or as may be
+ otherwise permitted by applicable law, if You Reproduce, Distribute or
+ Publicly Perform the Work either by itself or as part of any
+ Adaptations or Collections, You must not distort, mutilate, modify or
+ take other derogatory action in relation to the Work which would be
+ prejudicial to the Original Author's honor or reputation. Licensor
+ agrees that in those jurisdictions (e.g. Japan), in which any exercise
+ of the right granted in Section 3(b) of this License (the right to
+ make Adaptations) would be deemed to be a distortion, mutilation,
+ modification or other derogatory action prejudicial to the Original
+ Author's honor and reputation, the Licensor will waive or not assert,
+ as appropriate, this Section, to the fullest extent permitted by the
+ applicable national law, to enable You to reasonably exercise Your
+ right under Section 3(b) of this License (right to make Adaptations)
+ but not otherwise.
+
+5. Representations, Warranties and Disclaimer
+
+UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR
+OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY
+KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE,
+INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY,
+FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF
+LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS,
+WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION
+OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
+
+6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE
+LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR
+ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES
+ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS
+BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+7. Termination
+
+ a. This License and the rights granted hereunder will terminate
+ automatically upon any breach by You of the terms of this License.
+ Individuals or entities who have received Adaptations or Collections
+ from You under this License, however, will not have their licenses
+ terminated provided such individuals or entities remain in full
+ compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will
+ survive any termination of this License.
+ b. Subject to the above terms and conditions, the license granted here is
+ perpetual (for the duration of the applicable copyright in the Work).
+ Notwithstanding the above, Licensor reserves the right to release the
+ Work under different license terms or to stop distributing the Work at
+ any time; provided, however that any such election will not serve to
+ withdraw this License (or any other license that has been, or is
+ required to be, granted under the terms of this License), and this
+ License will continue in full force and effect unless terminated as
+ stated above.
+
+8. Miscellaneous
+
+ a. Each time You Distribute or Publicly Perform the Work or a Collection,
+ the Licensor offers to the recipient a license to the Work on the same
+ terms and conditions as the license granted to You under this License.
+ b. Each time You Distribute or Publicly Perform an Adaptation, Licensor
+ offers to the recipient a license to the original Work on the same
+ terms and conditions as the license granted to You under this License.
+ c. If any provision of this License is invalid or unenforceable under
+ applicable law, it shall not affect the validity or enforceability of
+ the remainder of the terms of this License, and without further action
+ by the parties to this agreement, such provision shall be reformed to
+ the minimum extent necessary to make such provision valid and
+ enforceable.
+ d. No term or provision of this License shall be deemed waived and no
+ breach consented to unless such waiver or consent shall be in writing
+ and signed by the party to be charged with such waiver or consent.
+ e. This License constitutes the entire agreement between the parties with
+ respect to the Work licensed here. There are no understandings,
+ agreements or representations with respect to the Work not specified
+ here. Licensor shall not be bound by any additional provisions that
+ may appear in any communication from You. This License may not be
+ modified without the mutual written agreement of the Licensor and You.
+ f. The rights granted under, and the subject matter referenced, in this
+ License were drafted utilizing the terminology of the Berne Convention
+ for the Protection of Literary and Artistic Works (as amended on
+ September 28, 1979), the Rome Convention of 1961, the WIPO Copyright
+ Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996
+ and the Universal Copyright Convention (as revised on July 24, 1971).
+ These rights and subject matter take effect in the relevant
+ jurisdiction in which the License terms are sought to be enforced
+ according to the corresponding provisions of the implementation of
+ those treaty provisions in the applicable national law. If the
+ standard suite of rights granted under applicable copyright law
+ includes additional rights not granted under this License, such
+ additional rights are deemed to be included in the License; this
+ License is not intended to restrict the license of any rights under
+ applicable law.
+
+
+Creative Commons Notice
+
+ Creative Commons is not a party to this License, and makes no warranty
+ whatsoever in connection with the Work. Creative Commons will not be
+ liable to You or any party on any legal theory for any damages
+ whatsoever, including without limitation any general, special,
+ incidental or consequential damages arising in connection to this
+ license. Notwithstanding the foregoing two (2) sentences, if Creative
+ Commons has expressly identified itself as the Licensor hereunder, it
+ shall have all rights and obligations of Licensor.
+
+ Except for the limited purpose of indicating to the public that the
+ Work is licensed under the CCPL, Creative Commons does not authorize
+ the use by either party of the trademark "Creative Commons" or any
+ related trademark or logo of Creative Commons without the prior
+ written consent of Creative Commons. Any permitted use will be in
+ compliance with Creative Commons' then-current trademark usage
+ guidelines, as may be published on its website or otherwise made
+ available upon request from time to time. For the avoidance of doubt,
+ this trademark restriction does not form part of the License.
+
+ Creative Commons may be contacted at http://creativecommons.org/.
\ No newline at end of file
diff --git a/data/sites/entity-coref-dbpedia/src/main/appended-resources/META-INF/NOTICE b/data/sites/entity-coref-dbpedia/src/main/appended-resources/META-INF/NOTICE
new file mode 100644
index 0000000..2ac4478
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/src/main/appended-resources/META-INF/NOTICE
@@ -0,0 +1,3 @@
+This product contains an index from DBPedia licensed under
+the Creative Commons Attribution-ShareAlike 3.0 Unported License
+http://creativecommons.org/licenses/by-sa/3.0/
\ No newline at end of file
diff --git a/data/sites/entity-coref-dbpedia/src/main/resources/README b/data/sites/entity-coref-dbpedia/src/main/resources/README
new file mode 100644
index 0000000..fab9d22
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/src/main/resources/README
@@ -0,0 +1,25 @@
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+The following files are provided under the Apache License, Version 2.0:
+
+org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/entity-coref-dbpedia.solrindex.ref
+org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.core.site.CacheImpl-entity-coref-dbpedia.config
+org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.site.referencedSite-entity-coref-dbpedia.config
+org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-entity-coref-dbpedia.config
+
+The following file is licensed under Creative Commons Attribution-ShareAlike 3.0 Unported License:
+
+org/apache/stanbol/data/site/dbpedia/default/index/entity-coref-dbpedia.solrindex.zip
diff --git a/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/entity-coref-dbpedia.solrindex.ref b/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/entity-coref-dbpedia.solrindex.ref
new file mode 100644
index 0000000..0b4382c
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/entity-coref-dbpedia.solrindex.ref
@@ -0,0 +1,5 @@
+#Tue Feb 03 20:20:31 EET 2015
+Name=SolrIndex for entity-coref-dbpedia
+Synchronized=true
+Description=DBpedia.org
+Index-Archive=entity-coref-dbpedia.solrindex.zip
diff --git a/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.core.site.CacheImpl-entity-coref-dbpedia.config b/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.core.site.CacheImpl-entity-coref-dbpedia.config
new file mode 100644
index 0000000..0cca7fc
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.core.site.CacheImpl-entity-coref-dbpedia.config
@@ -0,0 +1,4 @@
+org.apache.stanbol.entityhub.yard.name="entity-coref-dbpedia\ Cache"
+org.apache.stanbol.entityhub.yard.cacheYardId="entity-coref-dbpediaIndex"
+org.apache.stanbol.entityhub.yard.id="entity-coref-dbpediaIndex"
+org.apache.stanbol.entityhub.yard.description="Cache\ for\ the\ entity-coref-dbpedia\ Referenced\ Site\ using\ the\ entity-coref-dbpediaIndex."
diff --git a/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.site.referencedSite-entity-coref-dbpedia.config b/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.site.referencedSite-entity-coref-dbpedia.config
new file mode 100644
index 0000000..220f697
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.site.referencedSite-entity-coref-dbpedia.config
@@ -0,0 +1,9 @@
+org.apache.stanbol.entityhub.site.defaultExpireDuration=I"0"
+org.apache.stanbol.entityhub.site.cacheId="entity-coref-dbpediaIndex"
+org.apache.stanbol.entityhub.site.defaultSymbolState="proposed"
+org.apache.stanbol.entityhub.site.name="entity-coref-dbpedia"
+org.apache.stanbol.entityhub.site.id="entity-coref-dbpedia"
+org.apache.stanbol.entityhub.site.description="DBpedia.org\ "
+org.apache.stanbol.entityhub.site.defaultMappedEntityState="proposed"
+org.apache.stanbol.entityhub.site.fieldMappings=("#\ Licensed\ to\ the\ Apache\ Software\ Foundation\ (ASF)\ under\ one\ or\ more","#\ contributor\ license\ agreements.\ \ See\ the\ NOTICE\ file\ distributed\ with","#\ this\ work\ for\ additional\ information\ regarding\ copyright\ ownership.","#\ The\ ASF\ licenses\ this\ file\ to\ You\ under\ the\ Apache\ License,\ Version\ 2.0","#\ (the\ \"License\");\ you\ may\ not\ use\ this\ file\ except\ in\ compliance\ with","#\ the\ License.\ \ You\ may\ obtain\ a\ copy\ of\ the\ License\ at","#","#\ \ \ \ \ http://www.apache.org/licenses/LICENSE-2.0","#","#\ Unless\ required\ by\ applicable\ law\ or\ agreed\ to\ in\ writing,\ software","#\ distributed\ under\ the\ License\ is\ distributed\ on\ an\ \"AS\ IS\"\ BASIS,","#\ WITHOUT\ WARRANTIES\ OR\ CONDITIONS\ OF\ ANY\ KIND,\ either\ express\ or\ implied.","#\ See\ the\ License\ for\ the\ specific\ language\ governing\ permissions\ and","#\ limitations\ under\ the\ License.","#","#NOTE:\ THIS\ IS\ A\ DEFAULT\ MAPPING\ SPECIFICATION\ THAT\ INCLUDES\ MAPPINGS\ FOR","#\ \ \ \ \ \ COMMON\ ONTOLOGIES.\ USERS\ MIGHT\ WANT\ TO\ ADAPT\ THIS\ CONFIGURATION\ BY","#\ \ \ \ \ \ COMMENTING/UNCOMMENTING\ AND/OR\ ADDING\ NEW\ MAPPINGS","","#\ ---\ Define\ the\ Languages\ for\ all\ fields\ ---","#\ to\ restrict\ languages\ to\ be\ imported\ (for\ all\ fields)","#|\ @\=null;en;de;fr;it","","#NOTE:\ null\ is\ used\ to\ import\ labels\ with\ no\ specified\ language","","#\ ---\ Define\ the\ Languages\ for\ all\ fields\ ---","#\ Uncomment\ to\ restrict\ indexing\ to\ a\ specific\ list\ of\ languages,\ otherwise\ all","#\ languages\ are\ indexed","#|\ @\=null;en;de;fr;it","","#\ ---\ RDF\ RDFS\ and\ OWL\ Mappings\ ---","#\ This\ configuration\ only\ index\ properties\ that\ are\ typically\ used\ to\ store","#\ instance\ data\ defined\ by\ such\ namespaces.\ This\ excludes\ ontology\ definitions","","#\ NOTE\ that\ nearly\ all\ other\ ontologies\ are\ are\ using\ properties\ of\ these\ three","#\ \ \ \ \ \ schemas,\ therefore\ it\ is\ strongly\ recommended\ to\ include\ such\ information!","","#rdf:type\ |\ d\=entityhub:ref","","rdfs:label\ ","#rdfs:comment","#rdfs:seeAlso\ |\ d\=entityhub:ref","","","#owl:sameAs\ |\ d\=entityhub:ref","","#If\ one\ likes\ to\ also\ index\ ontologies\ one\ should\ add\ the\ following\ statements","#owl:*","#rdfs:*","","#\ ---\ Dublin\ Core\ (DC)\ ---","#\ The\ default\ configuration\ imports\ all\ dc-terms\ data\ and\ copies\ values\ for\ the","#\ old\ dc-elements\ standard\ over\ to\ the\ according\ properties\ of\ the\ dc-terms","#\ standard.","","#\ NOTE\ that\ a\ lot\ of\ other\ ontologies\ are\ also\ using\ DC\ for\ some\ of\ there\ data","#\ \ \ \ \ \ therefore\ it\ is\ strongly\ recommended\ to\ include\ such\ information!","","#mapping\ for\ all\ dc-terms\ properties","#dc:*","","#\ copy\ dc:title\ to\ rdfs:label","#dc:title\ >\ rdfs:label","","#\ deactivated\ by\ default,\ because\ such\ mappings\ are\ mapped\ to\ dc-terms","#dc-elements:*","","#\ mappings\ for\ the\ dc-elements\ properties\ to\ the\ dc-terms","#dc-elements:contributor\ >\ dc:contributor","#dc-elements:coverage\ >\ dc:coverage","#dc-elements:creator\ >\ dc:creator","#dc-elements:date\ >\ dc:date","#dc-elements:description\ >\ dc:description","#dc-elements:format\ >\ dc:format","#dc-elements:identifier\ >\ dc:identifier","#dc-elements:language\ >\ dc:language","#dc-elements:publisher\ >\ dc:publisher","#dc-elements:relation\ >\ dc:relation","#dc-elements:rights\ >\ dc:rights","#dc-elements:source\ >\ dc:source","#dc-elements:subject\ >\ dc:subject","#dc-elements:title\ >\ dc:title","#dc-elements:type\ >\ dc:type","#also\ use\ dc-elements:title\ as\ label","#dc-elements:title\ >\ rdfs:label","","#\ ---\ Social\ Networks\ (via\ foaf)\ ---","#The\ Friend\ of\ a\ Friend\ schema\ is\ often\ used\ to\ describe\ social\ relations\ between\ people","#foaf:*","","#\ copy\ the\ name\ of\ a\ person\ over\ to\ rdfs:label","#foaf:name\ >\ rdfs:label","","#\ additional\ data\ types\ checks","#foaf:knows\ |\ d\=entityhub:ref","#foaf:made\ |\ d\=entityhub:ref","#foaf:maker\ |\ d\=entityhub:ref","#foaf:member\ |\ d\=entityhub:ref","#foaf:homepage\ |\ d\=xsd:anyURI","#foaf:depiction\ |\ d\=xsd:anyURI","#foaf:img\ |\ d\=xsd:anyURI","#foaf:logo\ |\ d\=xsd:anyURI","#page\ about\ the\ entity","#foaf:page\ |\ d\=xsd:anyURI","","","#\ ---\ Schema.org\ --","","#\ Defines\ an\ Ontology\ used\ by\ search\ engines\ (Google,\ Yahoo\ and\ Bing)\ for\ ","#\ indexing\ websites.","","#schema:*","#\ Copy\ all\ names\ of\ schema\ instances\ over\ to\ rdfs:label","#schema:name\ >\ rdfs:label","","#\ ---\ Simple\ Knowledge\ Organization\ System\ (SKOS)\ ---","","#\ A\ common\ data\ model\ for\ sharing\ and\ linking\ knowledge\ organization\ systems\ ","#\ via\ the\ Semantic\ Web.\ Typically\ used\ to\ encode\ controlled\ vocabularies\ as","#\ a\ thesaurus\ \ ","#skos:*","","#\ copy\ all\ SKOS\ labels\ (preferred,\ alternative\ and\ hidden)\ over\ to\ rdfs:label","#skos:prefLabel\ >\ rdfs:label","#skos:altLabel\ >\ rdfs:label","#skos:hiddenLabel\ >\ rdfs:label","","#\ copy\ values\ of\ **Match\ relations\ to\ the\ according\ related,\ broader\ and\ narrower","#skos:relatedMatch\ >\ skos:related","#skos:broadMatch\ >\ skos:broader","#skos:narrowMatch\ >\ skos:skos:narrower","","#similar\ mappings\ for\ transitive\ variants\ are\ not\ contained,\ because\ transitive","#reasoning\ is\ not\ directly\ supported\ by\ the\ Entityhub.","","#\ Some\ SKOS\ thesaurus\ do\ use\ \"skos:transitiveBroader\"\ and\ \"skos:transitiveNarrower\"","#\ however\ such\ properties\ are\ only\ intended\ to\ be\ used\ by\ reasoners\ to","#\ calculate\ transitive\ closures\ over\ broader/narrower\ hierarchies.","#\ see\ http://www.w3.org/TR/skos-reference/#L2413\ for\ details","#\ to\ correct\ such\ cases\ we\ will\ copy\ transitive\ relations\ to\ their\ counterpart","#skos:narrowerTransitive\ >\ skos:narrower","#skos:broaderTransitive\ >\ skos:broader","","","#\ ---\ Semantically-Interlinked\ Online\ Communities\ (SIOC)\ ---","","#\ An\ ontology\ for\ describing\ the\ information\ in\ online\ communities.\ ","#\ This\ information\ can\ be\ used\ to\ export\ information\ from\ online\ communities\ ","#\ and\ to\ link\ them\ together.\ The\ scope\ of\ the\ application\ areas\ that\ SIOC\ can\ ","#\ be\ used\ for\ includes\ (and\ is\ not\ limited\ to)\ weblogs,\ message\ boards,\ ","#\ mailing\ lists\ and\ chat\ channels.","#sioc:*","","#\ ---\ biographical\ information\ (bio)","#\ A\ vocabulary\ for\ describing\ biographical\ information\ about\ people,\ both\ living","#\ and\ dead.\ (see\ http://vocab.org/bio/0.1/)","#bio:*","","#\ ---\ Rich\ Site\ Summary\ (rss)\ ---","#rss:*","","#\ ---\ GoodRelations\ (gr)\ ---","#\ GoodRelations\ is\ a\ standardised\ vocabulary\ for\ product,\ price,\ and\ company\ data","#gr:*","","#\ ---\ Creative\ Commons\ Rights\ Expression\ Language\ (cc)","#\ The\ Creative\ Commons\ Rights\ Expression\ Language\ (CC\ REL)\ lets\ you\ describe\ ","#\ copyright\ licenses\ in\ RDF.","#cc:*","","","","","","")
+org.apache.stanbol.entityhub.site.cacheStrategy="all"
diff --git a/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-entity-coref-dbpedia.config b/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-entity-coref-dbpedia.config
new file mode 100644
index 0000000..f346063
--- /dev/null
+++ b/data/sites/entity-coref-dbpedia/src/main/resources/org/apache/stanbol/data/site/entity-coref-dbpedia/default/config/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-entity-coref-dbpedia.config
@@ -0,0 +1,7 @@
+org.apache.stanbol.entityhub.yard.solr.solrUri="entity-coref-dbpedia"
+org.apache.stanbol.entityhub.yard.name="entity-coref-dbpedia\ Index"
+org.apache.stanbol.entityhub.yard.solr.multiYardIndexLayout=B"false"
+org.apache.stanbol.entityhub.yard.solr.useDefaultConfig=B"false"
+org.apache.stanbol.entityhub.yard.id="entity-coref-dbpediaIndex"
+http://stanbol.apache.org/ontology/entityhub/entityhub#entityRank="http://stanbol.apache.org/ontology/entityhub/entityhub#entityRank"
+org.apache.stanbol.entityhub.yard.description="Full\ local\ index\ for\ the\ Referenced\ Site\ \"entity-coref-dbpedia\"."
diff --git a/enhancement-engines/entitycoreference/README.md b/enhancement-engines/entitycoreference/README.md
new file mode 100644
index 0000000..2ef9311
--- /dev/null
+++ b/enhancement-engines/entitycoreference/README.md
@@ -0,0 +1,42 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Entity Co-reference Engine
+
+The Entity co-reference Engine performs co-reference resolution of Named Entities in a given text.
+The co-references will be noun phrases which refer to those Named Entities by having a minimal set of attributes which match
+contextual information (rdf:type of the entity and spatial and object function giving info) from entity repositories
+such as Dbpedia and Yago for that Named Entity.
+
+We have the following text as an example : "Microsoft has posted its 2013 earnings. The software company did better than expected. ... The Redmond-based company will hire 500 new developers this year."
+The enhancement engine will link "Microsoft" with "The software company" and "The Redmond-based company".
+
+## (1) Configuring the Engine
+TODO
+
+## (2) Running the Entity co-reference engine in Stanbol.
+
+In order to run the engine you need to add it to a chain that also contains the following engine types:
+- a language detection engine
+- a sentence detection engine (like opennlp-sentence)
+- a token detection engine (like opennlp-token)
+- a NER detection engine (like opennlp-ner)
+- a noun phrase detection engine (like pos-chunker)
+
+The default data bundle which contains dbpedia and yago data with which the coreferencing is done is the entity-coref-dbpedia data bundle.
+You can find the bundle in data/sites/entity-coref-dbpedia. Install this bundle into Stanbol.
+You can create your own data bundle but be sure to input the correct attributes when configuring the engine(see point no 1).
diff --git a/enhancement-engines/entitycoreference/pom.xml b/enhancement-engines/entitycoreference/pom.xml
new file mode 100644
index 0000000..e54db7a
--- /dev/null
+++ b/enhancement-engines/entitycoreference/pom.xml
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>apache-stanbol-enhancement-engines</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitycoreference</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ <packaging>bundle</packaging>
+
+ <name>Apache Stanbol Enhancement Engine : Entity Co-Reference</name>
+ <description>
+ An Engine that finds co-references of Named Entities based on
+ dbpedia/yago concepts.
+ </description>
+
+ <inceptionYear>2014</inceptionYear>
+
+ <scm>
+ <connection>
+ scm:svn:http://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/entitycoreference/
+ </connection>
+ <developerConnection>
+ scm:svn:https://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/entitycoreference/
+ </developerConnection>
+ <url>http://stanbol.apache.org/</url>
+ </scm>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Import-Package>
+ org.apache.stanbol.enhancer.servicesapi; provide:=true; version="[0.11,1.1)",
+ org.apache.stanbol.enhancer.engines.entitylinking;version=${project.version}; provide:=true,
+ *
+ </Import-Package>
+ <Private-Package>
+ org.apache.stanbol.enhancer.engines.entitycoreference.*;version=${project.version}
+ </Private-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.commons.namespaceprefix.service</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitylinking.engine</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.servicesapi</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.scr.annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency> <!-- used for debug level logging during tests -->
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
diff --git a/enhancement-engines/entitycoreference/src/license/THIRD-PARTY.properties b/enhancement-engines/entitycoreference/src/license/THIRD-PARTY.properties
new file mode 100644
index 0000000..21c81d0
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/license/THIRD-PARTY.properties
@@ -0,0 +1,24 @@
+# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
+#-------------------------------------------------------------------------------
+# Already used licenses in project :
+# - Apache Software License
+# - Apache Software License, Version 2.0
+# - BSD License
+# - Common Development And Distribution License (CDDL), Version 1.0
+# - Common Development And Distribution License (CDDL), Version 1.1
+# - Common Public License, Version 1.0
+# - Eclipse Public License, Version 1.0
+# - GNU General Public License (GPL), Version 2 with classpath exception
+# - GNU Lesser General Public License (LGPL)
+# - GNU Lesser General Public License (LGPL), Version 2.1
+# - ICU License
+# - MIT License
+# - New BSD License
+# - Public Domain License
+#-------------------------------------------------------------------------------
+# Please fill the missing licenses for dependencies :
+#
+#
+#Tue Jul 23 16:41:35 CEST 2013
+org.osgi--org.osgi.compendium--4.1.0=The Apache Software License, Version 2.0
+org.osgi--org.osgi.core--4.1.0=The Apache Software License, Version 2.0
diff --git a/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/Constants.java b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/Constants.java
new file mode 100644
index 0000000..cbefffd
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/Constants.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitycoreference;
+
+/**
+ * Constants used engine wide.
+ *
+ * @author Cristian Petroaca
+ *
+ */
+public final class Constants {
+ /**
+ * The main config folder of the engine
+ */
+ public final static String CONFIG_FOLDER = "/config";
+
+ /**
+ * The main data folder
+ */
+ public final static String DATA_FOLDER = "/data";
+
+ /**
+ * The path to the pos config folder.
+ */
+ public final static String POS_CONFIG_FOLDER = CONFIG_FOLDER + "/pos";
+
+ /**
+ * The path to the place adjectivals folder.
+ */
+ public final static String PLACE_ADJECTIVALS_FOLDER = DATA_FOLDER
+ + "/place_adjectivals";
+
+ public static final int MAX_DISTANCE_DEFAULT_VALUE = 1;
+
+ public static final int MAX_DISTANCE_NO_CONSTRAINT = -1;
+
+ public final static String DEFAULT_SPATIAL_ATTR_FOR_PERSON = "http://dbpedia.org/ontology/birthPlace,"
+ + "http://dbpedia.org/ontology/region,http://dbpedia.org/ontology/nationality,http://dbpedia.org/ontology/country";
+
+ public final static String DEFAULT_SPATIAL_ATTR_FOR_ORGANIZATION = "http://dbpedia.org/ontology/foundationPlace,"
+ + "http://dbpedia.org/ontology/locationCity,http://dbpedia.org/ontology/location,http://dbpedia.org/ontology/hometown";
+
+ public final static String DEFAULT_SPATIAL_ATTR_FOR_PLACE = "http://dbpedia.org/ontology/country,"
+ + "http://dbpedia.org/ontology/subdivisionName,http://dbpedia.org/ontology/location";
+
+ public final static String DEFAULT_ENTITY_CLASSES_TO_EXCLUDE = "http://dbpedia.org/ontology/Person,"
+ + "http://dbpedia.org/class/yago/LivingThing100004258,http://dbpedia.org/class/yago/PhysicalEntity100001930,"
+ + "http://dbpedia.org/class/yago/Abstraction100002137,http://dbpedia.org/class/yago/Organism100004475,"
+ + "http://dbpedia.org/class/yago/Location100027167,http://schema.org/Place,http://dbpedia.org/class/yago/Object100002684,"
+ + "http://dbpedia.org/class/yago/YagoGeoEntity,http://www.w3.org/2002/07/owl#Thing,"
+ + "http://dbpedia.org/class/yago/YagoPermanentlyLocatedEntity";
+
+ public final static String DEFAULT_ORG_ATTR_FOR_PERSON = "http://dbpedia.org/ontology/occupation,"
+ + "http://dbpedia.org/ontology/associatedBand,http://dbpedia.org/ontology/employer";
+
+ private Constants() {
+ }
+}
diff --git a/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/EntityCoReferenceEngine.java b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/EntityCoReferenceEngine.java
new file mode 100644
index 0000000..efc3814
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/EntityCoReferenceEngine.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitycoreference;
+
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.getLanguage;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Dictionary;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Properties;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.enhancer.engines.entitycoreference.datamodel.NounPhrase;
+import org.apache.stanbol.enhancer.engines.entitycoreference.impl.CoreferenceFinder;
+import org.apache.stanbol.enhancer.engines.entitycoreference.impl.NounPhraseFilterer;
+import org.apache.stanbol.enhancer.nlp.NlpAnnotations;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
+import org.apache.stanbol.enhancer.nlp.model.Section;
+import org.apache.stanbol.enhancer.nlp.model.Span;
+import org.apache.stanbol.enhancer.nlp.model.SpanTypeEnum;
+import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
+import org.apache.stanbol.enhancer.nlp.ner.NerTag;
+import org.apache.stanbol.enhancer.nlp.phrase.PhraseTag;
+import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
+import org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
+import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine;
+import org.apache.stanbol.entityhub.servicesapi.Entityhub;
+import org.apache.stanbol.entityhub.servicesapi.site.SiteManager;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This engine extracts references in the given text of noun phrases which point to NERs. The coreference is
+ * performed based on matching several of the named entity's dbpedia/yago properties to the noun phrase
+ * tokens.
+ *
+ * TODO - Be able to detect possessive coreferences such as Germany's prime minister
+ * TODO - be able to detect products and their developer such as Iphone 7 and Apple's new device.
+ * TODO - provide the ability via config for the user to also allow coreferencing of 1 word noun phrases based
+ * solely on comparison with entity class type?
+ *
+ * @author Cristian Petroaca
+ *
+ */
+@Component(immediate = true, metatype = true)
+@Service(value = EnhancementEngine.class)
+@Properties(value = {
+ @Property(name = EnhancementEngine.PROPERTY_NAME, value = "entity-coreference"),
+ @Property(name = EntityCoReferenceEngine.CONFIG_LANGUAGES, value = "en"),
+ @Property(name = EntityCoReferenceEngine.REFERENCED_SITE_ID, value = "entity-coref-dbpedia"),
+ @Property(name = EntityCoReferenceEngine.ENTITY_URI_BASE, value = "http://dbpedia.org/resource/"),
+ @Property(name = EntityCoReferenceEngine.MAX_DISTANCE, intValue = Constants.MAX_DISTANCE_DEFAULT_VALUE),
+ @Property(name = EntityCoReferenceEngine.SPATIAL_ATTR_FOR_PERSON, value = Constants.DEFAULT_SPATIAL_ATTR_FOR_PERSON),
+ @Property(name = EntityCoReferenceEngine.SPATIAL_ATTR_FOR_ORGANIZATION, value = Constants.DEFAULT_SPATIAL_ATTR_FOR_ORGANIZATION),
+ @Property(name = EntityCoReferenceEngine.SPATIAL_ATTR_FOR_PLACE, value = Constants.DEFAULT_SPATIAL_ATTR_FOR_PLACE),
+ @Property(name = EntityCoReferenceEngine.ORG_ATTR_FOR_PERSON, value = Constants.DEFAULT_ORG_ATTR_FOR_PERSON),
+ @Property(name = EntityCoReferenceEngine.ENTITY_CLASSES_TO_EXCLUDE, value = Constants.DEFAULT_ENTITY_CLASSES_TO_EXCLUDE)})
+public class EntityCoReferenceEngine extends AbstractEnhancementEngine<RuntimeException,RuntimeException>
+ implements EnhancementEngine, ServiceProperties {
+
+ private static final Integer ENGINE_ORDERING = ServiceProperties.ORDERING_POST_PROCESSING + 91;
+
+ /**
+ * Language configuration. Takes a list of ISO language codes of supported languages. Currently supported
+ * are the languages given as default value.
+ */
+ protected static final String CONFIG_LANGUAGES = "enhancer.engine.entitycoreference.languages";
+
+ /**
+ * Referenced site configuration. Defaults to dbpedia.
+ */
+ protected static final String REFERENCED_SITE_ID = "enhancer.engine.entitycoreference.referencedSiteId";
+
+ /**
+ *
+ */
+ protected static final String ENTITY_URI_BASE = "enhancer.engine.entitycoreference.entity.uri.base";
+
+ /**
+ * Maximum sentence distance between the ner and the noun phrase which mentions it. -1 means no distance
+ * constraint.
+ */
+ protected static final String MAX_DISTANCE = "enhancer.engine.entitycoreference.maxDistance";
+
+ /**
+ * Attributes used for spatial coreference when dealing with a person entity.
+ */
+ protected static final String SPATIAL_ATTR_FOR_PERSON = "enhancer.engine.entitycoreference.spatial.attr.person";
+
+ /**
+ * Attributes used for spatial coreference when dealing with an organization entity.
+ */
+ protected static final String SPATIAL_ATTR_FOR_ORGANIZATION = "enhancer.engine.entitycoreference.spatial.attr.org";
+
+ /**
+ * Attributes used for spatial coreference when dealing with a place entity.
+ */
+ protected static final String SPATIAL_ATTR_FOR_PLACE = "enhancer.engine.entitycoreference.spatial.attr.place";
+
+ /**
+ * Attributes used for organisational membership coreference when dealing with a person entity.
+ */
+ protected static final String ORG_ATTR_FOR_PERSON = "enhancer.engine.entitycoreference.org.attr.person";
+
+ /**
+ * Entity classes which will be excluded when doing the entity class type matching
+ * because they are too general in nature.
+ */
+ protected static final String ENTITY_CLASSES_TO_EXCLUDE = "enhancer.engine.entitycoreference.entity.classes.excluded";
+
+ /**
+ * Logger
+ */
+ private final Logger log = LoggerFactory.getLogger(EntityCoReferenceEngine.class);
+
+ /**
+ * Service of the Entityhub that manages all the active referenced Site. This Service is used to lookup
+ * the configured Referenced Site when we need to enhance a content item.
+ */
+ @Reference
+ protected SiteManager siteManager;
+
+ /**
+ * Used to lookup Entities if the {@link #REFERENCED_SITE_ID} property is set to "entityhub" or "local"
+ */
+ @Reference
+ protected Entityhub entityhub;
+
+ /**
+ * Specialized class which filters out bad noun phrases based on the language.
+ */
+ private NounPhraseFilterer nounPhraseFilterer;
+
+ /**
+ * Performs the logic needed to find corefs based on the NERs and noun phrases in the text.
+ */
+ private CoreferenceFinder corefFinder;
+
+ @SuppressWarnings("unchecked")
+ @Activate
+ protected void activate(ComponentContext ctx) throws ConfigurationException {
+ super.activate(ctx);
+
+ Dictionary<String,Object> config = ctx.getProperties();
+
+ /* Step 1 - initialize the {@link NounPhraseFilterer} with the language config */
+ String languages = (String) config.get(CONFIG_LANGUAGES);
+
+ if (languages == null || languages.isEmpty()) {
+ throw new ConfigurationException(CONFIG_LANGUAGES,
+ "The Languages Config is a required Parameter and MUST NOT be NULL or an empty String!");
+ }
+
+ nounPhraseFilterer = new NounPhraseFilterer(languages.split(","));
+
+ /* Step 2 - initialize the {@link CoreferenceFinder} */
+ String referencedSiteID = null;
+ Object referencedSiteIDfromConfig = config.get(REFERENCED_SITE_ID);
+
+ if (referencedSiteIDfromConfig == null) {
+ throw new ConfigurationException(REFERENCED_SITE_ID,
+ "The ID of the Referenced Site is a required Parameter and MUST NOT be NULL!");
+ }
+
+ referencedSiteID = referencedSiteIDfromConfig.toString();
+ if (referencedSiteID.isEmpty()) {
+ throw new ConfigurationException(REFERENCED_SITE_ID,
+ "The ID of the Referenced Site is a required Parameter and MUST NOT be an empty String!");
+ }
+
+ if (Entityhub.ENTITYHUB_IDS.contains(referencedSiteID.toLowerCase())) {
+ log.debug("Init NamedEntityTaggingEngine instance for the Entityhub");
+ referencedSiteID = null;
+ }
+
+ int maxDistance;
+ Object maxDistanceFromConfig = config.get(MAX_DISTANCE);
+
+ if (maxDistanceFromConfig == null) {
+ maxDistance = Constants.MAX_DISTANCE_DEFAULT_VALUE;
+ } else if (maxDistanceFromConfig instanceof Number) {
+ maxDistance = ((Number) maxDistanceFromConfig).intValue();
+ } else {
+ try {
+ maxDistance = Integer.parseInt(maxDistanceFromConfig.toString());
+ } catch (NumberFormatException nfe) {
+ throw new ConfigurationException(MAX_DISTANCE, "The Max Distance parameter must be a number");
+ }
+ }
+
+ if (maxDistance < -1) {
+ throw new ConfigurationException(MAX_DISTANCE,
+ "The Max Distance parameter must not be smaller than -1");
+ }
+
+ String entityUriBase = (String) config.get(ENTITY_URI_BASE);
+ if (entityUriBase == null || entityUriBase.isEmpty()) {
+ throw new ConfigurationException(ENTITY_URI_BASE, "The Entity Uri Base parameter cannot be empty");
+ }
+
+ String spatialAttrForPerson = (String) config.get(SPATIAL_ATTR_FOR_PERSON);
+ String spatialAttrForOrg = (String) config.get(SPATIAL_ATTR_FOR_ORGANIZATION);
+ String spatialAttrForPlace = (String) config.get(SPATIAL_ATTR_FOR_PLACE);
+ String orgAttrForPerson = (String) config.get(ORG_ATTR_FOR_PERSON);
+ String entityClassesToExclude = (String) config.get(ENTITY_CLASSES_TO_EXCLUDE);
+
+ corefFinder = new CoreferenceFinder(languages.split(","), siteManager, entityhub, referencedSiteID,
+ maxDistance, entityUriBase, spatialAttrForPerson, spatialAttrForOrg,
+ spatialAttrForPlace, orgAttrForPerson, entityClassesToExclude);
+
+ log.info("activate {}[name:{}]", getClass().getSimpleName(), getName());
+ }
+
+ @Override
+ public Map<String,Object> getServiceProperties() {
+ return Collections.unmodifiableMap(Collections.singletonMap(ENHANCEMENT_ENGINE_ORDERING,
+ (Object) ENGINE_ORDERING));
+ }
+
+ @Override
+ public int canEnhance(ContentItem ci) throws EngineException {
+ String language = getLanguage(this, ci, false);
+ if (language == null) {
+ log.debug("Engine {} ignores ContentItem {} becuase language {} is not detected.",
+ new Object[] {getName(), ci.getUri(), language});
+ return CANNOT_ENHANCE;
+ }
+
+ if (!nounPhraseFilterer.supportsLanguage(language)) {
+ log.debug("Engine {} does not support language {}.", new Object[] {getName(), language});
+ return CANNOT_ENHANCE;
+ }
+
+ return ENHANCE_SYNCHRONOUS;
+ }
+
+ @Override
+ public void computeEnhancements(ContentItem ci) throws EngineException {
+ /*
+ * Step 1 - Build the NER list and the noun phrase list.
+ *
+ * TODO - the noun phrases need to be lemmatized.
+ */
+ Map<Integer,List<Span>> ners = new HashMap<Integer,List<Span>>();
+ List<NounPhrase> nounPhrases = new ArrayList<NounPhrase>();
+ extractNersAndNounPhrases(ci, ners, nounPhrases);
+
+ /*
+ * If there are no NERs to reference there's nothing to do but exit.
+ */
+ if (ners.size() == 0) {
+ log.info("Did not find any NERs for which to do the coreferencing");
+ return;
+ }
+
+ /*
+ * Step 2 - Filter out bad noun phrases.
+ */
+ String language = getLanguage(this, ci, false);
+ if (language == null) {
+ log.info("Could not detect the language of the text");
+ return;
+ }
+
+ nounPhraseFilterer.filter(nounPhrases, language);
+
+ /*
+ * If there are no good noun phrases there's nothing to do but exit.
+ */
+ if (nounPhrases.size() == 0) {
+ log.info("Did not find any noun phrases with which to do the coreferencing");
+ return;
+ }
+
+ /*
+ * Step 3 - Extract corefs and write them as {@link NlpAnnotations.COREF_ANNOTATION}s in the {@link
+ * Span}s
+ */
+ corefFinder.extractCorefs(ners, nounPhrases, language);
+ }
+
+ @Deactivate
+ protected void deactivate(ComponentContext ctx) {
+ log.info("deactivate {}[name:{}]", getClass().getSimpleName(), getName());
+
+ nounPhraseFilterer = null;
+ corefFinder = null;
+
+ super.deactivate(ctx);
+ }
+
+ /**
+ * Extracts the NERs and the noun phrases from the given text and puts them in the given lists.
+ *
+ * @param ci
+ * @param ners
+ * @param nounPhrases
+ */
+ private void extractNersAndNounPhrases(ContentItem ci,
+ Map<Integer,List<Span>> ners,
+ List<NounPhrase> nounPhrases) {
+ AnalysedText at = NlpEngineHelper.getAnalysedText(this, ci, true);
+ Iterator<? extends Section> sections = at.getSentences();
+ if (!sections.hasNext()) { // process as single sentence
+ sections = Collections.singleton(at).iterator();
+ }
+
+ int sentenceCnt = 0;
+ while (sections.hasNext()) {
+ sentenceCnt++;
+ Section section = sections.next();
+ List<NounPhrase> sectionNounPhrases = new ArrayList<NounPhrase>();
+ List<Span> sectionNers = new ArrayList<Span>();
+
+ Iterator<Span> chunks = section.getEnclosed(EnumSet.of(SpanTypeEnum.Chunk));
+ while (chunks.hasNext()) {
+ Span chunk = chunks.next();
+
+ Value<NerTag> ner = chunk.getAnnotation(NlpAnnotations.NER_ANNOTATION);
+ if (ner != null) {
+ sectionNers.add(chunk);
+ }
+
+ Value<PhraseTag> phrase = chunk.getAnnotation(NlpAnnotations.PHRASE_ANNOTATION);
+ if (phrase != null && phrase.value().getCategory() == LexicalCategory.Noun) {
+ sectionNounPhrases.add(new NounPhrase(chunk, sentenceCnt));
+ }
+ }
+
+ for (NounPhrase nounPhrase : sectionNounPhrases) {
+ Iterator<Span> tokens = section.getEnclosed(EnumSet.of(SpanTypeEnum.Token));
+
+ while (tokens.hasNext()) {
+ Span token = tokens.next();
+
+ if (nounPhrase.containsSpan(token)) {
+ nounPhrase.addToken(token);
+ }
+ }
+
+ for (Span sectionNer : sectionNers) {
+ if (nounPhrase.containsSpan(sectionNer)) {
+ nounPhrase.addNerChunk(sectionNer);
+ }
+ }
+ }
+
+ nounPhrases.addAll(sectionNounPhrases);
+
+ if (!sectionNers.isEmpty()) {
+ ners.put(sentenceCnt, sectionNers);
+ }
+ }
+ }
+}
diff --git a/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/datamodel/NounPhrase.java b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/datamodel/NounPhrase.java
new file mode 100644
index 0000000..9b916b6
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/datamodel/NounPhrase.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitycoreference.datamodel;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.stanbol.enhancer.nlp.model.Span;
+
+/**
+ * Encapsulates span and sentence information about a noun phrase.
+ *
+ * @author Cristian Petroaca
+ *
+ */
+public class NounPhrase {
+ /**
+ * The {@link Span} which represents this noun phrase.
+ */
+ private Span chunk;
+
+ /*
+ * TODO - should use Set instead?
+ */
+ /**
+ * The {@link Span}s - tokens - which make up this noun phrase.
+ */
+ private List<Span> tokens;
+
+ /**
+ * The {@link Span}s contained in this noun phrase which represent Ners.
+ */
+ private List<Span> nerChunks;
+
+ /**
+ * The sentence index in which this noun phrase is found.
+ */
+ private int sentenceNo;
+
+ public NounPhrase(Span chunk, int sentenceNo) {
+ if (chunk == null) {
+ throw new IllegalArgumentException("Chunk cannot be null");
+ }
+
+ this.chunk = chunk;
+ this.tokens = new ArrayList<Span>();
+ this.nerChunks = new ArrayList<Span>();
+ this.sentenceNo = sentenceNo;
+ }
+
+ /**
+ * Gets the chunk representing this noun phrase.
+ *
+ * @return
+ */
+ public Span getChunk() {
+ return chunk;
+ }
+
+ /**
+ * Adds a new token which is found in this noun phrase.
+ *
+ * @param token
+ */
+ public void addToken(Span token) {
+ /*
+ * TODO - validate token boundaries within this noun phrase.
+ */
+ tokens.add(token);
+ }
+
+ /**
+ * Gets the list of tokens which make up this noun phrase.
+ *
+ * @return
+ */
+ public List<Span> getTokens() {
+ return tokens;
+ }
+
+ /**
+ * Adds a new NER chunk which is found within this noun phrase.
+ *
+ * @param chunk
+ */
+ public void addNerChunk(Span chunk) {
+ /*
+ * TODO - validate NER boundaries within this noun phrase.
+ */
+ nerChunks.add(chunk);
+ }
+
+ /**
+ * Gets the list of NERs within this noun phrase.
+ *
+ * @return
+ */
+ public List<Span> getNerChunks() {
+ return nerChunks;
+ }
+
+ /**
+ * Determines whether this noun phrase's {@link Span} contains the given {@link Span}.
+ *
+ * @param span
+ * @return
+ */
+ public boolean containsSpan(Span span) {
+ return (span.getStart() >= chunk.getStart() && span.getEnd() <= chunk.getEnd());
+ }
+
+ /**
+ * Determines whether this noun phrase has NERs.
+ *
+ * @return
+ */
+ public boolean hasNers() {
+ return nerChunks.size() > 0;
+ }
+
+ /**
+ * Returns the sentence index in which this noun phrase is found.
+ *
+ * @return
+ */
+ public int getSentenceNo() {
+ return this.sentenceNo;
+ }
+
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + chunk.hashCode();
+ result = prime * result + tokens.hashCode();
+ result = prime * result + nerChunks.hashCode();
+
+ return result;
+ }
+
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null) return false;
+ if (getClass() != obj.getClass()) return false;
+
+ NounPhrase other = (NounPhrase) obj;
+
+ return chunk.equals(other.chunk) && tokens.equals(other.tokens) && nerChunks.equals(other.nerChunks)
+ && sentenceNo == other.sentenceNo;
+ }
+}
diff --git a/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/datamodel/PlaceAdjectival.java b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/datamodel/PlaceAdjectival.java
new file mode 100644
index 0000000..2b59330
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/datamodel/PlaceAdjectival.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitycoreference.datamodel;
+
+import org.apache.clerezza.rdf.core.UriRef;
+
+/**
+ * Represents a place adjectival inside a {@link Span}.
+ *
+ * @author Cristian Petroaca
+ *
+ */
+public class PlaceAdjectival {
+ /**
+ * The start index in the {@link Span}.
+ */
+ private int startIdx;
+
+ /**
+ * The end index in the {@link Span}.
+ */
+ private int endIdx;
+
+ /**
+ * The {@link UriRef} in the {@link SiteManager} or {@link Entityhub} that this place adjectival points
+ * to.
+ */
+ private UriRef placeUri;
+
+ public PlaceAdjectival(int startIdx, int endIdx, UriRef placeUri) {
+ this.startIdx = startIdx;
+ this.endIdx = endIdx;
+ this.placeUri = placeUri;
+ }
+
+ public UriRef getPlaceUri() {
+ return placeUri;
+ }
+
+ public int getStart() {
+ return this.startIdx;
+ }
+
+ public int getEnd() {
+ return this.endIdx;
+ }
+
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + startIdx;
+ result = prime * result + endIdx;
+ result = prime * result + placeUri.hashCode();
+
+ return result;
+ }
+
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null) return false;
+ if (getClass() != obj.getClass()) return false;
+
+ PlaceAdjectival other = (PlaceAdjectival) obj;
+
+ return this.startIdx == other.startIdx && this.endIdx == other.endIdx
+ && this.placeUri.equals(other.placeUri);
+ }
+}
diff --git a/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/CoreferenceFinder.java b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/CoreferenceFinder.java
new file mode 100644
index 0000000..bcaf662
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/CoreferenceFinder.java
@@ -0,0 +1,404 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitycoreference.impl;
+
+import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.COREF_ANNOTATION;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDFS_LABEL;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.stanbol.enhancer.engines.entitycoreference.Constants;
+import org.apache.stanbol.enhancer.engines.entitycoreference.datamodel.NounPhrase;
+import org.apache.stanbol.enhancer.engines.entitycoreference.datamodel.PlaceAdjectival;
+import org.apache.stanbol.enhancer.nlp.NlpAnnotations;
+import org.apache.stanbol.enhancer.nlp.coref.CorefFeature;
+import org.apache.stanbol.enhancer.nlp.model.Span;
+import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
+import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.rdf.OntologicalClasses;
+import org.apache.stanbol.entityhub.servicesapi.Entityhub;
+import org.apache.stanbol.entityhub.servicesapi.model.Entity;
+import org.apache.stanbol.entityhub.servicesapi.model.Text;
+import org.apache.stanbol.entityhub.servicesapi.query.Constraint;
+import org.apache.stanbol.entityhub.servicesapi.query.FieldQuery;
+import org.apache.stanbol.entityhub.servicesapi.query.FieldQueryFactory;
+import org.apache.stanbol.entityhub.servicesapi.query.QueryResultList;
+import org.apache.stanbol.entityhub.servicesapi.query.ReferenceConstraint;
+import org.apache.stanbol.entityhub.servicesapi.query.TextConstraint;
+import org.apache.stanbol.entityhub.servicesapi.site.Site;
+import org.apache.stanbol.entityhub.servicesapi.site.SiteManager;
+import org.osgi.service.cm.ConfigurationException;
+
+/**
+ * Uses the list of NERs and the list of {@link NounPhrase}s found in the analyzed text to find possible
+ * co-references.
+ *
+ * @author Cristian Petroaca
+ *
+ */
+public class CoreferenceFinder {
+ /**
+ * The configured {@link SiteManager} for {@link Entity} storage.
+ */
+ private SiteManager siteManager;
+
+ /**
+ * The default {@link Entity} storage.
+ */
+ private Entityhub entityHub;
+
+ /**
+ * The name of the configured site for the {@link SiteManager}.
+ */
+ private String referencedSiteID;
+
+ /**
+ * In memory cache storing {@link Entity} types which are often used.
+ */
+ private InMemoryEntityTypeIndex entityTypeIndex;
+
+ /**
+ * Class holding configuration params.
+ */
+ private CoreferenceFinderConfig config;
+
+ /**
+ * Holds vocabulary.dictionary info such as the list of place adjectivals by language.
+ */
+ private Dictionaries dictionaries;
+
+ public CoreferenceFinder(String[] languages,
+ SiteManager siteManager,
+ Entityhub entityHub,
+ String referencedSiteID,
+ int maxDistance,
+ String entityUriBase,
+ String spatialAttrForPerson,
+ String spatialAttrForOrg,
+ String spatialAttrForPlace,
+ String orgAttributesForPerson,
+ String entityClassesToExclude) throws ConfigurationException {
+ this.siteManager = siteManager;
+ this.entityHub = entityHub;
+ this.referencedSiteID = referencedSiteID;
+ this.entityTypeIndex = new InMemoryEntityTypeIndex();
+ this.config = new CoreferenceFinderConfig(maxDistance, spatialAttrForPerson,
+ spatialAttrForOrg, spatialAttrForPlace, orgAttributesForPerson, entityClassesToExclude);
+ this.dictionaries = new Dictionaries(languages, entityUriBase);
+ }
+
+ /**
+ * Performs the actual coreference resolution by iterating through all the NERs and all the
+ * {@link NounPhrase}s which are after the given Ner in the text. If any coreferences are found they are
+ * written as {@link NlpAnnotation}s in the NER and noun phrase {@link Span}s.
+ *
+ * @param ners
+ * @param nounPhrases
+ * @param language
+ * @throws EngineException
+ */
+ public void extractCorefs(Map<Integer,List<Span>> ners, List<NounPhrase> nounPhrases, String language) throws EngineException {
+ for (Map.Entry<Integer,List<Span>> entry : ners.entrySet()) {
+ int nerSentenceNo = entry.getKey();
+ List<Span> nerSpans = entry.getValue();
+ int maxDistance = this.config.getMaxDistance();
+
+ for (Span ner : nerSpans) {
+ Entity entity = null;
+ Set<String> typeLabels = null;
+ Set<Span> corefs = new HashSet<Span>();
+
+ for (NounPhrase nounPhrase : nounPhrases) {
+ int nounPhraseSentenceNo = nounPhrase.getSentenceNo();
+
+ if (nounPhrase.getChunk().getStart() > ner.getStart()
+ && (maxDistance != Constants.MAX_DISTANCE_NO_CONSTRAINT
+ && nounPhraseSentenceNo > nerSentenceNo && nounPhraseSentenceNo - nerSentenceNo <= maxDistance)) {
+
+ if (entity == null) {
+ entity = lookupEntity(ner, language);
+
+ /*
+ * If the entity is still null there's nothing to do but go to the next ner.
+ */
+ if (entity == null) break;
+
+ if (typeLabels == null) {
+ typeLabels = buildEntityTypeLabels(entity, language);
+ }
+ }
+
+ if (isCoreferent(typeLabels, entity, ner, nounPhrase, language)) {
+ Set<Span> coreferencedNer = new HashSet<Span>();
+ coreferencedNer.add(ner);
+ Span chunk = nounPhrase.getChunk();
+
+ chunk.addAnnotation(COREF_ANNOTATION,
+ Value.value(new CorefFeature(false, coreferencedNer)));
+ corefs.add(chunk);
+ }
+ }
+ }
+
+ if (corefs.size() > 0) {
+ ner.addAnnotation(COREF_ANNOTATION, Value.value(new CorefFeature(true, corefs)));
+ }
+ }
+ }
+ }
+
+ /**
+ * Gets an Entity from the configured {@link Site} based on the NER text and type.
+ *
+ * @param ner
+ * @param language
+ * @return
+ * @throws EngineException
+ */
+ private Entity lookupEntity(Span ner, String language) throws EngineException {
+ Site site = getReferencedSite();
+ FieldQueryFactory queryFactory = site == null ? entityHub.getQueryFactory() : site.getQueryFactory();
+ FieldQuery query = queryFactory.createFieldQuery();
+
+ Constraint labelConstraint;
+ String namedEntityLabel = ner.getSpan();
+ labelConstraint = new TextConstraint(namedEntityLabel, false, language, null);
+ query.setConstraint(RDFS_LABEL.getUnicodeString(), labelConstraint);
+ query.setConstraint(RDF_TYPE.getUnicodeString(),
+ new ReferenceConstraint(ner.getAnnotation(NlpAnnotations.NER_ANNOTATION).value().getType()
+ .getUnicodeString()));
+ query.setLimit(1);
+ QueryResultList<Entity> results = site == null ? // if site is NULL
+ entityHub.findEntities(query)
+ : // use the Entityhub
+ site.findEntities(query); // else the referenced site
+
+ if (results.isEmpty()) return null;
+
+ // We set the limit to 1 so if it found anything it should contain just 1 entry
+ return results.iterator().next();
+ }
+
+ /**
+ * Performs the coreference matching rules: 1. Match the entity type. 2. If the {@link NounPhrase}
+ * contains any NERs match the NER to any spatial/org membership/functional Entity properties from the
+ * {@link Site}. 3. If {@link NounPhrase} contains any place adjectivals perform spatial co-reference
+ * based on the entity spatial properties.
+ *
+ * @param typeLabels
+ * - a list of types (classes) that the given entity has.
+ * @param entity
+ * - the entity for which we want to do the coref.
+ * @param ner
+ * - the ner in the text for which we want to do the coref.
+ * @param nounPhrase
+ * - the {@link NounPhrase} which we want to test for coref.
+ * @param language
+ * - the language of the text.
+ * @return
+ * @throws EngineException
+ */
+ private boolean isCoreferent(Set<String> typeLabels,
+ Entity entity,
+ Span ner,
+ NounPhrase nounPhrase,
+ String language) throws EngineException {
+ /*
+ * 1. Try to match the entity class to the noun phrase.
+ */
+ String matchedClass = null;
+ String nounPhraseText = nounPhrase.getChunk().getSpan().toLowerCase();
+ int classStart = 0;
+ int classEnd = 0;
+
+ for (String label : typeLabels) {
+ if (nounPhraseText.matches(".*\\b" + label + "\\b.*")
+ && (matchedClass == null || label.split("\\s").length > matchedClass.split("\\s").length)) {
+ matchedClass = label;
+ classStart = nounPhrase.getChunk().getStart() + nounPhraseText.indexOf(label);
+ classEnd = classStart + label.length();
+ }
+ }
+
+ if (matchedClass == null) return false;
+
+ /*
+ * 2. See if there are any NERs in the noun phrase to further identify the coref. Any NERs found
+ * should be separate words from the class matches from point 1.
+ */
+ /*
+ * TODO - devise a coref confidence scheme?
+ */
+ if (nounPhrase.hasNers()) {
+ List<Span> npNers = nounPhrase.getNerChunks();
+ UriRef nerType = ner.getAnnotation(NlpAnnotations.NER_ANNOTATION).value().getType();
+
+ for (Span npNer : npNers) {
+ /*
+ * Don't go any further if for some reason it turns out that the ner text is the same as the
+ * entity class text.
+ */
+ if ((npNer.getStart() >= classStart && npNer.getStart() <= classEnd)
+ || (npNer.getEnd() >= classStart && npNer.getEnd() <= classEnd)) continue;
+
+ Entity npEntity = lookupEntity(npNer, language);
+
+ if (npEntity != null) {
+ UriRef npNerType = npNer.getAnnotation(NlpAnnotations.NER_ANNOTATION).value().getType();
+ Set<String> rulesOntologyAttr = new HashSet<String>();
+
+ if (OntologicalClasses.DBPEDIA_PLACE.equals(npNerType)) {
+ rulesOntologyAttr = this.config.getSpatialAttributes(nerType);
+ } else if (OntologicalClasses.DBPEDIA_ORGANISATION.equals(npNerType)) {
+ rulesOntologyAttr = this.config.getOrgMembershipAttributes(nerType);
+ }
+
+ if (valueExistsInEntityAttributes(rulesOntologyAttr, entity, npEntity.getId())) {
+ return true;
+ }
+ }
+ }
+ }
+
+ /*
+ * 3. Detect any place adjectivals in noun phrases and use them for spatial coreference. Any place
+ * adjectivals found should be separate words from the class matches from point 1.
+ */
+ PlaceAdjectival placeAdjectival = this.dictionaries.findPlaceAdjectival(language, nounPhrase);
+
+ if (placeAdjectival != null
+ && (placeAdjectival.getEnd() < classStart || placeAdjectival.getStart() > classEnd)) {
+ /*
+ * We use the same spatial rules ontology attributes as before.
+ */
+ Set<String> rulesOntologyAttr = this.config.getSpatialAttributes(ner
+ .getAnnotation(NlpAnnotations.NER_ANNOTATION).value().getType());
+
+ if (valueExistsInEntityAttributes(rulesOntologyAttr, entity, placeAdjectival.getPlaceUri()
+ .getUnicodeString())) {
+ return true;
+ }
+ }
+
+ /*
+ * If there was no additional info to do the coref and if the entity class matched and has more than 1
+ * word then we consider this a good enough coreference.
+ */
+ if (matchedClass.split("\\s").length > 1) return true;
+
+ return false;
+ }
+
+ /**
+ * Builds a Set of Entity Type labels given the Entity type uris.
+ *
+ * @param entity
+ * @param language
+ * @return
+ * @throws EngineException
+ */
+ private Set<String> buildEntityTypeLabels(Entity entity, String language) throws EngineException {
+ Iterator<Object> typeUris = entity.getRepresentation().get(RDF_TYPE.getUnicodeString());
+ Set<String> allTypeLabels = new HashSet<String>();
+
+ while (typeUris.hasNext()) {
+ String typeUri = typeUris.next().toString();
+
+ if (this.config.shouldExcludeClass(typeUri)) continue;
+
+ // First try the in memory index
+ Set<String> labels = this.entityTypeIndex.lookupEntityType(new UriRef(typeUri), language);
+
+ if (labels == null) {
+ Site site = getReferencedSite();
+ Entity entityType = (site == null) ? this.entityHub.getEntity(typeUri) : site
+ .getEntity(typeUri);
+
+ if (entityType != null) {
+ labels = new HashSet<String>();
+ Iterator<Text> labelIterator = entityType.getRepresentation().get(
+ RDFS_LABEL.getUnicodeString(), language);
+
+ while (labelIterator.hasNext()) {
+ labels.add(labelIterator.next().getText());
+ }
+
+ this.entityTypeIndex.addEntityType(new UriRef(typeUri), language, labels);
+ }
+ }
+
+ if (labels != null) allTypeLabels.addAll(labels);
+ }
+
+ return allTypeLabels;
+ }
+
+ /**
+ * Checks whether any of the attributes in rulesOntologyAttr from the given Entity contain the given
+ * value.
+ *
+ * @param rulesOntologyAttr
+ * @param entity
+ * @param value
+ * @return
+ */
+ private boolean valueExistsInEntityAttributes(Set<String> rulesOntologyAttr, Entity entity, String value) {
+ for (String attribute : rulesOntologyAttr) {
+ Iterator<Object> entityAttributes = entity.getRepresentation().get(attribute);
+
+ while (entityAttributes.hasNext()) {
+ Object entityAttribute = entityAttributes.next();
+
+ if (entityAttribute.toString().equals(value)) {
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Retrieves the configured {@link Site} which holds the NER properties.
+ *
+ * @return
+ * @throws EngineException
+ */
+ private Site getReferencedSite() throws EngineException {
+ Site site = null;
+
+ if (referencedSiteID != null) { // lookup the referenced site
+ site = siteManager.getSite(referencedSiteID);
+ // ensure that it is present
+ if (site == null) {
+ String msg = String
+ .format("Unable to enhance because Referenced Site %s is currently not active!",
+ referencedSiteID);
+
+ throw new EngineException(msg);
+ }
+ }
+
+ return site;
+ }
+}
diff --git a/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/CoreferenceFinderConfig.java b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/CoreferenceFinderConfig.java
new file mode 100644
index 0000000..1e5e17b
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/CoreferenceFinderConfig.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitycoreference.impl;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.stanbol.enhancer.engines.entitycoreference.datamodel.NounPhrase;
+import org.apache.stanbol.enhancer.servicesapi.rdf.OntologicalClasses;
+import org.osgi.service.cm.ConfigurationException;
+
+/**
+ * Contains configuration parameters for the {@link CoreferenceFinder}.
+ *
+ * @author Cristian Petroaca
+ *
+ */
+public class CoreferenceFinderConfig {
+ /**
+ * The maximum distance (in sentence numbers) between a NER and a {@link NounPhrase} for which we look for
+ * a coreference.
+ */
+ private int maxDistance;
+
+ /**
+ * The Uris for spatial properties for the NER to be inspected when doing the coref spatial match.
+ */
+ private Map<UriRef,Set<String>> spatialAttributes;
+
+ /**
+ * The Uris for org membership properties for the NER to be inspected when doing the coref match.
+ */
+ private Map<UriRef,Set<String>> orgMembershipAttributes;
+
+ /**
+ * Entity classes which will not be used for coreference because they are too general.
+ */
+ private Set<String> entityClassesToExclude;
+
+ public CoreferenceFinderConfig(int maxDistance,
+ String spatialAttrForPerson,
+ String spatialAttrForOrg,
+ String spatialAttrForPlace,
+ String orgAttrForPerson,
+ String entityClassesToExclude) throws ConfigurationException {
+ this.maxDistance = maxDistance;
+
+ this.spatialAttributes = new HashMap<UriRef,Set<String>>();
+ this.orgMembershipAttributes = new HashMap<UriRef, Set<String>>();
+
+ if (spatialAttrForPerson != null) {
+ Set<String> attributes = new HashSet<String>();
+ for (String attribute : spatialAttrForPerson.split(",")) {
+ attributes.add(attribute);
+ }
+ this.spatialAttributes.put(OntologicalClasses.DBPEDIA_PERSON, attributes);
+ }
+
+ if (spatialAttrForOrg != null) {
+ Set<String> attributes = new HashSet<String>();
+ for (String attribute : spatialAttrForOrg.split(",")) {
+ attributes.add(attribute);
+ }
+ this.spatialAttributes.put(OntologicalClasses.DBPEDIA_ORGANISATION, attributes);
+ }
+
+
+ if (spatialAttrForPlace != null) {
+ Set<String> attributes = new HashSet<String>();
+ for (String attribute : spatialAttrForPlace.split(",")) {
+ attributes.add(attribute);
+ }
+ this.spatialAttributes.put(OntologicalClasses.DBPEDIA_PLACE, attributes);
+ }
+
+ if (orgAttrForPerson != null) {
+ Set<String> attributes = new HashSet<String>();
+ for (String attribute : orgAttrForPerson.split(",")) {
+ attributes.add(attribute);
+ }
+
+ this.orgMembershipAttributes.put(OntologicalClasses.DBPEDIA_PERSON, attributes);
+ }
+
+ if (entityClassesToExclude != null) {
+ this.entityClassesToExclude = new HashSet<String>();
+
+ for (String clazz : entityClassesToExclude.split(",")) {
+ this.entityClassesToExclude.add(clazz);
+ }
+ }
+ }
+
+ /**
+ * Gets the max distance parameter.
+ *
+ * @return
+ */
+ public int getMaxDistance() {
+ return maxDistance;
+ }
+
+ /**
+ * Gets the URIs for the spatial properties for a given Entity Type.
+ *
+ * @param uri
+ * of the Entity type for which we want to get the ontology.
+ * @return
+ */
+ public Set<String> getSpatialAttributes(UriRef uri) {
+ return this.spatialAttributes.get(uri);
+ }
+
+ /**
+ * Gets the URIs for the org membership properties for a given Entity Type.
+ *
+ * @param uri
+ * of the Entity type for which we want to get the ontology.
+ * @return
+ */
+ public Set<String> getOrgMembershipAttributes(UriRef uri) {
+ return this.orgMembershipAttributes.get(uri);
+ }
+
+ /**
+ * Checks whether we should exclude the given class based on our config.
+ *
+ * @param clazz
+ * @return
+ */
+ public boolean shouldExcludeClass(String clazz) {
+ return this.entityClassesToExclude.contains(clazz);
+ }
+}
diff --git a/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/Dictionaries.java b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/Dictionaries.java
new file mode 100644
index 0000000..a222370
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/Dictionaries.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitycoreference.impl;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.stanbol.enhancer.engines.entitycoreference.Constants;
+import org.apache.stanbol.enhancer.engines.entitycoreference.datamodel.NounPhrase;
+import org.apache.stanbol.enhancer.engines.entitycoreference.datamodel.PlaceAdjectival;
+import org.apache.stanbol.enhancer.nlp.model.Span;
+import org.osgi.service.cm.ConfigurationException;
+
+/**
+ * Contains information about several terms and properties of words we use in the {@link CoreferenceFinder}.
+ *
+ * @author Cristian Petroaca
+ *
+ */
+class Dictionaries {
+ /**
+ * Contains the list of place adjectivals in the form: language -> adjectival -> UriRef -> adjectival ->
+ * UriRef There are Places that have multiple adjectivals so in this map there are adjectivals that point
+ * to the same UriRef but that ensures a fast lookup.
+ */
+ private Map<String,Map<String,UriRef>> placeAdjectivalsMap;
+
+ public Dictionaries(String[] languages, String entityUriBase) throws ConfigurationException {
+ placeAdjectivalsMap = new HashMap<>();
+
+ for (String language : languages) {
+ String line = null;
+ Map<String,UriRef> languagePlaceAdjMap = new HashMap<>();
+ InputStream langIn = null;
+ BufferedReader reader = null;
+
+ try {
+ langIn = Dictionaries.class.getResourceAsStream(Constants.PLACE_ADJECTIVALS_FOLDER + "/"
+ + language);
+ reader = new BufferedReader(new InputStreamReader(langIn));
+
+ while ((line = reader.readLine()) != null) {
+ String[] splittedLine = line.split("\t");
+ String place = splittedLine[0];
+ String adjectivals = splittedLine[1];
+ UriRef ref = new UriRef(entityUriBase + place.trim());
+ String[] adjectivalsArray = adjectivals.split(",");
+
+ for (String adjectival : adjectivalsArray) {
+ languagePlaceAdjMap.put(adjectival.trim().toLowerCase(), ref);
+ }
+ }
+
+ placeAdjectivalsMap.put(language, languagePlaceAdjMap);
+ } catch (IOException ioe) {
+ throw new ConfigurationException("", "Could not read " + Constants.PLACE_ADJECTIVALS_FOLDER
+ + "/" + language, ioe);
+ } finally {
+ if (langIn != null) {
+ try {
+ langIn.close();
+ } catch (IOException e) {}
+ }
+
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {}
+ }
+ }
+ }
+ }
+
+ /**
+ * Checks whether a {@link NounPhrase} contains a place adjectival and returns it.
+ *
+ * @param language
+ * @param nounPhrase
+ * @return the {@link PlaceAdjectival} if the {@link NounPhrase} contains one or null if not.
+ */
+ public PlaceAdjectival findPlaceAdjectival(String language, NounPhrase nounPhrase) {
+ List<Span> tokens = nounPhrase.getTokens();
+ Map<String,UriRef> langPlaceAdjectivalsMap = placeAdjectivalsMap.get(language);
+ /*
+ * Go through all 1-grams and 2-grams and see if we have a match in the place adjectivals map. 2-grams
+ * should be good enough since there are no 3-gram places at least from what I saw.
+ */
+ for (int i = 0; i < tokens.size(); i++) {
+ Span currentToken = tokens.get(i);
+ String currentTokenString = currentToken.getSpan().toLowerCase();
+ // First the current 1-gram
+ if (langPlaceAdjectivalsMap.containsKey(currentTokenString)) {
+ return new PlaceAdjectival(currentToken.getStart(), currentToken.getEnd(),
+ langPlaceAdjectivalsMap.get(currentTokenString));
+ }
+
+ // Then use the 2-gram with the token before it
+ StringBuilder concatTokens = new StringBuilder();
+ String concatTokensString = null;
+
+ if (i > 0) {
+ Span previousToken = tokens.get(i - 1);
+ String previousTokenString = previousToken.getSpan().toLowerCase();
+ concatTokens = new StringBuilder();
+ concatTokens.append(previousTokenString);
+ concatTokens.append(" ");
+ concatTokens.append(currentTokenString);
+ concatTokensString = concatTokens.toString();
+
+ if (langPlaceAdjectivalsMap.containsKey(concatTokensString.toLowerCase())) {
+ return new PlaceAdjectival(previousToken.getStart(), currentToken.getEnd(),
+ langPlaceAdjectivalsMap.get(concatTokensString));
+ }
+ }
+
+ // Now use the 2-gram with the token after it
+ if (i < tokens.size() - 1) {
+ Span nextToken = tokens.get(i + 1);
+ String nextTokenString = nextToken.getSpan().toLowerCase();
+ concatTokens = new StringBuilder();
+ concatTokens.append(currentTokenString);
+ concatTokens.append(" ");
+ concatTokens.append(nextTokenString);
+
+ concatTokensString = concatTokens.toString();
+
+ if (langPlaceAdjectivalsMap.containsKey(concatTokens.toString())) {
+ return new PlaceAdjectival(currentToken.getStart(), nextToken.getEnd(),
+ langPlaceAdjectivalsMap.get(concatTokensString));
+ }
+ }
+ }
+
+ return null;
+ }
+}
diff --git a/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/InMemoryEntityTypeIndex.java b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/InMemoryEntityTypeIndex.java
new file mode 100644
index 0000000..3142291
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/InMemoryEntityTypeIndex.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitycoreference.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.clerezza.rdf.core.UriRef;
+
+/**
+ * Memory cache for storing often used Entity Type (Class) information.
+ *
+ * @author Cristian Petroaca
+ *
+ */
+public class InMemoryEntityTypeIndex {
+ /**
+ * The index having as key the Uri of the class and the value the set of labels ordered by language.
+ */
+ private Map<UriRef,Map<String,Set<String>>> index;
+
+ public InMemoryEntityTypeIndex() {
+ index = new HashMap<UriRef,Map<String,Set<String>>>();
+ }
+
+ /**
+ * Searches for a given class URI for the given language.
+ *
+ * @param uri
+ * @param language
+ * @return
+ */
+ public Set<String> lookupEntityType(UriRef uri, String language) {
+ Map<String,Set<String>> langMap = index.get(uri);
+
+ if (langMap != null) {
+ return langMap.get(language);
+ }
+
+ return null;
+ }
+
+ /**
+ * Adds a new class URI's labels for the given language.
+ *
+ * @param uri
+ * @param language
+ * @param labels
+ */
+ public void addEntityType(UriRef uri, String language, Set<String> labels) {
+ Map<String,Set<String>> langMap = index.get(uri);
+
+ if (langMap == null) {
+ langMap = new HashMap<String,Set<String>>();
+ index.put(uri, langMap);
+ }
+
+ langMap.put(language, labels);
+ }
+}
diff --git a/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/NounPhraseFilterer.java b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/NounPhraseFilterer.java
new file mode 100644
index 0000000..da72dc8
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/java/org/apache/stanbol/enhancer/engines/entitycoreference/impl/NounPhraseFilterer.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitycoreference.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.stanbol.enhancer.engines.entitycoreference.Constants;
+import org.apache.stanbol.enhancer.engines.entitycoreference.datamodel.NounPhrase;
+import org.apache.stanbol.enhancer.nlp.NlpAnnotations;
+import org.apache.stanbol.enhancer.nlp.model.Span;
+import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
+import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
+import org.apache.stanbol.enhancer.nlp.pos.Pos;
+import org.apache.stanbol.enhancer.nlp.pos.PosTag;
+import org.osgi.service.cm.ConfigurationException;
+
+/**
+ * Filters out bad {@link NounPhrase}s based on pos information.
+ *
+ * @author Cristian Petroaca
+ *
+ */
+/*
+ * TODO - create a NounPhraseFilterer interface with multiple implementations to separate languages with
+ * appositional definite article from the others.
+ */
+public class NounPhraseFilterer {
+ private final static String WITHIN_TEXT_DET_PROP = "within.text.referencing.determiners";
+ private final static short MIN_POS_NUMBER = 2;
+
+ /**
+ * Set of determiners based on language which make a {@link NounPhrase} valid for being a coref mention.
+ */
+ private Map<String,Set<String>> withinTextRefDeterminers;
+
+ public NounPhraseFilterer(String[] languages) throws ConfigurationException {
+ withinTextRefDeterminers = new HashMap<String,Set<String>>();
+
+ for (String language : languages) {
+ Properties props = new Properties();
+ String propertiesFile = Constants.POS_CONFIG_FOLDER + "/" + language + ".properties";
+ InputStream in = null;
+
+ try {
+ in = NounPhraseFilterer.class.getResourceAsStream(propertiesFile);
+ props.load(in);
+ } catch (IOException e) {
+ throw new ConfigurationException("", "Could not read " + propertiesFile);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {}
+ }
+ }
+
+ String determinersProperty = props.getProperty(WITHIN_TEXT_DET_PROP);
+
+ if (determinersProperty == null) {
+ throw new ConfigurationException(WITHIN_TEXT_DET_PROP, "Missing property in "
+ + propertiesFile);
+ }
+
+ Set<String> langDeterminerSet = new HashSet<String>();
+ for (String determiner : determinersProperty.split(",")) {
+ langDeterminerSet.add(determiner);
+ }
+
+ withinTextRefDeterminers.put(language, langDeterminerSet);
+ }
+ }
+
+ /**
+ * Filters out noun phrases which do not contain a determiner from the given config and do not a token
+ * count bigger than 2 - TODO : should this be configurable to be able to also include 1 word noun
+ * phrases?
+ *
+ * @param nounPhrases
+ * @param language
+ */
+ public void filter(List<NounPhrase> nounPhrases, String language) {
+ Set<String> langDeterminerSet = withinTextRefDeterminers.get(language);
+ Iterator<NounPhrase> it = nounPhrases.iterator();
+
+ while (it.hasNext()) {
+ NounPhrase nounPhrase = it.next();
+ boolean hasGoodDeterminer = false;
+ short nounNo = 0;
+
+ for (Span token : nounPhrase.getTokens()) {
+ Value<PosTag> pos = token.getAnnotation(NlpAnnotations.POS_ANNOTATION);
+
+ if (pos != null) {
+ PosTag posTag = pos.value();
+
+ if (posTag.hasCategory(LexicalCategory.Noun)
+ || posTag.hasCategory(LexicalCategory.Adjective)) {
+ nounNo++;
+ }
+
+ if (!hasGoodDeterminer && posTag.hasPos(Pos.Determiner)
+ && langDeterminerSet.contains(token.getSpan().toLowerCase())) {
+ hasGoodDeterminer = true;
+ }
+ }
+ }
+
+ if (!hasGoodDeterminer || nounNo < MIN_POS_NUMBER) {
+ it.remove();
+ }
+ }
+ }
+
+ public boolean supportsLanguage(String language) {
+ return withinTextRefDeterminers.containsKey(language);
+ }
+}
diff --git a/enhancement-engines/entitycoreference/src/main/resources/OSGI-INF/metatype/metatype.properties b/enhancement-engines/entitycoreference/src/main/resources/OSGI-INF/metatype/metatype.properties
new file mode 100644
index 0000000..40b18f0
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/resources/OSGI-INF/metatype/metatype.properties
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stanbol.enhancer.engine.name.name=Name
+stanbol.enhancer.engine.name.description=The name of the enhancement engine as \
+used in the RESTful interface '/engine/<name>'
+
+service.ranking.name=Ranking
+service.ranking.description=If two enhancement engines with the same name are active the \
+one with the higher ranking will be used to process parsed content items.
+
+
+#===============================================================================
+#Properties and Options used to configure
+#===============================================================================
+org.apache.stanbol.enhancer.engines.entitycoreference.EntityCoReferenceEngine.name=Apache \
+Stanbol Enhancer Engine: Entity Co-Reference
+org.apache.stanbol.enhancer.engines.entitycoreference.EntityCoReferenceEngine.description=An Engine that finds \
+co-references of Named Entities based on dbpedia/yago concepts.
+
+enhancer.engine.entitycoreference.languages.name=Language configuration
+enhancer.engine.entitycoreference.languages.description=Takes a list of ISO \
+ language codes. '*' is the Wildcard; '!{lang}' to exclude a language
+enhancer.engine.entitycoreference.referencedSiteId.name=Referenced Site
+enhancer.engine.entitycoreference.referencedSiteId.description=The ID of the \
+Entityhub Referenced Site holding the Entity Index.
+enhancer.engine.entitycoreference.entity.uri.base.name=Entity URI base
+enhancer.engine.entitycoreference.entity.uri.base.description=The base uri which \
+is used to represent an Entity.
+enhancer.engine.entitycoreference.maxDistance.name=Max sentence distance
+enhancer.engine.entitycoreference.maxDistance.description=The maximum sentence distance between the Ner \
+and the noun phrase which mentions it. -1 means no distance constraint.
+
+enhancer.engine.entitycoreference.spatial.attr.person.name=Spatial Attributes for Person
+enhancer.engine.entitycoreference.spatial.attr.person.description=Attributes used for spatial \
+coreference when dealing with a person entity.
+enhancer.engine.entitycoreference.spatial.attr.org.name=Spatial Attributes for Organization
+enhancer.engine.entitycoreference.spatial.attr.org.description=Attributes used for spatial \
+coreference when dealing with an organization entity.
+enhancer.engine.entitycoreference.spatial.attr.place.name=Spatial Attributes for Place
+enhancer.engine.entitycoreference.spatial.attr.place.description=Attributes used for spatial \
+coreference when dealing with a place entity.
+enhancer.engine.entitycoreference.org.attr.person.name=Organisational Membership Attributes for Person
+enhancer.engine.entitycoreference.org.attr.person.description=Attributes used for organisational \
+membership coreference when dealing with a person entity.
+
+enhancer.engine.entitycoreference.entity.classes.excluded.name=Entity classes to be excluded
+enhancer.engine.entitycoreference.entity.classes.excluded.description=Entity classes which will \
+be excluded when doing the entity class type matching because they are too general in nature.
\ No newline at end of file
diff --git a/enhancement-engines/entitycoreference/src/main/resources/config/pos/en.properties b/enhancement-engines/entitycoreference/src/main/resources/config/pos/en.properties
new file mode 100644
index 0000000..f754216
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/resources/config/pos/en.properties
@@ -0,0 +1,2 @@
+# Determiners of a noun phrase which determine that the noun phrase is a good candidate for coref.
+within.text.referencing.determiners=the,this,these
\ No newline at end of file
diff --git a/enhancement-engines/entitycoreference/src/main/resources/data/place_adjectivals/en b/enhancement-engines/entitycoreference/src/main/resources/data/place_adjectivals/en
new file mode 100644
index 0000000..effc95b
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/main/resources/data/place_adjectivals/en
@@ -0,0 +1,236 @@
+Abkhazia Abkhaz, Abkhazian
+Afghanistan Afghan
+Albania Albanian
+Algeria Algerian
+American_Samoa American Samoan
+Andorra Andorran
+Angola Angolan
+Anguilla Anguillan
+Antigua_and_Barbuda Antiguan, Barbudan
+Argentina Argentine, Argentinean, Argentinian
+Armenia Armenian
+Aruba Aruban
+Australia Australian
+Austria Austrian
+Azerbaijan Azerbaijani, Azeri
+Bahamas Bahamian
+Bahrain Bahraini
+Bangladesh Bangladeshi
+Barbados Barbadian
+Belarus Belarusian
+Belgium Belgian
+Belize Belizean
+Benin Beninese, Beninois
+Bermuda Bermudian, Bermudan
+Bhutan Bhutanese
+Bolivia Bolivian
+Bosnia_and_Herzegovina Bosnian, Bosniak, Herzegovinian
+Botswana Motswana, Botswanan
+Brazil Brazilian
+British Virgin Islands British Virgin Island
+Brunei Bruneian
+Bulgaria Bulgarian
+Burkina_Fasoa Burkinabè
+Burmab Burmese
+Burundi Burundian
+Cambodia Cambodian
+Cameroon Cameroonian
+Canada Canadian
+Cape_Verde Cape Verdean
+Cayman_Islands Caymanian
+Central_African_Republic Central African
+Chad Chadian
+Chile Chilean
+China Chinese
+Christmas_Island Christmas Island
+Cocos_Islands Cocos Island
+Colombia Colombian
+Comoros Comorian
+Congo Congolese, Congo
+Cook_Islands Cook Island, Cook Islands
+Costa_Rica Costa Rican
+Côte_d'Ivoire Ivorian
+Croatia Croatian
+Cuba Cuban
+Cyprus Cypriot
+Czech_Republic Czech
+Denmark Danish
+Djibouti Djiboutian
+Dominica Dominicand
+Dominican_Republic Dominicane
+East_Timor Timorese
+Ecuador Ecuadorian
+Egypt Egyptian
+El_Salvador Salvadoran
+England English
+Equatorial_Guinea Equatorial Guinean, Equatoguinean
+Eritrea Eritrean
+Estonia Estonian
+Ethiopia Ethiopian
+Falkland_Islands Falkland Island
+Faroe_Islands Faroese
+Fiji Fijian
+Finland Finnish
+France French
+French_Guiana French Guianese
+French_Polynesia French Polynesian
+Gabon Gabonese
+Gambia Gambian
+Georgia Georgian
+Germany German
+Ghana Ghanaian
+Gibraltar Gibraltar
+Great_Britain British
+Greece Greek, Greciang, Hellenic
+Greenland Greenlandic
+Grenada Grenadian
+Guadeloupe Guadeloupe
+Guam Guamanian, Guambat
+Guatemala Guatemalan
+Guinea Guinean
+Guyana Guyanese
+Haiti Haitian
+Honduras Honduran
+Hong_Kong Hong Kong, Hongkongese
+Hungary Hungarian, Magyar
+Iceland Icelandic
+India Indian
+Indonesia Indonesian
+Iran Iranian, Persian
+Iraq Iraqi
+Ireland Irish
+Isle_of_Man Manx
+Israel Israeli
+Italy Italian, Italic
+Jamaica Jamaican
+Japan Japanese
+Jordan Jordanian
+Kazakhstan Kazakh, Kazakhstani
+Kenya Kenyan
+Kiribati I-Kiribati
+North_Korea North Korean
+South_Korea South Korean
+Kosovo Kosovar, Kosovan
+Kuwait Kuwaiti
+Kyrgyzstan Kyrgyzstani, Kyrgyz, Kirgiz, Kirghiz
+Laos Laotian, Lao
+Latvia Latvian
+Lebanon Lebanese
+Lesotho Basotho
+Liberia Liberian
+Libya Libyan
+Liechtenstein Liechtenstein
+Lithuania Lithuanian
+Luxembourg Luxembourg, Luxembourgish
+Macau Macanese, Chinese
+Macedonia Macedonian
+Madagascar Malagasy
+Malawi Malawian
+Malaysia Malaysian
+Maldives Maldivian
+Mali Malian
+Malta Maltese
+Marshall Islands Marshallese
+Martinique Martiniquais, Martinican
+Mauritania Mauritanian
+Mauritius Mauritian
+Mayotte Mahoran
+Mexico Mexican
+Micronesia Micronesian
+Moldova Moldovan
+Monaco Monégasque, Monacan
+Mongolia Mongolian
+Montenegro Montenegrin
+Montserrat Montserratian
+Morocco Moroccan
+Mozambique Mozambican
+Namibia Namibian
+Nauru Nauruan
+Nepal Nepalese, Nepali
+Netherlands Dutch, Netherlandic
+New_Caledonia New Caledonian
+New_Zealand New Zealand, NZ
+Nicaragua Nicaraguan
+Niue Niuean
+Niger Nigerien
+Nigeria Nigerian
+Norway Norwegian
+Northern_Ireland Northern Irish, Irish
+Northern_Marianas Northern Marianan
+Oman Omani
+Pakistan Pakistani
+Palestine Palestinian
+Palau Palauan
+Panama Panamanian
+Papua_New_Guinea Papua New Guinean, Papuan
+Paraguay Paraguayan
+Peru Peruvian
+Philippines Philippine, Filipino
+Pitcairn_Island Pitcairn Island
+Poland Polish
+Portugal Portuguese
+Puerto_Rico Puerto Rican
+Qatar Qatari
+Ireland Irish
+Réunion Réunionese, Réunionnais
+Romania Romanian
+Russia Russian
+Rwanda Rwandan
+St._Helena St. Helenian
+St._Kitts_and_Nevis Kittitian, Nevisian
+St._Lucia St. Lucian
+Saint-Pierre_and_Miquelon Saint-Pierrais, Miquelonnais
+St._Vincent_and_the_Grenadines St. Vincentian, Vincentian
+Samoa Samoan
+San_Marino Sammarinese
+São_Tomé_and_Príncipe São Toméan
+Saudi_Arabia Saudi, Saudi Arabian
+Scotland Scots, Scottish, Scotchi
+Senegal Senegalese
+Serbia Serbian
+Seychelles Seychellois
+Sierra_Leone Sierra Leonean
+Singapore Singaporean
+Slovakia Slovak
+Slovenia Slovenian, Slovene
+Solomon_Islands Solomon Island
+Somalia Somali, Somalian
+South_Africa South African
+South_Ossetia South Ossetian
+South_Sudan South Sudanese
+Spain Spanish
+Sri_Lanka Sri Lankan
+Sudan Sudanese
+Surinam Surinamese
+Swaziland Swazi
+Sweden Swedish
+Switzerland Swiss
+Syria Syrian
+Taiwan Taiwanese
+Tajikistan Tajikistani
+Tanzania Tanzanian
+Thailand Thai
+Togo Togolese
+Tonga Tongan
+Trinidad_and_Tobago Trinidadian, Tobagonian
+Tunisia Tunisian
+Turkey Turkish
+Turkmenistan Turkmen
+Tuvalu Tuvaluan
+Uganda Ugandan
+Ukraine Ukrainian
+United_Arab_Emirates Emirati, Emirian
+United_Kingdom British, UK
+United_States American, US
+Uruguay Uruguayan
+Uzbekistan Uzbekistani, Uzbek
+Vanuatu Ni-Vanuatu, Vanuatuan
+Venezuela Venezuelan
+Vietnam Vietnamese
+Virgin_Islands Virgin Island
+Wales Welsh
+Wallis_and_Futuna Wallisian, Futunan
+Western_Sahara Sahraw, Sahrawian, Sahraouian
+Yemen Yemeni
+Zambia Zambian
+Zimbabwe Zimbabwean
\ No newline at end of file
diff --git a/enhancement-engines/entitycoreference/src/test/resources/log4j.properties b/enhancement-engines/entitycoreference/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a7d5b65
--- /dev/null
+++ b/enhancement-engines/entitycoreference/src/test/resources/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Root logger option
+log4j.rootLogger=INFO, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n
+log4j.logger.org.apache.stanbol.enhancer.engines.keywordextraction=DEBUG
\ No newline at end of file
diff --git a/enhancement-engines/pom.xml b/enhancement-engines/pom.xml
index bb0f2e2..9c8b9d8 100644
--- a/enhancement-engines/pom.xml
+++ b/enhancement-engines/pom.xml
@@ -114,9 +114,8 @@
<module>geonames</module> <!-- http://geonames.org -->
<module>opencalais</module> <!-- http://opencalais.com/ -->
<module>zemanta</module> <!-- htt://zemanta.com -->
-
- <!-- The entity co-refernece engine is not yet in trunk -->
- <!-- module>entitycoreference</module -->
+
+ <module>entitycoreference</module>
</modules>
<build>
