LUCENE-6852: merge trunk
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene6852@1710430 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java b/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java
index ce48d5c..78ad691 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java
@@ -32,12 +32,10 @@
import org.apache.lucene.classification.Classifier;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.StoredDocument;
-import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NamedThreadFactory;
@@ -152,6 +150,7 @@
/**
* get the linearized confusion matrix as a {@link Map}
+ *
* @return a {@link Map} whose keys are the correct classification answers and whose values are the actual answers'
* counts
*/
@@ -160,6 +159,61 @@
}
/**
+ * calculate precision on the given class
+ *
+ * @param klass the class to calculate the precision for
+ * @return the precision for the given class
+ */
+ public double getPrecision(String klass) {
+ Map<String, Long> classifications = linearizedMatrix.get(klass);
+ double tp = 0;
+ double fp = 0;
+ for (Map.Entry<String, Long> entry : classifications.entrySet()) {
+ if (klass.equals(entry.getKey())) {
+ tp += entry.getValue();
+ }
+ }
+ for (Map<String, Long> values : linearizedMatrix.values()) {
+ if (values.containsKey(klass)) {
+ fp += values.get(klass);
+ }
+ }
+ return tp / (tp + fp);
+ }
+
+ /**
+ * calculate recall on the given class
+ *
+ * @param klass the class to calculate the recall for
+ * @return the recall for the given class
+ */
+ public double getRecall(String klass) {
+ Map<String, Long> classifications = linearizedMatrix.get(klass);
+ double tp = 0;
+ double fn = 0;
+ for (Map.Entry<String, Long> entry : classifications.entrySet()) {
+ if (klass.equals(entry.getKey())) {
+ tp += entry.getValue();
+ } else {
+ fn += entry.getValue();
+ }
+ }
+ return tp / (tp + fn);
+ }
+
+ /**
+ * get the F-1 measure of the given class
+ *
+ * @param klass the class to calculate the F-1 measure for
+ * @return the F-1 measure for the given class
+ */
+ public double getF1Measure(String klass) {
+ double recall = getRecall(klass);
+ double precision = getPrecision(klass);
+ return 2 * precision * recall / (precision + recall);
+ }
+
+ /**
* Calculate accuracy on this confusion matrix using the formula:
* {@literal accuracy = correctly-classified / (correctly-classified + wrongly-classified)}
*
@@ -199,6 +253,7 @@
/**
* get the average classification time in milliseconds
+ *
* @return the avg classification time
*/
public double getAvgClassificationTime() {
@@ -207,6 +262,7 @@
/**
* get the no. of documents evaluated while generating this confusion matrix
+ *
* @return the no. of documents evaluated
*/
public int getNumberOfEvaluatedDocs() {
diff --git a/lucene/classification/src/test/org/apache/lucene/classification/BooleanPerceptronClassifierTest.java b/lucene/classification/src/test/org/apache/lucene/classification/BooleanPerceptronClassifierTest.java
index 4f81b2d..4b83dce 100644
--- a/lucene/classification/src/test/org/apache/lucene/classification/BooleanPerceptronClassifierTest.java
+++ b/lucene/classification/src/test/org/apache/lucene/classification/BooleanPerceptronClassifierTest.java
@@ -93,8 +93,9 @@
assertTrue("evaluation took more than 1m: " + evaluationTime / 1000 + "s", evaluationTime < 60000);
double avgClassificationTime = confusionMatrix.getAvgClassificationTime();
assertTrue(5000 > avgClassificationTime);
- double accuracy = confusionMatrix.getAccuracy();
- assertTrue(accuracy > 0d);
+ // accuracy check disabled until LUCENE-6853 is fixed
+// double accuracy = confusionMatrix.getAccuracy();
+// assertTrue(accuracy > 0d);
} finally {
leafReader.close();
}
diff --git a/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java b/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java
index 23d08fb..faae083 100644
--- a/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java
+++ b/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java
@@ -145,6 +145,12 @@
assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs());
assertTrue(confusionMatrix.getAvgClassificationTime() >= 0d);
assertTrue(confusionMatrix.getAccuracy() > 0d);
+ assertTrue(confusionMatrix.getPrecision("true") > 0d);
+ assertTrue(confusionMatrix.getPrecision("false") > 0d);
+ assertTrue(confusionMatrix.getRecall("true") > 0d);
+ assertTrue(confusionMatrix.getRecall("false") > 0d);
+ assertTrue(confusionMatrix.getF1Measure("true") > 0d);
+ assertTrue(confusionMatrix.getF1Measure("false") > 0d);
} finally {
if (reader != null) {
reader.close();
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 8d129a9..24849b1 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -291,7 +291,10 @@
* SOLR-8189: eTag calculation during HTTP Cache Validation uses unsynchronized WeakHashMap causing
threads to be stuck in runnable state. (shalin)
-
+
+* SOLR-7993: Raw json output for fields stopped working in 5.3.0 when requested fields do not include
+ the unique key field name. (Bill Bell, Ryan McKinley via shalin)
+
Optimizations
----------------------
@@ -373,6 +376,8 @@
* SOLR-8116: SearchGroupsResultTransformer tweaks (String literals, list/map initialCapacity)
(Christine Poerschke)
+* SOLR-8074: LoadAdminUIServlet directly references admin.html (Mark Miller, Upayavira)
+
================== 5.3.1 ==================
Bug Fixes
diff --git a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
index a1462db..374a0db 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
@@ -63,9 +63,9 @@
public abstract void transform(SolrDocument doc, int docid, float score) throws IOException;
/**
- * When a transformer needs access to fields that are not automaticaly derived from the
+ * When a transformer needs access to fields that are not automatically derived from the
* input fields names, this option lets us explicitly say the field names that we hope
- * will be in the SolrDocument. These fields will be requestd from the
+ * will be in the SolrDocument. These fields will be requested from the
* {@link SolrIndexSearcher} but may or may not be returned in the final
* {@link QueryResponseWriter}
*
diff --git a/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
index 8abb050..5e1f81a 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
@@ -124,6 +124,11 @@
doc.setField(display, new WriteableStringValue(val));
}
}
+
+ @Override
+ public String[] getExtraRequestFields() {
+ return new String[] {this.field};
+ }
}
public static class WriteableStringValue extends WriteableValue {
diff --git a/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java b/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
index 98524ef..d079e4c 100644
--- a/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
+++ b/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
@@ -47,8 +47,9 @@
response.addHeader("X-Frame-Options", "DENY"); // security: SOLR-7966 - avoid clickjacking for admin interface
// This attribute is set by the SolrDispatchFilter
+ String admin = request.getRequestURI().substring(request.getContextPath().length());
CoreContainer cores = (CoreContainer) request.getAttribute("org.apache.solr.CoreContainer");
- InputStream in = getServletContext().getResourceAsStream("/admin.html");
+ InputStream in = getServletContext().getResourceAsStream(admin);
if(in != null && cores != null) {
try {
response.setCharacterEncoding("UTF-8");
diff --git a/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java b/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java
index 228a169..7c0dede 100644
--- a/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java
+++ b/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java
@@ -27,6 +27,7 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.lucene.util.WeakIdentityMap;
import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
@@ -56,7 +57,7 @@
*
* @see #calcEtag
*/
- private static Map<SolrCore, EtagCacheVal> etagCoreCache = Collections.synchronizedMap(new WeakHashMap<>());
+ private static WeakIdentityMap<SolrCore, EtagCacheVal> etagCoreCache = WeakIdentityMap.newConcurrentHashMap();
/** @see #etagCoreCache */
private static class EtagCacheVal {
diff --git a/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java b/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
index 2fcea1e..1e9d6dd 100644
--- a/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
+++ b/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
@@ -67,6 +67,9 @@
String url = getFullUrl(req.node.getUrl());
ConcurrentUpdateSolrClient client = solrClients.get(url);
if (client == null) {
+ // NOTE: increasing to more than 1 threadCount for the client could cause updates to be reordered
+ // on a greater scale since the current behavior is to only increase the number of connections/Runners when
+ // the queue is more than half full.
client = new ConcurrentUpdateSolrClient(url, httpClient, 100, 1, updateExecutor, true) {
@Override
public void handleError(Throwable ex) {
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
index 99fd3b1..7e6a80a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
@@ -29,6 +29,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrQuery;
@@ -44,6 +45,7 @@
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.util.BadZookeeperThreadsFilter;
import org.apache.solr.util.RevertDefaultThreadHandlerRule;
+import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -87,6 +89,11 @@
new SystemPropertiesRestoreRule()).around(
new RevertDefaultThreadHandlerRule());
+ @BeforeClass
+ public static void betterNotBeJava9() {
+ assumeFalse("FIXME: SOLR-8182: This test fails under Java 9", Constants.JRE_IS_MINIMUM_JAVA9);
+ }
+
@Override
public void setUp() throws Exception {
savedLocale = KerberosTestUtil.overrideLocaleIfNotSpportedByMiniKdc();
diff --git a/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java b/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java
new file mode 100644
index 0000000..d8b66fd
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java
@@ -0,0 +1,71 @@
+package org.apache.solr.response;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests Raw JSON output for fields when used with and without the unique key field.
+ *
+ * See SOLR-7993
+ */
+public class TestRawTransformer extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-doctransformers.xml", "schema.xml");
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ assertU(delQ("*:*"));
+ assertU(commit());
+ }
+
+ @Test
+ public void testCustomTransformer() throws Exception {
+ // Build a simple index
+ int max = 10;
+ for (int i = 0; i < max; i++) {
+ SolrInputDocument sdoc = new SolrInputDocument();
+ sdoc.addField("id", i);
+ sdoc.addField("subject", "{poffL:[{offL:[{oGUID:\"79D5A31D-B3E4-4667-B812-09DF4336B900\",oID:\"OO73XRX\",prmryO:1,oRank:1,addTp:\"Office\",addCd:\"AA4GJ5T\",ad1:\"102 S 3rd St Ste 100\",city:\"Carson City\",st:\"MI\",zip:\"48811\",lat:43.176885,lng:-84.842919,phL:[\"(989) 584-1308\"],faxL:[\"(989) 584-6453\"]}]}]}");
+ sdoc.addField("title", "title_" + i);
+ updateJ(jsonAdd(sdoc), null);
+ }
+ assertU(commit());
+ assertQ(req("q", "*:*"), "//*[@numFound='" + max + "']");
+
+ SolrQueryRequest req = req("q", "*:*", "fl", "subject:[json]", "wt", "json");
+ String strResponse = h.query(req);
+ assertTrue("response does not contain right JSON encoding: " + strResponse,
+ strResponse.contains("\"subject\":[{poffL:[{offL:[{oGUID:\"7"));
+
+ req = req("q", "*:*", "fl", "id,subject", "wt", "json");
+ strResponse = h.query(req);
+ assertTrue("response does not contain right JSON encoding: " + strResponse,
+ strResponse.contains("subject\":[\""));
+ }
+
+}
+
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
index c3ea976..083e895 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
@@ -535,8 +535,8 @@
DirectUpdateHandler2.commitOnClose = true;
double runtime = runTimer.getTime()/1000.0f;
- if (runtime > 30 && stops.get() == 0) {
- LuceneTestCase.fail("The Monkey ran for over 30 seconds and no jetties were stopped - this is worth investigating!");
+ if (runtime > 45 && stops.get() == 0) {
+ LuceneTestCase.fail("The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!");
}
}
diff --git a/solr/webapp/web/WEB-INF/web.xml b/solr/webapp/web/WEB-INF/web.xml
index 01aca18..24a27ae 100644
--- a/solr/webapp/web/WEB-INF/web.xml
+++ b/solr/webapp/web/WEB-INF/web.xml
@@ -141,7 +141,12 @@
<servlet-mapping>
<servlet-name>LoadAdminUI</servlet-name>
- <url-pattern>/admin.html</url-pattern>
+ <url-pattern>/old.html</url-pattern>
+ </servlet-mapping>
+
+ <servlet-mapping>
+ <servlet-name>LoadAdminUI</servlet-name>
+ <url-pattern>/index.html</url-pattern>
</servlet-mapping>
<servlet-mapping>
@@ -156,7 +161,7 @@
</mime-mapping>
<welcome-file-list>
- <welcome-file>admin.html</welcome-file>
+ <welcome-file>index.html</welcome-file>
</welcome-file-list>
</web-app>
diff --git a/solr/webapp/web/css/angular/common.css b/solr/webapp/web/css/angular/common.css
index 41e3bca..1a3b087 100644
--- a/solr/webapp/web/css/angular/common.css
+++ b/solr/webapp/web/css/angular/common.css
@@ -755,8 +755,23 @@
top: -20px;
}
-.other-ui-link span {
+.other-ui-link span,
+.new-ui-warning span.help {
background-image: url( ../../img/ico/information-white.png );
right: 0px;
padding-left: 16px;
-}
\ No newline at end of file
+}
+
+.new-ui-warning {
+ position: absolute;
+ left: 150px;
+ top: -20px;
+ align: center;
+ color: red;
+ font-weight: bold;
+}
+.new-ui-warning a.ul {
+ color: red;
+ font-weight: bold;
+ text-decoration: underline;
+}
diff --git a/solr/webapp/web/index.html b/solr/webapp/web/index.html
index df85ef4..d64002e 100644
--- a/solr/webapp/web/index.html
+++ b/solr/webapp/web/index.html
@@ -121,8 +121,10 @@
<div class="exception">{{exception.msg}}</div>
</div>
- <div class="other-ui-link">
- <a href="/solr/">Original UI</a><a target="_blank" href="http://wiki.apache.org/solr/AngularUI"> <span class="help"></span></a>
+ <div class="new-ui-warning">
+ This is an experimental UI. Report bugs <a class="ul" target="_blank" href="http://issues.apache.org/jira/browse/SOLR">here</a>.
+ For the old UI click <a class="ul" href="/solr/old.html">here</a>
+ <a target="_blank" href="http://wiki.apache.org/solr/AngularUI"> <span class="help"></span></a>
</div>
<div id="content-wrapper">
diff --git a/solr/webapp/web/admin.html b/solr/webapp/web/old.html
similarity index 97%
rename from solr/webapp/web/admin.html
rename to solr/webapp/web/old.html
index 4f20a97..880b1c5 100644
--- a/solr/webapp/web/admin.html
+++ b/solr/webapp/web/old.html
@@ -80,7 +80,7 @@
</div>
<div class="other-ui-link">
- <a href="/solr/index.html">New UI</a><a target="_blank" href="http://wiki.apache.org/solr/AngularUI"> <span class="help"></span></a>
+ <a href="/solr/">New UI</a><a target="_blank" href="http://wiki.apache.org/solr/AngularUI"> <span class="help"></span></a>
</div>
<div id="content-wrapper">
