SOLR-14186: Introduce gitattributes to manage EOL
diff --git a/solr/.gitattributes b/solr/.gitattributes
new file mode 100644
index 0000000..b343da9
--- /dev/null
+++ b/solr/.gitattributes
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Handles all files not treated particularly below
+* text=auto
+
+
+# -nix specific
+bin/solr text eol=lf
+bin/init.d/solr text eol=lf
+bin/post text eol=lf
+*.bash text eol=lf
+*.sh text eol=lf
+
+# Windows specific
+*.bat text eol=crlf
+*.cmd text eol=crlf
+*.ps1 text eol=crlf
+
+# Graphics - no eol normalization
+*.png binary
+*.jpg binary
+*.jpeg binary
+*.gif binary
+*.tif binary
+*.tiff binary
+*.ico binary
+*.svg binary
+*.eps binary
+
+# Other files to ignore from eol normalization
+licenses/* -text
+solr-ref-guide/src/fonts/Inconsolata/OFL.txt -text
diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd
index e462336..20f42fe 100755
--- a/solr/bin/solr.in.cmd
+++ b/solr/bin/solr.in.cmd
@@ -1,177 +1,177 @@
-@REM
-@REM Licensed to the Apache Software Foundation (ASF) under one or more
-@REM contributor license agreements. See the NOTICE file distributed with
-@REM this work for additional information regarding copyright ownership.
-@REM The ASF licenses this file to You under the Apache License, Version 2.0
-@REM (the "License"); you may not use this file except in compliance with
-@REM the License. You may obtain a copy of the License at
-@REM
-@REM http://www.apache.org/licenses/LICENSE-2.0
-@REM
-@REM Unless required by applicable law or agreed to in writing, software
-@REM distributed under the License is distributed on an "AS IS" BASIS,
-@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@REM See the License for the specific language governing permissions and
-@REM limitations under the License.
-
-@echo off
-
-REM Settings here will override settings in existing env vars or in bin/solr. The default shipped state
-REM of this file is completely commented.
-
-REM By default the script will use JAVA_HOME to determine which java
-REM to use, but you can set a specific path for Solr to use without
-REM affecting other Java applications on your server/workstation.
-REM set SOLR_JAVA_HOME=
-
-REM Increase Java Min/Max Heap as needed to support your indexing / query needs
-REM set SOLR_JAVA_MEM=-Xms512m -Xmx512m
-
-REM Configure verbose GC logging:
-REM For Java 8: if this is set, additional params will be added to specify the log file & rotation
-REM For Java 9 or higher: GC_LOG_OPTS is currently not supported. If you set it, the startup script will exit with failure.
-REM set GC_LOG_OPTS=-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime
-
-REM Various GC settings have shown to work well for a number of common Solr workloads.
-REM See solr.cmd GC_TUNE for the default list.
-REM set GC_TUNE=-XX:SurvivorRatio=4
-REM set GC_TUNE=%GC_TUNE% -XX:TargetSurvivorRatio=90
-REM set GC_TUNE=%GC_TUNE% -XX:MaxTenuringThreshold=8
-REM set GC_TUNE=%GC_TUNE% -XX:+UseConcMarkSweepGC
-REM set GC_TUNE=%GC_TUNE% -XX:ConcGCThreads=4
-REM set GC_TUNE=%GC_TUNE% -XX:ParallelGCThreads=4
-REM set GC_TUNE=%GC_TUNE% -XX:+CMSScavengeBeforeRemark
-REM set GC_TUNE=%GC_TUNE% -XX:PretenureSizeThreshold=64m
-REM set GC_TUNE=%GC_TUNE% -XX:+UseCMSInitiatingOccupancyOnly
-REM set GC_TUNE=%GC_TUNE% -XX:CMSInitiatingOccupancyFraction=50
-REM set GC_TUNE=%GC_TUNE% -XX:CMSMaxAbortablePrecleanTime=6000
-REM set GC_TUNE=%GC_TUNE% -XX:+CMSParallelRemarkEnabled
-REM set GC_TUNE=%GC_TUNE% -XX:+ParallelRefProcEnabled
-REM set GC_TUNE=%GC_TUNE% -XX:-OmitStackTraceInFastThrow etc.
-
-REM Set the ZooKeeper connection string if using an external ZooKeeper ensemble
-REM e.g. host1:2181,host2:2181/chroot
-REM Leave empty if not using SolrCloud
-REM set ZK_HOST=
-
-REM Set the ZooKeeper client timeout (for SolrCloud mode)
-REM set ZK_CLIENT_TIMEOUT=15000
-
-REM By default the start script uses "localhost"; override the hostname here
-REM for production SolrCloud environments to control the hostname exposed to cluster state
-REM set SOLR_HOST=192.168.1.1
-
-REM By default Solr will try to connect to Zookeeper with 30 seconds in timeout; override the timeout if needed
-REM set SOLR_WAIT_FOR_ZK=30
-
-REM By default the start script uses UTC; override the timezone if needed
-REM set SOLR_TIMEZONE=UTC
-
-REM Set to true to activate the JMX RMI connector to allow remote JMX client applications
-REM to monitor the JVM hosting Solr; set to "false" to disable that behavior
-REM (false is recommended in production environments)
-REM set ENABLE_REMOTE_JMX_OPTS=false
-
-REM The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here
-REM set RMI_PORT=18983
-
-REM Anything you add to the SOLR_OPTS variable will be included in the java
-REM start command line as-is, in ADDITION to other options. If you specify the
-REM -a option on start script, those options will be appended as well. Examples:
-REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.autoSoftCommit.maxTime=3000
-REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.autoCommit.maxTime=60000
-REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.clustering.enabled=true
-
-REM Path to a directory for Solr to store cores and their data. By default, Solr will use server\solr
-REM If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
-REM set SOLR_HOME=
-
-REM Path to a directory that Solr will use as root for data folders for each core.
-REM If not set, defaults to <instance_dir>/data. Overridable per core through 'dataDir' core property
-REM set SOLR_DATA_HOME=
-
-REM Changes the logging level. Valid values: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF. Default is INFO
-REM This is an alternative to changing the rootLogger in log4j2.xml
-REM set SOLR_LOG_LEVEL=INFO
-
-REM Location where Solr should write logs to. Absolute or relative to solr start dir
-REM set SOLR_LOGS_DIR=logs
-
-REM Enables log rotation before starting Solr. Setting SOLR_LOG_PRESTART_ROTATION=true will let Solr take care of pre
-REM start rotation of logs. This is false by default as log4j2 handles this for us. If you choose to use another log
-REM framework that cannot do startup rotation, you may want to enable this to let Solr rotate logs on startup.
-REM set SOLR_LOG_PRESTART_ROTATION=false
-
-REM Set the host interface to listen on. Jetty will listen on all interfaces (0.0.0.0) by default.
-REM This must be an IPv4 ("a.b.c.d") or bracketed IPv6 ("[x::y]") address, not a hostname!
-REM set SOLR_JETTY_HOST=0.0.0.0
-
-REM Sets the port Solr binds to, default is 8983
-REM set SOLR_PORT=8983
-
-REM Enables HTTPS. It is implictly true if you set SOLR_SSL_KEY_STORE. Use this config
-REM to enable https module with custom jetty configuration.
-REM set SOLR_SSL_ENABLED=true
-REM Uncomment to set SSL-related system properties
-REM Be sure to update the paths to the correct keystore for your environment
-REM set SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks
-REM set SOLR_SSL_KEY_STORE_PASSWORD=secret
-REM set SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks
-REM set SOLR_SSL_TRUST_STORE_PASSWORD=secret
-REM Require clients to authenticate
-REM set SOLR_SSL_NEED_CLIENT_AUTH=false
-REM Enable clients to authenticate (but not require)
-REM set SOLR_SSL_WANT_CLIENT_AUTH=false
-REM Verify client hostname during SSL handshake
-REM set SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
-REM SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
-REM this to false can be useful to disable these checks when re-using a certificate on many hosts
-REM set SOLR_SSL_CHECK_PEER_NAME=true
-REM Override Key/Trust Store types if necessary
-REM set SOLR_SSL_KEY_STORE_TYPE=JKS
-REM set SOLR_SSL_TRUST_STORE_TYPE=JKS
-
-REM Uncomment if you want to override previously defined SSL values for HTTP client
-REM otherwise keep them commented and the above values will automatically be set for HTTP clients
-REM set SOLR_SSL_CLIENT_KEY_STORE=
-REM set SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=
-REM set SOLR_SSL_CLIENT_TRUST_STORE=
-REM set SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
-REM set SOLR_SSL_CLIENT_KEY_STORE_TYPE=
-REM set SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
-
-REM Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
-REM enables usage of credential store.
-REM Credential provider should store the following keys:
-REM * solr.jetty.keystore.password
-REM * solr.jetty.truststore.password
-REM Set the two below if you want to set specific store passwords for HTTP client
-REM * javax.net.ssl.keyStorePassword
-REM * javax.net.ssl.trustStorePassword
-REM More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
-REM set SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
-REM set SOLR_OPTS=" -Dsolr.ssl.credential.provider.chain=hadoop"
-
-REM Settings for authentication
-REM Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
-REM set SOLR_AUTHENTICATION_CLIENT_BUILDER=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory
-REM set SOLR_AUTH_TYPE=basic
-REM set SOLR_AUTHENTICATION_OPTS="-Dbasicauth=solr:SolrRocks"
-
-REM Settings for ZK ACL
-REM set SOLR_ZK_CREDS_AND_ACLS=-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider ^
-REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider ^
-REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^
-REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD
-REM set SOLR_OPTS=%SOLR_OPTS% %SOLR_ZK_CREDS_AND_ACLS%
-
-REM When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the
-REM list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can be configured in solr.xml,
-REM or if you are using the OOTB solr.xml, can be specified using the system property "solr.shardsWhitelist". Alternatively
-REM host checking can be disabled by using the system property "solr.disable.shardsWhitelist"
-REM set SOLR_OPTS="%SOLR_OPTS% -Dsolr.shardsWhitelist=http://localhost:8983,http://localhost:8984"
-
-REM For a visual indication in the Admin UI of what type of environment this cluster is, configure
-REM a -Dsolr.environment property below. Valid values are prod, stage, test, dev, with an optional
-REM label or color, e.g. -Dsolr.environment=test,label=Functional+test,color=brown
-REM SOLR_OPTS="$SOLR_OPTS -Dsolr.environment=prod"
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one or more
+@REM contributor license agreements. See the NOTICE file distributed with
+@REM this work for additional information regarding copyright ownership.
+@REM The ASF licenses this file to You under the Apache License, Version 2.0
+@REM (the "License"); you may not use this file except in compliance with
+@REM the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing, software
+@REM distributed under the License is distributed on an "AS IS" BASIS,
+@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM See the License for the specific language governing permissions and
+@REM limitations under the License.
+
+@echo off
+
+REM Settings here will override settings in existing env vars or in bin/solr. The default shipped state
+REM of this file is completely commented.
+
+REM By default the script will use JAVA_HOME to determine which java
+REM to use, but you can set a specific path for Solr to use without
+REM affecting other Java applications on your server/workstation.
+REM set SOLR_JAVA_HOME=
+
+REM Increase Java Min/Max Heap as needed to support your indexing / query needs
+REM set SOLR_JAVA_MEM=-Xms512m -Xmx512m
+
+REM Configure verbose GC logging:
+REM For Java 8: if this is set, additional params will be added to specify the log file & rotation
+REM For Java 9 or higher: GC_LOG_OPTS is currently not supported. If you set it, the startup script will exit with failure.
+REM set GC_LOG_OPTS=-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime
+
+REM Various GC settings have shown to work well for a number of common Solr workloads.
+REM See solr.cmd GC_TUNE for the default list.
+REM set GC_TUNE=-XX:SurvivorRatio=4
+REM set GC_TUNE=%GC_TUNE% -XX:TargetSurvivorRatio=90
+REM set GC_TUNE=%GC_TUNE% -XX:MaxTenuringThreshold=8
+REM set GC_TUNE=%GC_TUNE% -XX:+UseConcMarkSweepGC
+REM set GC_TUNE=%GC_TUNE% -XX:ConcGCThreads=4
+REM set GC_TUNE=%GC_TUNE% -XX:ParallelGCThreads=4
+REM set GC_TUNE=%GC_TUNE% -XX:+CMSScavengeBeforeRemark
+REM set GC_TUNE=%GC_TUNE% -XX:PretenureSizeThreshold=64m
+REM set GC_TUNE=%GC_TUNE% -XX:+UseCMSInitiatingOccupancyOnly
+REM set GC_TUNE=%GC_TUNE% -XX:CMSInitiatingOccupancyFraction=50
+REM set GC_TUNE=%GC_TUNE% -XX:CMSMaxAbortablePrecleanTime=6000
+REM set GC_TUNE=%GC_TUNE% -XX:+CMSParallelRemarkEnabled
+REM set GC_TUNE=%GC_TUNE% -XX:+ParallelRefProcEnabled
+REM set GC_TUNE=%GC_TUNE% -XX:-OmitStackTraceInFastThrow etc.
+
+REM Set the ZooKeeper connection string if using an external ZooKeeper ensemble
+REM e.g. host1:2181,host2:2181/chroot
+REM Leave empty if not using SolrCloud
+REM set ZK_HOST=
+
+REM Set the ZooKeeper client timeout (for SolrCloud mode)
+REM set ZK_CLIENT_TIMEOUT=15000
+
+REM By default the start script uses "localhost"; override the hostname here
+REM for production SolrCloud environments to control the hostname exposed to cluster state
+REM set SOLR_HOST=192.168.1.1
+
+REM By default Solr will try to connect to Zookeeper with 30 seconds in timeout; override the timeout if needed
+REM set SOLR_WAIT_FOR_ZK=30
+
+REM By default the start script uses UTC; override the timezone if needed
+REM set SOLR_TIMEZONE=UTC
+
+REM Set to true to activate the JMX RMI connector to allow remote JMX client applications
+REM to monitor the JVM hosting Solr; set to "false" to disable that behavior
+REM (false is recommended in production environments)
+REM set ENABLE_REMOTE_JMX_OPTS=false
+
+REM The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here
+REM set RMI_PORT=18983
+
+REM Anything you add to the SOLR_OPTS variable will be included in the java
+REM start command line as-is, in ADDITION to other options. If you specify the
+REM -a option on start script, those options will be appended as well. Examples:
+REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.autoSoftCommit.maxTime=3000
+REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.autoCommit.maxTime=60000
+REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.clustering.enabled=true
+
+REM Path to a directory for Solr to store cores and their data. By default, Solr will use server\solr
+REM If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
+REM set SOLR_HOME=
+
+REM Path to a directory that Solr will use as root for data folders for each core.
+REM If not set, defaults to <instance_dir>/data. Overridable per core through 'dataDir' core property
+REM set SOLR_DATA_HOME=
+
+REM Changes the logging level. Valid values: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF. Default is INFO
+REM This is an alternative to changing the rootLogger in log4j2.xml
+REM set SOLR_LOG_LEVEL=INFO
+
+REM Location where Solr should write logs to. Absolute or relative to solr start dir
+REM set SOLR_LOGS_DIR=logs
+
+REM Enables log rotation before starting Solr. Setting SOLR_LOG_PRESTART_ROTATION=true will let Solr take care of pre
+REM start rotation of logs. This is false by default as log4j2 handles this for us. If you choose to use another log
+REM framework that cannot do startup rotation, you may want to enable this to let Solr rotate logs on startup.
+REM set SOLR_LOG_PRESTART_ROTATION=false
+
+REM Set the host interface to listen on. Jetty will listen on all interfaces (0.0.0.0) by default.
+REM This must be an IPv4 ("a.b.c.d") or bracketed IPv6 ("[x::y]") address, not a hostname!
+REM set SOLR_JETTY_HOST=0.0.0.0
+
+REM Sets the port Solr binds to, default is 8983
+REM set SOLR_PORT=8983
+
+REM Enables HTTPS. It is implictly true if you set SOLR_SSL_KEY_STORE. Use this config
+REM to enable https module with custom jetty configuration.
+REM set SOLR_SSL_ENABLED=true
+REM Uncomment to set SSL-related system properties
+REM Be sure to update the paths to the correct keystore for your environment
+REM set SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks
+REM set SOLR_SSL_KEY_STORE_PASSWORD=secret
+REM set SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks
+REM set SOLR_SSL_TRUST_STORE_PASSWORD=secret
+REM Require clients to authenticate
+REM set SOLR_SSL_NEED_CLIENT_AUTH=false
+REM Enable clients to authenticate (but not require)
+REM set SOLR_SSL_WANT_CLIENT_AUTH=false
+REM Verify client hostname during SSL handshake
+REM set SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
+REM SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
+REM this to false can be useful to disable these checks when re-using a certificate on many hosts
+REM set SOLR_SSL_CHECK_PEER_NAME=true
+REM Override Key/Trust Store types if necessary
+REM set SOLR_SSL_KEY_STORE_TYPE=JKS
+REM set SOLR_SSL_TRUST_STORE_TYPE=JKS
+
+REM Uncomment if you want to override previously defined SSL values for HTTP client
+REM otherwise keep them commented and the above values will automatically be set for HTTP clients
+REM set SOLR_SSL_CLIENT_KEY_STORE=
+REM set SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=
+REM set SOLR_SSL_CLIENT_TRUST_STORE=
+REM set SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
+REM set SOLR_SSL_CLIENT_KEY_STORE_TYPE=
+REM set SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
+
+REM Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
+REM enables usage of credential store.
+REM Credential provider should store the following keys:
+REM * solr.jetty.keystore.password
+REM * solr.jetty.truststore.password
+REM Set the two below if you want to set specific store passwords for HTTP client
+REM * javax.net.ssl.keyStorePassword
+REM * javax.net.ssl.trustStorePassword
+REM More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
+REM set SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
+REM set SOLR_OPTS=" -Dsolr.ssl.credential.provider.chain=hadoop"
+
+REM Settings for authentication
+REM Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
+REM set SOLR_AUTHENTICATION_CLIENT_BUILDER=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory
+REM set SOLR_AUTH_TYPE=basic
+REM set SOLR_AUTHENTICATION_OPTS="-Dbasicauth=solr:SolrRocks"
+
+REM Settings for ZK ACL
+REM set SOLR_ZK_CREDS_AND_ACLS=-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider ^
+REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider ^
+REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^
+REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD
+REM set SOLR_OPTS=%SOLR_OPTS% %SOLR_ZK_CREDS_AND_ACLS%
+
+REM When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the
+REM list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can be configured in solr.xml,
+REM or if you are using the OOTB solr.xml, can be specified using the system property "solr.shardsWhitelist". Alternatively
+REM host checking can be disabled by using the system property "solr.disable.shardsWhitelist"
+REM set SOLR_OPTS="%SOLR_OPTS% -Dsolr.shardsWhitelist=http://localhost:8983,http://localhost:8984"
+
+REM For a visual indication in the Admin UI of what type of environment this cluster is, configure
+REM a -Dsolr.environment property below. Valid values are prod, stage, test, dev, with an optional
+REM label or color, e.g. -Dsolr.environment=test,label=Functional+test,color=brown
+REM SOLR_OPTS="$SOLR_OPTS -Dsolr.environment=prod"
diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java
index 5d4ec17..71e3fe4 100644
--- a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java
+++ b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java
@@ -1,86 +1,86 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud;
-
-import java.lang.invoke.MethodHandles;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.solr.client.solrj.cloud.ShardTerms;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.core.SolrCore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Start recovery of a core if its term is less than leader's term
- */
-public class RecoveringCoreTermWatcher implements ZkShardTerms.CoreTermWatcher {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private final CoreDescriptor coreDescriptor;
- private final CoreContainer coreContainer;
- // used to prevent the case when term of other replicas get changed, we redo recovery
- // the idea here is with a specific term of a replica, we only do recovery one
- private final AtomicLong lastTermDoRecovery;
-
- RecoveringCoreTermWatcher(CoreDescriptor coreDescriptor, CoreContainer coreContainer) {
- this.coreDescriptor = coreDescriptor;
- this.coreContainer = coreContainer;
- this.lastTermDoRecovery = new AtomicLong(-1);
- }
-
- @Override
- public boolean onTermChanged(ShardTerms terms) {
- if (coreContainer.isShutDown()) return false;
-
- try (SolrCore solrCore = coreContainer.getCore(coreDescriptor.getName())) {
- if (solrCore == null || solrCore.isClosed()) {
- return false;
- }
-
- if (solrCore.getCoreDescriptor() == null || solrCore.getCoreDescriptor().getCloudDescriptor() == null) return true;
- String coreNodeName = solrCore.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
- if (terms.haveHighestTermValue(coreNodeName)) return true;
- if (lastTermDoRecovery.get() < terms.getTerm(coreNodeName)) {
- log.info("Start recovery on {} because core's term is less than leader's term", coreNodeName);
- lastTermDoRecovery.set(terms.getTerm(coreNodeName));
- solrCore.getUpdateHandler().getSolrCoreState().doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor());
- }
- } catch (Exception e) {
- log.info("Failed to watch term of core {}", coreDescriptor.getName(), e);
- return false;
- }
-
- return true;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- RecoveringCoreTermWatcher that = (RecoveringCoreTermWatcher) o;
-
- return coreDescriptor.getName().equals(that.coreDescriptor.getName());
- }
-
- @Override
- public int hashCode() {
- return coreDescriptor.getName().hashCode();
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.solr.client.solrj.cloud.ShardTerms;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Start recovery of a core if its term is less than leader's term
+ */
+public class RecoveringCoreTermWatcher implements ZkShardTerms.CoreTermWatcher {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final CoreDescriptor coreDescriptor;
+ private final CoreContainer coreContainer;
+ // used to prevent the case when term of other replicas get changed, we redo recovery
+ // the idea here is with a specific term of a replica, we only do recovery one
+ private final AtomicLong lastTermDoRecovery;
+
+ RecoveringCoreTermWatcher(CoreDescriptor coreDescriptor, CoreContainer coreContainer) {
+ this.coreDescriptor = coreDescriptor;
+ this.coreContainer = coreContainer;
+ this.lastTermDoRecovery = new AtomicLong(-1);
+ }
+
+ @Override
+ public boolean onTermChanged(ShardTerms terms) {
+ if (coreContainer.isShutDown()) return false;
+
+ try (SolrCore solrCore = coreContainer.getCore(coreDescriptor.getName())) {
+ if (solrCore == null || solrCore.isClosed()) {
+ return false;
+ }
+
+ if (solrCore.getCoreDescriptor() == null || solrCore.getCoreDescriptor().getCloudDescriptor() == null) return true;
+ String coreNodeName = solrCore.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
+ if (terms.haveHighestTermValue(coreNodeName)) return true;
+ if (lastTermDoRecovery.get() < terms.getTerm(coreNodeName)) {
+ log.info("Start recovery on {} because core's term is less than leader's term", coreNodeName);
+ lastTermDoRecovery.set(terms.getTerm(coreNodeName));
+ solrCore.getUpdateHandler().getSolrCoreState().doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor());
+ }
+ } catch (Exception e) {
+ log.info("Failed to watch term of core {}", coreDescriptor.getName(), e);
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ RecoveringCoreTermWatcher that = (RecoveringCoreTermWatcher) o;
+
+ return coreDescriptor.getName().equals(that.coreDescriptor.getName());
+ }
+
+ @Override
+ public int hashCode() {
+ return coreDescriptor.getName().hashCode();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java b/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
index b232f9b..629f2bc 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
@@ -1,65 +1,65 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.util.ObjectReleaseTracker;
-import org.apache.solr.core.CoreDescriptor;
-
-/**
- * Used to manage all ZkShardTerms of a collection
- */
-class ZkCollectionTerms implements AutoCloseable {
- private final String collection;
- private final Map<String, ZkShardTerms> terms;
- private final SolrZkClient zkClient;
-
- ZkCollectionTerms(String collection, SolrZkClient client) {
- this.collection = collection;
- this.terms = new HashMap<>();
- this.zkClient = client;
- ObjectReleaseTracker.track(this);
- }
-
-
- public ZkShardTerms getShard(String shardId) {
- synchronized (terms) {
- if (!terms.containsKey(shardId)) terms.put(shardId, new ZkShardTerms(collection, shardId, zkClient));
- return terms.get(shardId);
- }
- }
-
- public void remove(String shardId, CoreDescriptor coreDescriptor) {
- synchronized (terms) {
- if (getShard(shardId).removeTerm(coreDescriptor)) {
- terms.remove(shardId).close();
- }
- }
- }
-
- public void close() {
- synchronized (terms) {
- terms.values().forEach(ZkShardTerms::close);
- }
- ObjectReleaseTracker.release(this);
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.apache.solr.core.CoreDescriptor;
+
+/**
+ * Used to manage all ZkShardTerms of a collection
+ */
+class ZkCollectionTerms implements AutoCloseable {
+ private final String collection;
+ private final Map<String, ZkShardTerms> terms;
+ private final SolrZkClient zkClient;
+
+ ZkCollectionTerms(String collection, SolrZkClient client) {
+ this.collection = collection;
+ this.terms = new HashMap<>();
+ this.zkClient = client;
+ ObjectReleaseTracker.track(this);
+ }
+
+
+ public ZkShardTerms getShard(String shardId) {
+ synchronized (terms) {
+ if (!terms.containsKey(shardId)) terms.put(shardId, new ZkShardTerms(collection, shardId, zkClient));
+ return terms.get(shardId);
+ }
+ }
+
+ public void remove(String shardId, CoreDescriptor coreDescriptor) {
+ synchronized (terms) {
+ if (getShard(shardId).removeTerm(coreDescriptor)) {
+ terms.remove(shardId).close();
+ }
+ }
+ }
+
+ public void close() {
+ synchronized (terms) {
+ terms.values().forEach(ZkShardTerms::close);
+ }
+ ObjectReleaseTracker.release(this);
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java
index 2c2a24a..0cab1a8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java
@@ -1,425 +1,425 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud;
-
-import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.solr.client.solrj.cloud.ShardTerms;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.ObjectReleaseTracker;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.CoreDescriptor;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Class used for interact with a ZK term node.
- * Each ZK term node relates to a shard of a collection and have this format (in json)
- * <p>
- * <code>
- * {
- * "replicaNodeName1" : 1,
- * "replicaNodeName2" : 2,
- * ..
- * }
- * </code>
- * <p>
- * The values correspond to replicas are called terms.
- * Only replicas with highest term value are considered up to date and be able to become leader and serve queries.
- * <p>
- * Terms can only updated in two strict ways:
- * <ul>
- * <li>A replica sets its term equals to leader's term
- * <li>The leader increase its term and some other replicas by 1
- * </ul>
- * This class should not be reused after {@link org.apache.zookeeper.Watcher.Event.KeeperState#Expired} event
- */
-public class ZkShardTerms implements AutoCloseable{
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private final Object writingLock = new Object();
- private final String collection;
- private final String shard;
- private final String znodePath;
- private final SolrZkClient zkClient;
- private final Set<CoreTermWatcher> listeners = new HashSet<>();
- private final AtomicBoolean isClosed = new AtomicBoolean(false);
-
- private ShardTerms terms;
-
- // Listener of a core for shard's term change events
- interface CoreTermWatcher {
- // return true if the listener wanna to be triggered in the next time
- boolean onTermChanged(ShardTerms terms);
- }
-
- public ZkShardTerms(String collection, String shard, SolrZkClient zkClient) {
- this.znodePath = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/terms/" + shard;
- this.collection = collection;
- this.shard = shard;
- this.zkClient = zkClient;
- ensureTermNodeExist();
- refreshTerms();
- retryRegisterWatcher();
- ObjectReleaseTracker.track(this);
- }
-
- /**
- * Ensure that leader's term is higher than some replica's terms
- * @param leader coreNodeName of leader
- * @param replicasNeedingRecovery set of replicas in which their terms should be lower than leader's term
- */
- public void ensureTermsIsHigher(String leader, Set<String> replicasNeedingRecovery) {
- if (replicasNeedingRecovery.isEmpty()) return;
-
- ShardTerms newTerms;
- while( (newTerms = terms.increaseTerms(leader, replicasNeedingRecovery)) != null) {
- if (forceSaveTerms(newTerms)) return;
- }
- }
-
- public ShardTerms getShardTerms() {
- return terms;
- }
- /**
- * Can this replica become leader?
- * @param coreNodeName of the replica
- * @return true if this replica can become leader, false if otherwise
- */
- public boolean canBecomeLeader(String coreNodeName) {
- return terms.canBecomeLeader(coreNodeName);
- }
-
- /**
- * Should leader skip sending updates to this replica?
- * @param coreNodeName of the replica
- * @return true if this replica has term equals to leader's term, false if otherwise
- */
- public boolean skipSendingUpdatesTo(String coreNodeName) {
- return !terms.haveHighestTermValue(coreNodeName);
- }
-
- /**
- * Did this replica registered its term? This is a sign to check f
- * @param coreNodeName of the replica
- * @return true if this replica registered its term, false if otherwise
- */
- public boolean registered(String coreNodeName) {
- return terms.getTerm(coreNodeName) != null;
- }
-
- public void close() {
- // no watcher will be registered
- isClosed.set(true);
- synchronized (listeners) {
- listeners.clear();
- }
- ObjectReleaseTracker.release(this);
- }
-
- // package private for testing, only used by tests
- Map<String, Long> getTerms() {
- synchronized (writingLock) {
- return terms.getTerms();
- }
- }
-
- /**
- * Add a listener so the next time the shard's term get updated, listeners will be called
- */
- void addListener(CoreTermWatcher listener) {
- synchronized (listeners) {
- listeners.add(listener);
- }
- }
-
- /**
- * Remove the coreNodeName from terms map and also remove any expired listeners
- * @return Return true if this object should not be reused
- */
- boolean removeTerm(CoreDescriptor cd) {
- int numListeners;
- synchronized (listeners) {
- // solrcore already closed
- listeners.removeIf(coreTermWatcher -> !coreTermWatcher.onTermChanged(terms));
- numListeners = listeners.size();
- }
- return removeTerm(cd.getCloudDescriptor().getCoreNodeName()) || numListeners == 0;
- }
-
- // package private for testing, only used by tests
- // return true if this object should not be reused
- boolean removeTerm(String coreNodeName) {
- ShardTerms newTerms;
- while ( (newTerms = terms.removeTerm(coreNodeName)) != null) {
- try {
- if (saveTerms(newTerms)) return false;
- } catch (KeeperException.NoNodeException e) {
- return true;
- }
- }
- return true;
- }
-
- /**
- * Register a replica's term (term value will be 0).
- * If a term is already associate with this replica do nothing
- * @param coreNodeName of the replica
- */
- void registerTerm(String coreNodeName) {
- ShardTerms newTerms;
- while ( (newTerms = terms.registerTerm(coreNodeName)) != null) {
- if (forceSaveTerms(newTerms)) break;
- }
- }
-
- /**
- * Set a replica's term equals to leader's term, and remove recovering flag of a replica.
- * This call should only be used by {@link org.apache.solr.common.params.CollectionParams.CollectionAction#FORCELEADER}
- * @param coreNodeName of the replica
- */
- public void setTermEqualsToLeader(String coreNodeName) {
- ShardTerms newTerms;
- while ( (newTerms = terms.setTermEqualsToLeader(coreNodeName)) != null) {
- if (forceSaveTerms(newTerms)) break;
- }
- }
-
- public void setTermToZero(String coreNodeName) {
- ShardTerms newTerms;
- while ( (newTerms = terms.setTermToZero(coreNodeName)) != null) {
- if (forceSaveTerms(newTerms)) break;
- }
- }
-
- /**
- * Mark {@code coreNodeName} as recovering
- */
- public void startRecovering(String coreNodeName) {
- ShardTerms newTerms;
- while ( (newTerms = terms.startRecovering(coreNodeName)) != null) {
- if (forceSaveTerms(newTerms)) break;
- }
- }
-
- /**
- * Mark {@code coreNodeName} as finished recovering
- */
- public void doneRecovering(String coreNodeName) {
- ShardTerms newTerms;
- while ( (newTerms = terms.doneRecovering(coreNodeName)) != null) {
- if (forceSaveTerms(newTerms)) break;
- }
- }
-
- public boolean isRecovering(String name) {
- return terms.isRecovering(name);
- }
-
- /**
- * When first updates come in, all replicas have some data now,
- * so we must switch from term 0 (registered) to 1 (have some data)
- */
- public void ensureHighestTermsAreNotZero() {
- ShardTerms newTerms;
- while ( (newTerms = terms.ensureHighestTermsAreNotZero()) != null) {
- if (forceSaveTerms(newTerms)) break;
- }
- }
-
- public long getHighestTerm() {
- return terms.getMaxTerm();
- }
-
- public long getTerm(String coreNodeName) {
- Long term = terms.getTerm(coreNodeName);
- return term == null? -1 : term;
- }
-
- // package private for testing, only used by tests
- int getNumListeners() {
- synchronized (listeners) {
- return listeners.size();
- }
- }
-
- /**
- * Set new terms to ZK.
- * In case of correspond ZK term node is not created, create it
- * @param newTerms to be set
- * @return true if terms is saved successfully to ZK, false if otherwise
- */
- private boolean forceSaveTerms(ShardTerms newTerms) {
- try {
- return saveTerms(newTerms);
- } catch (KeeperException.NoNodeException e) {
- ensureTermNodeExist();
- return false;
- }
- }
-
- /**
- * Set new terms to ZK, the version of new terms must match the current ZK term node
- * @param newTerms to be set
- * @return true if terms is saved successfully to ZK, false if otherwise
- * @throws KeeperException.NoNodeException correspond ZK term node is not created
- */
- private boolean saveTerms(ShardTerms newTerms) throws KeeperException.NoNodeException {
- byte[] znodeData = Utils.toJSON(newTerms);
- try {
- Stat stat = zkClient.setData(znodePath, znodeData, newTerms.getVersion(), true);
- setNewTerms(new ShardTerms(newTerms, stat.getVersion()));
- log.info("Successful update of terms at {} to {}", znodePath, newTerms);
- return true;
- } catch (KeeperException.BadVersionException e) {
- log.info("Failed to save terms, version is not a match, retrying");
- refreshTerms();
- } catch (KeeperException.NoNodeException e) {
- throw e;
- } catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while saving shard term for collection: " + collection, e);
- }
- return false;
- }
-
- /**
- * Create correspond ZK term node
- */
- private void ensureTermNodeExist() {
- String path = "/collections/" + collection + "/terms";
- try {
- path += "/" + shard;
-
- try {
- Map<String,Long> initialTerms = new HashMap<>();
- zkClient.makePath(path, Utils.toJSON(initialTerms), CreateMode.PERSISTENT, true);
- } catch (KeeperException.NodeExistsException e) {
- // it's okay if another beats us creating the node
- }
-
- } catch (InterruptedException e) {
- Thread.interrupted();
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Error creating shard term node in Zookeeper for collection: " + collection, e);
- } catch (KeeperException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Error creating shard term node in Zookeeper for collection: " + collection, e);
- }
- }
-
- /**
- * Fetch latest terms from ZK
- */
- public void refreshTerms() {
- ShardTerms newTerms;
- try {
- Stat stat = new Stat();
- byte[] data = zkClient.getData(znodePath, null, stat, true);
- newTerms = new ShardTerms((Map<String, Long>) Utils.fromJSON(data), stat.getVersion());
- } catch (KeeperException e) {
- Thread.interrupted();
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error updating shard term for collection: " + collection, e);
- } catch (InterruptedException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error updating shard term for collection: " + collection, e);
- }
-
- setNewTerms(newTerms);
- }
-
- /**
- * Retry register a watcher to the correspond ZK term node
- */
- private void retryRegisterWatcher() {
- while (!isClosed.get()) {
- try {
- registerWatcher();
- return;
- } catch (KeeperException.SessionExpiredException | KeeperException.AuthFailedException e) {
- isClosed.set(true);
- log.error("Failed watching shard term for collection: {} due to unrecoverable exception", collection, e);
- return;
- } catch (KeeperException e) {
- log.warn("Failed watching shard term for collection: {}, retrying!", collection, e);
- try {
- zkClient.getConnectionManager().waitForConnected(zkClient.getZkClientTimeout());
- } catch (TimeoutException te) {
- if (Thread.interrupted()) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error watching shard term for collection: " + collection, te);
- }
- }
- }
- }
- }
-
- /**
- * Register a watcher to the correspond ZK term node
- */
- private void registerWatcher() throws KeeperException {
- Watcher watcher = event -> {
- // session events are not change events, and do not remove the watcher
- if (Watcher.Event.EventType.None == event.getType()) {
- return;
- }
- retryRegisterWatcher();
- // Some events may be missed during register a watcher, so it is safer to refresh terms after registering watcher
- refreshTerms();
- };
- try {
- // exists operation is faster than getData operation
- zkClient.exists(znodePath, watcher, true);
- } catch (InterruptedException e) {
- Thread.interrupted();
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error watching shard term for collection: " + collection, e);
- }
- }
-
-
- /**
- * Atomically update {@link ZkShardTerms#terms} and call listeners
- * @param newTerms to be set
- */
- private void setNewTerms(ShardTerms newTerms) {
- boolean isChanged = false;
- synchronized (writingLock) {
- if (terms == null || newTerms.getVersion() > terms.getVersion()) {
- terms = newTerms;
- isChanged = true;
- }
- }
- if (isChanged) onTermUpdates(newTerms);
- }
-
- private void onTermUpdates(ShardTerms newTerms) {
- synchronized (listeners) {
- listeners.removeIf(coreTermWatcher -> !coreTermWatcher.onTermChanged(newTerms));
- }
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.solr.client.solrj.cloud.ShardTerms;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class used for interact with a ZK term node.
+ * Each ZK term node relates to a shard of a collection and have this format (in json)
+ * <p>
+ * <code>
+ * {
+ * "replicaNodeName1" : 1,
+ * "replicaNodeName2" : 2,
+ * ..
+ * }
+ * </code>
+ * <p>
+ * The values correspond to replicas are called terms.
+ * Only replicas with highest term value are considered up to date and be able to become leader and serve queries.
+ * <p>
+ * Terms can only updated in two strict ways:
+ * <ul>
+ * <li>A replica sets its term equals to leader's term
+ * <li>The leader increase its term and some other replicas by 1
+ * </ul>
+ * This class should not be reused after {@link org.apache.zookeeper.Watcher.Event.KeeperState#Expired} event
+ */
+public class ZkShardTerms implements AutoCloseable{
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final Object writingLock = new Object();
+ private final String collection;
+ private final String shard;
+ private final String znodePath;
+ private final SolrZkClient zkClient;
+ private final Set<CoreTermWatcher> listeners = new HashSet<>();
+ private final AtomicBoolean isClosed = new AtomicBoolean(false);
+
+ private ShardTerms terms;
+
+ // Listener of a core for shard's term change events
+ interface CoreTermWatcher {
+ // return true if the listener wanna to be triggered in the next time
+ boolean onTermChanged(ShardTerms terms);
+ }
+
+ public ZkShardTerms(String collection, String shard, SolrZkClient zkClient) {
+ this.znodePath = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/terms/" + shard;
+ this.collection = collection;
+ this.shard = shard;
+ this.zkClient = zkClient;
+ ensureTermNodeExist();
+ refreshTerms();
+ retryRegisterWatcher();
+ ObjectReleaseTracker.track(this);
+ }
+
+ /**
+ * Ensure that leader's term is higher than some replica's terms
+ * @param leader coreNodeName of leader
+ * @param replicasNeedingRecovery set of replicas in which their terms should be lower than leader's term
+ */
+ public void ensureTermsIsHigher(String leader, Set<String> replicasNeedingRecovery) {
+ if (replicasNeedingRecovery.isEmpty()) return;
+
+ ShardTerms newTerms;
+ while( (newTerms = terms.increaseTerms(leader, replicasNeedingRecovery)) != null) {
+ if (forceSaveTerms(newTerms)) return;
+ }
+ }
+
+ public ShardTerms getShardTerms() {
+ return terms;
+ }
+ /**
+ * Can this replica become leader?
+ * @param coreNodeName of the replica
+ * @return true if this replica can become leader, false if otherwise
+ */
+ public boolean canBecomeLeader(String coreNodeName) {
+ return terms.canBecomeLeader(coreNodeName);
+ }
+
+ /**
+ * Should leader skip sending updates to this replica?
+ * @param coreNodeName of the replica
+ * @return true if this replica has term equals to leader's term, false if otherwise
+ */
+ public boolean skipSendingUpdatesTo(String coreNodeName) {
+ return !terms.haveHighestTermValue(coreNodeName);
+ }
+
+ /**
+ * Did this replica registered its term? This is a sign to check f
+ * @param coreNodeName of the replica
+ * @return true if this replica registered its term, false if otherwise
+ */
+ public boolean registered(String coreNodeName) {
+ return terms.getTerm(coreNodeName) != null;
+ }
+
+ public void close() {
+ // no watcher will be registered
+ isClosed.set(true);
+ synchronized (listeners) {
+ listeners.clear();
+ }
+ ObjectReleaseTracker.release(this);
+ }
+
+ // package private for testing, only used by tests
+ Map<String, Long> getTerms() {
+ synchronized (writingLock) {
+ return terms.getTerms();
+ }
+ }
+
+ /**
+ * Add a listener so the next time the shard's term get updated, listeners will be called
+ */
+ void addListener(CoreTermWatcher listener) {
+ synchronized (listeners) {
+ listeners.add(listener);
+ }
+ }
+
+ /**
+ * Remove the coreNodeName from terms map and also remove any expired listeners
+ * @return Return true if this object should not be reused
+ */
+ boolean removeTerm(CoreDescriptor cd) {
+ int numListeners;
+ synchronized (listeners) {
+ // solrcore already closed
+ listeners.removeIf(coreTermWatcher -> !coreTermWatcher.onTermChanged(terms));
+ numListeners = listeners.size();
+ }
+ return removeTerm(cd.getCloudDescriptor().getCoreNodeName()) || numListeners == 0;
+ }
+
+ // package private for testing, only used by tests
+ // return true if this object should not be reused
+ boolean removeTerm(String coreNodeName) {
+ ShardTerms newTerms;
+ while ( (newTerms = terms.removeTerm(coreNodeName)) != null) {
+ try {
+ if (saveTerms(newTerms)) return false;
+ } catch (KeeperException.NoNodeException e) {
+ return true;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Register a replica's term (term value will be 0).
+ * If a term is already associate with this replica do nothing
+ * @param coreNodeName of the replica
+ */
+ void registerTerm(String coreNodeName) {
+ ShardTerms newTerms;
+ while ( (newTerms = terms.registerTerm(coreNodeName)) != null) {
+ if (forceSaveTerms(newTerms)) break;
+ }
+ }
+
+ /**
+ * Set a replica's term equals to leader's term, and remove recovering flag of a replica.
+ * This call should only be used by {@link org.apache.solr.common.params.CollectionParams.CollectionAction#FORCELEADER}
+ * @param coreNodeName of the replica
+ */
+ public void setTermEqualsToLeader(String coreNodeName) {
+ ShardTerms newTerms;
+ while ( (newTerms = terms.setTermEqualsToLeader(coreNodeName)) != null) {
+ if (forceSaveTerms(newTerms)) break;
+ }
+ }
+
+ public void setTermToZero(String coreNodeName) {
+ ShardTerms newTerms;
+ while ( (newTerms = terms.setTermToZero(coreNodeName)) != null) {
+ if (forceSaveTerms(newTerms)) break;
+ }
+ }
+
+ /**
+ * Mark {@code coreNodeName} as recovering
+ */
+ public void startRecovering(String coreNodeName) {
+ ShardTerms newTerms;
+ while ( (newTerms = terms.startRecovering(coreNodeName)) != null) {
+ if (forceSaveTerms(newTerms)) break;
+ }
+ }
+
+ /**
+ * Mark {@code coreNodeName} as finished recovering
+ */
+ public void doneRecovering(String coreNodeName) {
+ ShardTerms newTerms;
+ while ( (newTerms = terms.doneRecovering(coreNodeName)) != null) {
+ if (forceSaveTerms(newTerms)) break;
+ }
+ }
+
+ public boolean isRecovering(String name) {
+ return terms.isRecovering(name);
+ }
+
+ /**
+ * When first updates come in, all replicas have some data now,
+ * so we must switch from term 0 (registered) to 1 (have some data)
+ */
+ public void ensureHighestTermsAreNotZero() {
+ ShardTerms newTerms;
+ while ( (newTerms = terms.ensureHighestTermsAreNotZero()) != null) {
+ if (forceSaveTerms(newTerms)) break;
+ }
+ }
+
+ public long getHighestTerm() {
+ return terms.getMaxTerm();
+ }
+
+ public long getTerm(String coreNodeName) {
+ Long term = terms.getTerm(coreNodeName);
+ return term == null? -1 : term;
+ }
+
+ // package private for testing, only used by tests
+ int getNumListeners() {
+ synchronized (listeners) {
+ return listeners.size();
+ }
+ }
+
+ /**
+ * Set new terms to ZK.
+ * In case of correspond ZK term node is not created, create it
+ * @param newTerms to be set
+ * @return true if terms is saved successfully to ZK, false if otherwise
+ */
+ private boolean forceSaveTerms(ShardTerms newTerms) {
+ try {
+ return saveTerms(newTerms);
+ } catch (KeeperException.NoNodeException e) {
+ ensureTermNodeExist();
+ return false;
+ }
+ }
+
+ /**
+ * Set new terms to ZK, the version of new terms must match the current ZK term node
+ * @param newTerms to be set
+ * @return true if terms is saved successfully to ZK, false if otherwise
+ * @throws KeeperException.NoNodeException correspond ZK term node is not created
+ */
+ private boolean saveTerms(ShardTerms newTerms) throws KeeperException.NoNodeException {
+ byte[] znodeData = Utils.toJSON(newTerms);
+ try {
+ Stat stat = zkClient.setData(znodePath, znodeData, newTerms.getVersion(), true);
+ setNewTerms(new ShardTerms(newTerms, stat.getVersion()));
+ log.info("Successful update of terms at {} to {}", znodePath, newTerms);
+ return true;
+ } catch (KeeperException.BadVersionException e) {
+ log.info("Failed to save terms, version is not a match, retrying");
+ refreshTerms();
+ } catch (KeeperException.NoNodeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while saving shard term for collection: " + collection, e);
+ }
+ return false;
+ }
+
+ /**
+ * Create correspond ZK term node
+ */
+ private void ensureTermNodeExist() {
+ String path = "/collections/" + collection + "/terms";
+ try {
+ path += "/" + shard;
+
+ try {
+ Map<String,Long> initialTerms = new HashMap<>();
+ zkClient.makePath(path, Utils.toJSON(initialTerms), CreateMode.PERSISTENT, true);
+ } catch (KeeperException.NodeExistsException e) {
+ // it's okay if another beats us creating the node
+ }
+
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Error creating shard term node in Zookeeper for collection: " + collection, e);
+ } catch (KeeperException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Error creating shard term node in Zookeeper for collection: " + collection, e);
+ }
+ }
+
+ /**
+ * Fetch latest terms from ZK
+ */
+ public void refreshTerms() {
+ ShardTerms newTerms;
+ try {
+ Stat stat = new Stat();
+ byte[] data = zkClient.getData(znodePath, null, stat, true);
+ newTerms = new ShardTerms((Map<String, Long>) Utils.fromJSON(data), stat.getVersion());
+ } catch (KeeperException e) {
+ Thread.interrupted();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error updating shard term for collection: " + collection, e);
+ } catch (InterruptedException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error updating shard term for collection: " + collection, e);
+ }
+
+ setNewTerms(newTerms);
+ }
+
+ /**
+ * Retry register a watcher to the correspond ZK term node
+ */
+ private void retryRegisterWatcher() {
+ while (!isClosed.get()) {
+ try {
+ registerWatcher();
+ return;
+ } catch (KeeperException.SessionExpiredException | KeeperException.AuthFailedException e) {
+ isClosed.set(true);
+ log.error("Failed watching shard term for collection: {} due to unrecoverable exception", collection, e);
+ return;
+ } catch (KeeperException e) {
+ log.warn("Failed watching shard term for collection: {}, retrying!", collection, e);
+ try {
+ zkClient.getConnectionManager().waitForConnected(zkClient.getZkClientTimeout());
+ } catch (TimeoutException te) {
+ if (Thread.interrupted()) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error watching shard term for collection: " + collection, te);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Register a watcher to the correspond ZK term node
+ */
+ private void registerWatcher() throws KeeperException {
+ Watcher watcher = event -> {
+ // session events are not change events, and do not remove the watcher
+ if (Watcher.Event.EventType.None == event.getType()) {
+ return;
+ }
+ retryRegisterWatcher();
+ // Some events may be missed during register a watcher, so it is safer to refresh terms after registering watcher
+ refreshTerms();
+ };
+ try {
+ // exists operation is faster than getData operation
+ zkClient.exists(znodePath, watcher, true);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error watching shard term for collection: " + collection, e);
+ }
+ }
+
+
+ /**
+ * Atomically update {@link ZkShardTerms#terms} and call listeners
+ * @param newTerms to be set
+ */
+ private void setNewTerms(ShardTerms newTerms) {
+ boolean isChanged = false;
+ synchronized (writingLock) {
+ if (terms == null || newTerms.getVersion() > terms.getVersion()) {
+ terms = newTerms;
+ isChanged = true;
+ }
+ }
+ if (isChanged) onTermUpdates(newTerms);
+ }
+
+ private void onTermUpdates(ShardTerms newTerms) {
+ synchronized (listeners) {
+ listeners.removeIf(coreTermWatcher -> !coreTermWatcher.onTermChanged(newTerms));
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/highlight/LuceneRegexFragmenter.java b/solr/core/src/java/org/apache/solr/highlight/LuceneRegexFragmenter.java
index 0dc3340..61b6cff 100644
--- a/solr/core/src/java/org/apache/solr/highlight/LuceneRegexFragmenter.java
+++ b/solr/core/src/java/org/apache/solr/highlight/LuceneRegexFragmenter.java
@@ -1,217 +1,217 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.highlight;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.search.highlight.Fragmenter;
-
-/**
- * Fragmenter that tries to produce snippets that "look" like a regular
- * expression.
- *
- * NOTE: the default for <code>maxAnalyzedChars</code> is much lower for this
- * fragmenter. After this limit is exhausted, fragments are produced in the
- * same way as <code>GapFragmenter</code>
- */
-class LuceneRegexFragmenter implements Fragmenter
-{
- // ** defaults
- public static final int DEFAULT_FRAGMENT_SIZE = 70;
- public static final int DEFAULT_INCREMENT_GAP = 50;
- public static final float DEFAULT_SLOP = 0.6f;
- public static final int DEFAULT_MAX_ANALYZED_CHARS = 10000;
-
- // ** settings
-
- // desired length of fragments, in characters
- protected int targetFragChars;
- // increment gap which indicates a new fragment should occur
- // (often due to multi-valued fields)
- protected int incrementGapThreshold;
- // factor by which we are allowed to bend the frag size (larger or smaller)
- protected float slop;
- // analysis limit (ensures we don't waste too much time on long fields)
- protected int maxAnalyzedChars;
- // default desirable pattern for text fragments.
- protected Pattern textRE;
-
-
- // ** state
- protected int currentNumFrags;
- protected int currentOffset;
- protected int targetOffset;
- protected int[] hotspots;
-
- private PositionIncrementAttribute posIncAtt;
- private OffsetAttribute offsetAtt;
-
- // ** other
- // note: could dynamically change size of sentences extracted to match
- // target frag size
- public static final String
- DEFAULT_PATTERN_RAW = "[-\\w ,\\n\"']{20,200}";
- public static final Pattern
- DEFAULT_PATTERN = Pattern.compile(DEFAULT_PATTERN_RAW);
-
-
- public LuceneRegexFragmenter() {
- this(DEFAULT_FRAGMENT_SIZE,
- DEFAULT_INCREMENT_GAP,
- DEFAULT_SLOP,
- DEFAULT_MAX_ANALYZED_CHARS);
- }
- public LuceneRegexFragmenter(int targetFragChars) {
- this(targetFragChars,
- DEFAULT_INCREMENT_GAP,
- DEFAULT_SLOP,
- DEFAULT_MAX_ANALYZED_CHARS);
- }
-
- public LuceneRegexFragmenter(int targetFragChars,
- int incrementGapThreshold,
- float slop,
- int maxAnalyzedChars ) {
- this(targetFragChars, incrementGapThreshold, slop, maxAnalyzedChars,
- DEFAULT_PATTERN);
-
- }
-
- public LuceneRegexFragmenter(int targetFragChars,
- int incrementGapThreshold,
- float slop,
- int maxAnalyzedChars,
- Pattern targetPattern) {
- this.targetFragChars = targetFragChars;
- this.incrementGapThreshold = incrementGapThreshold;
- this.slop = slop;
- this.maxAnalyzedChars = maxAnalyzedChars;
- this.textRE = targetPattern;
- }
-
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.highlight.TextFragmenter#start(java.lang.String)
- */
- @Override
- public void start(String originalText, TokenStream tokenStream) {
- currentNumFrags = 1;
- currentOffset = 0;
- addHotSpots(originalText);
- posIncAtt = tokenStream.getAttribute(PositionIncrementAttribute.class);
- offsetAtt = tokenStream.getAttribute(OffsetAttribute.class);
- }
-
- ////////////////////////////////////
- // pre-analysis
- ////////////////////////////////////
-
- protected void addHotSpots(String text) {
- //System.out.println("hot spotting");
- ArrayList<Integer> temphs = new ArrayList<>(
- text.length() / targetFragChars);
- Matcher match = textRE.matcher(text);
- int cur = 0;
- while(match.find() && cur < maxAnalyzedChars) {
- int start=match.start(), end=match.end();
- temphs.add(start);
- temphs.add(end);
- cur = end;
- //System.out.println("Matched " + match.group());
- }
- hotspots = new int[temphs.size()];
- for(int i = 0; i < temphs.size(); i++) {
- hotspots[i] = temphs.get(i);
- }
- // perhaps not necessary--I don't know if re matches are non-overlapping
- Arrays.sort(hotspots);
- }
-
- ////////////////////////////////////
- // fragmenting
- ////////////////////////////////////
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.highlight.TextFragmenter#isNewFragment(org.apache.lucene.analysis.Token)
- */
- @Override
- public boolean isNewFragment()
- {
- boolean isNewFrag = false;
- int minFragLen = (int)((1.0f - slop)*targetFragChars);
- int endOffset = offsetAtt.endOffset();
-
- // ** determin isNewFrag
- if(posIncAtt.getPositionIncrement() > incrementGapThreshold) {
- // large position gaps always imply new fragments
- isNewFrag = true;
-
- } else if(endOffset - currentOffset < minFragLen) {
- // we're not in our range of flexibility
- isNewFrag = false;
-
- } else if(targetOffset > 0) {
- // we've already decided on a target
- isNewFrag = endOffset > targetOffset;
-
- } else {
- // we might be able to do something
- int minOffset = currentOffset + minFragLen;
- int maxOffset = (int)(currentOffset + (1.0f + slop)*targetFragChars);
- int hotIndex;
-
- // look for a close hotspot
- hotIndex = Arrays.binarySearch(hotspots, endOffset);
- if(hotIndex < 0) hotIndex = -hotIndex;
- if(hotIndex >= hotspots.length) {
- // no more hotspots in this input stream
- targetOffset = currentOffset + targetFragChars;
-
- } else if(hotspots[hotIndex] > maxOffset) {
- // no hotspots within slop
- targetOffset = currentOffset + targetFragChars;
-
- } else {
- // try to find hotspot in slop
- int goal = hotspots[hotIndex];
- while(goal < minOffset && hotIndex < hotspots.length) {
- hotIndex++;
- goal = hotspots[hotIndex];
- }
- targetOffset = goal <= maxOffset ? goal : currentOffset + targetFragChars;
- }
-
- isNewFrag = endOffset > targetOffset;
- }
-
- // ** operate on isNewFrag
- if(isNewFrag) {
- currentNumFrags++;
- currentOffset = endOffset;
- targetOffset = -1;
- }
- return isNewFrag;
- }
-
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.highlight;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.search.highlight.Fragmenter;
+
+/**
+ * Fragmenter that tries to produce snippets that "look" like a regular
+ * expression.
+ *
+ * NOTE: the default for <code>maxAnalyzedChars</code> is much lower for this
+ * fragmenter. After this limit is exhausted, fragments are produced in the
+ * same way as <code>GapFragmenter</code>
+ */
+class LuceneRegexFragmenter implements Fragmenter
+{
+ // ** defaults
+ public static final int DEFAULT_FRAGMENT_SIZE = 70;
+ public static final int DEFAULT_INCREMENT_GAP = 50;
+ public static final float DEFAULT_SLOP = 0.6f;
+ public static final int DEFAULT_MAX_ANALYZED_CHARS = 10000;
+
+ // ** settings
+
+ // desired length of fragments, in characters
+ protected int targetFragChars;
+ // increment gap which indicates a new fragment should occur
+ // (often due to multi-valued fields)
+ protected int incrementGapThreshold;
+ // factor by which we are allowed to bend the frag size (larger or smaller)
+ protected float slop;
+ // analysis limit (ensures we don't waste too much time on long fields)
+ protected int maxAnalyzedChars;
+ // default desirable pattern for text fragments.
+ protected Pattern textRE;
+
+
+ // ** state
+ protected int currentNumFrags;
+ protected int currentOffset;
+ protected int targetOffset;
+ protected int[] hotspots;
+
+ private PositionIncrementAttribute posIncAtt;
+ private OffsetAttribute offsetAtt;
+
+ // ** other
+ // note: could dynamically change size of sentences extracted to match
+ // target frag size
+ public static final String
+ DEFAULT_PATTERN_RAW = "[-\\w ,\\n\"']{20,200}";
+ public static final Pattern
+ DEFAULT_PATTERN = Pattern.compile(DEFAULT_PATTERN_RAW);
+
+
+ public LuceneRegexFragmenter() {
+ this(DEFAULT_FRAGMENT_SIZE,
+ DEFAULT_INCREMENT_GAP,
+ DEFAULT_SLOP,
+ DEFAULT_MAX_ANALYZED_CHARS);
+ }
+ public LuceneRegexFragmenter(int targetFragChars) {
+ this(targetFragChars,
+ DEFAULT_INCREMENT_GAP,
+ DEFAULT_SLOP,
+ DEFAULT_MAX_ANALYZED_CHARS);
+ }
+
+ public LuceneRegexFragmenter(int targetFragChars,
+ int incrementGapThreshold,
+ float slop,
+ int maxAnalyzedChars ) {
+ this(targetFragChars, incrementGapThreshold, slop, maxAnalyzedChars,
+ DEFAULT_PATTERN);
+
+ }
+
+ public LuceneRegexFragmenter(int targetFragChars,
+ int incrementGapThreshold,
+ float slop,
+ int maxAnalyzedChars,
+ Pattern targetPattern) {
+ this.targetFragChars = targetFragChars;
+ this.incrementGapThreshold = incrementGapThreshold;
+ this.slop = slop;
+ this.maxAnalyzedChars = maxAnalyzedChars;
+ this.textRE = targetPattern;
+ }
+
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.TextFragmenter#start(java.lang.String)
+ */
+ @Override
+ public void start(String originalText, TokenStream tokenStream) {
+ currentNumFrags = 1;
+ currentOffset = 0;
+ addHotSpots(originalText);
+ posIncAtt = tokenStream.getAttribute(PositionIncrementAttribute.class);
+ offsetAtt = tokenStream.getAttribute(OffsetAttribute.class);
+ }
+
+ ////////////////////////////////////
+ // pre-analysis
+ ////////////////////////////////////
+
+ protected void addHotSpots(String text) {
+ //System.out.println("hot spotting");
+ ArrayList<Integer> temphs = new ArrayList<>(
+ text.length() / targetFragChars);
+ Matcher match = textRE.matcher(text);
+ int cur = 0;
+ while(match.find() && cur < maxAnalyzedChars) {
+ int start=match.start(), end=match.end();
+ temphs.add(start);
+ temphs.add(end);
+ cur = end;
+ //System.out.println("Matched " + match.group());
+ }
+ hotspots = new int[temphs.size()];
+ for(int i = 0; i < temphs.size(); i++) {
+ hotspots[i] = temphs.get(i);
+ }
+ // perhaps not necessary--I don't know if re matches are non-overlapping
+ Arrays.sort(hotspots);
+ }
+
+ ////////////////////////////////////
+ // fragmenting
+ ////////////////////////////////////
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.TextFragmenter#isNewFragment(org.apache.lucene.analysis.Token)
+ */
+ @Override
+ public boolean isNewFragment()
+ {
+ boolean isNewFrag = false;
+ int minFragLen = (int)((1.0f - slop)*targetFragChars);
+ int endOffset = offsetAtt.endOffset();
+
+ // ** determin isNewFrag
+ if(posIncAtt.getPositionIncrement() > incrementGapThreshold) {
+ // large position gaps always imply new fragments
+ isNewFrag = true;
+
+ } else if(endOffset - currentOffset < minFragLen) {
+ // we're not in our range of flexibility
+ isNewFrag = false;
+
+ } else if(targetOffset > 0) {
+ // we've already decided on a target
+ isNewFrag = endOffset > targetOffset;
+
+ } else {
+ // we might be able to do something
+ int minOffset = currentOffset + minFragLen;
+ int maxOffset = (int)(currentOffset + (1.0f + slop)*targetFragChars);
+ int hotIndex;
+
+ // look for a close hotspot
+ hotIndex = Arrays.binarySearch(hotspots, endOffset);
+ if(hotIndex < 0) hotIndex = -hotIndex;
+ if(hotIndex >= hotspots.length) {
+ // no more hotspots in this input stream
+ targetOffset = currentOffset + targetFragChars;
+
+ } else if(hotspots[hotIndex] > maxOffset) {
+ // no hotspots within slop
+ targetOffset = currentOffset + targetFragChars;
+
+ } else {
+ // try to find hotspot in slop
+ int goal = hotspots[hotIndex];
+ while(goal < minOffset && hotIndex < hotspots.length) {
+ hotIndex++;
+ goal = hotspots[hotIndex];
+ }
+ targetOffset = goal <= maxOffset ? goal : currentOffset + targetFragChars;
+ }
+
+ isNewFrag = endOffset > targetOffset;
+ }
+
+ // ** operate on isNewFrag
+ if(isNewFrag) {
+ currentNumFrags++;
+ currentOffset = endOffset;
+ targetOffset = -1;
+ }
+ return isNewFrag;
+ }
+
}
\ No newline at end of file
diff --git a/solr/server/scripts/cloud-scripts/zkcli.bat b/solr/server/scripts/cloud-scripts/zkcli.bat
index 8b10b19..0dcf817 100644
--- a/solr/server/scripts/cloud-scripts/zkcli.bat
+++ b/solr/server/scripts/cloud-scripts/zkcli.bat
@@ -1,21 +1,21 @@
-@echo off
-REM You can override pass the following parameters to this script:
-REM
-
-set JVM=java
-
-REM Find location of this script
-
-set SDIR=%~dp0
-if "%SDIR:~-1%"=="\" set SDIR=%SDIR:~0,-1%
-
-set "LOG4J_CONFIG=file:///%SDIR%\..\..\resources\log4j2-console.xml"
-
-REM Settings for ZK ACL
-REM set SOLR_ZK_CREDS_AND_ACLS=-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider ^
-REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider ^
-REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^
-REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD
-
-"%JVM%" %SOLR_ZK_CREDS_AND_ACLS% %ZKCLI_JVM_FLAGS% -Dlog4j.configurationFile="%LOG4J_CONFIG%" ^
--classpath "%SDIR%\..\..\solr-webapp\webapp\WEB-INF\lib\*;%SDIR%\..\..\lib\ext\*;%SDIR%\..\..\lib\*" org.apache.solr.cloud.ZkCLI %*
+@echo off
+REM You can override pass the following parameters to this script:
+REM
+
+set JVM=java
+
+REM Find location of this script
+
+set SDIR=%~dp0
+if "%SDIR:~-1%"=="\" set SDIR=%SDIR:~0,-1%
+
+set "LOG4J_CONFIG=file:///%SDIR%\..\..\resources\log4j2-console.xml"
+
+REM Settings for ZK ACL
+REM set SOLR_ZK_CREDS_AND_ACLS=-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider ^
+REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider ^
+REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^
+REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD
+
+"%JVM%" %SOLR_ZK_CREDS_AND_ACLS% %ZKCLI_JVM_FLAGS% -Dlog4j.configurationFile="%LOG4J_CONFIG%" ^
+-classpath "%SDIR%\..\..\solr-webapp\webapp\WEB-INF\lib\*;%SDIR%\..\..\lib\ext\*;%SDIR%\..\..\lib\*" org.apache.solr.cloud.ZkCLI %*
