| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.solr.handler.admin; |
| |
| import java.io.IOException; |
| import java.lang.invoke.MethodHandles; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collection; |
| import java.util.Date; |
| import java.util.LinkedHashMap; |
| import java.util.List; |
| import java.util.function.BiConsumer; |
| import java.util.stream.Collectors; |
| |
| import org.apache.lucene.index.DocValuesType; |
| import org.apache.lucene.index.FieldInfo; |
| import org.apache.lucene.index.FieldInfos; |
| import org.apache.lucene.index.FilterLeafReader; |
| import org.apache.lucene.index.IndexOptions; |
| import org.apache.lucene.index.IndexWriter; |
| import org.apache.lucene.index.LeafMetaData; |
| import org.apache.lucene.index.LeafReader; |
| import org.apache.lucene.index.LeafReaderContext; |
| import org.apache.lucene.index.MergePolicy; |
| import org.apache.lucene.index.MergePolicy.MergeSpecification; |
| import org.apache.lucene.index.MergePolicy.OneMerge; |
| import org.apache.lucene.index.MergeTrigger; |
| import org.apache.lucene.index.SegmentCommitInfo; |
| import org.apache.lucene.index.SegmentInfos; |
| import org.apache.lucene.index.SegmentReader; |
| import org.apache.lucene.index.Terms; |
| import org.apache.lucene.store.Directory; |
| import org.apache.lucene.util.Accountable; |
| import org.apache.lucene.util.RamUsageEstimator; |
| import org.apache.lucene.util.Version; |
| import org.apache.solr.common.luke.FieldFlag; |
| import org.apache.solr.common.util.Pair; |
| import org.apache.solr.common.util.SimpleOrderedMap; |
| import org.apache.solr.core.SolrCore; |
| import org.apache.solr.handler.RequestHandlerBase; |
| import org.apache.solr.request.SolrQueryRequest; |
| import org.apache.solr.response.SolrQueryResponse; |
| import org.apache.solr.schema.IndexSchema; |
| import org.apache.solr.schema.SchemaField; |
| import org.apache.solr.search.SolrIndexSearcher; |
| import org.apache.solr.update.SolrIndexWriter; |
| import org.apache.solr.util.RefCounted; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import static org.apache.lucene.index.IndexOptions.DOCS; |
| import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS; |
| import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; |
| import static org.apache.solr.common.params.CommonParams.NAME; |
| |
| /** |
| * This handler exposes information about last commit generation segments |
| */ |
| public class SegmentsInfoRequestHandler extends RequestHandlerBase { |
| private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); |
| |
| public static final String FIELD_INFO_PARAM = "fieldInfo"; |
| public static final String CORE_INFO_PARAM = "coreInfo"; |
| public static final String SIZE_INFO_PARAM = "sizeInfo"; |
| public static final String RAW_SIZE_PARAM = "rawSize"; |
| public static final String RAW_SIZE_SUMMARY_PARAM = "rawSizeSummary"; |
| public static final String RAW_SIZE_DETAILS_PARAM = "rawSizeDetails"; |
| public static final String RAW_SIZE_SAMPLING_PERCENT_PARAM = "rawSizeSamplingPercent"; |
| |
| private static final List<String> FI_LEGEND; |
| |
| static { |
| FI_LEGEND = Arrays.asList( |
| FieldFlag.INDEXED.toString(), |
| FieldFlag.DOC_VALUES.toString(), |
| "xxx - DocValues type", |
| FieldFlag.TERM_VECTOR_STORED.toString(), |
| FieldFlag.OMIT_NORMS.toString(), |
| FieldFlag.OMIT_TF.toString(), |
| FieldFlag.OMIT_POSITIONS.toString(), |
| FieldFlag.STORE_OFFSETS_WITH_POSITIONS.toString(), |
| "p - field has payloads", |
| "s - field uses soft deletes", |
| ":x:x:x - point data dim : index dim : num bytes"); |
| } |
| |
| @Override |
| public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) |
| throws Exception { |
| getSegmentsInfo(req, rsp); |
| rsp.setHttpCaching(false); |
| } |
| |
| private static final double GB = 1024.0 * 1024.0 * 1024.0; |
| |
| private void getSegmentsInfo(SolrQueryRequest req, SolrQueryResponse rsp) |
| throws Exception { |
| boolean withFieldInfo = req.getParams().getBool(FIELD_INFO_PARAM, false); |
| boolean withCoreInfo = req.getParams().getBool(CORE_INFO_PARAM, false); |
| boolean withSizeInfo = req.getParams().getBool(SIZE_INFO_PARAM, false); |
| boolean withRawSizeInfo = req.getParams().getBool(RAW_SIZE_PARAM, false); |
| boolean withRawSizeSummary = req.getParams().getBool(RAW_SIZE_SUMMARY_PARAM, false); |
| boolean withRawSizeDetails = req.getParams().getBool(RAW_SIZE_DETAILS_PARAM, false); |
| if (withRawSizeSummary || withRawSizeDetails) { |
| withRawSizeInfo = true; |
| } |
| SolrIndexSearcher searcher = req.getSearcher(); |
| |
| SegmentInfos infos = |
| SegmentInfos.readLatestCommit(searcher.getIndexReader().directory()); |
| |
| SimpleOrderedMap<Object> segmentInfos = new SimpleOrderedMap<>(); |
| |
| SolrCore core = req.getCore(); |
| SimpleOrderedMap<Object> infosInfo = new SimpleOrderedMap<>(); |
| Version minVersion = infos.getMinSegmentLuceneVersion(); |
| if (minVersion != null) { |
| infosInfo.add("minSegmentLuceneVersion", minVersion.toString()); |
| } |
| Version commitVersion = infos.getCommitLuceneVersion(); |
| if (commitVersion != null) { |
| infosInfo.add("commitLuceneVersion", commitVersion.toString()); |
| } |
| infosInfo.add("numSegments", infos.size()); |
| infosInfo.add("segmentsFileName", infos.getSegmentsFileName()); |
| infosInfo.add("totalMaxDoc", infos.totalMaxDoc()); |
| infosInfo.add("userData", infos.userData); |
| if (withCoreInfo) { |
| SimpleOrderedMap<Object> coreInfo = new SimpleOrderedMap<>(); |
| infosInfo.add("core", coreInfo); |
| coreInfo.add("startTime", core.getStartTimeStamp().getTime() + "(" + core.getStartTimeStamp() + ")"); |
| coreInfo.add("dataDir", core.getDataDir()); |
| coreInfo.add("indexDir", core.getIndexDir()); |
| coreInfo.add("sizeInGB", (double)core.getIndexSize() / GB); |
| |
| RefCounted<IndexWriter> iwRef = core.getSolrCoreState().getIndexWriter(core); |
| if (iwRef != null) { |
| try { |
| IndexWriter iw = iwRef.get(); |
| String iwConfigStr = iw.getConfig().toString(); |
| SimpleOrderedMap<Object> iwConfig = new SimpleOrderedMap<>(); |
| // meh ... |
| String[] lines = iwConfigStr.split("\\n"); |
| for (String line : lines) { |
| String[] parts = line.split("="); |
| if (parts.length < 2) { |
| continue; |
| } |
| iwConfig.add(parts[0], parts[1]); |
| } |
| coreInfo.add("indexWriterConfig", iwConfig); |
| } finally { |
| iwRef.decref(); |
| } |
| } |
| } |
| SimpleOrderedMap<Object> segmentInfo = null; |
| List<SegmentCommitInfo> sortable = new ArrayList<>(infos.asList()); |
| // Order by the number of live docs. The display is logarithmic so it is a little jumbled visually |
| sortable.sort((s1, s2) -> |
| (s2.info.maxDoc() - s2.getDelCount()) - (s1.info.maxDoc() - s1.getDelCount()) |
| ); |
| |
| List<String> mergeCandidates = new ArrayList<>(); |
| SimpleOrderedMap<Object> runningMerges = getMergeInformation(req, infos, mergeCandidates); |
| List<LeafReaderContext> leafContexts = searcher.getIndexReader().leaves(); |
| IndexSchema schema = req.getSchema(); |
| for (SegmentCommitInfo segmentCommitInfo : sortable) { |
| segmentInfo = getSegmentInfo(segmentCommitInfo, withSizeInfo, withFieldInfo, leafContexts, schema); |
| if (mergeCandidates.contains(segmentCommitInfo.info.name)) { |
| segmentInfo.add("mergeCandidate", true); |
| } |
| segmentInfos.add((String) segmentInfo.get(NAME), segmentInfo); |
| } |
| |
| rsp.add("info", infosInfo); |
| if (runningMerges.size() > 0) { |
| rsp.add("runningMerges", runningMerges); |
| } |
| if (withFieldInfo) { |
| rsp.add("fieldInfoLegend", FI_LEGEND); |
| } |
| rsp.add("segments", segmentInfos); |
| if (withRawSizeInfo) { |
| IndexSizeEstimator estimator = new IndexSizeEstimator(searcher.getRawReader(), 20, 100, withRawSizeSummary, withRawSizeDetails); |
| Object samplingPercentVal = req.getParams().get(RAW_SIZE_SAMPLING_PERCENT_PARAM); |
| if (samplingPercentVal != null) { |
| estimator.setSamplingPercent(Float.parseFloat(String.valueOf(samplingPercentVal))); |
| } |
| IndexSizeEstimator.Estimate estimate = estimator.estimate(); |
| SimpleOrderedMap<Object> estimateMap = new SimpleOrderedMap<>(); |
| // make the units more user-friendly |
| estimateMap.add(IndexSizeEstimator.FIELDS_BY_SIZE, estimate.getHumanReadableFieldsBySize()); |
| estimateMap.add(IndexSizeEstimator.TYPES_BY_SIZE, estimate.getHumanReadableTypesBySize()); |
| if (estimate.getSummary() != null) { |
| estimateMap.add(IndexSizeEstimator.SUMMARY, estimate.getSummary()); |
| } |
| if (estimate.getDetails() != null) { |
| estimateMap.add(IndexSizeEstimator.DETAILS, estimate.getDetails()); |
| } |
| rsp.add("rawSize", estimateMap); |
| } |
| } |
| |
| private SimpleOrderedMap<Object> getSegmentInfo( |
| SegmentCommitInfo segmentCommitInfo, boolean withSizeInfo, boolean withFieldInfos, |
| List<LeafReaderContext> leafContexts, IndexSchema schema) throws IOException { |
| SimpleOrderedMap<Object> segmentInfoMap = new SimpleOrderedMap<>(); |
| |
| segmentInfoMap.add(NAME, segmentCommitInfo.info.name); |
| segmentInfoMap.add("delCount", segmentCommitInfo.getDelCount()); |
| segmentInfoMap.add("softDelCount", segmentCommitInfo.getSoftDelCount()); |
| segmentInfoMap.add("hasFieldUpdates", segmentCommitInfo.hasFieldUpdates()); |
| segmentInfoMap.add("sizeInBytes", segmentCommitInfo.sizeInBytes()); |
| segmentInfoMap.add("size", segmentCommitInfo.info.maxDoc()); |
| Long timestamp = Long.parseLong(segmentCommitInfo.info.getDiagnostics() |
| .get("timestamp")); |
| segmentInfoMap.add("age", new Date(timestamp)); |
| segmentInfoMap.add("source", |
| segmentCommitInfo.info.getDiagnostics().get("source")); |
| segmentInfoMap.add("version", segmentCommitInfo.info.getVersion().toString()); |
| // don't open a new SegmentReader - try to find the right one from the leaf contexts |
| SegmentReader seg = null; |
| for (LeafReaderContext lrc : leafContexts) { |
| LeafReader leafReader = lrc.reader(); |
| leafReader = FilterLeafReader.unwrap(leafReader); |
| if (leafReader instanceof SegmentReader) { |
| SegmentReader sr = (SegmentReader)leafReader; |
| if (sr.getSegmentInfo().info.equals(segmentCommitInfo.info)) { |
| seg = sr; |
| break; |
| } |
| } |
| } |
| if (seg != null) { |
| LeafMetaData metaData = seg.getMetaData(); |
| if (metaData != null) { |
| segmentInfoMap.add("createdVersionMajor", metaData.getCreatedVersionMajor()); |
| segmentInfoMap.add("minVersion", metaData.getMinVersion().toString()); |
| if (metaData.getSort() != null) { |
| segmentInfoMap.add("sort", metaData.getSort().toString()); |
| } |
| } |
| } |
| if (!segmentCommitInfo.info.getDiagnostics().isEmpty()) { |
| segmentInfoMap.add("diagnostics", segmentCommitInfo.info.getDiagnostics()); |
| } |
| if (!segmentCommitInfo.info.getAttributes().isEmpty()) { |
| segmentInfoMap.add("attributes", segmentCommitInfo.info.getAttributes()); |
| } |
| if (withSizeInfo) { |
| Directory dir = segmentCommitInfo.info.dir; |
| List<Pair<String, Long>> files = segmentCommitInfo.files().stream() |
| .map(f -> { |
| long size = -1; |
| try { |
| size = dir.fileLength(f); |
| } catch (IOException e) { |
| } |
| return new Pair<String, Long>(f, size); |
| }).sorted((p1, p2) -> { |
| if (p1.second() > p2.second()) { |
| return -1; |
| } else if (p1.second() < p2.second()) { |
| return 1; |
| } else { |
| return 0; |
| } |
| }).collect(Collectors.toList()); |
| if (!files.isEmpty()) { |
| SimpleOrderedMap<Object> topFiles = new SimpleOrderedMap<>(); |
| for (int i = 0; i < Math.min(files.size(), 5); i++) { |
| Pair<String, Long> p = files.get(i); |
| topFiles.add(p.first(), RamUsageEstimator.humanReadableUnits(p.second())); |
| } |
| segmentInfoMap.add("largestFiles", topFiles); |
| } |
| } |
| if (seg != null && withSizeInfo) { |
| SimpleOrderedMap<Object> ram = new SimpleOrderedMap<>(); |
| ram.add("total", seg.ramBytesUsed()); |
| for (Accountable ac : seg.getChildResources()) { |
| accountableToMap(ac, ram::add); |
| } |
| segmentInfoMap.add("ramBytesUsed", ram); |
| } |
| if (withFieldInfos) { |
| if (seg == null) { |
| log.debug("Skipping segment info - not available as a SegmentReader: {}", segmentCommitInfo); |
| } else { |
| FieldInfos fis = seg.getFieldInfos(); |
| SimpleOrderedMap<Object> fields = new SimpleOrderedMap<>(); |
| for (FieldInfo fi : fis) { |
| fields.add(fi.name, getFieldInfo(seg, fi, schema)); |
| } |
| segmentInfoMap.add("fields", fields); |
| } |
| } |
| |
| return segmentInfoMap; |
| } |
| |
| private void accountableToMap(Accountable accountable, BiConsumer<String, Object> consumer) { |
| Collection<Accountable> children = accountable.getChildResources(); |
| if (children != null && !children.isEmpty()) { |
| LinkedHashMap<String, Object> map = new LinkedHashMap<>(); |
| map.put("total", accountable.ramBytesUsed()); |
| for (Accountable child : children) { |
| accountableToMap(child, map::put); |
| } |
| consumer.accept(accountable.toString(), map); |
| } else { |
| consumer.accept(accountable.toString(), accountable.ramBytesUsed()); |
| } |
| } |
| |
| private SimpleOrderedMap<Object> getFieldInfo(SegmentReader reader, FieldInfo fi, IndexSchema schema) { |
| SimpleOrderedMap<Object> fieldFlags = new SimpleOrderedMap<>(); |
| StringBuilder flags = new StringBuilder(); |
| IndexOptions opts = fi.getIndexOptions(); |
| flags.append( (opts != IndexOptions.NONE) ? FieldFlag.INDEXED.getAbbreviation() : '-' ); |
| DocValuesType dvt = fi.getDocValuesType(); |
| if (dvt != DocValuesType.NONE) { |
| flags.append(FieldFlag.DOC_VALUES.getAbbreviation()); |
| switch (dvt) { |
| case NUMERIC: |
| flags.append("num"); |
| break; |
| case BINARY: |
| flags.append("bin"); |
| break; |
| case SORTED: |
| flags.append("srt"); |
| break; |
| case SORTED_NUMERIC: |
| flags.append("srn"); |
| break; |
| case SORTED_SET: |
| flags.append("srs"); |
| break; |
| default: |
| flags.append("???"); // should not happen |
| } |
| } else { |
| flags.append("----"); |
| } |
| flags.append( (fi.hasVectors()) ? FieldFlag.TERM_VECTOR_STORED.getAbbreviation() : '-' ); |
| flags.append( (fi.omitsNorms()) ? FieldFlag.OMIT_NORMS.getAbbreviation() : '-' ); |
| |
| flags.append( (DOCS == opts ) ? |
| FieldFlag.OMIT_TF.getAbbreviation() : '-' ); |
| |
| flags.append((DOCS_AND_FREQS == opts) ? |
| FieldFlag.OMIT_POSITIONS.getAbbreviation() : '-'); |
| |
| flags.append((DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS == opts) ? |
| FieldFlag.STORE_OFFSETS_WITH_POSITIONS.getAbbreviation() : '-'); |
| |
| flags.append( (fi.hasPayloads() ? "p" : "-")); |
| flags.append( (fi.isSoftDeletesField() ? "s" : "-")); |
| if (fi.getPointDimensionCount() > 0 || fi.getPointIndexDimensionCount() > 0) { |
| flags.append(":"); |
| flags.append(fi.getPointDimensionCount()).append(':'); |
| flags.append(fi.getPointIndexDimensionCount()).append(':'); |
| flags.append(fi.getPointNumBytes()); |
| } |
| |
| fieldFlags.add("flags", flags.toString()); |
| try { |
| Terms terms = reader.terms(fi.name); |
| if (terms != null) { |
| fieldFlags.add("docCount", terms.getDocCount()); |
| fieldFlags.add("sumDocFreq", terms.getSumDocFreq()); |
| fieldFlags.add("sumTotalTermFreq", terms.getSumTotalTermFreq()); |
| } |
| } catch (Exception e) { |
| log.debug("Exception retrieving term stats for field {}", fi.name, e); |
| } |
| |
| // probably too much detail? |
| // Map<String, String> attributes = fi.attributes(); |
| // if (!attributes.isEmpty()) { |
| // fieldFlags.add("attributes", attributes); |
| // } |
| |
| // check compliance of the index with the current schema |
| SchemaField sf = schema.getFieldOrNull(fi.name); |
| boolean hasPoints = fi.getPointDimensionCount() > 0 || fi.getPointIndexDimensionCount() > 0; |
| |
| if (sf != null) { |
| fieldFlags.add("schemaType", sf.getType().getTypeName()); |
| SimpleOrderedMap<Object> nonCompliant = new SimpleOrderedMap<>(); |
| if (sf.hasDocValues() && |
| fi.getDocValuesType() == DocValuesType.NONE && |
| fi.getIndexOptions() != IndexOptions.NONE) { |
| nonCompliant.add("docValues", "schema=" + sf.getType().getUninversionType(sf) + ", segment=false"); |
| } |
| if (!sf.hasDocValues() && |
| fi.getDocValuesType() != DocValuesType.NONE) { |
| nonCompliant.add("docValues", "schema=false, segment=" + fi.getDocValuesType().toString()); |
| } |
| if (!sf.isPolyField()) { // difficult to find all sub-fields in a general way |
| if (sf.indexed() != ((fi.getIndexOptions() != IndexOptions.NONE) || hasPoints)) { |
| nonCompliant.add("indexed", "schema=" + sf.indexed() + ", segment=" + fi.getIndexOptions()); |
| } |
| } |
| if (!hasPoints && (sf.omitNorms() != fi.omitsNorms())) { |
| nonCompliant.add("omitNorms", "schema=" + sf.omitNorms() + ", segment=" + fi.omitsNorms()); |
| } |
| if (sf.storeTermVector() != fi.hasVectors()) { |
| nonCompliant.add("termVectors", "schema=" + sf.storeTermVector() + ", segment=" + fi.hasVectors()); |
| } |
| if (sf.storeOffsetsWithPositions() != (fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)) { |
| nonCompliant.add("storeOffsetsWithPositions", "schema=" + sf.storeOffsetsWithPositions() + ", segment=" + fi.getIndexOptions()); |
| } |
| |
| if (nonCompliant.size() > 0) { |
| nonCompliant.add("schemaField", sf.toString()); |
| fieldFlags.add("nonCompliant", nonCompliant); |
| } |
| } else { |
| fieldFlags.add("schemaType", "(UNKNOWN)"); |
| } |
| return fieldFlags; |
| } |
| |
| // returns a map of currently running merges, and populates a list of candidate segments for merge |
| private SimpleOrderedMap<Object> getMergeInformation(SolrQueryRequest req, SegmentInfos infos, List<String> mergeCandidates) throws IOException { |
| SimpleOrderedMap<Object> result = new SimpleOrderedMap<>(); |
| RefCounted<IndexWriter> refCounted = req.getCore().getSolrCoreState().getIndexWriter(req.getCore()); |
| try { |
| IndexWriter indexWriter = refCounted.get(); |
| if (indexWriter instanceof SolrIndexWriter) { |
| result.addAll(((SolrIndexWriter)indexWriter).getRunningMerges()); |
| } |
| //get chosen merge policy |
| MergePolicy mp = indexWriter.getConfig().getMergePolicy(); |
| //Find merges |
| MergeSpecification findMerges = mp.findMerges(MergeTrigger.EXPLICIT, infos, indexWriter); |
| if (findMerges != null && findMerges.merges != null && findMerges.merges.size() > 0) { |
| for (OneMerge merge : findMerges.merges) { |
| //TODO: add merge grouping |
| for (SegmentCommitInfo mergeSegmentInfo : merge.segments) { |
| mergeCandidates.add(mergeSegmentInfo.info.name); |
| } |
| } |
| } |
| |
| return result; |
| } finally { |
| refCounted.decref(); |
| } |
| } |
| |
| @Override |
| public String getDescription() { |
| return "Lucene segments info."; |
| } |
| |
| @Override |
| public Category getCategory() { |
| return Category.ADMIN; |
| } |
| } |
