docs/attachments/LUCENE-5813/LUCENE-5813.patch

Index: lucene/CHANGES.txt
===================================================================
--- lucene/CHANGES.txt	(revision 1611404)
+++ lucene/CHANGES.txt	(working copy)
@@ -111,6 +111,8 @@
 
 * LUCENE-5826: Support proper hunspell case handling, LANG, KEEPCASE, NEEDAFFIX,
   and ONLYINCOMPOUND flags.  (Robert Muir)
+
+* LUCENE-5813: Directory implements Accountable. (Adrien Grand)
   
 API Changes
 
Index: lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java	(working copy)
@@ -259,6 +259,11 @@
   }
 
   @Override
+  public long ramBytesUsed() {
+    return directory.ramBytesUsed();
+  }
+
+  @Override
   public String toString() {
     return "CompoundFileDirectory(file=\"" + fileName + "\" in dir=" + directory + ")";
   }
Index: lucene/core/src/java/org/apache/lucene/store/Directory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/Directory.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/Directory.java	(working copy)
@@ -23,6 +23,7 @@
 import java.nio.file.NoSuchFileException;
 import java.util.Collection; // for javadocs
 
+import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.IOUtils;
 
 /** A Directory is a flat list of files.  Files may be written once, when they
@@ -41,7 +42,7 @@
  * instance using {@link #setLockFactory}.
  *
  */
-public abstract class Directory implements Closeable {
+public abstract class Directory implements Accountable, Closeable {
 
   /**
    * Returns an array of strings, one for each file in the directory.
Index: lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java	(working copy)
@@ -166,4 +166,9 @@
   public IndexInput openInput(String name, IOContext context) throws IOException {
     return getDirectory(name).openInput(name, context);
   }
+
+  @Override
+  public long ramBytesUsed() {
+    return primaryDir.ramBytesUsed() + secondaryDir.ramBytesUsed();
+  }
 }
Index: lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java	(working copy)
@@ -110,4 +110,8 @@
     return getClass().getSimpleName() + "(" + in.toString() + ")";
   }
 
+  @Override
+  public long ramBytesUsed() {
+    return in.ramBytesUsed();
+  }
 }
Index: lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java	(working copy)
@@ -285,4 +285,9 @@
       }
     }
   };
+
+  @Override
+  public long ramBytesUsed() {
+    return 0;
+  }
 }
Index: lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java	(working copy)
@@ -193,4 +193,9 @@
     @Override
     protected void seekInternal(long pos) throws IOException {}
   }
+
+  @Override
+  public long ramBytesUsed() {
+    return 0;
+  }
 }
Index: lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java	(working copy)
@@ -64,7 +64,7 @@
  * @lucene.experimental
  */
 
-public class NRTCachingDirectory extends FilterDirectory implements Accountable {
+public class NRTCachingDirectory extends FilterDirectory {
 
   private final RAMDirectory cache = new RAMDirectory();
 
@@ -259,6 +259,6 @@
 
   @Override
   public long ramBytesUsed() {
-    return cache.ramBytesUsed();
+    return super.ramBytesUsed() + cache.ramBytesUsed();
   }
 }
Index: lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java	(working copy)
@@ -47,7 +47,7 @@
  * implementation working directly on the file system cache of the
  * operating system, so copying data to Java heap space is not useful.
  */
-public class RAMDirectory extends BaseDirectory implements Accountable {
+public class RAMDirectory extends BaseDirectory {
   protected final Map<String,RAMFile> fileMap = new ConcurrentHashMap<>();
   protected final AtomicLong sizeInBytes = new AtomicLong();
   
Index: lucene/core/src/java/org/apache/lucene/store/RAMFile.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/RAMFile.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/RAMFile.java	(working copy)
@@ -20,11 +20,15 @@
 import java.util.ArrayList;
 
 import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /** 
  * Represents a file in RAM as a list of byte[] buffers.
  * @lucene.internal */
 public class RAMFile implements Accountable {
+
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(RAMFile.class);
+
   protected ArrayList<byte[]> buffers = new ArrayList<>();
   long length;
   RAMDirectory directory;
@@ -31,10 +35,14 @@
   protected long sizeInBytes;
 
   // File used as buffer, in no RAMDirectory
-  public RAMFile() {}
+  public RAMFile() {
+    sizeInBytes = BASE_RAM_BYTES_USED;
+  }
   
   RAMFile(RAMDirectory directory) {
+    this();
     this.directory = directory;
+    directory.sizeInBytes.getAndAdd(sizeInBytes);
   }
 
   // For non-stream access from thread that might be concurrent with writing
@@ -48,13 +56,14 @@
 
   protected final byte[] addBuffer(int size) {
     byte[] buffer = newBuffer(size);
+    final long ramBytesUsed = RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.sizeOf(buffer);
     synchronized(this) {
       buffers.add(buffer);
-      sizeInBytes += size;
+      sizeInBytes += ramBytesUsed;
     }
 
     if (directory != null) {
-      directory.sizeInBytes.getAndAdd(size);
+      directory.sizeInBytes.getAndAdd(ramBytesUsed);
     }
     return buffer;
   }
Index: lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java	(revision 1611404)
+++ lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java	(working copy)
@@ -159,4 +159,9 @@
       return file.getFD().valid();
     }
   }
+
+  @Override
+  public long ramBytesUsed() {
+    return 0;
+  }
 }
Index: lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java	(revision 1611404)
+++ lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java	(working copy)
@@ -140,6 +140,10 @@
     public void close() throws IOException {
       fsDir.close();
     }
+    @Override
+    public long ramBytesUsed() {
+      return fsDir.ramBytesUsed();
+    }
   }
 
   private static class FaultyIndexInput extends BufferedIndexInput {
Index: lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java	(revision 1611404)
+++ lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java	(working copy)
@@ -297,6 +297,11 @@
         dir = new SimpleFSDirectory(path, null);
       }
 
+      @Override
+      public long ramBytesUsed() {
+        return dir.ramBytesUsed();
+      }
+
       public void tweakBufferSizes() {
         //int count = 0;
         for (final IndexInput ip : allIndexInputs) {
Index: lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
===================================================================
--- lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java	(revision 1611404)
+++ lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java	(working copy)
@@ -120,6 +120,11 @@
   }  
 
   @Override
+  public long ramBytesUsed() {
+    return delegate.ramBytesUsed();
+  }
+
+  @Override
   public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
     if (context.context != Context.MERGE || context.mergeInfo.estimatedMergeBytes < minBytesDirect || fileLength(name) < minBytesDirect) {
Index: lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.java
===================================================================
--- lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.java	(revision 1611404)
+++ lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.java	(working copy)
@@ -70,6 +70,11 @@
   }
 
   @Override
+  public long ramBytesUsed() {
+    return 0;
+  }
+
+  @Override
   public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
     return new WindowsIndexInput(new File(getDirectory(), name), Math.max(BufferedIndexInput.bufferSize(context), DEFAULT_BUFFERSIZE));
Index: lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
===================================================================
--- lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java	(revision 1611404)
+++ lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java	(working copy)
@@ -21,11 +21,17 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.lang.reflect.Field;
 import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.zip.CRC32;
 
 import org.apache.lucene.index.DirectoryReader;
@@ -32,6 +38,7 @@
 import org.apache.lucene.index.IndexNotFoundException;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.RamUsageTester;
 import org.apache.lucene.util.TestUtil;
 
 /** Base class for per-Directory tests. */
@@ -914,5 +921,94 @@
     input.close();
     dir.close();
   }
+
+  private static void createRandomFile(Directory dir, String name, long size) throws IOException {
+    try (final IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) {
+      final byte[] bytes = new byte[1024];
+      for (long written = 0; written < size; ) {
+        random().nextBytes(bytes);
+        final int toWrite = (int) Math.min(bytes.length, size - written);
+        out.writeBytes(bytes, 0, toWrite);
+        written += toWrite;
+      }
+    }
+  }
+
+  private static final Field FS_DIRECTORY_STALE_FILES;
+  static {
+    try {
+      FS_DIRECTORY_STALE_FILES = FSDirectory.class.getDeclaredField("staleFiles");
+    } catch (NoSuchFieldException | SecurityException e) {
+      throw new Error(e);
+    }
+  }
+  private static final RamUsageTester.Accumulator DIRECTORY_ACCUMULATOR = new RamUsageTester.Accumulator() {
+    
+    public long accumulateObject(Object o, long shallowSize, Map<Field, Object> fieldValues, Collection<Object> queue) {
+      if (o instanceof MockDirectoryWrapper) {
+        // MockDirectoryWrapper keeps lots of references on open inputs, not sync'd files, etc.
+        // just take the wrapped directory into account
+        final Map<Field, Object> newFieldValues = new HashMap<>();
+        for (Map.Entry<Field, Object> entry : fieldValues.entrySet()) {
+          if (entry.getValue() instanceof Directory) {
+            newFieldValues.put(entry.getKey(), entry.getValue());
+          }
+        }
+        fieldValues = newFieldValues;
+      }
+      if (o instanceof FSDirectory) {
+        fieldValues = new HashMap<>(fieldValues);
+        fieldValues.remove(FS_DIRECTORY_STALE_FILES);
+      }
+      return super.accumulateObject(o, shallowSize, fieldValues, queue);
+    }
+    
+  };
+
+  public void testRamBytesUsed() throws IOException {
+    Directory dir = getDirectory(createTempDir("testBytes"));
+
+    // Create one large file and lots of small files
+    List<String> files = new ArrayList<>();
+    createRandomFile(dir, "large.bin", TestUtil.nextInt(random(), 1 << 18, 1 << 20));
+    files.add("large.bin");
+    final int numSmallFiles = TestUtil.nextInt(random(), 10, 1000);
+    for (int i = 0; i < numSmallFiles; ++i) {
+      final String fileName = "small" + i + ".bin";
+      createRandomFile(dir, fileName, TestUtil.nextInt(random(), 10, 1024));
+      files.add(fileName);
+    }
+    if (random().nextBoolean()) {
+      dir.sync(files);
+    }
+
+    Set<IndexInput> openInputs = new HashSet<>();
+    if (random().nextBoolean()) {
+      openInputs.add(dir.openInput("large.bin", IOContext.READ));
+    }
+    for (int i = 0; i < numSmallFiles; ++i) {
+      if (random().nextBoolean()) {
+        final String fileName = "small" + i + ".bin";
+        openInputs.add(dir.openInput(fileName, IOContext.READ));
+      }
+    }
+    for (IndexInput input : openInputs) {
+      if (input.length() >= 2) {
+        input.seek(random().nextInt((int) input.length() - 1));
+        input.readByte();
+      }
+    }
+
+    final long estimatedBytes = dir.ramBytesUsed();
+    final long actualBytes = RamUsageTester.sizeOf(dir, DIRECTORY_ACCUMULATOR);
+    final double absoluteError = Math.abs(estimatedBytes - actualBytes);
+    final double relativeError = absoluteError / actualBytes;
+    assertTrue("Absolute error: " + absoluteError + ", relative error: " + relativeError,
+        relativeError < 0.20 || absoluteError < 4096);
+    for (IndexInput in : openInputs) {
+      in.close();
+    }
+    dir.close();
+  }
 }
 
Index: solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java
===================================================================
--- solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java	(revision 1611404)
+++ solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java	(working copy)
@@ -21,6 +21,8 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.lucene.util.Accountable;
+
 import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
 import com.googlecode.concurrentlinkedhashmap.EvictionListener;
 
@@ -27,7 +29,7 @@
 /**
  * @lucene.experimental
  */
-public class BlockCache {
+public class BlockCache implements Accountable {
   
   public static final int _128M = 134217728;
   public static final int _32K = 32768;
@@ -79,7 +81,12 @@
         .maximumWeightedCapacity(maxEntries).listener(listener).build();
     this.blockSize = blockSize;
   }
-  
+
+  @Override
+  public long ramBytesUsed() {
+    return (long) numberOfBlocksPerBank * blockSize * banks.length;
+  }
+
   private void releaseLocation(BlockCacheLocation location) {
     if (location == null) {
       return;
Index: solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
===================================================================
--- solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java	(revision 1611404)
+++ solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java	(working copy)
@@ -57,7 +57,11 @@
   }
   
   public static Cache NO_CACHE = new Cache() {
-    
+
+    public long ramBytesUsed() {
+      return 0;
+    }
+
     @Override
     public void update(String name, long blockId, int blockOffset,
         byte[] buffer, int offset, int length) {}
@@ -114,7 +118,12 @@
       setLockFactory(directory.getLockFactory());
     }
   }
-  
+
+  @Override
+  public long ramBytesUsed() {
+    return directory.ramBytesUsed() + cache.ramBytesUsed();
+  }
+
   private IndexInput openInput(String name, int bufferSize, IOContext context)
       throws IOException {
     final IndexInput source = directory.openInput(name, context);
Index: solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java
===================================================================
--- solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java	(revision 1611404)
+++ solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java	(working copy)
@@ -36,7 +36,12 @@
     this.path = path;
     this.metrics = metrics;
   }
-  
+
+  @Override
+  public long ramBytesUsed() {
+    return blockCache.ramBytesUsed();
+  }
+
   /**
    * Expert: mostly for tests
    * 
Index: solr/core/src/java/org/apache/solr/store/blockcache/Cache.java
===================================================================
--- solr/core/src/java/org/apache/solr/store/blockcache/Cache.java	(revision 1611404)
+++ solr/core/src/java/org/apache/solr/store/blockcache/Cache.java	(working copy)
@@ -1,5 +1,7 @@
 package org.apache.solr.store.blockcache;
 
+import org.apache.lucene.util.Accountable;
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -20,7 +22,7 @@
 /**
  * @lucene.experimental
  */
-public interface Cache {
+public interface Cache extends Accountable {
   
   /**
    * Remove a file from the cache.
Index: solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
===================================================================
--- solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java	(revision 1611404)
+++ solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java	(working copy)
@@ -89,8 +89,13 @@
       }
     }
   }
-  
+
   @Override
+  public long ramBytesUsed() {
+    return 0;
+  }
+
+  @Override
   public void close() throws IOException {
     LOG.info("Closing hdfs directory {}", hdfsDirPath);
     fileSystem.close();
Index: solr/core/src/test/org/apache/solr/store/blockcache/BlockDirectoryTest.java
===================================================================
--- solr/core/src/test/org/apache/solr/store/blockcache/BlockDirectoryTest.java	(revision 1611404)
+++ solr/core/src/test/org/apache/solr/store/blockcache/BlockDirectoryTest.java	(working copy)
@@ -22,7 +22,6 @@
 import java.util.Map;
 import java.util.Random;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.IOContext;
@@ -29,10 +28,8 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.MergeInfo;
-import org.apache.lucene.util.LuceneTestCase;
-
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.store.hdfs.HdfsDirectory;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -45,6 +42,15 @@
     public Map<String, byte[]> map = new ConcurrentLinkedHashMap.Builder<String, byte[]>().maximumWeightedCapacity(8).build();
 
     @Override
+    public long ramBytesUsed() {
+      long ramBytesUsed = map.size() * 2L * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+      for (byte[] array : map.values()) {
+        ramBytesUsed += RamUsageEstimator.sizeOf(array);
+      }
+      return ramBytesUsed;
+    }
+
+    @Override
     public void update(String name, long blockId, int blockOffset, byte[] buffer, int offset, int length) {
       byte[] cached = map.get(name + blockId);
       if (cached != null) {