| package org.apache.commons.jcs.auxiliary.disk.block; |
| |
| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| import java.io.BufferedInputStream; |
| import java.io.BufferedOutputStream; |
| import java.io.EOFException; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileOutputStream; |
| import java.io.IOException; |
| import java.io.ObjectInputStream; |
| import java.io.ObjectOutputStream; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.Map; |
| import java.util.Map.Entry; |
| import java.util.Set; |
| import java.util.TreeMap; |
| import java.util.concurrent.atomic.AtomicInteger; |
| |
| import org.apache.commons.jcs.auxiliary.disk.behavior.IDiskCacheAttributes.DiskLimitType; |
| import org.apache.commons.jcs.io.ObjectInputStreamClassLoaderAware; |
| import org.apache.commons.jcs.utils.struct.AbstractLRUMap; |
| import org.apache.commons.jcs.utils.struct.LRUMap; |
| import org.apache.commons.jcs.utils.timing.ElapsedTimer; |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| |
| /** |
| * This is responsible for storing the keys. |
| * <p> |
| * |
| * @author Aaron Smuts |
| */ |
| public class BlockDiskKeyStore<K> |
| { |
| /** The logger */ |
| private static final Log log = LogFactory.getLog(BlockDiskKeyStore.class); |
| |
| /** Attributes governing the behavior of the block disk cache. */ |
| private final BlockDiskCacheAttributes blockDiskCacheAttributes; |
| |
| /** The key to block map */ |
| private Map<K, int[]> keyHash; |
| |
| /** The file where we persist the keys */ |
| private final File keyFile; |
| |
| /** The name to prefix log messages with. */ |
| protected final String logCacheName; |
| |
| /** Name of the file where we persist the keys */ |
| private final String fileName; |
| |
| /** The maximum number of keys to store in memory */ |
| private final int maxKeySize; |
| |
| /** |
| * we need this so we can communicate free blocks to the data store when |
| * keys fall off the LRU |
| */ |
| protected final BlockDiskCache<K, ?> blockDiskCache; |
| |
| private DiskLimitType diskLimitType = DiskLimitType.COUNT; |
| |
| private int blockSize; |
| |
| /** |
| * Set the configuration options. |
| * <p> |
| * |
| * @param cacheAttributes |
| * @param blockDiskCache |
| * used for freeing |
| */ |
| public BlockDiskKeyStore(BlockDiskCacheAttributes cacheAttributes, BlockDiskCache<K, ?> blockDiskCache) |
| { |
| this.blockDiskCacheAttributes = cacheAttributes; |
| this.logCacheName = "Region [" + this.blockDiskCacheAttributes.getCacheName() + "] "; |
| this.fileName = this.blockDiskCacheAttributes.getCacheName(); |
| this.maxKeySize = cacheAttributes.getMaxKeySize(); |
| this.blockDiskCache = blockDiskCache; |
| this.diskLimitType = cacheAttributes.getDiskLimitType(); |
| this.blockSize = cacheAttributes.getBlockSizeBytes(); |
| |
| File rootDirectory = cacheAttributes.getDiskPath(); |
| |
| if (log.isInfoEnabled()) |
| { |
| log.info(logCacheName + "Cache file root directory [" + rootDirectory + "]"); |
| } |
| |
| this.keyFile = new File(rootDirectory, fileName + ".key"); |
| |
| if (log.isInfoEnabled()) |
| { |
| log.info(logCacheName + "Key File [" + this.keyFile.getAbsolutePath() + "]"); |
| } |
| |
| if (keyFile.length() > 0) |
| { |
| loadKeys(); |
| if (!verify()) |
| { |
| log.warn(logCacheName + "Key File is invalid. Resetting file."); |
| initKeyMap(); |
| reset(); |
| } |
| } |
| else |
| { |
| initKeyMap(); |
| } |
| } |
| |
| /** |
| * Saves key file to disk. This gets the LRUMap entry set and write the |
| * entries out one by one after putting them in a wrapper. |
| */ |
| protected void saveKeys() |
| { |
| try |
| { |
| ElapsedTimer timer = new ElapsedTimer(); |
| int numKeys = keyHash.size(); |
| if (log.isInfoEnabled()) |
| { |
| log.info(logCacheName + "Saving keys to [" + this.keyFile.getAbsolutePath() + "], key count [" + numKeys + "]"); |
| } |
| |
| synchronized (keyFile) |
| { |
| FileOutputStream fos = new FileOutputStream(keyFile); |
| BufferedOutputStream bos = new BufferedOutputStream(fos, 65536); |
| ObjectOutputStream oos = new ObjectOutputStream(bos); |
| try |
| { |
| if (!verify()) |
| { |
| throw new IOException("Inconsistent key file"); |
| } |
| // don't need to synchronize, since the underlying |
| // collection makes a copy |
| for (Map.Entry<K, int[]> entry : keyHash.entrySet()) |
| { |
| BlockDiskElementDescriptor<K> descriptor = new BlockDiskElementDescriptor<K>(); |
| descriptor.setKey(entry.getKey()); |
| descriptor.setBlocks(entry.getValue()); |
| // stream these out in the loop. |
| oos.writeUnshared(descriptor); |
| } |
| } |
| finally |
| { |
| oos.flush(); |
| oos.close(); |
| } |
| } |
| |
| if (log.isInfoEnabled()) |
| { |
| log.info(logCacheName + "Finished saving keys. It took " + timer.getElapsedTimeString() + " to store " + numKeys |
| + " keys. Key file length [" + keyFile.length() + "]"); |
| } |
| } |
| catch (IOException e) |
| { |
| log.error(logCacheName + "Problem storing keys.", e); |
| } |
| } |
| |
| /** |
| * Resets the file and creates a new key map. |
| */ |
| protected void reset() |
| { |
| synchronized (keyFile) |
| { |
| clearMemoryMap(); |
| saveKeys(); |
| } |
| } |
| |
| /** |
| * This is mainly used for testing. It leave the disk in tact, and just |
| * clears memory. |
| */ |
| protected void clearMemoryMap() |
| { |
| this.keyHash.clear(); |
| } |
| |
| /** |
| * Create the map for keys that contain the index position on disk. |
| */ |
| private void initKeyMap() |
| { |
| keyHash = null; |
| if (maxKeySize >= 0) |
| { |
| if (this.diskLimitType == DiskLimitType.SIZE) |
| { |
| keyHash = new LRUMapSizeLimited(maxKeySize); |
| } |
| else |
| { |
| keyHash = new LRUMapCountLimited(maxKeySize); |
| } |
| if (log.isInfoEnabled()) |
| { |
| log.info(logCacheName + "Set maxKeySize to: '" + maxKeySize + "'"); |
| } |
| } |
| else |
| { |
| // If no max size, use a plain map for memory and processing |
| // efficiency. |
| keyHash = new HashMap<K, int[]>(); |
| // keyHash = Collections.synchronizedMap( new HashMap() ); |
| if (log.isInfoEnabled()) |
| { |
| log.info(logCacheName + "Set maxKeySize to unlimited'"); |
| } |
| } |
| } |
| |
| /** |
| * Loads the keys from the .key file. The keys are stored individually on |
| * disk. They are added one by one to an LRUMap.. |
| */ |
| protected void loadKeys() |
| { |
| if (log.isInfoEnabled()) |
| { |
| log.info(logCacheName + "Loading keys for " + keyFile.toString()); |
| } |
| |
| try |
| { |
| // create a key map to use. |
| initKeyMap(); |
| |
| HashMap<K, int[]> keys = new HashMap<K, int[]>(); |
| |
| synchronized (keyFile) |
| { |
| FileInputStream fis = new FileInputStream(keyFile); |
| BufferedInputStream bis = new BufferedInputStream(fis, 65536); |
| ObjectInputStream ois = new ObjectInputStreamClassLoaderAware(bis, null); |
| try |
| { |
| while (true) |
| { |
| @SuppressWarnings("unchecked") |
| // Need to cast from Object |
| BlockDiskElementDescriptor<K> descriptor = (BlockDiskElementDescriptor<K>) ois.readObject(); |
| if (descriptor != null) |
| { |
| keys.put(descriptor.getKey(), descriptor.getBlocks()); |
| } |
| } |
| } |
| catch (EOFException eof) |
| { |
| // nothing |
| } |
| finally |
| { |
| ois.close(); |
| } |
| } |
| |
| if (!keys.isEmpty()) |
| { |
| keyHash.putAll(keys); |
| |
| if (log.isDebugEnabled()) |
| { |
| log.debug(logCacheName + "Found " + keys.size() + " in keys file."); |
| } |
| |
| if (log.isInfoEnabled()) |
| { |
| log.info(logCacheName + "Loaded keys from [" + fileName + "], key count: " + keyHash.size() + "; up to " |
| + maxKeySize + " will be available."); |
| } |
| } |
| } |
| catch (Exception e) |
| { |
| log.error(logCacheName + "Problem loading keys for file " + fileName, e); |
| } |
| } |
| |
| /** |
| * Gets the entry set. |
| * <p> |
| * |
| * @return entry set. |
| */ |
| public Set<Map.Entry<K, int[]>> entrySet() |
| { |
| return this.keyHash.entrySet(); |
| } |
| |
| /** |
| * Gets the key set. |
| * <p> |
| * |
| * @return key set. |
| */ |
| public Set<K> keySet() |
| { |
| return this.keyHash.keySet(); |
| } |
| |
| /** |
| * Gets the size of the key hash. |
| * <p> |
| * |
| * @return the number of keys. |
| */ |
| public int size() |
| { |
| return this.keyHash.size(); |
| } |
| |
| /** |
| * gets the object for the key. |
| * <p> |
| * |
| * @param key |
| * @return Object |
| */ |
| public int[] get(K key) |
| { |
| return this.keyHash.get(key); |
| } |
| |
| /** |
| * Puts a int[] in the keyStore. |
| * <p> |
| * |
| * @param key |
| * @param value |
| */ |
| public void put(K key, int[] value) |
| { |
| this.keyHash.put(key, value); |
| } |
| |
| /** |
| * Remove by key. |
| * <p> |
| * |
| * @param key |
| * @return BlockDiskElementDescriptor if it was present, else null |
| */ |
| public int[] remove(K key) |
| { |
| return this.keyHash.remove(key); |
| } |
| |
| /** |
| * Verify key store integrity |
| * |
| * @return true if key store is valid |
| */ |
| private boolean verify() |
| { |
| Map<Integer, Set<K>> blockAllocationMap = new TreeMap<Integer, Set<K>>(); |
| for (Entry<K, int[]> e : keyHash.entrySet()) |
| { |
| for (int block : e.getValue()) |
| { |
| Set<K> keys = blockAllocationMap.get(block); |
| if (keys == null) |
| { |
| keys = new HashSet<K>(); |
| blockAllocationMap.put(block, keys); |
| } |
| else if (!log.isDebugEnabled()) |
| { |
| // keys are not null, and no debug - fail fast |
| return false; |
| } |
| keys.add(e.getKey()); |
| } |
| } |
| boolean ok = true; |
| if (log.isDebugEnabled()) |
| { |
| for (Entry<Integer, Set<K>> e : blockAllocationMap.entrySet()) |
| { |
| log.debug("Block " + e.getKey() + ":" + e.getValue()); |
| if (e.getValue().size() > 1) |
| { |
| ok = false; |
| } |
| } |
| return ok; |
| } |
| else |
| { |
| return ok; |
| } |
| } |
| |
| /** |
| * Class for recycling and lru. This implements the LRU size overflow |
| * callback, so we can mark the blocks as free. |
| */ |
| public class LRUMapSizeLimited extends AbstractLRUMap<K, int[]> |
| { |
| /** |
| * <code>tag</code> tells us which map we are working on. |
| */ |
| public final static String TAG = "orig-lru-size"; |
| |
| // size of the content in kB |
| private AtomicInteger contentSize; |
| private int maxSize; |
| |
| /** |
| * Default |
| */ |
| public LRUMapSizeLimited() |
| { |
| this(-1); |
| } |
| |
| /** |
| * @param maxSize |
| * maximum cache size in kB |
| */ |
| public LRUMapSizeLimited(int maxSize) |
| { |
| super(); |
| this.maxSize = maxSize; |
| this.contentSize = new AtomicInteger(0); |
| } |
| |
| // keep the content size in kB, so 2^31 kB is reasonable value |
| private void subLengthFromCacheSize(int[] value) |
| { |
| contentSize.addAndGet(value.length * blockSize / -1024 - 1); |
| } |
| |
| // keep the content size in kB, so 2^31 kB is reasonable value |
| private void addLengthToCacheSize(int[] value) |
| { |
| contentSize.addAndGet(value.length * blockSize / 1024 + 1); |
| } |
| |
| @Override |
| public int[] put(K key, int[] value) |
| { |
| int[] oldValue = null; |
| |
| try |
| { |
| oldValue = super.put(key, value); |
| } |
| finally |
| { |
| if (value != null) |
| { |
| addLengthToCacheSize(value); |
| } |
| if (oldValue != null) |
| { |
| subLengthFromCacheSize(oldValue); |
| } |
| } |
| |
| return oldValue; |
| } |
| |
| @Override |
| public int[] remove(Object key) |
| { |
| int[] value = null; |
| |
| try |
| { |
| value = super.remove(key); |
| return value; |
| } |
| finally |
| { |
| if (value != null) |
| { |
| subLengthFromCacheSize(value); |
| } |
| } |
| } |
| |
| /** |
| * This is called when the may key size is reached. The least recently |
| * used item will be passed here. We will store the position and size of |
| * the spot on disk in the recycle bin. |
| * <p> |
| * |
| * @param key |
| * @param value |
| */ |
| @Override |
| protected void processRemovedLRU(K key, int[] value) |
| { |
| blockDiskCache.freeBlocks(value); |
| if (log.isDebugEnabled()) |
| { |
| log.debug(logCacheName + "Removing key: [" + key + "] from key store."); |
| log.debug(logCacheName + "Key store size: [" + super.size() + "]."); |
| } |
| |
| if (value != null) |
| { |
| subLengthFromCacheSize(value); |
| } |
| } |
| |
| @Override |
| protected boolean shouldRemove() |
| { |
| return maxSize > 0 && contentSize.get() > maxSize && this.size() > 1; |
| } |
| } |
| |
| /** |
| * Class for recycling and lru. This implements the LRU overflow callback, |
| * so we can mark the blocks as free. |
| */ |
| public class LRUMapCountLimited extends LRUMap<K, int[]> |
| { |
| /** |
| * <code>tag</code> tells us which map we are working on. |
| */ |
| public final static String TAG = "orig-lru-count"; |
| |
| public LRUMapCountLimited(int maxKeySize) |
| { |
| super(maxKeySize); |
| } |
| |
| /** |
| * This is called when the may key size is reached. The least recently |
| * used item will be passed here. We will store the position and size of |
| * the spot on disk in the recycle bin. |
| * <p> |
| * |
| * @param key |
| * @param value |
| */ |
| @Override |
| protected void processRemovedLRU(K key, int[] value) |
| { |
| blockDiskCache.freeBlocks(value); |
| if (log.isDebugEnabled()) |
| { |
| log.debug(logCacheName + "Removing key: [" + key + "] from key store."); |
| log.debug(logCacheName + "Key store size: [" + super.size() + "]."); |
| } |
| } |
| } |
| } |