[MERCURY-123] - trying to cache results from DependencyProcessor() to enable once-only processing on POMs. This should be linked to session-like behavior because next session may happen in quite some time and data should be properly aged
git-svn-id: https://svn.apache.org/repos/asf/maven/mercury/trunk@776133 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java b/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
index a14152a..f46f496 100644
--- a/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
+++ b/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
@@ -62,6 +62,16 @@
public static final String EVENT_SAVE_RAW = "save.raw";
+// public static final String SYSTEM_PROPERTY_CACHE_METADATA = "mercury.cache.metadata";
+//
+// /** by default - cache metadata in memory */
+// private static final boolean cacheMetadata = Boolean.valueOf( System.getProperty( SYSTEM_PROPERTY_CACHE_METADATA, "true" ) );
+
+ public static final String SYSTEM_PROPERTY_CACHE_RAW = "mercury.cache.raw";
+
+ /** by default - do not cache raw data in memory to preserve RAM */
+ private static final boolean cacheRaw = Boolean.valueOf( System.getProperty( SYSTEM_PROPERTY_CACHE_RAW, "false" ) );
+
private static final Language LANG = new DefaultLanguage( RepositoryGAVMetadata.class );
static volatile Map<String, MetadataCacheFs> fsCaches =
@@ -72,11 +82,12 @@
(Map<String, RepositoryGAMetadata>) Collections.synchronizedMap( new HashMap<String, RepositoryGAMetadata>( 512 ) );
private volatile Map<String, RepositoryGAVMetadata> gavCache =
- (Map<String, RepositoryGAVMetadata>) Collections.synchronizedMap( new HashMap<String, RepositoryGAVMetadata>(
- 1024 ) );
+ (Map<String, RepositoryGAVMetadata>) Collections.synchronizedMap( new HashMap<String, RepositoryGAVMetadata>( 1024 ) );
- private volatile Map<String, byte[]> rawCache =
- (Map<String, byte[]>) Collections.synchronizedMap( new HashMap<String, byte[]>( 1024 ) );
+ private volatile Map<String, byte[]> rawCache = cacheRaw
+ ? (Map<String, byte[]>) Collections.synchronizedMap( new HashMap<String, byte[]>( 1024 ) )
+ : null
+ ;
private File root;
@@ -374,7 +385,10 @@
if ( _eventManager != null )
event = new GenericEvent( EventTypeEnum.fsCache, EVENT_FIND_RAW, rawKey );
- byte[] res = rawCache.get( rawKey );
+ byte[] res = cacheRaw
+ ? rawCache.get( rawKey )
+ : null
+ ;
if ( res != null )
{
@@ -394,7 +408,8 @@
res = FileUtil.readRawData( f );
- rawCache.put( rawKey, res );
+ if( cacheRaw)
+ rawCache.put( rawKey, res );
if ( _eventManager != null )
event.setResult( "found on disk" );
@@ -428,7 +443,8 @@
if ( _eventManager != null )
event = new GenericEvent( EventTypeEnum.fsCache, EVENT_SAVE_RAW, rawKey );
- rawCache.put( rawKey, rawBytes );
+ if( cacheRaw )
+ rawCache.put( rawKey, rawBytes );
File f =
new File( getGAVDir( md.getEffectiveCoordinates() ), md.getArtifactId() + FileUtil.DASH
@@ -519,7 +535,9 @@
public void clearSession()
throws MetadataCacheException
{
- rawCache.clear();
+ if( cacheRaw )
+ rawCache.clear();
+
gaCache.clear();
gavCache.clear();
}
diff --git a/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java b/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
index 9b55cf1..57ff627 100644
--- a/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
+++ b/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
@@ -57,6 +57,8 @@
import org.apache.maven.mercury.repository.cache.fs.MetadataCacheFs;
import org.apache.maven.mercury.repository.remote.m2.RemoteRepositoryM2;
import org.apache.maven.mercury.repository.remote.m2.RemoteRepositoryReaderM2;
+import org.apache.maven.mercury.util.LruMemCache;
+import org.apache.maven.mercury.util.MemCache;
import org.apache.maven.mercury.util.Util;
import org.codehaus.plexus.lang.DefaultLanguage;
import org.codehaus.plexus.lang.Language;
@@ -119,6 +121,15 @@
private boolean _initialized = false;
private EventManager _eventManager;
+
+ public static final String SYSTEM_PROPERTY_VERSION_CACHE_SIZE = "mercury.version.cache.size";
+
+ private static final int _versionCacheSize = Integer.valueOf( System.getProperty( SYSTEM_PROPERTY_VERSION_CACHE_SIZE, "1024" ) );
+
+ private static final MemCache<ArtifactMetadata, List<ArtifactMetadata> > _cachedVersions =
+ _versionCacheSize == 0 ? null
+ : new LruMemCache<ArtifactMetadata, List<ArtifactMetadata>>( _versionCacheSize )
+ ;
// ----------------------------------------------------------------------------------------------------------------------------
public VirtualRepositoryReader( Collection<Repository> repositories )
@@ -301,6 +312,59 @@
}
}
+ //----------------------------------------------------------------------------------------------------------------------------
+ private MetadataResults readCachedVersions( Collection<ArtifactMetadata> query, List<ArtifactMetadata> leftOvers )
+ {
+ if( _cachedVersions == null )
+ {
+ leftOvers.addAll( query );
+ return null;
+ }
+
+ MetadataResults res = null;
+
+ for( ArtifactMetadata key : query )
+ {
+ List<ArtifactMetadata> vl = _cachedVersions.get( key );
+ if( Util.isEmpty( vl ) )
+ leftOvers.add( key );
+ else
+ {
+ if( res == null )
+ res = new MetadataResults( key, vl );
+ else
+ res.add( key, vl );
+ }
+ }
+
+ return res;
+ }
+ //----------------------------------------------------------------------------------------------------------------------------
+ private MetadataResults cacheVersions( MetadataResults res )
+ {
+ if(true)
+ return res;
+
+ if( _cachedVersions == null || res == null || res.hasExceptions() || ! res.hasResults() )
+ {
+ return res;
+ }
+
+ for( ArtifactMetadata key :res.getResults().keySet() )
+ {
+ List<ArtifactMetadata> vl = res.getResult( key );
+
+ if( Util.isEmpty( vl ) )
+ continue;
+
+ for( ArtifactMetadata md : vl )
+ md.setTracker( null );
+
+ _cachedVersions.put( key, vl );
+ }
+
+ return res;
+ }
// ----------------------------------------------------------------------------------------------------------------------------
public MetadataResults readVersions( Collection<ArtifactMetadata> query )
throws IllegalArgumentException, RepositoryException
@@ -321,13 +385,16 @@
event = new GenericEvent( EventTypeEnum.virtualRepositoryReader, EVENT_READ_VERSIONS );
}
- MetadataResults res = null;
ArtifactListProcessor tp = _processors == null ? null : _processors.get( ArtifactListProcessor.FUNCTION_TP );
GenericEvent eventRead = null;
List<ArtifactMetadata> qList = new ArrayList<ArtifactMetadata>( query.size() );
- qList.addAll( query );
+// qList.addAll( query );
+ MetadataResults res = readCachedVersions( query, qList );
+
+ if( Util.isEmpty( qList ) )
+ return res;
for ( RepositoryReader rr : _repositoryReaders )
{
@@ -425,7 +492,7 @@
processSingletons( res );
}
- return res;
+ return cacheVersions( res );
}
finally
{
diff --git a/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java b/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
new file mode 100644
index 0000000..64f7710
--- /dev/null
+++ b/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
@@ -0,0 +1,71 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+package org.apache.maven.mercury.util;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ *
+ *
+ * @author Oleg Gusakov
+ * @version $Id$
+ *
+ */
+public class LruMemCache<K, V>
+ implements MemCache<K, V>
+{
+ ConcurrentHashMap<K, V> _cache;
+
+ K _lastKey;
+
+ int _sz;
+
+ public LruMemCache()
+ {
+ this( DEFAULT_CACHE_SIZE );
+ }
+
+ public LruMemCache( int sz )
+ {
+ _sz = sz;
+ _cache = new ConcurrentHashMap<K, V>( sz );
+ }
+
+ public V get( K key )
+ {
+ return _cache.get( key );
+ }
+
+ public void put( K key, V val )
+ {
+ // MRU is easier - do it first ..
+ synchronized( _cache )
+ {
+ if( _cache.size() == _sz )
+ {// free up one slot
+ _cache.remove( _lastKey );
+ _lastKey = key;
+ }
+ }
+
+ _cache.put( key, val );
+ }
+
+}
diff --git a/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java b/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
new file mode 100644
index 0000000..2fba1cb
--- /dev/null
+++ b/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
@@ -0,0 +1,36 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+package org.apache.maven.mercury.util;
+
+/**
+ *
+ *
+ * @author Oleg Gusakov
+ * @version $Id$
+ *
+ */
+public interface MemCache<K extends Object, V extends Object>
+{
+ public static final int DEFAULT_CACHE_SIZE = 1024;
+
+ void put( K key, V val );
+
+ V get( K key );
+}
diff --git a/mercury-it/pom.xml b/mercury-it/pom.xml
index 7bd4fcc..b1f3bf9 100644
--- a/mercury-it/pom.xml
+++ b/mercury-it/pom.xml
@@ -182,7 +182,7 @@
</dependencies>
<build>
-
+ <!--
<filters>
<filter>src/test/filters/filter.txt</filter>
</filters>
@@ -209,7 +209,8 @@
</testResources>
-
+ -->
+
<plugins>
<plugin>
diff --git a/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1 b/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
index d7a8a9a..45d10af 100644
--- a/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
+++ b/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
@@ -1 +1 @@
-526bfebc865ac047ff3fa7d77924a4edff7ba468
\ No newline at end of file
+5ab8cde950b8b29bc5c8d79b8df5a804a76e573e
