| package org.apache.archiva.proxy; |
| |
| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| import org.apache.archiva.admin.model.RepositoryAdminException; |
| import org.apache.archiva.admin.model.beans.NetworkProxy; |
| import org.apache.archiva.admin.model.beans.ProxyConnectorRuleType; |
| import org.apache.archiva.admin.model.beans.RemoteRepository; |
| import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin; |
| import org.apache.archiva.common.filelock.FileLockException; |
| import org.apache.archiva.common.filelock.FileLockManager; |
| import org.apache.archiva.common.filelock.FileLockTimeoutException; |
| import org.apache.archiva.common.filelock.Lock; |
| import org.apache.archiva.configuration.ArchivaConfiguration; |
| import org.apache.archiva.configuration.Configuration; |
| import org.apache.archiva.configuration.ConfigurationNames; |
| import org.apache.archiva.configuration.NetworkProxyConfiguration; |
| import org.apache.archiva.configuration.ProxyConnectorConfiguration; |
| import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration; |
| import org.apache.archiva.model.ArtifactReference; |
| import org.apache.archiva.model.Keys; |
| import org.apache.archiva.model.RepositoryURL; |
| import org.apache.archiva.policies.DownloadErrorPolicy; |
| import org.apache.archiva.policies.DownloadPolicy; |
| import org.apache.archiva.policies.PolicyConfigurationException; |
| import org.apache.archiva.policies.PolicyViolationException; |
| import org.apache.archiva.policies.PostDownloadPolicy; |
| import org.apache.archiva.policies.PreDownloadPolicy; |
| import org.apache.archiva.policies.ProxyDownloadException; |
| import org.apache.archiva.policies.urlcache.UrlFailureCache; |
| import org.apache.archiva.proxy.common.WagonFactory; |
| import org.apache.archiva.proxy.common.WagonFactoryException; |
| import org.apache.archiva.proxy.common.WagonFactoryRequest; |
| import org.apache.archiva.proxy.model.ProxyFetchResult; |
| import org.apache.archiva.proxy.model.ProxyConnector; |
| import org.apache.archiva.proxy.model.RepositoryProxyConnectors; |
| import org.apache.archiva.redback.components.registry.Registry; |
| import org.apache.archiva.redback.components.registry.RegistryListener; |
| import org.apache.archiva.redback.components.taskqueue.TaskQueueException; |
| import org.apache.archiva.repository.ManagedRepositoryContent; |
| import org.apache.archiva.repository.RemoteRepositoryContent; |
| import org.apache.archiva.repository.RepositoryContentFactory; |
| import org.apache.archiva.repository.RepositoryException; |
| import org.apache.archiva.repository.RepositoryNotFoundException; |
| import org.apache.archiva.repository.metadata.MetadataTools; |
| import org.apache.archiva.repository.metadata.RepositoryMetadataException; |
| import org.apache.archiva.scheduler.ArchivaTaskScheduler; |
| import org.apache.archiva.scheduler.repository.model.RepositoryTask; |
| import org.apache.commons.collections.CollectionUtils; |
| import org.apache.commons.io.FileUtils; |
| import org.apache.commons.io.FilenameUtils; |
| import org.apache.commons.lang.StringUtils; |
| import org.apache.commons.lang.SystemUtils; |
| import org.apache.maven.wagon.ConnectionException; |
| import org.apache.maven.wagon.ResourceDoesNotExistException; |
| import org.apache.maven.wagon.Wagon; |
| import org.apache.maven.wagon.WagonException; |
| import org.apache.maven.wagon.authentication.AuthenticationException; |
| import org.apache.maven.wagon.authentication.AuthenticationInfo; |
| import org.apache.maven.wagon.proxy.ProxyInfo; |
| import org.apache.maven.wagon.repository.Repository; |
| import org.apache.tools.ant.types.selectors.SelectorUtils; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| import org.slf4j.MarkerFactory; |
| import org.springframework.stereotype.Service; |
| |
| import javax.annotation.PostConstruct; |
| import javax.inject.Inject; |
| import javax.inject.Named; |
| import java.io.File; |
| import java.io.IOException; |
| import java.nio.file.Files; |
| import java.util.ArrayList; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.LinkedHashMap; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Map.Entry; |
| import java.util.Properties; |
| import java.util.concurrent.ConcurrentHashMap; |
| import java.util.concurrent.ConcurrentMap; |
| |
| /** |
| * DefaultRepositoryProxyConnectors |
| * TODO exception handling needs work - "not modified" is not really an exceptional case, and it has more layers than |
| * your average brown onion |
| */ |
| @Service("repositoryProxyConnectors#default") |
| public class DefaultRepositoryProxyConnectors |
| implements RepositoryProxyConnectors, RegistryListener |
| { |
| private Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyConnectors.class ); |
| |
| @Inject |
| @Named(value = "archivaConfiguration#default") |
| private ArchivaConfiguration archivaConfiguration; |
| |
| @Inject |
| @Named(value = "repositoryContentFactory#default") |
| private RepositoryContentFactory repositoryFactory; |
| |
| @Inject |
| @Named(value = "metadataTools#default") |
| private MetadataTools metadataTools; |
| |
| @Inject |
| private Map<String, PreDownloadPolicy> preDownloadPolicies; |
| |
| @Inject |
| private Map<String, PostDownloadPolicy> postDownloadPolicies; |
| |
| @Inject |
| private Map<String, DownloadErrorPolicy> downloadErrorPolicies; |
| |
| @Inject |
| private UrlFailureCache urlFailureCache; |
| |
| private ConcurrentMap<String, List<ProxyConnector>> proxyConnectorMap = new ConcurrentHashMap<>(); |
| |
| private ConcurrentMap<String, ProxyInfo> networkProxyMap = new ConcurrentHashMap<>(); |
| |
| @Inject |
| private WagonFactory wagonFactory; |
| |
| @Inject |
| @Named(value = "archivaTaskScheduler#repository") |
| private ArchivaTaskScheduler scheduler; |
| |
| @Inject |
| private NetworkProxyAdmin networkProxyAdmin; |
| |
| @Inject |
| @Named(value = "fileLockManager#default") |
| private FileLockManager fileLockManager; |
| |
| @PostConstruct |
| public void initialize() |
| { |
| initConnectorsAndNetworkProxies(); |
| archivaConfiguration.addChangeListener( this ); |
| |
| } |
| |
| @SuppressWarnings("unchecked") |
| private void initConnectorsAndNetworkProxies() |
| { |
| |
| ProxyConnectorOrderComparator proxyOrderSorter = new ProxyConnectorOrderComparator(); |
| this.proxyConnectorMap.clear(); |
| |
| Configuration configuration = archivaConfiguration.getConfiguration(); |
| |
| List<ProxyConnectorRuleConfiguration> allProxyConnectorRuleConfigurations = |
| configuration.getProxyConnectorRuleConfigurations(); |
| |
| List<ProxyConnectorConfiguration> proxyConfigs = configuration.getProxyConnectors(); |
| for ( ProxyConnectorConfiguration proxyConfig : proxyConfigs ) |
| { |
| String key = proxyConfig.getSourceRepoId(); |
| |
| try |
| { |
| // Create connector object. |
| ProxyConnector connector = new ProxyConnector(); |
| |
| connector.setSourceRepository( |
| repositoryFactory.getManagedRepositoryContent( proxyConfig.getSourceRepoId() ) ); |
| connector.setTargetRepository( |
| repositoryFactory.getRemoteRepositoryContent( proxyConfig.getTargetRepoId() ) ); |
| |
| connector.setProxyId( proxyConfig.getProxyId() ); |
| connector.setPolicies( proxyConfig.getPolicies() ); |
| connector.setOrder( proxyConfig.getOrder() ); |
| connector.setDisabled( proxyConfig.isDisabled() ); |
| |
| // Copy any blacklist patterns. |
| List<String> blacklist = new ArrayList<>( 0 ); |
| if ( CollectionUtils.isNotEmpty( proxyConfig.getBlackListPatterns() ) ) |
| { |
| blacklist.addAll( proxyConfig.getBlackListPatterns() ); |
| } |
| connector.setBlacklist( blacklist ); |
| |
| // Copy any whitelist patterns. |
| List<String> whitelist = new ArrayList<>( 0 ); |
| if ( CollectionUtils.isNotEmpty( proxyConfig.getWhiteListPatterns() ) ) |
| { |
| whitelist.addAll( proxyConfig.getWhiteListPatterns() ); |
| } |
| connector.setWhitelist( whitelist ); |
| |
| List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = |
| findProxyConnectorRules( connector.getSourceRepository().getId(), |
| connector.getTargetRepository().getId(), |
| allProxyConnectorRuleConfigurations ); |
| |
| if ( !proxyConnectorRuleConfigurations.isEmpty() ) |
| { |
| for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : proxyConnectorRuleConfigurations ) |
| { |
| if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(), |
| ProxyConnectorRuleType.BLACK_LIST.getRuleType() ) ) |
| { |
| connector.getBlacklist().add( proxyConnectorRuleConfiguration.getPattern() ); |
| } |
| |
| if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(), |
| ProxyConnectorRuleType.WHITE_LIST.getRuleType() ) ) |
| { |
| connector.getWhitelist().add( proxyConnectorRuleConfiguration.getPattern() ); |
| } |
| } |
| } |
| |
| // Get other connectors |
| List<ProxyConnector> connectors = this.proxyConnectorMap.get( key ); |
| if ( connectors == null ) |
| { |
| // Create if we are the first. |
| connectors = new ArrayList<>( 1 ); |
| } |
| |
| // Add the connector. |
| connectors.add( connector ); |
| |
| // Ensure the list is sorted. |
| Collections.sort( connectors, proxyOrderSorter ); |
| |
| // Set the key to the list of connectors. |
| this.proxyConnectorMap.put( key, connectors ); |
| } |
| catch ( RepositoryNotFoundException e ) |
| { |
| log.warn( "Unable to use proxy connector: {}", e.getMessage(), e ); |
| } |
| catch ( RepositoryException e ) |
| { |
| log.warn( "Unable to use proxy connector: {}", e.getMessage(), e ); |
| } |
| |
| |
| } |
| |
| this.networkProxyMap.clear(); |
| |
| List<NetworkProxyConfiguration> networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies(); |
| for ( NetworkProxyConfiguration networkProxyConfig : networkProxies ) |
| { |
| String key = networkProxyConfig.getId(); |
| |
| ProxyInfo proxy = new ProxyInfo(); |
| |
| proxy.setType( networkProxyConfig.getProtocol() ); |
| proxy.setHost( networkProxyConfig.getHost() ); |
| proxy.setPort( networkProxyConfig.getPort() ); |
| proxy.setUserName( networkProxyConfig.getUsername() ); |
| proxy.setPassword( networkProxyConfig.getPassword() ); |
| |
| this.networkProxyMap.put( key, proxy ); |
| } |
| |
| } |
| |
| private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules( String sourceRepository, |
| String targetRepository, |
| List<ProxyConnectorRuleConfiguration> all ) |
| { |
| List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>(); |
| |
| for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all ) |
| { |
| for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() ) |
| { |
| if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals( |
| targetRepository, proxyConnector.getTargetRepoId() ) ) |
| { |
| proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration ); |
| } |
| } |
| } |
| |
| return proxyConnectorRuleConfigurations; |
| } |
| |
| @Override |
| public File fetchFromProxies( ManagedRepositoryContent repository, ArtifactReference artifact ) |
| throws ProxyDownloadException |
| { |
| File localFile = toLocalFile( repository, artifact ); |
| |
| Properties requestProperties = new Properties(); |
| requestProperties.setProperty( "filetype", "artifact" ); |
| requestProperties.setProperty( "version", artifact.getVersion() ); |
| requestProperties.setProperty( "managedRepositoryId", repository.getId() ); |
| |
| List<ProxyConnector> connectors = getProxyConnectors( repository ); |
| Map<String, Exception> previousExceptions = new LinkedHashMap<>(); |
| for ( ProxyConnector connector : connectors ) |
| { |
| if ( connector.isDisabled() ) |
| { |
| continue; |
| } |
| |
| RemoteRepositoryContent targetRepository = connector.getTargetRepository(); |
| requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() ); |
| |
| String targetPath = targetRepository.toPath( artifact ); |
| |
| if ( SystemUtils.IS_OS_WINDOWS ) |
| { |
| // toPath use system PATH_SEPARATOR so on windows url are \ which doesn't work very well :-) |
| targetPath = FilenameUtils.separatorsToUnix( targetPath ); |
| } |
| |
| try |
| { |
| File downloadedFile = |
| transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties, |
| true ); |
| |
| if ( fileExists( downloadedFile ) ) |
| { |
| log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() ); |
| return downloadedFile; |
| } |
| } |
| catch ( NotFoundException e ) |
| { |
| log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ), |
| targetRepository.getRepository().getId() ); |
| } |
| catch ( NotModifiedException e ) |
| { |
| log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ), |
| targetRepository.getRepository().getId() ); |
| } |
| catch ( ProxyException | RepositoryAdminException e ) |
| { |
| validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact, |
| targetRepository, localFile, e, previousExceptions ); |
| } |
| } |
| |
| if ( !previousExceptions.isEmpty() ) |
| { |
| throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories", |
| previousExceptions ); |
| } |
| |
| log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) ); |
| |
| return null; |
| } |
| |
| @Override |
| public File fetchFromProxies( ManagedRepositoryContent repository, String path ) |
| { |
| File localFile = new File( repository.getRepoRoot(), path ); |
| |
| // no update policies for these paths |
| if ( localFile.exists() ) |
| { |
| return null; |
| } |
| |
| Properties requestProperties = new Properties(); |
| requestProperties.setProperty( "filetype", "resource" ); |
| requestProperties.setProperty( "managedRepositoryId", repository.getId() ); |
| |
| List<ProxyConnector> connectors = getProxyConnectors( repository ); |
| for ( ProxyConnector connector : connectors ) |
| { |
| if ( connector.isDisabled() ) |
| { |
| continue; |
| } |
| |
| RemoteRepositoryContent targetRepository = connector.getTargetRepository(); |
| requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() ); |
| |
| String targetPath = path; |
| |
| try |
| { |
| File downloadedFile = |
| transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties, |
| false ); |
| |
| if ( fileExists( downloadedFile ) ) |
| { |
| log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() ); |
| return downloadedFile; |
| } |
| } |
| catch ( NotFoundException e ) |
| { |
| log.debug( "Resource {} not found on repository \"{}\".", path, |
| targetRepository.getRepository().getId() ); |
| } |
| catch ( NotModifiedException e ) |
| { |
| log.debug( "Resource {} not updated on repository \"{}\".", path, |
| targetRepository.getRepository().getId() ); |
| } |
| catch ( ProxyException e ) |
| { |
| log.warn( |
| "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}", |
| targetRepository.getRepository().getId(), path, e.getMessage() ); |
| log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), |
| "Transfer error from repository \"" + targetRepository.getRepository().getId() |
| + "\" for resource " + path + ", continuing to next repository. Error message: {}", |
| e.getMessage(), e |
| ); |
| } |
| catch ( RepositoryAdminException e ) |
| { |
| log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), |
| "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}", |
| targetRepository.getRepository().getId(), path, e.getMessage(), e ); |
| log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), "Full stack trace", e ); |
| } |
| } |
| |
| log.debug( "Exhausted all target repositories, resource {} not found.", path ); |
| |
| return null; |
| } |
| |
| @Override |
| public ProxyFetchResult fetchMetadataFromProxies( ManagedRepositoryContent repository, String logicalPath ) |
| { |
| File localFile = new File( repository.getRepoRoot(), logicalPath ); |
| |
| Properties requestProperties = new Properties(); |
| requestProperties.setProperty( "filetype", "metadata" ); |
| boolean metadataNeedsUpdating = false; |
| long originalTimestamp = getLastModified( localFile ); |
| |
| List<ProxyConnector> connectors = new ArrayList<>( getProxyConnectors( repository ) ); |
| for ( ProxyConnector connector : connectors ) |
| { |
| if ( connector.isDisabled() ) |
| { |
| continue; |
| } |
| |
| RemoteRepositoryContent targetRepository = connector.getTargetRepository(); |
| |
| File localRepoFile = toLocalRepoFile( repository, targetRepository, logicalPath ); |
| long originalMetadataTimestamp = getLastModified( localRepoFile ); |
| |
| try |
| { |
| transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties, |
| true ); |
| |
| if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) ) |
| { |
| metadataNeedsUpdating = true; |
| } |
| } |
| catch ( NotFoundException e ) |
| { |
| |
| log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath, |
| targetRepository.getRepository().getId(), e ); |
| |
| } |
| catch ( NotModifiedException e ) |
| { |
| |
| log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath, |
| targetRepository.getRepository().getId(), e ); |
| |
| } |
| catch ( ProxyException | RepositoryAdminException e ) |
| { |
| log.warn( |
| "Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}", |
| targetRepository.getRepository().getId(), logicalPath, e.getMessage() ); |
| log.debug( "Full stack trace", e ); |
| } |
| } |
| |
| if ( hasBeenUpdated( localFile, originalTimestamp ) ) |
| { |
| metadataNeedsUpdating = true; |
| } |
| |
| if ( metadataNeedsUpdating || !localFile.exists() ) |
| { |
| try |
| { |
| metadataTools.updateMetadata( repository, logicalPath ); |
| } |
| catch ( RepositoryMetadataException e ) |
| { |
| log.warn( "Unable to update metadata {}:{}", localFile.getAbsolutePath(), e.getMessage(), e ); |
| } |
| |
| } |
| |
| if ( fileExists( localFile ) ) |
| { |
| return new ProxyFetchResult( localFile, metadataNeedsUpdating ); |
| } |
| |
| return new ProxyFetchResult( null, false ); |
| } |
| |
| /** |
| * @param connector |
| * @param remoteRepository |
| * @param tmpMd5 |
| * @param tmpSha1 |
| * @param tmpResource |
| * @param url |
| * @param remotePath |
| * @param resource |
| * @param workingDirectory |
| * @param repository |
| * @throws ProxyException |
| * @throws NotModifiedException |
| * @throws org.apache.archiva.admin.model.RepositoryAdminException |
| */ |
| protected void transferResources( ProxyConnector connector, RemoteRepositoryContent remoteRepository, File tmpMd5, |
| File tmpSha1, File tmpResource, String url, String remotePath, File resource, |
| File workingDirectory, ManagedRepositoryContent repository ) |
| throws ProxyException, NotModifiedException, RepositoryAdminException |
| { |
| Wagon wagon = null; |
| try |
| { |
| RepositoryURL repoUrl = remoteRepository.getURL(); |
| String protocol = repoUrl.getProtocol(); |
| NetworkProxy networkProxy = null; |
| if ( StringUtils.isNotBlank( connector.getProxyId() ) ) |
| { |
| networkProxy = networkProxyAdmin.getNetworkProxy( connector.getProxyId() ); |
| } |
| WagonFactoryRequest wagonFactoryRequest = new WagonFactoryRequest( "wagon#" + protocol, |
| remoteRepository.getRepository().getExtraHeaders() ).networkProxy( |
| networkProxy ); |
| wagon = wagonFactory.getWagon( wagonFactoryRequest ); |
| if ( wagon == null ) |
| { |
| throw new ProxyException( "Unsupported target repository protocol: " + protocol ); |
| } |
| |
| if ( wagon == null ) |
| { |
| throw new ProxyException( "Unsupported target repository protocol: " + protocol ); |
| } |
| |
| boolean connected = connectToRepository( connector, wagon, remoteRepository ); |
| if ( connected ) |
| { |
| transferArtifact( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, |
| tmpResource ); |
| |
| // TODO: these should be used to validate the download based on the policies, not always downloaded |
| // to |
| // save on connections since md5 is rarely used |
| transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".sha1", |
| tmpSha1 ); |
| transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".md5", |
| tmpMd5 ); |
| } |
| } |
| catch ( NotFoundException e ) |
| { |
| urlFailureCache.cacheFailure( url ); |
| throw e; |
| } |
| catch ( NotModifiedException e ) |
| { |
| // Do not cache url here. |
| throw e; |
| } |
| catch ( ProxyException e ) |
| { |
| urlFailureCache.cacheFailure( url ); |
| throw e; |
| } |
| catch ( WagonFactoryException e ) |
| { |
| throw new ProxyException( e.getMessage(), e ); |
| } |
| finally |
| { |
| if ( wagon != null ) |
| { |
| try |
| { |
| wagon.disconnect(); |
| } |
| catch ( ConnectionException e ) |
| { |
| log.warn( "Unable to disconnect wagon.", e ); |
| } |
| } |
| } |
| } |
| |
| private void transferArtifact( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath, |
| ManagedRepositoryContent repository, File resource, File tmpDirectory, |
| File destFile ) |
| throws ProxyException |
| { |
| transferSimpleFile( wagon, remoteRepository, remotePath, repository, resource, destFile ); |
| } |
| |
| private long getLastModified( File file ) |
| { |
| if ( !file.exists() || !file.isFile() ) |
| { |
| return 0; |
| } |
| |
| return file.lastModified(); |
| } |
| |
| private boolean hasBeenUpdated( File file, long originalLastModified ) |
| { |
| if ( !file.exists() || !file.isFile() ) |
| { |
| return false; |
| } |
| |
| long currentLastModified = getLastModified( file ); |
| return ( currentLastModified > originalLastModified ); |
| } |
| |
| private File toLocalRepoFile( ManagedRepositoryContent repository, RemoteRepositoryContent targetRepository, |
| String targetPath ) |
| { |
| String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath ); |
| return new File( repository.getRepoRoot(), repoPath ); |
| } |
| |
| /** |
| * Test if the provided ManagedRepositoryContent has any proxies configured for it. |
| */ |
| @Override |
| public boolean hasProxies( ManagedRepositoryContent repository ) |
| { |
| synchronized ( this.proxyConnectorMap ) |
| { |
| return this.proxyConnectorMap.containsKey( repository.getId() ); |
| } |
| } |
| |
| private File toLocalFile( ManagedRepositoryContent repository, ArtifactReference artifact ) |
| { |
| return repository.toFile( artifact ); |
| } |
| |
| /** |
| * Simple method to test if the file exists on the local disk. |
| * |
| * @param file the file to test. (may be null) |
| * @return true if file exists. false if the file param is null, doesn't exist, or is not of type File. |
| */ |
| private boolean fileExists( File file ) |
| { |
| if ( file == null ) |
| { |
| return false; |
| } |
| |
| if ( !file.exists() ) |
| { |
| return false; |
| } |
| |
| return file.isFile(); |
| } |
| |
| /** |
| * Perform the transfer of the file. |
| * |
| * @param connector the connector configuration to use. |
| * @param remoteRepository the remote repository get the resource from. |
| * @param remotePath the path in the remote repository to the resource to get. |
| * @param repository the managed repository that will hold the file |
| * @param resource the local file to place the downloaded resource into |
| * @param requestProperties the request properties to utilize for policy handling. |
| * @param executeConsumers whether to execute the consumers after proxying |
| * @return the local file that was downloaded, or null if not downloaded. |
| * @throws NotFoundException if the file was not found on the remote repository. |
| * @throws NotModifiedException if the localFile was present, and the resource was present on remote repository, but |
| * the remote resource is not newer than the local File. |
| * @throws ProxyException if transfer was unsuccessful. |
| */ |
| private File transferFile( ProxyConnector connector, RemoteRepositoryContent remoteRepository, String remotePath, |
| ManagedRepositoryContent repository, File resource, Properties requestProperties, |
| boolean executeConsumers ) |
| throws ProxyException, NotModifiedException, RepositoryAdminException |
| { |
| String url = remoteRepository.getURL().getUrl(); |
| if ( !url.endsWith( "/" ) ) |
| { |
| url = url + "/"; |
| } |
| url = url + remotePath; |
| requestProperties.setProperty( "url", url ); |
| |
| // Is a whitelist defined? |
| if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) ) |
| { |
| // Path must belong to whitelist. |
| if ( !matchesPattern( remotePath, connector.getWhitelist() ) ) |
| { |
| log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).", |
| remotePath, remoteRepository.getRepository().getName() ); |
| return null; |
| } |
| } |
| |
| // Is target path part of blacklist? |
| if ( matchesPattern( remotePath, connector.getBlacklist() ) ) |
| { |
| log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath, |
| remoteRepository.getRepository().getName() ); |
| return null; |
| } |
| |
| // Handle pre-download policy |
| try |
| { |
| validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource ); |
| } |
| catch ( PolicyViolationException e ) |
| { |
| String emsg = "Transfer not attempted on " + url + " : " + e.getMessage(); |
| if ( fileExists( resource ) ) |
| { |
| log.debug( "{} : using already present local file.", emsg ); |
| return resource; |
| } |
| |
| log.debug( emsg ); |
| return null; |
| } |
| |
| File workingDirectory = createWorkingDirectory( repository ); |
| File tmpResource = new File( workingDirectory, resource.getName() ); |
| File tmpMd5 = new File( workingDirectory, resource.getName() + ".md5" ); |
| File tmpSha1 = new File( workingDirectory, resource.getName() + ".sha1" ); |
| |
| try |
| { |
| |
| transferResources( connector, remoteRepository, tmpMd5, tmpSha1, tmpResource, url, remotePath, resource, |
| workingDirectory, repository ); |
| |
| // Handle post-download policies. |
| try |
| { |
| validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource ); |
| } |
| catch ( PolicyViolationException e ) |
| { |
| log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() ); |
| executeConsumers = false; |
| if ( !fileExists( tmpResource ) ) |
| { |
| resource = null; |
| } |
| } |
| |
| if ( resource != null ) |
| { |
| synchronized ( resource.getAbsolutePath().intern() ) |
| { |
| File directory = resource.getParentFile(); |
| moveFileIfExists( tmpMd5, directory ); |
| moveFileIfExists( tmpSha1, directory ); |
| moveFileIfExists( tmpResource, directory ); |
| } |
| } |
| } |
| finally |
| { |
| FileUtils.deleteQuietly( workingDirectory ); |
| } |
| |
| if ( executeConsumers ) |
| { |
| // Just-in-time update of the index and database by executing the consumers for this artifact |
| //consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource ); |
| queueRepositoryTask( connector.getSourceRepository().getRepository().getId(), resource ); |
| } |
| |
| return resource; |
| } |
| |
| private void queueRepositoryTask( String repositoryId, File localFile ) |
| { |
| RepositoryTask task = new RepositoryTask(); |
| task.setRepositoryId( repositoryId ); |
| task.setResourceFile( localFile ); |
| task.setUpdateRelatedArtifacts( true ); |
| task.setScanAll( true ); |
| |
| try |
| { |
| scheduler.queueTask( task ); |
| } |
| catch ( TaskQueueException e ) |
| { |
| log.error( "Unable to queue repository task to execute consumers on resource file ['" + localFile.getName() |
| + "']." ); |
| } |
| } |
| |
| /** |
| * Moves the file into repository location if it exists |
| * |
| * @param fileToMove this could be either the main artifact, sha1 or md5 checksum file. |
| * @param directory directory to write files to |
| */ |
| private void moveFileIfExists( File fileToMove, File directory ) |
| throws ProxyException |
| { |
| if ( fileToMove != null && fileToMove.exists() ) |
| { |
| File newLocation = new File( directory, fileToMove.getName() ); |
| moveTempToTarget( fileToMove, newLocation ); |
| } |
| } |
| |
| /** |
| * <p> |
| * Quietly transfer the checksum file from the remote repository to the local file. |
| * </p> |
| * |
| * @param wagon the wagon instance (should already be connected) to use. |
| * @param remoteRepository the remote repository to transfer from. |
| * @param remotePath the remote path to the resource to get. |
| * @param repository the managed repository that will hold the file |
| * @param resource the local file that should contain the downloaded contents |
| * @param tmpDirectory the temporary directory to download to |
| * @param ext the type of checksum to transfer (example: ".md5" or ".sha1") |
| * @throws ProxyException if copying the downloaded file into place did not succeed. |
| */ |
| private void transferChecksum( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath, |
| ManagedRepositoryContent repository, File resource, File tmpDirectory, String ext, |
| File destFile ) |
| throws ProxyException |
| { |
| String url = remoteRepository.getURL().getUrl() + remotePath + ext; |
| |
| // Transfer checksum does not use the policy. |
| if ( urlFailureCache.hasFailedBefore( url ) ) |
| { |
| return; |
| } |
| |
| try |
| { |
| transferSimpleFile( wagon, remoteRepository, remotePath + ext, repository, resource, destFile ); |
| log.debug( "Checksum {} Downloaded: {} to move to {}", url, destFile, resource ); |
| } |
| catch ( NotFoundException e ) |
| { |
| urlFailureCache.cacheFailure( url ); |
| log.debug( "Transfer failed, checksum not found: {}", url ); |
| // Consume it, do not pass this on. |
| } |
| catch ( NotModifiedException e ) |
| { |
| log.debug( "Transfer skipped, checksum not modified: {}", url ); |
| // Consume it, do not pass this on. |
| } |
| catch ( ProxyException e ) |
| { |
| urlFailureCache.cacheFailure( url ); |
| log.warn( "Transfer failed on checksum: {} : {}", url, e.getMessage(), e ); |
| // Critical issue, pass it on. |
| throw e; |
| } |
| } |
| |
| /** |
| * Perform the transfer of the remote file to the local file specified. |
| * |
| * @param wagon the wagon instance to use. |
| * @param remoteRepository the remote repository to use |
| * @param remotePath the remote path to attempt to get |
| * @param repository the managed repository that will hold the file |
| * @param origFile the local file to save to |
| * @throws ProxyException if there was a problem moving the downloaded file into place. |
| */ |
| private void transferSimpleFile( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath, |
| ManagedRepositoryContent repository, File origFile, File destFile ) |
| throws ProxyException |
| { |
| assert ( remotePath != null ); |
| |
| // Transfer the file. |
| try |
| { |
| boolean success = false; |
| |
| if ( !origFile.exists() ) |
| { |
| log.debug( "Retrieving {} from {}", remotePath, remoteRepository.getRepository().getName() ); |
| wagon.get( addParameters( remotePath, remoteRepository.getRepository() ), destFile ); |
| success = true; |
| |
| // You wouldn't get here on failure, a WagonException would have been thrown. |
| log.debug( "Downloaded successfully." ); |
| } |
| else |
| { |
| log.debug( "Retrieving {} from {} if updated", remotePath, remoteRepository.getRepository().getName() ); |
| success = wagon.getIfNewer( addParameters( remotePath, remoteRepository.getRepository() ), destFile, |
| origFile.lastModified() ); |
| if ( !success ) |
| { |
| throw new NotModifiedException( |
| "Not downloaded, as local file is newer than remote side: " + origFile.getAbsolutePath() ); |
| } |
| |
| if ( destFile.exists() ) |
| { |
| log.debug( "Downloaded successfully." ); |
| } |
| } |
| } |
| catch ( ResourceDoesNotExistException e ) |
| { |
| throw new NotFoundException( |
| "Resource [" + remoteRepository.getURL() + "/" + remotePath + "] does not exist: " + e.getMessage(), |
| e ); |
| } |
| catch ( WagonException e ) |
| { |
| // TODO: shouldn't have to drill into the cause, but TransferFailedException is often not descriptive enough |
| |
| String msg = |
| "Download failure on resource [" + remoteRepository.getURL() + "/" + remotePath + "]:" + e.getMessage(); |
| if ( e.getCause() != null ) |
| { |
| msg += " (cause: " + e.getCause() + ")"; |
| } |
| throw new ProxyException( msg, e ); |
| } |
| } |
| |
| /** |
| * Apply the policies. |
| * |
| * @param policies the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects) |
| * @param settings the map of settings for the policies to execute. (Map of String policy keys, to String policy |
| * setting) |
| * @param request the request properties (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)} |
| * ) |
| * @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)}) |
| * @throws PolicyViolationException |
| */ |
| private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<String, String> settings, |
| Properties request, File localFile ) |
| throws PolicyViolationException |
| { |
| for ( Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() ) |
| { |
| // olamy with spring rolehint is now downloadPolicy#hint |
| // so substring after last # to get the hint as with plexus |
| String key = StringUtils.substringAfterLast( entry.getKey(), "#" ); |
| DownloadPolicy policy = entry.getValue(); |
| String defaultSetting = policy.getDefaultOption(); |
| |
| String setting = StringUtils.defaultString( settings.get( key ), defaultSetting ); |
| |
| log.debug( "Applying [{}] policy with [{}]", key, setting ); |
| try |
| { |
| policy.applyPolicy( setting, request, localFile ); |
| } |
| catch ( PolicyConfigurationException e ) |
| { |
| log.error( e.getMessage(), e ); |
| } |
| } |
| } |
| |
| private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<String, String> settings, |
| Properties request, ArtifactReference artifact, RemoteRepositoryContent content, |
| File localFile, Exception exception, Map<String, Exception> previousExceptions ) |
| throws ProxyDownloadException |
| { |
| boolean process = true; |
| for ( Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() ) |
| { |
| |
| // olamy with spring rolehint is now downloadPolicy#hint |
| // so substring after last # to get the hint as with plexus |
| String key = StringUtils.substringAfterLast( entry.getKey(), "#" ); |
| DownloadErrorPolicy policy = entry.getValue(); |
| String defaultSetting = policy.getDefaultOption(); |
| String setting = StringUtils.defaultString( settings.get( key ), defaultSetting ); |
| |
| log.debug( "Applying [{}] policy with [{}]", key, setting ); |
| try |
| { |
| // all policies must approve the exception, any can cancel |
| process = policy.applyPolicy( setting, request, localFile, exception, previousExceptions ); |
| if ( !process ) |
| { |
| break; |
| } |
| } |
| catch ( PolicyConfigurationException e ) |
| { |
| log.error( e.getMessage(), e ); |
| } |
| } |
| |
| if ( process ) |
| { |
| // if the exception was queued, don't throw it |
| if ( !previousExceptions.containsKey( content.getId() ) ) |
| { |
| throw new ProxyDownloadException( |
| "An error occurred in downloading from the remote repository, and the policy is to fail immediately", |
| content.getId(), exception ); |
| } |
| } |
| else |
| { |
| // if the exception was queued, but cancelled, remove it |
| previousExceptions.remove( content.getId() ); |
| } |
| |
| log.warn( |
| "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}", |
| content.getRepository().getId(), Keys.toKey( artifact ), exception.getMessage() ); |
| log.debug( "Full stack trace", exception ); |
| } |
| |
| /** |
| * Creates a working directory |
| * |
| * @param repository |
| * @return file location of working directory |
| */ |
| private File createWorkingDirectory( ManagedRepositoryContent repository ) |
| { |
| try |
| { |
| return Files.createTempDirectory( "temp" ).toFile(); |
| } |
| catch ( IOException e ) |
| { |
| throw new RuntimeException( e.getMessage(), e ); |
| } |
| |
| } |
| |
| /** |
| * Used to move the temporary file to its real destination. This is patterned from the way WagonManager handles its |
| * downloaded files. |
| * |
| * @param temp The completed download file |
| * @param target The final location of the downloaded file |
| * @throws ProxyException when the temp file cannot replace the target file |
| */ |
| private void moveTempToTarget( File temp, File target ) |
| throws ProxyException |
| { |
| |
| // TODO file lock library |
| Lock lock = null; |
| try |
| { |
| lock = fileLockManager.writeFileLock( target ); |
| if ( lock.getFile().exists() && !lock.getFile().delete() ) |
| { |
| throw new ProxyException( "Unable to overwrite existing target file: " + target.getAbsolutePath() ); |
| } |
| |
| lock.getFile().getParentFile().mkdirs(); |
| |
| if ( !temp.renameTo( lock.getFile() ) ) |
| { |
| log.warn( "Unable to rename tmp file to its final name... resorting to copy command." ); |
| |
| try |
| { |
| FileUtils.copyFile( temp, lock.getFile() ); |
| } |
| catch ( IOException e ) |
| { |
| if ( lock.getFile().exists() ) |
| { |
| log.debug( "Tried to copy file {} to {} but file with this name already exists.", |
| temp.getName(), lock.getFile().getAbsolutePath() ); |
| } |
| else |
| { |
| throw new ProxyException( |
| "Cannot copy tmp file " + temp.getAbsolutePath() + " to its final location", e ); |
| } |
| } |
| finally |
| { |
| FileUtils.deleteQuietly( temp ); |
| } |
| } |
| } |
| catch ( FileLockException e ) |
| { |
| throw new ProxyException( e.getMessage(), e ); |
| } |
| catch ( FileLockTimeoutException e ) |
| { |
| throw new ProxyException( e.getMessage(), e ); |
| } |
| } |
| |
| /** |
| * Using wagon, connect to the remote repository. |
| * |
| * @param connector the connector configuration to utilize (for obtaining network proxy configuration from) |
| * @param wagon the wagon instance to establish the connection on. |
| * @param remoteRepository the remote repository to connect to. |
| * @return true if the connection was successful. false if not connected. |
| */ |
| private boolean connectToRepository( ProxyConnector connector, Wagon wagon, |
| RemoteRepositoryContent remoteRepository ) |
| { |
| boolean connected = false; |
| |
| final ProxyInfo networkProxy = |
| connector.getProxyId() == null ? null : this.networkProxyMap.get( connector.getProxyId() ); |
| |
| if ( log.isDebugEnabled() ) |
| { |
| if ( networkProxy != null ) |
| { |
| // TODO: move to proxyInfo.toString() |
| String msg = "Using network proxy " + networkProxy.getHost() + ":" + networkProxy.getPort() |
| + " to connect to remote repository " + remoteRepository.getURL(); |
| if ( networkProxy.getNonProxyHosts() != null ) |
| { |
| msg += "; excluding hosts: " + networkProxy.getNonProxyHosts(); |
| } |
| if ( StringUtils.isNotBlank( networkProxy.getUserName() ) ) |
| { |
| msg += "; as user: " + networkProxy.getUserName(); |
| } |
| log.debug( msg ); |
| } |
| } |
| |
| AuthenticationInfo authInfo = null; |
| String username = remoteRepository.getRepository().getUserName(); |
| String password = remoteRepository.getRepository().getPassword(); |
| |
| if ( StringUtils.isNotBlank( username ) && StringUtils.isNotBlank( password ) ) |
| { |
| log.debug( "Using username {} to connect to remote repository {}", username, remoteRepository.getURL() ); |
| authInfo = new AuthenticationInfo(); |
| authInfo.setUserName( username ); |
| authInfo.setPassword( password ); |
| } |
| |
| // Convert seconds to milliseconds |
| int timeoutInMilliseconds = remoteRepository.getRepository().getTimeout() * 1000; |
| |
| // Set timeout read and connect |
| // FIXME olamy having 2 config values |
| wagon.setReadTimeout( timeoutInMilliseconds ); |
| wagon.setTimeout( timeoutInMilliseconds ); |
| |
| try |
| { |
| Repository wagonRepository = |
| new Repository( remoteRepository.getId(), remoteRepository.getURL().toString() ); |
| wagon.connect( wagonRepository, authInfo, networkProxy ); |
| connected = true; |
| } |
| catch ( ConnectionException | AuthenticationException e ) |
| { |
| log.warn( "Could not connect to {}: {}", remoteRepository.getRepository().getName(), e.getMessage() ); |
| connected = false; |
| } |
| |
| return connected; |
| } |
| |
| /** |
| * Tests whitelist and blacklist patterns against path. |
| * |
| * @param path the path to test. |
| * @param patterns the list of patterns to check. |
| * @return true if the path matches at least 1 pattern in the provided patterns list. |
| */ |
| private boolean matchesPattern( String path, List<String> patterns ) |
| { |
| if ( CollectionUtils.isEmpty( patterns ) ) |
| { |
| return false; |
| } |
| |
| if ( !path.startsWith( "/" ) ) |
| { |
| path = "/" + path; |
| } |
| |
| for ( String pattern : patterns ) |
| { |
| if ( !pattern.startsWith( "/" ) ) |
| { |
| pattern = "/" + pattern; |
| } |
| |
| if ( SelectorUtils.matchPath( pattern, path, false ) ) |
| { |
| return true; |
| } |
| } |
| |
| return false; |
| } |
| |
| /** |
| * TODO: Ensure that list is correctly ordered based on configuration. See MRM-477 |
| */ |
| @Override |
| public List<ProxyConnector> getProxyConnectors( ManagedRepositoryContent repository ) |
| { |
| |
| if ( !this.proxyConnectorMap.containsKey( repository.getId() ) ) |
| { |
| return Collections.emptyList(); |
| } |
| List<ProxyConnector> ret = new ArrayList<>( this.proxyConnectorMap.get( repository.getId() ) ); |
| |
| Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() ); |
| return ret; |
| |
| } |
| |
| @Override |
| public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue ) |
| { |
| if ( ConfigurationNames.isNetworkProxy( propertyName ) || ConfigurationNames.isManagedRepositories( |
| propertyName ) || ConfigurationNames.isRemoteRepositories( propertyName ) |
| || ConfigurationNames.isProxyConnector( propertyName ) ) |
| { |
| initConnectorsAndNetworkProxies(); |
| } |
| } |
| |
| protected String addParameters( String path, RemoteRepository remoteRepository ) |
| { |
| if ( remoteRepository.getExtraParameters().isEmpty() ) |
| { |
| return path; |
| } |
| |
| boolean question = false; |
| |
| StringBuilder res = new StringBuilder( path == null ? "" : path ); |
| |
| for ( Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() ) |
| { |
| if ( !question ) |
| { |
| res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() ); |
| } |
| } |
| |
| return res.toString(); |
| } |
| |
| |
| @Override |
| public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue ) |
| { |
| /* do nothing */ |
| } |
| |
| public ArchivaConfiguration getArchivaConfiguration() |
| { |
| return archivaConfiguration; |
| } |
| |
| public void setArchivaConfiguration( ArchivaConfiguration archivaConfiguration ) |
| { |
| this.archivaConfiguration = archivaConfiguration; |
| } |
| |
| public RepositoryContentFactory getRepositoryFactory() |
| { |
| return repositoryFactory; |
| } |
| |
| public void setRepositoryFactory( RepositoryContentFactory repositoryFactory ) |
| { |
| this.repositoryFactory = repositoryFactory; |
| } |
| |
| public MetadataTools getMetadataTools() |
| { |
| return metadataTools; |
| } |
| |
| public void setMetadataTools( MetadataTools metadataTools ) |
| { |
| this.metadataTools = metadataTools; |
| } |
| |
| public UrlFailureCache getUrlFailureCache() |
| { |
| return urlFailureCache; |
| } |
| |
| public void setUrlFailureCache( UrlFailureCache urlFailureCache ) |
| { |
| this.urlFailureCache = urlFailureCache; |
| } |
| |
| public WagonFactory getWagonFactory() |
| { |
| return wagonFactory; |
| } |
| |
| public void setWagonFactory( WagonFactory wagonFactory ) |
| { |
| this.wagonFactory = wagonFactory; |
| } |
| |
| public Map<String, PreDownloadPolicy> getPreDownloadPolicies() |
| { |
| return preDownloadPolicies; |
| } |
| |
| public void setPreDownloadPolicies( Map<String, PreDownloadPolicy> preDownloadPolicies ) |
| { |
| this.preDownloadPolicies = preDownloadPolicies; |
| } |
| |
| public Map<String, PostDownloadPolicy> getPostDownloadPolicies() |
| { |
| return postDownloadPolicies; |
| } |
| |
| public void setPostDownloadPolicies( Map<String, PostDownloadPolicy> postDownloadPolicies ) |
| { |
| this.postDownloadPolicies = postDownloadPolicies; |
| } |
| |
| public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies() |
| { |
| return downloadErrorPolicies; |
| } |
| |
| public void setDownloadErrorPolicies( Map<String, DownloadErrorPolicy> downloadErrorPolicies ) |
| { |
| this.downloadErrorPolicies = downloadErrorPolicies; |
| } |
| } |
