Add Proxy Site cache support for squid proxy delivery

api/src/main/java/org/commonjava/maven/galley/proxy/NoOpProxySitesCache.java

@@ -0,0 +1,59 @@
+/**
+ * Copyright (C) 2013 Red Hat, Inc. (https://github.com/Commonjava/galley)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commonjava.maven.galley.proxy;
+
+import org.commonjava.maven.galley.spi.proxy.ProxySitesCache;
+
+import javax.enterprise.inject.Alternative;
+import javax.inject.Named;
+import java.util.Collections;
+import java.util.Set;
+
+@Named
+@Alternative
+public class NoOpProxySitesCache
+        implements ProxySitesCache
+{
+    @Override
+    public Set<String> getProxySites()
+    {
+        return Collections.emptySet();
+    }
+
+    @Override
+    public boolean isProxySite( String site )
+    {
+        return false;
+    }
+
+    @Override
+    public void saveProxySite( String site )
+    {
+
+    }
+
+    @Override
+    public void deleteProxySite( String site )
+    {
+
+    }
+
+    @Override
+    public void deleteAllProxySites()
+    {
+
+    }
+}

api/src/main/java/org/commonjava/maven/galley/spi/proxy/ProxySitesCache.java

@@ -0,0 +1,32 @@
+/**
+ * Copyright (C) 2013 Red Hat, Inc. (https://github.com/Commonjava/galley)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commonjava.maven.galley.spi.proxy;
+
+import java.util.Set;
+
+public interface ProxySitesCache
+{
+    Set<String> getProxySites();
+
+    boolean isProxySite( String site );
+
+    void saveProxySite( String site );
+
+    void deleteProxySite( String site );
+
+    void deleteAllProxySites();
+
+}

cdi-embedder/src/main/java/org/commonjava/maven/galley/embed/EmbeddableCDIProducer.java

@@ -29,11 +29,13 @@
 import org.commonjava.maven.galley.maven.spi.defaults.MavenPluginDefaults;
 import org.commonjava.maven.galley.maven.spi.defaults.MavenPluginImplications;
 import org.commonjava.maven.galley.nfc.MemoryNotFoundCache;
+import org.commonjava.maven.galley.proxy.NoOpProxySitesCache;
 import org.commonjava.maven.galley.spi.auth.PasswordManager;
 import org.commonjava.maven.galley.spi.event.FileEventManager;
 import org.commonjava.maven.galley.spi.io.PathGenerator;
 import org.commonjava.maven.galley.spi.io.TransferDecorator;
 import org.commonjava.maven.galley.spi.nfc.NotFoundCache;
+import org.commonjava.maven.galley.spi.proxy.ProxySitesCache;
 import org.commonjava.maven.galley.spi.transport.TransportManager;
 import org.commonjava.maven.galley.transport.htcli.Http;
 import org.commonjava.maven.galley.transport.htcli.HttpImpl;
@@ -59,6 +61,8 @@ public class EmbeddableCDIProducer
 
     private NotFoundCache nfc;
 
+    private ProxySitesCache proxySitesCache;
+
     private ObjectMapper objectMapper;
 
     private Http http;
@@ -86,6 +90,7 @@ public void postConstruct()
         transferDecorator = new NoOpTransferDecorator();
         pathGenerator = new HashedLocationPathGenerator();
         nfc = new MemoryNotFoundCache();
+        proxySitesCache = new NoOpProxySitesCache();
         transportManagerConfig = new TransportManagerConfig();
 
         passwordManager = new MemoryPasswordManager();
@@ -161,6 +166,13 @@ public NotFoundCache getNotFoundCache()
         return nfc;
     }
 
+    @Default
+    @Produces
+    public ProxySitesCache getProxySitesCache()
+    {
+        return proxySitesCache;
+    }
+
     @Default
     @Produces
     public Http getHttp()

transports/httpclient/src/main/java/org/commonjava/maven/galley/transport/htcli/HttpClientTransport.java

@@ -23,6 +23,7 @@
 import org.commonjava.maven.galley.model.ConcreteResource;
 import org.commonjava.maven.galley.model.Location;
 import org.commonjava.maven.galley.model.Transfer;
+import org.commonjava.maven.galley.spi.proxy.ProxySitesCache;
 import org.commonjava.maven.galley.spi.transport.DownloadJob;
 import org.commonjava.maven.galley.spi.transport.ExistenceJob;
 import org.commonjava.maven.galley.spi.transport.ListingJob;
@@ -78,6 +79,9 @@ public class HttpClientTransport
     @Inject
     private TransportMetricConfig metricConfig;
 
+    @Inject
+    private ProxySitesCache proxySitesCache;
+
     protected HttpClientTransport()
     {
     }
@@ -117,7 +121,8 @@ public DownloadJob createDownloadJob( final ConcreteResource resource, final Tra
         throws TransferException
     {
         return new HttpDownload( getUrl( resource ), getHttpLocation( resource.getLocation(), download ), target,
-                                 transferSizes, eventMetadata, http, mapper, metricRegistry, metricConfig );
+                                 transferSizes, eventMetadata, http, mapper, metricRegistry, metricConfig,
+                                 proxySitesCache );
     }
 
     @Override
@@ -160,7 +165,7 @@ public ListingJob createListingJob( final ConcreteResource resource, final Trans
     {
         return new HttpListing( getUrl( resource ),
                                 new ConcreteResource( getHttpLocation( resource.getLocation(), listing ),
-                                                      resource.getPath() ), http );
+                                                      resource.getPath() ), http, proxySitesCache );
     }
 
     private HttpLocation getHttpLocation( final Location repository, HttpJobType httpJobType )
@@ -184,7 +189,7 @@ public ExistenceJob createExistenceJob( final ConcreteResource resource, final T
         throws TransferException
     {
         return new HttpExistence( getUrl( resource ), getHttpLocation( resource.getLocation(), existence ), target,
-                                  http, mapper );
+                                  http, mapper, proxySitesCache );
     }
 
     private String getUrl( final ConcreteResource resource )

transports/httpclient/src/main/java/org/commonjava/maven/galley/transport/htcli/internal/AbstractHttpJob.java

@@ -33,6 +33,7 @@
 import org.commonjava.maven.galley.io.checksum.ChecksumAlgorithm;
 import org.commonjava.maven.galley.model.Transfer;
 import org.commonjava.maven.galley.model.TransferOperation;
+import org.commonjava.maven.galley.spi.proxy.ProxySitesCache;
 import org.commonjava.maven.galley.transport.htcli.Http;
 import org.commonjava.maven.galley.transport.htcli.internal.util.TransferResponseUtils;
 import org.commonjava.maven.galley.transport.htcli.model.HttpExchangeMetadata;
@@ -72,12 +73,15 @@ public abstract class AbstractHttpJob
 
     protected boolean success;
 
+    private final ProxySitesCache proxySitesCache;
+
     protected AbstractHttpJob( final String url, final HttpLocation location, final Http http,
-                               final Integer... successStatuses )
+                               ProxySitesCache proxySitesCache, final Integer... successStatuses )
     {
         this.url = url;
         this.location = location;
         this.http = http;
+        this.proxySitesCache = proxySitesCache;
 
         if ( successStatuses.length < 1 )
         {
@@ -104,13 +108,19 @@ protected boolean executeHttp()
     {
         int tries = 1;
         boolean doProxy = false;
+        String site = location.getHost();
         try
         {
             while ( tries > 0 )
             {
                 tries--;
                 try
                 {
+                    if ( proxySitesCache != null && proxySitesCache.isProxySite( site ) )
+                    {
+                        doProxy = true;
+                        logger.debug( "Access with proxy in cache, site: {}", site );
+                    }
                     client = http.createClient( location, doProxy );
                     response = client.execute( request, http.createContext( location ) );
 
@@ -144,24 +154,29 @@ else if ( !doProxy ) // never do with proxy, retry with proxy
                     {
                         tries = 1;
                         doProxy = true;
-                        logger.debug( "Retry to execute with global proxy for {}", url );
+                        if ( proxySitesCache != null )
+                        {
+                            proxySitesCache.saveProxySite( site );
+                        }
+                        logger.debug( "Retry to execute with global proxy for {} and add into proxy cache, site: {}",
+                                      url, site );
                     }
                     else // already did proxy, still timeout
                     {
                         addFieldToActiveSpan( "target-error-reason", "timeout" );
                         addFieldToActiveSpan( "target-error", e.getClass().getSimpleName() );
                         throw new TransferTimeoutException( location, url,
-                                                            "Repository remote request failed for: {}. Reason: {}", e,
-                                                            url, e.getMessage() );
+                                                            "Retried with proxy, repository remote request timeout failed for: {}. Reason: {}",
+                                                            e, url, e.getMessage() );
                     }
                 }
                 catch ( final IOException e )
                 {
                     addFieldToActiveSpan( "target-error-reason", "I/O" );
                     addFieldToActiveSpan( "target-error", e.getClass().getSimpleName() );
                     throw new TransferLocationException( location,
-                                                         "Repository remote request failed for: {}. Reason: {}", e, url,
-                                                         e.getMessage() );
+                                                         "Repository remote request IO failed for: {}. Reason: {}", e,
+                                                         url, e.getMessage() );
                 }
                 catch ( TransferLocationException e )
                 {
@@ -173,7 +188,7 @@ else if ( !doProxy ) // never do with proxy, retry with proxy
                 {
                     addFieldToActiveSpan( "target-error-reason", "unknown" );
                     addFieldToActiveSpan( "target-error", e.getClass().getSimpleName() );
-                    throw new TransferException( "Repository remote request failed for: {}. Reason: {}", e, url,
+                    throw new TransferException( "Repository remote request Galley failed for: {}. Reason: {}", e, url,
                                                  e.getMessage() );
                 }
             }
@@ -200,7 +215,6 @@ else if ( !doProxy ) // never do with proxy, retry with proxy
     }
 
     /* for GET/HEAD request, need to override below two methods for writeMetadata() */
-
     protected Transfer getTransfer()
     {
         return null;
@@ -306,5 +320,4 @@ protected void cleanup()
         request = null;
         response = null;
     }
-
 }

transports/httpclient/src/main/java/org/commonjava/maven/galley/transport/htcli/internal/HttpDownload.java

@@ -21,13 +21,14 @@
 import org.apache.http.util.EntityUtils;
 import org.commonjava.maven.galley.TransferContentException;
 import org.commonjava.maven.galley.TransferException;
+import org.commonjava.maven.galley.config.TransportMetricConfig;
 import org.commonjava.maven.galley.event.EventMetadata;
 import org.commonjava.maven.galley.model.ConcreteResource;
 import org.commonjava.maven.galley.model.Transfer;
 import org.commonjava.maven.galley.model.TransferOperation;
+import org.commonjava.maven.galley.spi.proxy.ProxySitesCache;
 import org.commonjava.maven.galley.spi.transport.DownloadJob;
 import org.commonjava.maven.galley.transport.htcli.Http;
-import org.commonjava.maven.galley.config.TransportMetricConfig;
 import org.commonjava.maven.galley.transport.htcli.model.HttpLocation;
 import org.commonjava.maven.galley.transport.htcli.util.HttpUtil;
 import org.commonjava.o11yphant.metrics.api.MetricRegistry;
@@ -67,7 +68,7 @@ public HttpDownload( final String url, final HttpLocation location, final Transf
                          final Map<Transfer, Long> transferSizes, final EventMetadata eventMetadata, final Http http,
                          final ObjectMapper mapper )
     {
-        this( url, location, target, transferSizes, eventMetadata, http, mapper, true, null, null );
+        this( url, location, target, transferSizes, eventMetadata, http, mapper, true, null, null, null );
     }
 
     public HttpDownload( final String url, final HttpLocation location, final Transfer target,
@@ -76,15 +77,25 @@ public HttpDownload( final String url, final HttpLocation location, final Transf
                          final TransportMetricConfig transportMetricConfig )
     {
         this( url, location, target, transferSizes, eventMetadata, http, mapper, true, metricRegistry,
-              transportMetricConfig );
+              transportMetricConfig, null );
+    }
+
+    public HttpDownload( final String url, final HttpLocation location, final Transfer target,
+                         final Map<Transfer, Long> transferSizes, final EventMetadata eventMetadata, final Http http,
+                         final ObjectMapper mapper, final MetricRegistry metricRegistry,
+                         final TransportMetricConfig transportMetricConfig, ProxySitesCache proxySitesCache )
+    {
+        this( url, location, target, transferSizes, eventMetadata, http, mapper, true, metricRegistry,
+              transportMetricConfig, proxySitesCache );
     }
 
     public HttpDownload( final String url, final HttpLocation location, final Transfer target,
                          final Map<Transfer, Long> transferSizes, final EventMetadata eventMetadata, final Http http,
                          final ObjectMapper mapper, final boolean deleteFilesOnPath,
-                         final MetricRegistry metricRegistry, final TransportMetricConfig transportMetricConfig)
+                         final MetricRegistry metricRegistry, final TransportMetricConfig transportMetricConfig,
+                         ProxySitesCache proxySitesCache )
     {
-        super( url, location, http );
+        super( url, location, http, proxySitesCache );
         this.request = new HttpGet( url );
         this.target = target;
         this.transferSizes = transferSizes;

transports/httpclient/src/main/java/org/commonjava/maven/galley/transport/htcli/internal/HttpExistence.java

@@ -15,20 +15,20 @@
  */
 package org.commonjava.maven.galley.transport.htcli.internal;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.http.client.methods.HttpHead;
 import org.commonjava.maven.galley.TransferException;
 import org.commonjava.maven.galley.model.Transfer;
+import org.commonjava.maven.galley.spi.proxy.ProxySitesCache;
 import org.commonjava.maven.galley.spi.transport.ExistenceJob;
 import org.commonjava.maven.galley.transport.htcli.Http;
 import org.commonjava.maven.galley.transport.htcli.model.HttpLocation;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-
 import static org.commonjava.o11yphant.trace.TraceManager.addFieldToActiveSpan;
 
 public final class HttpExistence
-    extends AbstractHttpJob
-    implements ExistenceJob
+        extends AbstractHttpJob
+        implements ExistenceJob
 {
 
     private final ObjectMapper mapper;
@@ -38,7 +38,13 @@ public final class HttpExistence
     public HttpExistence( final String url, final HttpLocation location, final Transfer transfer, final Http http,
                           final ObjectMapper mapper )
     {
-        super( url, location, http );
+        this( url, location, transfer, http, mapper, null );
+    }
+
+    public HttpExistence( final String url, final HttpLocation location, final Transfer transfer, final Http http,
+                          final ObjectMapper mapper, ProxySitesCache proxySitesCache )
+    {
+        super( url, location, http, proxySitesCache );
         this.transfer = transfer;
         this.mapper = mapper;
     }