Author: kwright
Date: Tue Dec 17 22:24:40 2013
New Revision: 1551737
URL: http://svn.apache.org/r1551737
Log:
Fix infinite loop issue with closing idle connections. Part of CONNECTORS-829.
Modified:
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java
Modified:
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1551737&r1=1551736&r2=1551737&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
(original)
+++
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Tue Dec 17 22:24:40 2013
@@ -2009,6 +2009,8 @@ public class ThrottledFetcher
connection.destroy();
connectionThrottler.noteConnectionDestroyed();
}
+ else
+ break;
}
}
Modified:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java?rev=1551737&r1=1551736&r2=1551737&view=diff
==============================================================================
---
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java
(original)
+++
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java
Tue Dec 17 22:24:40 2013
@@ -198,6 +198,19 @@ public class ConnectionBin
return CONNECTION_POOLEMPTY;
}
+ /** Check only if there's a pooled connection, and make moves to take it
from the pool.
+ */
+ public synchronized boolean hasPooledConnection(AtomicInteger poolCount)
+ {
+ int currentPoolCount = poolCount.get();
+ if (currentPoolCount > 0)
+ {
+ poolCount.set(currentPoolCount-1);
+ return true;
+ }
+ return false;
+ }
+
/** Undo the decision to destroy a pooled connection.
*/
public synchronized void undoPooledConnectionDecision(AtomicInteger
poolCount)
Modified:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java?rev=1551737&r1=1551736&r2=1551737&view=diff
==============================================================================
---
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java
(original)
+++
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java
Tue Dec 17 22:24:40 2013
@@ -583,6 +583,47 @@ public class Throttler
}
+ /** Connection expiration is tricky, because even though a connection may
be identified as
+ * being expired, at the very same moment it could be handed out in another
thread. So there
+ * is a natural race condition present.
+ * The way the connection throttler deals with that is to allow the caller
to reserve a connection
+ * for expiration. This must be called BEFORE the actual identified
connection is removed from the
+ * connection pool. If the value returned by this method is "true", then a
connection MUST be removed
+ * from the pool and destroyed, whether or not the identified connection is
actually still available for
+ * destruction or not.
+ *@return true if a connection from the pool can be expired. If true is
returned, noteConnectionDestruction()
+ * MUST be called once the connection has actually been destroyed.
+ */
+ public boolean checkExpireConnection(String[] binNames, AtomicInteger
poolCount)
+ {
+ synchronized (connectionBins)
+ {
+ int i = 0;
+ while (i < binNames.length)
+ {
+ String binName = binNames[i];
+ ConnectionBin bin = connectionBins.get(binName);
+ if (bin != null)
+ {
+ if (!bin.hasPooledConnection(poolCount))
+ {
+ // Give up now, and undo all the other bins
+ while (i > 0)
+ {
+ i--;
+ binName = binNames[i];
+ bin = connectionBins.get(binName);
+ bin.undoPooledConnectionDecision(poolCount);
+ }
+ return false;
+ }
+ }
+ i++;
+ }
+ return true;
+ }
+ }
+
public void noteConnectionReturnedToPool(String[] binNames, AtomicInteger
poolCount)
{
synchronized (connectionBins)
@@ -957,8 +998,7 @@ public class Throttler
@Override
public boolean checkExpireConnection()
{
- // MHL
- return false;
+ return parent.checkExpireConnection(binNames, poolCount);
}
/** Note that a connection has been returned to the pool. Call this
method after a connection has been
