Author: kwright
Date: Sun Aug 12 00:25:49 2012
New Revision: 1372040
URL: http://svn.apache.org/viewvc?rev=1372040&view=rev
Log:
Add first part of the management of HOPCOUNTREMOVE status documents.
Modified:
manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java
Modified:
manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1372040&r1=1372039&r2=1372040&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++
manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Sun Aug 12 00:25:49 2012
@@ -4097,10 +4097,17 @@ public class JobManager implements IJobM
// It was an insert
reorderedRval[z] = true;
else
+ {
// It was an existing row; do the update logic
+ // The hopcountChangesSeen array describes whether each reference
is a new one. This
+ // helps us determine whether we're going to need to "flip"
HOPCOUNTREMOVED documents
+ // to the PENDING state. If the new link ended in an existing
record, THEN we need to flip them all!
reorderedRval[z] =
jobQueue.updateExistingRecord(jr.getRecordID(),jr.getStatus(),jr.getCheckTimeValue(),
0L,currentTime,carrydownChangesSeen[z] ||
(hopcountChangesSeen!=null && hopcountChangesSeen[z]),
reorderedDocumentPriorities[z],reorderedDocumentPrerequisites[z]);
+ // Perform the flip
+ jobQueue.reactivateHopcountRemovedRecords(jobID);
+ }
z++;
}
@@ -6751,6 +6758,7 @@ public class JobManager implements IJobM
resetJobs.add(jobDesc);
// Label the job "finished"
+ jobQueue.deleteHopcountRemovedRecords(jobID);
jobs.finishJob(jobID,currentTime);
if (Logging.jobs.isDebugEnabled())
{
Modified:
manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1372040&r1=1372039&r2=1372040&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
(original)
+++
manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Sun Aug 12 00:25:49 2012
@@ -68,7 +68,7 @@ public class JobQueue extends org.apache
public final static int STATUS_ACTIVENEEDRESCANPURGATORY = 8;
public final static int STATUS_BEINGCLEANED = 9;
public final static int STATUS_ELIGIBLEFORDELETE = 10;
-
+ public final static int STATUS_HOPCOUNTREMOVED = 11;
// Action values
public final static int ACTION_RESCAN = 0;
public final static int ACTION_REMOVE = 1;
@@ -126,6 +126,7 @@ public class JobQueue extends org.apache
statusMap.put("a",new Integer(STATUS_ACTIVENEEDRESCAN));
statusMap.put("f",new Integer(STATUS_ACTIVENEEDRESCANPURGATORY));
statusMap.put("d",new Integer(STATUS_BEINGCLEANED));
+ statusMap.put("H",new Integer(STATUS_HOPCOUNTREMOVED));
}
protected static Map seedstatusMap;
@@ -364,6 +365,35 @@ public class JobQueue extends org.apache
unconditionallyAnalyzeTables();
}
+ /** Flip all records for a job that have status HOPCOUNTREMOVED back to
PENDING.
+ * NOTE: We need to actually schedule these!!! so the following can't really
work. ???
+ */
+ public void reactivateHopcountRemovedRecords(Long jobID)
+ throws ManifoldCFException
+ {
+ Map map = new HashMap();
+ // Map HOPCOUNTREMOVED to PENDING
+ map.put(statusField,statusToString(STATUS_PENDING));
+ map.put(checkTimeField,new Long(0L));
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(jobIDField,jobID),
+ new UnitaryClause(statusField,statusToString(STATUS_HOPCOUNTREMOVED))});
+ performUpdate(map,"WHERE "+query,list,null);
+ }
+
+ /** Delete all records for a job that have status HOPCOUNTREMOVED.
+ */
+ public void deleteHopcountRemovedRecords(Long jobID)
+ throws ManifoldCFException
+ {
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(jobIDField,jobID),
+ new UnitaryClause(statusField,statusToString(STATUS_HOPCOUNTREMOVED))});
+ performDelete("WHERE "+query,list,null);
+ }
+
/** Clear the failtimes for all documents associated with a job.
* This method is called when the system detects that a significant delaying
event has occurred,
* and therefore the "failure clock" needs to be reset.
@@ -1201,10 +1231,11 @@ public class JobQueue extends org.apache
/** Update an existing record (as the result of a reference add).
* The record is presumed to exist and have been locked, via "FOR UPDATE".
+ *@return true if the document priority slot has been retained, false if
freed.
*/
public boolean updateExistingRecord(Long recordID, int currentStatus, Long
checkTimeValue,
- long desiredExecuteTime, long currentTime, boolean otherChangesSeen,
double desiredPriority,
- String[] prereqEvents)
+ long desiredExecuteTime, long currentTime, boolean otherChangesSeen,
+ double desiredPriority, String[] prereqEvents)
throws ManifoldCFException
{
boolean rval = false;
@@ -1456,6 +1487,8 @@ public class JobQueue extends org.apache
return "f";
case STATUS_BEINGCLEANED:
return "d";
+ case STATUS_HOPCOUNTREMOVED:
+ return "H";
default:
throw new ManifoldCFException("Bad status value:
"+Integer.toString(status));
}
Modified:
manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java?rev=1372040&r1=1372039&r2=1372040&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java
(original)
+++
manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java
Sun Aug 12 00:25:49 2012
@@ -81,7 +81,7 @@ public class BigCrawlTester
job.setStartMethod(job.START_DISABLE);
job.setHopcountMode(job.HOPCOUNT_ACCURATE);
job.addHopCountFilter("link",new Long(2));
- job.addHopCountFilter("redirect",new Long(2));
+ //job.addHopCountFilter("redirect",new Long(2));
// Now, set up the document specification.
DocumentSpecification ds = job.getSpecification();
