Revision: 14788
http://gate.svn.sourceforge.net/gate/?rev=14788&view=rev
Author: valyt
Date: 2011-12-16 12:11:08 +0000 (Fri, 16 Dec 2011)
Log Message:
-----------
Removed some old scratch code.
Scratch code for exercising the new Query Runner.
Modified Paths:
--------------
mimir/trunk/mimir-test/src/gate/mimir/test/Scratch.java
Modified: mimir/trunk/mimir-test/src/gate/mimir/test/Scratch.java
===================================================================
--- mimir/trunk/mimir-test/src/gate/mimir/test/Scratch.java 2011-12-16
12:10:32 UTC (rev 14787)
+++ mimir/trunk/mimir-test/src/gate/mimir/test/Scratch.java 2011-12-16
12:11:08 UTC (rev 14788)
@@ -19,6 +19,7 @@
import java.io.InputStreamReader;
import java.text.NumberFormat;
import java.util.Arrays;
+import java.util.concurrent.Callable;
import gate.Gate;
import gate.mimir.search.QueryEngine;
@@ -83,7 +84,7 @@
* Version that exercises the scorers
* @param args
*/
- public static void mainTextConsole(String[] args) throws Exception {
+ public static void main(String[] args) throws Exception {
Gate.setGateHome(new File("gate-home"));
Gate.setUserConfigFile(new File("gate-home/user-gate.xml"));
Gate.init();
@@ -96,7 +97,20 @@
// load the measurements plugin
Gate.getCreoleRegister().registerDirectories(
new File("../plugins/measurements").toURI().toURL());
+ // load the SPARQL plugin
+ Gate.getCreoleRegister().registerDirectories(
+ new File("../plugins/sparql").toURI().toURL());
+
QueryEngine qEngine = new QueryEngine(new File(args[0]));
+ qEngine.setScorerSource(new Callable<MimirScorer>() {
+ @Override
+ public MimirScorer call() throws Exception {
+ return new BindingScorer(2, 0.9);
+ // return new DelegatingScoringQueryExecutor(new TfIdfScorer());
+ // return new DelegatingScoringQueryExecutor(new CountScorer());
+ // return new DelegatingScoringQueryExecutor(new BM25Scorer());
+ }
+ });
BufferedReader input = new BufferedReader(new
InputStreamReader(System.in));
String query = null;
do {
@@ -105,32 +119,30 @@
query = input.readLine();
long start = System.currentTimeMillis();
if(query == null || query.trim().length() == 0) break;
- QueryNode qNode = QueryParser.parse(query);
- QueryExecutor qExecutor = qNode.getQueryExecutor(qEngine);
- // MimirScorer scorer = new DelegatingScoringQueryExecutor(new
TfIdfScorer());
- // MimirScorer scorer = new DelegatingScoringQueryExecutor(new
CountScorer());
-// MimirScorer scorer = new DelegatingScoringQueryExecutor(new
BM25Scorer());
- MimirScorer scorer = new BindingScorer(2, 0.9);
- scorer.wrap(qExecutor);
+ QueryRunner qRunner = qEngine.getQueryRunner(query);
+
+ while(qRunner.getDocumentsCount() < 0) {
+ Thread.sleep(100);
+ }
double minScore = Double.MAX_VALUE;
double maxScore = Double.MIN_VALUE;
- int docCount = 0;
- // TestUtils.dumpResultsToFile(qExecutor, new File("results.txt"));
- int latestDoc = scorer.nextDocument();
- while(latestDoc >= 0) {
- docCount ++;
- double score = scorer.score();
+ int docCount = qRunner.getDocumentsCount();
+ for(int i = 0; i < docCount; i++) {
+ double score = qRunner.getDocumentScore(i);
if(score < minScore) minScore = score;
if(score > maxScore) maxScore = score;
- latestDoc = scorer.nextDocument(-1);
+ // exercise the runner
+ qRunner.getDocumentID(i);
+ qRunner.getDocumentTitle(i);
+ qRunner.getDocumentURI(i);
+ qRunner.getDocumentHits(i);
}
System.out.println(String.format(
"Matched %d documents, scores %02.4f - %02.4f, in %02.2f seconds",
docCount, minScore, maxScore,
(double)(System.currentTimeMillis() - start)/1000));
-
- qExecutor.close();
+ qRunner.close();
}catch(Exception e) {
e.printStackTrace(System.err);
}
@@ -138,164 +150,4 @@
qEngine.close();
}
-
- /**
- * Version that exercises the scorers
- * @param args
- */
- public static void main(String[] args) throws Exception {
- Gate.setGateHome(new File("gate-home"));
- Gate.setUserConfigFile(new File("gate-home/user-gate.xml"));
- Gate.init();
- // load the tokeniser plugin
- Gate.getCreoleRegister().registerDirectories(
- new File("gate-home/plugins/ANNIE-tokeniser").toURI().toURL());
- // load the DB plugin
- Gate.getCreoleRegister().registerDirectories(
- new File("../plugins/db-h2").toURI().toURL());
- // load the measurements plugin
- Gate.getCreoleRegister().registerDirectories(
- new File("../plugins/measurements").toURI().toURL());
- QueryEngine qEngine = new QueryEngine(new File(args[0]));
- QueryNode qNode = QueryParser.parse("invention");
- QueryExecutor qExecutor = qNode.getQueryExecutor(qEngine);
- QueryRunner qRunner =
- new RankingQueryRunnerImpl(qExecutor,
- new DelegatingScoringQueryExecutor(new CountScorer()));
-//
-// qEngine.close();
- }
-
- public static void mainPrioQueue(String[] args) throws Exception {
- // simulate a stream of documents, in increasing documentId order and
random scores
- int docCount = 10000;
- int[] inputDocIds = new int[docCount];
- final double[] inputDocScores = new double[docCount];
- for(int i = 0; i < inputDocIds.length; i++) {
- int prevDocID = i == 0 ? 0 : inputDocIds[i -1];
- inputDocIds[i] = (int)(prevDocID + (Math.random() * 10));
- inputDocScores[i] = Math.random() * 100;
- if(inputDocIds[i] % 5 == 0) {
- // multiples of 5 are good
- inputDocScores[i] += 100;
- }
- }
-
- // perform the filtering
- // how many documents are we keeping
- int maxSize = 10;
- // document IDs, sorted by score, highest first
- IntPriorityQueue topDocs = new IntHeapPriorityQueue(maxSize, new
IntComparator() {
- @Override
- public int compare(Integer o1, Integer o2) {
- return compare(o1.intValue(), o2.intValue());
- }
-
- @Override
- public int compare(int k1, int k2) {
- double d1 = inputDocScores[k1];
- double d2 = inputDocScores[k2];
- if(d1 > d2) return 1;
- else if(d1 < d2) return -1;
- return 0;
- }
- });
-
- for(int i = 0; i < inputDocIds.length; i++) {
- if(topDocs.size() < maxSize) {
- topDocs.enqueue(i);
- } else {
- if(inputDocScores[i] > inputDocScores[topDocs.first()]) {
- topDocs.dequeueInt();
- topDocs.enqueue(i);
- }
- }
- }
- // extract the results
- while(!topDocs.isEmpty()) {
- int smallestDoc = topDocs.dequeue();
- System.out.println("Doc id: " + inputDocIds[smallestDoc] + ", score: " +
inputDocScores[smallestDoc]);
- }
- }
-
-
- public static void mainSorting(String[] args) throws Exception {
- // simulate a stream of documents, in increasing documentId order and
random scores
- int docCount = 1000000;
- int[] inputDocIds = new int[docCount];
- double[] inputDocScores = new double[docCount];
- double maxScore = 0;
- for(int i = 0; i < inputDocIds.length; i++) {
- int prevDocID = i == 0 ? 0 : inputDocIds[i -1];
- inputDocIds[i] = (int)(prevDocID + 1 + (Math.random() * 10));
- inputDocScores[i] = Math.random() * 100;
- if(inputDocIds[i] % 5 == 0) {
- // multiples of 5 are good
- inputDocScores[i] += 100;
- }
- if(inputDocScores[i] > maxScore) maxScore = inputDocScores[i];
- }
-
- // how many documents are we keeping
- int maxSize = 10000;
-
- int docIds[] = new int[maxSize];
- final double docScores[] = new double[maxSize];
- int docIdWriteIndex = 0;
- int docsByScore[] = new int[maxSize];
- Arrays.fill(docIds, -1);
- Arrays.fill(docScores, 0.0d);
- Arrays.fill(docsByScore, -1);
-
- for(int i = 0; i < inputDocIds.length; i++) {
- int newDocId = inputDocIds[i];
- double newDocScore = inputDocScores[i];
- int smallestDoc = docsByScore[0];
- double smallestScore = smallestDoc == -1 ? 0.0 : docScores[smallestDoc];
- if(docIdWriteIndex < docIds.length || newDocScore > smallestScore) {
- // a new document to store
- if(docIdWriteIndex == docIds.length) {
- // we need to remove the smallest doc
- for(int j = smallestDoc; j < docIds.length -1; j++) {
- docIds[j] = docIds[j + 1];
- docScores[j] = docScores[j + 1];
- }
- docIds[docIds.length -1] = -1;
- docScores[docScores.length -1] = -1;
- for(int j = 0; j < docScores.length -1; j++){
- // the smallest scoring document disappears;
- int newIndex = docsByScore[j + 1];
- if(newIndex > smallestDoc) newIndex--;
- docsByScore[j] = newIndex;
- }
- docsByScore[docsByScore.length - 1] = -1;
- docIdWriteIndex--;
- }
- docIds[docIdWriteIndex] = newDocId;
- docScores[docIdWriteIndex] = newDocScore;
-
- // find the rank for the new doc
- int rank = 0;
- while(docsByScore[rank] >= 0 &&
- newDocScore > docScores[docsByScore[rank]]){
- rank++;
- }
- for(int j = docIdWriteIndex; j > rank; j--) {
- docsByScore[j] = docsByScore[j - 1];
- }
- docsByScore[rank] = docIdWriteIndex;
- docIdWriteIndex++;
- }
- }
-
- // extract the results
-// System.out.print("Retained document IDs:");
-// for(int docId : docIds) System.out.print(" " + docId);
- System.out.println("\nID\tScore");
- for(int i = docsByScore.length - 1; i >= docsByScore.length - 10 ; i--) {
- System.out.println(docIds[docsByScore[i]] + "\t" +
docScores[docsByScore[i]]);
- }
- System.out.println("Max score: " + maxScore);
- }
-
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Learn Windows Azure Live! Tuesday, Dec 13, 2011
Microsoft is holding a special Learn Windows Azure training event for
developers. It will provide a great way to learn Windows Azure and what it
provides. You can attend the event by watching it streamed LIVE online.
Learn more at http://p.sf.net/sfu/ms-windowsazure
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
