Revision: 16570
http://sourceforge.net/p/gate/code/16570
Author: nirajaswani
Date: 2013-03-07 17:38:30 +0000 (Thu, 07 Mar 2013)
Log Message:
-----------
when anonymous mode is disabled, user can now specify names of annotation sets
that refer to individual annotators (param name is
annotationSetNamesAsAnnotators)
Modified Paths:
--------------
gate/trunk/plugins/Teamware_Tools/src/gate/qa/QAForTeamwarePR.java
Modified: gate/trunk/plugins/Teamware_Tools/src/gate/qa/QAForTeamwarePR.java
===================================================================
--- gate/trunk/plugins/Teamware_Tools/src/gate/qa/QAForTeamwarePR.java
2013-03-07 17:16:18 UTC (rev 16569)
+++ gate/trunk/plugins/Teamware_Tools/src/gate/qa/QAForTeamwarePR.java
2013-03-07 17:38:30 UTC (rev 16570)
@@ -20,6 +20,8 @@
import gate.creole.metadata.Optional;
import gate.creole.metadata.RunTime;
import gate.persist.PersistenceException;
+import gate.qa.Measure;
+import gate.qa.QualityAssurancePR;
import gate.security.SecurityException;
import gate.util.OffsetComparator;
@@ -56,790 +58,878 @@
*/
@CreoleResource(name = "QA Summariser for Teamware", comment = "The Quality
Assurance PR for teamware", helpURL =
"http://gate.ac.uk/userguide/sec:eval:qaForTW";)
public class QAForTeamwarePR extends AbstractLanguageAnalyser implements
- ProcessingResource {
- /**
- * types of annotations to use
- */
- private List<String> annotationTypes;
+ ProcessingResource {
+ /**
+ * types of annotations to use
+ */
+ private List<String> annotationTypes;
- /**
- * features to use
- */
- private List<String> featureNames;
+ /**
+ * features to use
+ */
+ private List<String> featureNames;
- /**
- * which measure
- */
- private Measure measure;
+ /**
+ * which measure
+ */
+ private Measure measure;
- /**
- * folder where the output files need to be stored
- */
- private URL outputFolderUrl;
+ /**
+ * folder where the output files need to be stored
+ */
+ private URL outputFolderUrl;
- /**
- * number formatter
- */
- protected NumberFormat f = NumberFormat.getInstance(Locale.ENGLISH);
+ /**
+ * number formatter
+ */
+ protected NumberFormat f = NumberFormat.getInstance(Locale.ENGLISH);
- /**
- * Quality Assurance PR used internally
- */
- private QualityAssurancePR assurancePR;
+ /**
+ * Quality Assurance PR used internally
+ */
+ private QualityAssurancePR assurancePR;
- /**
- * used for keeping record of documents annotated by individual annotators
- */
- private Map<String, Set<Object>> annotatorToDocuments;
+ /**
+ * used for keeping record of documents annotated by individual
annotators
+ */
+ private Map<String, Set<Object>> annotatorToDocuments;
- /**
- * create combinations of annotators who have done annotations on same
- * documents
- */
- private Map<String, Set<Object>> annotatorsPairToDocuments;
+ /**
+ * create combinations of annotators who have done annotations on same
+ * documents
+ */
+ private Map<String, Set<Object>> annotatorsPairToDocuments;
- /**
- * Map used for storing score results
- */
- private Map<String, Map<String, Result>> results = null;
+ /**
+ * Map used for storing score results
+ */
+ private Map<String, Map<String, Result>> results = null;
- /**
- * ignore annotators
- */
- private List<String> annotatorsToIgnore = null;
+ /**
+ * ignore annotators
+ */
+ private List<String> annotatorsToIgnore = null;
- /**
- * Controller with QA PR as part of it.
- */
- private SerialAnalyserController controller;
+ /**
+ * If this parameter is provided, it is assumed that the annonymous
mode is
+ * not in place and therefore the names of annotation sets should be
+ * considered as author names.
+ */
+ private List<String> annotationSetNamesAsAnnotators = null;
- /**
- * name of the consensus annotation set
- */
- public static final String CONSENSUS_AS_NAME = "consensus";
+ /**
+ * Controller with QA PR as part of it.
+ */
+ private SerialAnalyserController controller;
- /**
- * Indicates if the consensus annotation set exists in documents
- */
- private boolean consensusExists = false;
+ /**
+ * name of the consensus annotation set
+ */
+ public static final String CONSENSUS_AS_NAME = "consensus";
- /** Initialise this resource, and return it. */
- public Resource init() throws ResourceInstantiationException {
- f.setMaximumFractionDigits(2); // format used for all decimal values
- f.setMinimumFractionDigits(2);
- // using QualityAssurancePR internally to calculate QA stats
- // but hiding this PR just in case
- FeatureMap hideParams = Factory.newFeatureMap();
- Gate.setHiddenAttribute(hideParams, true);
- assurancePR =
- (QualityAssurancePR)Factory.createResource("gate.qa.QualityAssurancePR",
- Factory.newFeatureMap(), hideParams);
- // we use controller to execute assurance PR
- controller =
- (SerialAnalyserController)Factory.createResource(
- SerialAnalyserController.class.getName(), Factory.newFeatureMap(),
- hideParams);
- controller.add(assurancePR);
- return this;
- } // init()
+ private static final String ANNOTATOR_PREFIX = "TW_Annotator_";
- /**
- * The execute method
- */
- public void execute() throws ExecutionException {
- // the corpus cannot be null or empty
- if(corpus == null || corpus.size() == 0) { throw new ExecutionException(
- "Corpus cannot be null or empty"); }
- // similarly user must provide annotation types that they want to
- // compare
- if(annotationTypes == null || annotationTypes.isEmpty())
- throw new ExecutionException(
- "Please provide at least one annotation type to compare");
- // also a measure to use for computation
- if(measure == null) { throw new ExecutionException("No measure selected");
}
- // check if we are processing the last document in the corpus
- Document firstDocument = (Document)corpus.get(0);
- if(firstDocument == document) {
- annotatorToDocuments = new HashMap<String, Set<Object>>();
- annotatorsPairToDocuments = new HashMap<String, Set<Object>>();
- results = new HashMap<String, Map<String, Result>>();
- consensusExists = false;
- }
- // checking if consensus annotation set exists
- consensusExists =
- (consensusExists || document.getNamedAnnotationSets().containsKey(
- CONSENSUS_AS_NAME));
- // annotators found in this document
- Set<String> annotators = new HashSet<String>();
- for(Object featureName : document.getFeatures().keySet()) {
- if(featureName instanceof String) {
- String fName = (String)featureName;
- if(fName.startsWith("safe.asname.")) {
- String annotatorName = (String)document.getFeatures().get(fName);
- annotators.add(annotatorName);
- }
- }
- }
- // we need to ignore some of the annotators
- if(annotatorsToIgnore != null && !annotatorsToIgnore.isEmpty()) {
- annotators.removeAll(annotatorsToIgnore);
- }
- // if no annotators found print a warning
- if(annotators.isEmpty() || annotators.size() == 1) {
- System.err
- .println("No annotators or only one annotator found for the document "
+
- document.getName() + "\n" +
- "Please make sure the document is annotated using Teamware and " +
- "annotated by atleast two annotators!");
- }
- // if documents are loaded from datastore, we store only the
- // persistence Ids
- // or the document object itself
- Object persistenceId = getDocument().getLRPersistenceId();
- Object toStore = document;
- if(persistenceId != null) {
- toStore = persistenceId;
- }
- // which annotator annotated what documents
- for(String annotatorName : annotators) {
- Set<Object> docs = annotatorToDocuments.get(annotatorName);
- if(docs == null) {
- docs = new HashSet<Object>();
- annotatorToDocuments.put(annotatorName, docs);
- }
- docs.add(toStore);
- }
- // given annotator names, we need to find out possible pairings
- List<String> sortedAnnNames = new ArrayList<String>(annotators);
- Collections.sort(sortedAnnNames);
- for(int i = 0; i < sortedAnnNames.size() - 1; i++) {
- String annName1 = sortedAnnNames.get(i);
- for(int j = i + 1; j < sortedAnnNames.size(); j++) {
- String annName2 = sortedAnnNames.get(j);
- String key = annName1 + ";" + annName2;
- Set<Object> docs = annotatorsPairToDocuments.get(key);
- if(docs == null) {
- docs = new HashSet<Object>();
- annotatorsPairToDocuments.put(key, docs);
- }
- docs.add(toStore);
- }
- }
- // check if we are processing the last document in the corpus
- Document lastDocument = (Document)corpus.get(corpus.size() - 1);
- if(lastDocument != document) { return; }
- // if documents are being loaded from a datastore, it should be
- // deleted
- // after it has been used
- boolean deleteDocs = false;
- // first iteration
- // this is for obtaining IAA between each individual annotator and
- // concensus
- // annotation sets
- for(String annName : annotatorToDocuments.keySet()) {
- // documents annotated by the current annotator
- Set<Object> docs = annotatorToDocuments.get(annName);
- Corpus corpus;
- try {
- corpus = Factory.newCorpus("qaCorpus");
- } catch(ResourceInstantiationException e1) {
- throw new ExecutionException(e1);
- }
- // one doc at a time
- for(Object aDoc : docs) {
- // load from ds if not in memory already
- Document gateDoc = null;
- if(!(aDoc instanceof Document)) {
- try {
- gateDoc =
- (Document)getCorpus().getDataStore().getLr(
- DocumentImpl.class.getName(), aDoc);
- deleteDocs = true;
- } catch(PersistenceException e) {
- throw new ExecutionException(e);
- } catch(SecurityException e) {
- throw new ExecutionException(e);
- }
- } else {
- gateDoc = (Document)aDoc;
- }
- // creating temporary annotation set with annotator's name
- // across all documents in the corpus and copying annotations
- // produced
- // by him in that document into his annotationset
- createAnnSet(gateDoc, getSetName(gateDoc, annName), annName);
- // add document to the corpus
- // doc will remain in memory until all computations are done
- corpus.add(gateDoc);
- }
- // calculating IAA stats between annotator's annotations and
- // consensus annotation set
- if(consensusExists) {
- calculateIAA(corpus, annName, CONSENSUS_AS_NAME, annName);
- }
- // deleting newly created annotation set and unloading documents
- // if necessary
- for(int k = corpus.size() - 1; k >= 0; k--) {
- Document aDoc = (Document)corpus.get(k);
- if(deleteDocs) {
- Factory.deleteResource(aDoc);
- continue;
- }
- aDoc.removeAnnotationSet(annName);
- }
- // delete the corpus as well
- Factory.deleteResource(corpus);
- }
- // preparing for second iteration
- deleteDocs = false;
- // second iteration
- // this is for obtaining IAA between individual annotators
- for(String annotatorsPair : annotatorsPairToDocuments.keySet()) {
- // documents annotated by this pair of annotators
- Set<Object> docs = annotatorsPairToDocuments.get(annotatorsPair);
- Corpus corpus;
- try {
- corpus = Factory.newCorpus("qaCorpus");
- } catch(ResourceInstantiationException e1) {
- throw new ExecutionException(e1);
- }
- // annotators in this pair
- String annotator1 =
- annotatorsPair.substring(0, annotatorsPair.indexOf(';'));
- String annotator2 =
- annotatorsPair.substring(annotatorsPair.indexOf(';') + 1);
- // one doc at a time
- for(Object aDoc : docs) {
- // load the doc from DS if needed
- Document gateDoc = null;
- if(!(aDoc instanceof Document)) {
- try {
- gateDoc =
- (Document)getCorpus().getDataStore().getLr(
- DocumentImpl.class.getName(), aDoc);
- deleteDocs = true;
- } catch(PersistenceException e) {
- throw new ExecutionException(e);
- } catch(SecurityException e) {
- throw new ExecutionException(e);
- }
- } else {
- gateDoc = (Document)aDoc;
- }
- // creating sets for individual annotators and copying
- // annotations from
- // their anonymous annotation sets into their own annotation set
- createAnnSet(gateDoc, getSetName(gateDoc, annotator1), annotator1);
- createAnnSet(gateDoc, getSetName(gateDoc, annotator2), annotator2);
- // adding doc to the corpus
- corpus.add(gateDoc);
- }
- // calculate IAA
- calculateIAA(corpus, annotator1, annotator2, annotator1 + "-" +
- annotator2);
- // deleting temporarily created annotation sets and deleting docs
- // from
- // memory if docs were loaded from the DS
- for(int k = corpus.size() - 1; k >= 0; k--) {
- Document aDoc = (Document)corpus.get(k);
- if(deleteDocs) {
- Factory.deleteResource(aDoc);
- continue;
- }
- aDoc.removeAnnotationSet(annotator1);
- aDoc.removeAnnotationSet(annotator2);
- }
- // delete the corpus
- Factory.deleteResource(corpus);
- }
- // generating summary
- // only accept the files with document-stats at the end
- File[] resultFiles =
- new File(outputFolderUrl.getFile()).listFiles(new FileFilter() {
- public boolean accept(File pathname) {
- return pathname.getAbsolutePath().endsWith("-document-stats.html");
- }
- });
- // authors found in the corpus
- List<String> columnAuthorNames = new ArrayList<String>();
- double consensusMacro = 0.0D;
- double consensusMicro = 0.0D;
- double annotatorMacro = 0.0D;
- double annotatorMicro = 0.0D;
- // no result file found so quitting the execution
- if(resultFiles == null) {
- System.err
- .println("WARNING: not enough information (most probably annotators)"
- + " found for the summary to be generated!");
- return;
- }
- // one file at a time
- for(File file : resultFiles) {
- // finding author names
- String fileName =
- file.getName().substring(0,
- file.getName().indexOf("-document-stats.html"));
- int index = fileName.indexOf('-');
- String author1 = null;
- String author2 = null;
- // if only one author found, it means this file is for author vs
- // consensus
- // IAA
- if(index < 0) {
- author1 = fileName;
- author2 = CONSENSUS_AS_NAME;
- } else {
- author1 = fileName.substring(0, index);
- author2 = fileName.substring(index + 1);
- }
- if(!columnAuthorNames.contains(author2)) {
- columnAuthorNames.add(author2);
- }
- if(!columnAuthorNames.contains(author1)) {
- columnAuthorNames.add(author1);
- }
- // loading the document-stats file as GATE document
- // and utilizing original markups to collect the needed
- // information
- // create a gate document
- Document aDoc = null;
- try {
- aDoc = Factory.newDocument(file.toURI().toURL());
- } catch(ResourceInstantiationException e) {
- throw new ExecutionException(e);
- } catch(MalformedURLException e) {
- throw new ExecutionException(e);
- }
- // we're interested in macro and micro averages figures only
- // these are the last two rows in the document-stats
- AnnotationSet omSet = aDoc.getAnnotations("Original markups");
- List<Annotation> rows = new ArrayList<Annotation>(omSet.get("tr"));
- Collections.sort(rows, new OffsetComparator());
- // temporary result object
- Result r = new Result();
- r.documentFileName = file.getName();
- // consider last row for micro summary
- Annotation row = rows.get(rows.size() - 1);
- List<Annotation> cols =
- new ArrayList<Annotation>(omSet.getContained(Utils.start(row),
- Utils.end(row)).get("td"));
- Collections.sort(cols, new OffsetComparator());
- // only interested in the last column
- r.micro =
- Double.parseDouble(Utils.stringFor(aDoc, cols.get(cols.size() - 1)));
- // consider second last row for macro summary
- row = rows.get(rows.size() - 2);
- cols =
- new ArrayList<Annotation>(omSet.getContained(Utils.start(row),
- Utils.end(row)).get("td"));
- Collections.sort(cols, new OffsetComparator());
- // only interested in the last column
- r.macro =
- Double.parseDouble(Utils.stringFor(aDoc, cols.get(cols.size() - 1)));
- // delete the document from GATe
- Factory.deleteResource(aDoc);
- // making two entries
- // i.e. author1-author2 and author2-author1
- Map<String, Result> authorResults = results.get(author1);
- if(authorResults == null) {
- authorResults = new HashMap<String, Result>();
- results.put(author1, authorResults);
- }
- authorResults.put(author2, r);
- authorResults = results.get(author2);
- if(authorResults == null) {
- authorResults = new HashMap<String, Result>();
- results.put(author2, authorResults);
- }
- authorResults.put(author1, r);
- }
- // collected all the results
- Collections.sort(columnAuthorNames);
- // generating html file contents
- StringBuffer buffer = new StringBuffer();
- buffer.append("<html>\n<title>Summary of IAA Results</title>\n<body>\n");
- buffer.append("<h1>Summary of IAA Results</h1>");
- buffer.append("<b>AnnotationTypes:</b> ");
- for(String aType : annotationTypes) {
- buffer.append(aType + ";");
- }
- buffer.append("<br>");
- buffer.append("<b>Features:</b> ");
- if(featureNames != null && !featureNames.isEmpty()) {
- for(String aFeature : featureNames) {
- buffer.append(aFeature + ";");
- }
- }
- buffer.append("<br>");
- buffer.append("<b>Measure:</b> " + measure.toString() + "<br>");
- buffer.append("<table border=\"1\">\n");
- // first row
- buffer.append("\t<tr>\n");
- buffer.append("\t\t<td><b>Author Names</b></td>\n");
- for(String author2 : columnAuthorNames) {
- buffer.append("\t\t<td colspan=\"2\"><b>" + author2 + "</b></td>\n");
- }
- buffer.append("\t\t<td colspan=\"2\"><b> Averages </b></td>\n");
- buffer.append("\t</tr>\n");
- // second row
- buffer.append("\t<tr>\n");
- buffer.append("\t\t<td> </td>\n");
- // additional columns for averages
- for(int i = 0; i <= columnAuthorNames.size(); i++) {
- buffer.append("\t\t<td><b>Macro</b></td>\n");
- buffer.append("\t\t<td><b>Micro</b></td>\n");
- }
- buffer.append("\t</tr>\n");
- // color coding with transperancy
- // 1.0 = dark green
- // ... as we proceed we lighten the green color
- // 0.5 = white
- // ... as we proceed we brighten the red color
- // 0.0 = red
- List<String> authorNamesList = new ArrayList<String>(results.keySet());
- Collections.sort(authorNamesList);
- // all rows onwards
- // producing matrix
- for(String author1 : authorNamesList) {
- double annMacro = 0.0D;
- double annMicro = 0.0D;
- int docs = 0;
- buffer.append("\t<tr>\n");
- buffer.append("\t\t<td><b>" + author1 + "</b></td>\n");
- Map<String, Result> resultsForAuthor1 = results.get(author1);
- // columns
- for(String author2 : columnAuthorNames) {
- Result r = resultsForAuthor1.get(author2);
- if(r == null) {
- buffer.append("\t\t<td colspan=\"2\"> </td>\n");
- } else {
- // append macro micro figures to the table
- appendMacroMicroFigures(buffer, r.macro, r.micro);
- if(author1.equals(CONSENSUS_AS_NAME)) {
- consensusMacro += r.macro;
- consensusMicro += r.micro;
- } else {
- annMacro += r.macro;
- annMicro += r.micro;
- docs++;
- }
- }
- }
- // adding averages to the last 2 columns
- if(author1.equals(CONSENSUS_AS_NAME)) {
- double caMacroAvg =
- (double)(consensusMacro / (double)(columnAuthorNames.size() - 1));
- double caMicroAvg =
- (double)(consensusMicro / (double)(columnAuthorNames.size() - 1));
- appendMacroMicroFigures(buffer, caMacroAvg, caMicroAvg);
- } else {
- double aaMacroAvg = (double)(annMacro / (double)(docs));
- double aaMicroAvg = (double)(annMicro / (double)(docs));
- annotatorMacro += aaMacroAvg;
- annotatorMicro += aaMicroAvg;
- appendMacroMicroFigures(buffer, aaMacroAvg, aaMicroAvg);
- }
- buffer.append("\t</tr>\n");
- buffer.append("\t<tr>\n");
- buffer.append("\t\t<td> </td>\n");
- for(String author2 : columnAuthorNames) {
- Result r = resultsForAuthor1.get(author2);
- if(r == null) {
- buffer.append("\t\t<td colspan=\"2\"> </td>\n");
- } else {
- buffer.append("\t\t<td colspan=\"2\"><a target=\"_blank\" href=\"" +
- r.documentFileName + "\">document</a></td>\n");
- }
- }
- buffer.append("\t\t<td colspan=\"2\"> </td>\n");
- }
- buffer.append("</table>\n");
- buffer.append("<hr>");
- if(consensusExists) {
- buffer.append("<br><b>Avg. " + CONSENSUS_AS_NAME + " macro avg:</b> " +
- (f.format((double)consensusMacro / (columnAuthorNames.size() - 1))));
- buffer.append("<br><b>Avg. " + CONSENSUS_AS_NAME + " micro avg:</b> " +
- (f.format((double)consensusMicro / (columnAuthorNames.size() - 1))));
- }
- // if consensus annotation set exists, make sure the number used in
- // demoninator does not include consensus as one annotator
- int totalAuthors =
- consensusExists ? columnAuthorNames.size() - 1 :
columnAuthorNames.size();
- buffer.append("<br><b>Avg. IAA macro avg:</b> " +
- (f.format((double)annotatorMacro / totalAuthors)));
- buffer.append("<br><b>Avg. IAA micro avg:</b> " +
- (f.format((double)annotatorMicro / totalAuthors)));
- buffer.append("</body>\n</html>");
- BufferedWriter bw = null;
- try {
- File indexFile = new File(outputFolderUrl.getFile(), "index.html");
- bw = new BufferedWriter(new FileWriter(indexFile));
- bw.write(buffer.toString());
- } catch(IOException ioe) {
- throw new ExecutionException(ioe);
- } finally {
- if(bw != null) {
- try {
- bw.close();
- } catch(IOException e) {
- throw new ExecutionException(e);
- }
- }
- }
- }
+ /**
+ * Indicates if the consensus annotation set exists in documents
+ */
+ private boolean consensusExists = false;
- /**
- * A method that adds two columns to the buffer - one for the macro figure
and
- * the other one for the micro figure. It also adds relevant style tag to
give
- * a proper color to each cell depending on the value of macro and micro
- * figures. See documentation of the getStyleTag(double) for more information
- * on how cell backgrounds are color coded.
- *
- * @param buffer
- * @param macro
- * @param micro
- */
- private void appendMacroMicroFigures(StringBuffer buffer, double macro,
- double micro) {
- String macCol = getStyleTag(macro);
- String micCol = getStyleTag(micro);
- buffer.append("\t\t<td " + macCol + " >" + f.format(macro) + "</td>\n");
- buffer.append("\t\t<td " + micCol + " >" + f.format(micro) + "</td>\n");
- }
+ /** Initialise this resource, and return it. */
+ public Resource init() throws ResourceInstantiationException {
+ f.setMaximumFractionDigits(2); // format used for all decimal
values
+ f.setMinimumFractionDigits(2);
+ // using QualityAssurancePR internally to calculate QA stats
+ // but hiding this PR just in case
+ FeatureMap hideParams = Factory.newFeatureMap();
+ Gate.setHiddenAttribute(hideParams, true);
+ assurancePR = (QualityAssurancePR) Factory.createResource(
+ "gate.qa.QualityAssurancePR",
Factory.newFeatureMap(),
+ hideParams);
+ // we use controller to execute assurance PR
+ controller = (SerialAnalyserController) Factory.createResource(
+ SerialAnalyserController.class.getName(),
+ Factory.newFeatureMap(), hideParams);
+ controller.add(assurancePR);
+ return this;
+ } // init()
- /**
- * Gets the style tag based on the score. The color green is used for a cell
- * background to indicate full agreement (i.e. 1.0). The background color
- * becomes lighter as the agreement reduces towards 0.5. At 0.5 agreement,
the
- * background color of a cell is fully white. From 0.5 downwards, the color
- * red is used and as the agreement reduces further, the color becomes darker
- * with dark red at 0.0 agreement.
- *
- * @param score
- * @return
- */
- private String getStyleTag(double score) {
- // two colors
- Color gc = Color.GREEN;
- Color rc = Color.RED;
- Color c = null;
- // if score is above .50, use the green color, otherwise the red one
- if(score > 0.50) {
- c =
- new Color(gc.getRed(), gc.getGreen(), gc.getBlue(),
- ((int)(score * 100) * 2) - 100);
- } else {
- c =
- new Color(rc.getRed(), rc.getGreen(), rc.getBlue(),
- 100 - ((int)(score * 100) * 2));
- }
- return "style=\"background-color: rgba(" + c.getRed() + "," + c.getGreen()
+
- "," + c.getBlue() + "," + ((double)c.getAlpha() / 100) + ")\"";
- }
+ /**
+ * The execute method
+ */
+ public void execute() throws ExecutionException {
+ // the corpus cannot be null or empty
+ if (corpus == null || corpus.size() == 0) {
+ throw new ExecutionException("Corpus cannot be null or
empty");
+ }
+ // similarly user must provide annotation types that they want
to
+ // compare
+ if (annotationTypes == null || annotationTypes.isEmpty())
+ throw new ExecutionException(
+ "Please provide at least one annotation
type to compare");
+ // also a measure to use for computation
+ if (measure == null) {
+ throw new ExecutionException("No measure selected");
+ }
+ // check if we are processing the last document in the corpus
+ Document firstDocument = (Document) corpus.get(0);
+ if (firstDocument == document) {
+ annotatorToDocuments = new HashMap<String,
Set<Object>>();
+ annotatorsPairToDocuments = new HashMap<String,
Set<Object>>();
+ results = new HashMap<String, Map<String, Result>>();
+ consensusExists = false;
+ }
+ // checking if consensus annotation set exists
+ consensusExists = (consensusExists ||
document.getNamedAnnotationSets()
+ .containsKey(CONSENSUS_AS_NAME));
- /**
- * Given the annotator name, the method finds out the annotation set which
- * contains annotations for that annotator
- *
- * @param doc
- * @param annotatorName
- * @return
- */
- private String getSetName(Document doc, String annotatorName) {
- String inputAS = null;
- for(Object key : doc.getFeatures().keySet()) {
- Object val = doc.getFeatures().get(key);
- if(val.equals(annotatorName)) {
- inputAS = key.toString().substring(12);
- break;
- }
- }
- return inputAS;
- }
+ // this is where we store names of annotators
+ Set<String> annotators = new HashSet<String>();
- /**
- * Creating a temporary annotation set
- *
- * @param doc
- * @param inputAS
- * @param outputAS
- */
- private void createAnnSet(Document doc, String inputAS, String outputAS) {
- AnnotationSet inAS =
- inputAS == null || inputAS.trim().length() == 0
- ? doc.getAnnotations()
- : doc.getAnnotations(inputAS);
- inAS = inAS.get(new HashSet<String>(annotationTypes));
- AnnotationSet outAS = doc.getAnnotations(outputAS);
- for(Annotation a : inAS) {
- outAS.add(a.getStartNode(), a.getEndNode(), a.getType(),
a.getFeatures());
- }
- }
+ // we ignore the following block if anonymous mode was disabled
+ // and we determine that by checking if user has provided values
+ // for annotationSetNamesAsAnnotators
+ if (annotationSetNamesAsAnnotators == null
+ || annotationSetNamesAsAnnotators.isEmpty()) {
+ // annotators found in this document
+ for (Object featureName :
document.getFeatures().keySet()) {
+ if (featureName instanceof String) {
+ String fName = (String) featureName;
+ if (fName.startsWith("safe.asname.")) {
+ String annotatorName = (String)
document.getFeatures()
+ .get(fName);
+ annotators.add(annotatorName);
+ }
+ }
+ }
- /**
- * A method that calculates IAA between the given annotation sets
- *
- * @param corpus
- * @param keyAS
- * @param responseAS
- * @param filePrefix
- * @throws ExecutionException
- */
- private void calculateIAA(Corpus corpus, String keyAS, String responseAS,
- String filePrefix) throws ExecutionException {
- try {
- assurancePR.reInit();
- } catch(ResourceInstantiationException rie) {
- throw new ExecutionException(rie);
- }
- // lets set the params on qualityAssurancePR
- assurancePR.setAnnotationTypes(annotationTypes);
- assurancePR.setFeatureNames(featureNames == null
- ? new ArrayList<String>()
- : featureNames);
- assurancePR.setMeasure(measure);
- assurancePR.setOutputFolderUrl(outputFolderUrl);
- // the remaining two params will be set later
- assurancePR.setKeyASName(keyAS);
- assurancePR.setResponseASName(responseAS);
- controller.setCorpus(corpus);
- controller.execute();
- // QA PR produces two stats file, one for corpus and one for
- // documents
- // for QAForTeamware, we don't need corpus one
- File corpusFile = new File(outputFolderUrl.getFile(), "corpus-stats.html");
- corpusFile.delete();
- File documentFile =
- new File(outputFolderUrl.getFile(), "document-stats.html");
- File documentOutFile =
- new File(outputFolderUrl.getFile(), filePrefix + "-document-stats.html");
- documentFile.renameTo(documentOutFile);
- }
+ // we need to ignore some of the annotators if we are
instructed to
+ // do
+ // so
+ if (annotatorsToIgnore != null &&
!annotatorsToIgnore.isEmpty()) {
+ annotators.removeAll(annotatorsToIgnore);
+ }
+ } else {
+ // find out which of the annotationSetNamesAsAnnotators
+ // has annotated this document
+ for (String annotatorName :
annotationSetNamesAsAnnotators) {
+ if (document.getNamedAnnotationSets()
+ .containsKey(annotatorName)) {
+ annotators.add(annotatorName);
+ }
+ }
+ }
- /**
- * Method called when a PR is unloaded.
- */
- public void cleanup() {
- super.cleanup();
- Factory.deleteResource(assurancePR);
- Factory.deleteResource(controller);
- }
+ // if no annotators found print a warning
+ if (annotators.isEmpty() || annotators.size() == 1) {
+ System.err
+ .println("No annotators or only one
annotator found for the document "
+ + document.getName()
+ + "\n"
+ + "Please make sure the
document is annotated using Teamware and "
+ + "annotated by atleast
two annotators!");
+ }
- /**
- * Annotation types for which the stats should be calculated
- *
- * @return
- */
- public List<String> getAnnotationTypes() {
- return annotationTypes;
- }
+ // if documents are loaded from datastore, we store only the
+ // persistence Ids
+ // or the document object itself
+ Object persistenceId = getDocument().getLRPersistenceId();
+ Object toStore = document;
+ if (persistenceId != null) {
+ toStore = persistenceId;
+ }
- /**
- * Annotation types for which the stats should be calculated
- *
- * @param annotationTypes
- */
- @RunTime
- @CreoleParameter
- public void setAnnotationTypes(List<String> annotationTypes) {
- this.annotationTypes = annotationTypes;
- }
+ // which annotator annotated what documents
+ for (String annotatorName : annotators) {
+ Set<Object> docs =
annotatorToDocuments.get(annotatorName);
+ if (docs == null) {
+ docs = new HashSet<Object>();
+ annotatorToDocuments.put(annotatorName, docs);
+ }
+ docs.add(toStore);
+ }
- /**
- * Features names for which the stats should be calculated
- *
- * @return
- */
- public List<String> getFeatureNames() {
- return featureNames;
- }
+ // given annotator names, we need to find out possible pairings
+ List<String> sortedAnnNames = new ArrayList<String>(annotators);
+ Collections.sort(sortedAnnNames);
+ for (int i = 0; i < sortedAnnNames.size() - 1; i++) {
+ String annName1 = sortedAnnNames.get(i);
+ for (int j = i + 1; j < sortedAnnNames.size(); j++) {
+ String annName2 = sortedAnnNames.get(j);
+ String key = annName1 + ";" + annName2;
+ Set<Object> docs =
annotatorsPairToDocuments.get(key);
+ if (docs == null) {
+ docs = new HashSet<Object>();
+ annotatorsPairToDocuments.put(key,
docs);
+ }
+ docs.add(toStore);
+ }
+ }
- /**
- * Features names for which the stats should be calculated
- *
- * @param featureNames
- */
- @RunTime
- @Optional
- @CreoleParameter
- public void setFeatureNames(List<String> featureNames) {
- this.featureNames = featureNames;
- }
+ // check if we are processing the last document in the corpus
+ Document lastDocument = (Document) corpus.get(corpus.size() -
1);
+ if (lastDocument != document) {
+ return;
+ }
- /**
- * Measure to use for stats calculation
- *
- * @return
- */
- public Measure getMeasure() {
- return measure;
- }
+ // if documents are being loaded from a datastore, it should be
+ // deleted
+ // after it has been used
+ boolean deleteDocs = false;
+ // first iteration
+ // this is for obtaining IAA between each individual annotator
and
+ // concensus
+ // annotation sets
+ for (String annName : annotatorToDocuments.keySet()) {
+ // documents annotated by the current annotator
+ Set<Object> docs = annotatorToDocuments.get(annName);
+ Corpus corpus;
+ try {
+ corpus = Factory.newCorpus("qaCorpus");
+ } catch (ResourceInstantiationException e1) {
+ throw new ExecutionException(e1);
+ }
+ // one doc at a time
+ for (Object aDoc : docs) {
+ // load from ds if not in memory already
+ Document gateDoc = null;
+ if (!(aDoc instanceof Document)) {
+ try {
+ gateDoc = (Document)
getCorpus().getDataStore().getLr(
+
DocumentImpl.class.getName(), aDoc);
+ deleteDocs = true;
+ } catch (PersistenceException e) {
+ throw new ExecutionException(e);
+ } catch (SecurityException e) {
+ throw new ExecutionException(e);
+ }
+ } else {
+ gateDoc = (Document) aDoc;
+ }
+ // creating temporary annotation set with
annotator's name
+ // across all documents in the corpus and
copying annotations
+ // produced
+ // by him in that document into his
annotationset
+ createAnnSet(gateDoc, getSetName(gateDoc,
annName), ANNOTATOR_PREFIX + annName);
- /**
- * Measure to use for stats calculation
- *
- * @param measure
- */
- @RunTime
- @CreoleParameter
- public void setMeasure(Measure measure) {
- this.measure = measure;
- }
+ // add document to the corpus
+ // doc will remain in memory until all
computations are done
+ corpus.add(gateDoc);
+ }
+ // calculating IAA stats between annotator's
annotations and
+ // consensus annotation set
+ if (consensusExists) {
+ calculateIAA(corpus, ANNOTATOR_PREFIX +
annName, CONSENSUS_AS_NAME, annName);
+ }
- /**
- * URL of the folder to store output files into
- *
- * @return
- */
- public URL getOutputFolderUrl() {
- return outputFolderUrl;
- }
+ // deleting newly created annotation set and unloading
documents
+ // if necessary
+ for (int k = corpus.size() - 1; k >= 0; k--) {
+ Document aDoc = (Document) corpus.get(k);
+ if (deleteDocs) {
+ Factory.deleteResource(aDoc);
+ continue;
+ }
- /**
- * URL of the folder to store output files into
- *
- * @param outputFolderUrl
- */
- @RunTime
- @CreoleParameter(suffixes = "html")
- public void setOutputFolderUrl(URL outputFolderUrl) {
- this.outputFolderUrl = outputFolderUrl;
- }
+ aDoc.removeAnnotationSet(ANNOTATOR_PREFIX +
annName);
+ }
+ // delete the corpus as well
+ Factory.deleteResource(corpus);
+ }
- /**
- * annotators to ignore when computing IAA
- *
- * @return
- */
- public List<String> getAnnotatorsToIgnore() {
- return annotatorsToIgnore;
- }
+ // preparing for second iteration
+ deleteDocs = false;
- /**
- * annotators to ingore when computing IAA
- *
- * @param annotatorsToIgnore
- */
- @RunTime
- @Optional
- @CreoleParameter
- public void setAnnotatorsToIgnore(List<String> annotatorsToIgnore) {
- this.annotatorsToIgnore = annotatorsToIgnore;
- }
+ // second iteration
+ // this is for obtaining IAA between individual annotators
+ for (String annotatorsPair :
annotatorsPairToDocuments.keySet()) {
+ // documents annotated by this pair of annotators
+ Set<Object> docs =
annotatorsPairToDocuments.get(annotatorsPair);
+ Corpus corpus;
+ try {
+ corpus = Factory.newCorpus("qaCorpus");
+ } catch (ResourceInstantiationException e1) {
+ throw new ExecutionException(e1);
+ }
+ // annotators in this pair
+ String annotator1 = annotatorsPair.substring(0,
+ annotatorsPair.indexOf(';'));
+ String annotator2 =
annotatorsPair.substring(annotatorsPair
+ .indexOf(';') + 1);
+ // one doc at a time
+ for (Object aDoc : docs) {
+ // load the doc from DS if needed
+ Document gateDoc = null;
+ if (!(aDoc instanceof Document)) {
+ try {
+ gateDoc = (Document)
getCorpus().getDataStore().getLr(
+
DocumentImpl.class.getName(), aDoc);
+ deleteDocs = true;
+ } catch (PersistenceException e) {
+ throw new ExecutionException(e);
+ } catch (SecurityException e) {
+ throw new ExecutionException(e);
+ }
+ } else {
+ gateDoc = (Document) aDoc;
+ }
+ // creating sets for individual annotators and
copying
+ // annotations from
+ // their anonymous annotation sets into their
own annotation set
+ createAnnSet(gateDoc, getSetName(gateDoc,
annotator1),
+ ANNOTATOR_PREFIX + annotator1);
+ createAnnSet(gateDoc, getSetName(gateDoc,
annotator2),
+ ANNOTATOR_PREFIX + annotator2);
+ // adding doc to the corpus
+ corpus.add(gateDoc);
+ }
+ // calculate IAA
+ calculateIAA(corpus, ANNOTATOR_PREFIX + annotator1,
+ ANNOTATOR_PREFIX + annotator2,
annotator1 + "-"
+ + annotator2);
+ // deleting temporarily created annotation sets and
deleting docs
+ // from
+ // memory if docs were loaded from the DS
+ for (int k = corpus.size() - 1; k >= 0; k--) {
+ Document aDoc = (Document) corpus.get(k);
+ if (deleteDocs) {
+ Factory.deleteResource(aDoc);
+ continue;
+ }
- /**
- * Storing individual results for each pair of annotators
- *
- * @author niraj
- */
- class Result {
- // macro average
- double macro = 0.0D;
+ aDoc.removeAnnotationSet(ANNOTATOR_PREFIX +
annotator1);
+ aDoc.removeAnnotationSet(ANNOTATOR_PREFIX +
annotator2);
+ }
+ // delete the corpus
+ Factory.deleteResource(corpus);
+ }
+ // generating summary
+ // only accept the files with document-stats at the end
+ File[] resultFiles = new File(outputFolderUrl.getFile())
+ .listFiles(new FileFilter() {
+ public boolean accept(File pathname) {
+ return
pathname.getAbsolutePath().endsWith(
+
"-document-stats.html");
+ }
+ });
+ // authors found in the corpus
+ List<String> columnAuthorNames = new ArrayList<String>();
+ double consensusMacro = 0.0D;
+ double consensusMicro = 0.0D;
+ double annotatorMacro = 0.0D;
+ double annotatorMicro = 0.0D;
+ // no result file found so quitting the execution
+ if (resultFiles == null) {
+ System.err
+ .println("WARNING: not enough
information (most probably annotators)"
+ + " found for the
summary to be generated!");
+ return;
+ }
+ // one file at a time
+ for (File file : resultFiles) {
+ // finding author names
+ String fileName = file.getName().substring(0,
+
file.getName().indexOf("-document-stats.html"));
+ int index = fileName.indexOf('-');
+ String author1 = null;
+ String author2 = null;
+ // if only one author found, it means this file is for
author vs
+ // consensus
+ // IAA
+ if (index < 0) {
+ author1 = fileName;
+ author2 = CONSENSUS_AS_NAME;
+ } else {
+ author1 = fileName.substring(0, index);
+ author2 = fileName.substring(index + 1);
+ }
+ if (!columnAuthorNames.contains(author2)) {
+ columnAuthorNames.add(author2);
+ }
+ if (!columnAuthorNames.contains(author1)) {
+ columnAuthorNames.add(author1);
+ }
+ // loading the document-stats file as GATE document
+ // and utilizing original markups to collect the needed
+ // information
+ // create a gate document
+ Document aDoc = null;
+ try {
+ aDoc =
Factory.newDocument(file.toURI().toURL());
+ } catch (ResourceInstantiationException e) {
+ throw new ExecutionException(e);
+ } catch (MalformedURLException e) {
+ throw new ExecutionException(e);
+ }
+ // we're interested in macro and micro averages figures
only
+ // these are the last two rows in the document-stats
+ AnnotationSet omSet = aDoc.getAnnotations("Original
markups");
+ List<Annotation> rows = new
ArrayList<Annotation>(omSet.get("tr"));
+ Collections.sort(rows, new OffsetComparator());
+ // temporary result object
+ Result r = new Result();
+ r.documentFileName = file.getName();
+ // consider last row for micro summary
+ Annotation row = rows.get(rows.size() - 1);
+ List<Annotation> cols = new ArrayList<Annotation>(omSet
+ .getContained(Utils.start(row),
Utils.end(row)).get("td"));
+ Collections.sort(cols, new OffsetComparator());
+ // only interested in the last column
+ r.micro = Double.parseDouble(Utils.stringFor(aDoc,
+ cols.get(cols.size() - 1)));
+ // consider second last row for macro summary
+ row = rows.get(rows.size() - 2);
+ cols = new ArrayList<Annotation>(omSet.getContained(
+ Utils.start(row),
Utils.end(row)).get("td"));
+ Collections.sort(cols, new OffsetComparator());
+ // only interested in the last column
+ r.macro = Double.parseDouble(Utils.stringFor(aDoc,
+ cols.get(cols.size() - 1)));
+ // delete the document from GATe
+ Factory.deleteResource(aDoc);
+ // making two entries
+ // i.e. author1-author2 and author2-author1
+ Map<String, Result> authorResults =
results.get(author1);
+ if (authorResults == null) {
+ authorResults = new HashMap<String, Result>();
+ results.put(author1, authorResults);
+ }
+ authorResults.put(author2, r);
+ authorResults = results.get(author2);
+ if (authorResults == null) {
+ authorResults = new HashMap<String, Result>();
+ results.put(author2, authorResults);
+ }
+ authorResults.put(author1, r);
+ }
+ // collected all the results
+ Collections.sort(columnAuthorNames);
+ // generating html file contents
+ StringBuffer buffer = new StringBuffer();
+ buffer.append("<html>\n<title>Summary of IAA
Results</title>\n<body>\n");
+ buffer.append("<h1>Summary of IAA Results</h1>");
+ buffer.append("<b>AnnotationTypes:</b> ");
+ for (String aType : annotationTypes) {
+ buffer.append(aType + ";");
+ }
+ buffer.append("<br>");
+ buffer.append("<b>Features:</b> ");
+ if (featureNames != null && !featureNames.isEmpty()) {
+ for (String aFeature : featureNames) {
+ buffer.append(aFeature + ";");
+ }
+ }
+ buffer.append("<br>");
+ buffer.append("<b>Measure:</b> " + measure.toString() + "<br>");
+ buffer.append("<table border=\"1\">\n");
+ // first row
+ buffer.append("\t<tr>\n");
+ buffer.append("\t\t<td><b>Author Names</b></td>\n");
+ for (String author2 : columnAuthorNames) {
+ buffer.append("\t\t<td colspan=\"2\"><b>" + author2 +
"</b></td>\n");
+ }
+ buffer.append("\t\t<td colspan=\"2\"><b> Averages </b></td>\n");
+ buffer.append("\t</tr>\n");
+ // second row
+ buffer.append("\t<tr>\n");
+ buffer.append("\t\t<td> </td>\n");
+ // additional columns for averages
+ for (int i = 0; i <= columnAuthorNames.size(); i++) {
+ buffer.append("\t\t<td><b>Macro</b></td>\n");
+ buffer.append("\t\t<td><b>Micro</b></td>\n");
+ }
+ buffer.append("\t</tr>\n");
+ // color coding with transperancy
+ // 1.0 = dark green
+ // ... as we proceed we lighten the green color
+ // 0.5 = white
+ // ... as we proceed we brighten the red color
+ // 0.0 = red
+ List<String> authorNamesList = new
ArrayList<String>(results.keySet());
+ Collections.sort(authorNamesList);
+ // all rows onwards
+ // producing matrix
+ for (String author1 : authorNamesList) {
+ double annMacro = 0.0D;
+ double annMicro = 0.0D;
+ int docs = 0;
+ buffer.append("\t<tr>\n");
+ buffer.append("\t\t<td><b>" + author1 + "</b></td>\n");
+ Map<String, Result> resultsForAuthor1 =
results.get(author1);
+ // columns
+ for (String author2 : columnAuthorNames) {
+ Result r = resultsForAuthor1.get(author2);
+ if (r == null) {
+ buffer.append("\t\t<td
colspan=\"2\"> </td>\n");
+ } else {
+ // append macro micro figures to the
table
+ appendMacroMicroFigures(buffer,
r.macro, r.micro);
+ if (author1.equals(CONSENSUS_AS_NAME)) {
+ consensusMacro += r.macro;
+ consensusMicro += r.micro;
+ } else {
+ annMacro += r.macro;
+ annMicro += r.micro;
+ docs++;
+ }
+ }
+ }
+ // adding averages to the last 2 columns
+ if (author1.equals(CONSENSUS_AS_NAME)) {
+ double caMacroAvg = (double) (consensusMacro /
(double) (columnAuthorNames
+ .size() - 1));
+ double caMicroAvg = (double) (consensusMicro /
(double) (columnAuthorNames
+ .size() - 1));
+ appendMacroMicroFigures(buffer, caMacroAvg,
caMicroAvg);
+ } else {
+ double aaMacroAvg = (double) (annMacro /
(double) (docs));
+ double aaMicroAvg = (double) (annMicro /
(double) (docs));
+ annotatorMacro += aaMacroAvg;
+ annotatorMicro += aaMicroAvg;
+ appendMacroMicroFigures(buffer, aaMacroAvg,
aaMicroAvg);
+ }
+ buffer.append("\t</tr>\n");
+ buffer.append("\t<tr>\n");
+ buffer.append("\t\t<td> </td>\n");
+ for (String author2 : columnAuthorNames) {
+ Result r = resultsForAuthor1.get(author2);
+ if (r == null) {
+ buffer.append("\t\t<td
colspan=\"2\"> </td>\n");
+ } else {
+ buffer.append("\t\t<td colspan=\"2\"><a
target=\"_blank\" href=\""
+ + r.documentFileName +
"\">document</a></td>\n");
+ }
+ }
+ buffer.append("\t\t<td colspan=\"2\"> </td>\n");
+ }
+ buffer.append("</table>\n");
+ buffer.append("<hr>");
+ if (consensusExists) {
+ buffer.append("<br><b>Avg. "
+ + CONSENSUS_AS_NAME
+ + " macro avg:</b> "
+ + (f.format((double) consensusMacro
+ /
(columnAuthorNames.size() - 1))));
+ buffer.append("<br><b>Avg. "
+ + CONSENSUS_AS_NAME
+ + " micro avg:</b> "
+ + (f.format((double) consensusMicro
+ /
(columnAuthorNames.size() - 1))));
+ }
+ // if consensus annotation set exists, make sure the number
used in
+ // demoninator does not include consensus as one annotator
+ int totalAuthors = consensusExists ? columnAuthorNames.size() -
1
+ : columnAuthorNames.size();
+ buffer.append("<br><b>Avg. IAA macro avg:</b> "
+ + (f.format((double) annotatorMacro /
totalAuthors)));
+ buffer.append("<br><b>Avg. IAA micro avg:</b> "
+ + (f.format((double) annotatorMicro /
totalAuthors)));
+ buffer.append("</body>\n</html>");
+ BufferedWriter bw = null;
+ try {
+ File indexFile = new File(outputFolderUrl.getFile(),
"index.html");
+ bw = new BufferedWriter(new FileWriter(indexFile));
+ bw.write(buffer.toString());
+ } catch (IOException ioe) {
+ throw new ExecutionException(ioe);
+ } finally {
+ if (bw != null) {
+ try {
+ bw.close();
+ } catch (IOException e) {
+ throw new ExecutionException(e);
+ }
+ }
+ }
+ }
- // micro average
- double micro = 0.0D;
+ /**
+ * A method that adds two columns to the buffer - one for the macro
figure
+ * and the other one for the micro figure. It also adds relevant style
tag
+ * to give a proper color to each cell depending on the value of macro
and
+ * micro figures. See documentation of the getStyleTag(double) for more
+ * information on how cell backgrounds are color coded.
+ *
+ * @param buffer
+ * @param macro
+ * @param micro
+ */
+ private void appendMacroMicroFigures(StringBuffer buffer, double macro,
+ double micro) {
+ String macCol = getStyleTag(macro);
+ String micCol = getStyleTag(micro);
+ buffer.append("\t\t<td " + macCol + " >" + f.format(macro) +
"</td>\n");
+ buffer.append("\t\t<td " + micCol + " >" + f.format(micro) +
"</td>\n");
+ }
- // name of the file to link to
- String documentFileName;
- }
+ /**
+ * Gets the style tag based on the score. The color green is used for a
cell
+ * background to indicate full agreement (i.e. 1.0). The background
color
+ * becomes lighter as the agreement reduces towards 0.5. At 0.5
agreement,
+ * the background color of a cell is fully white. From 0.5 downwards,
the
+ * color red is used and as the agreement reduces further, the color
becomes
+ * darker with dark red at 0.0 agreement.
+ *
+ * @param score
+ * @return
+ */
+ private String getStyleTag(double score) {
+ // two colors
+ Color gc = Color.GREEN;
+ Color rc = Color.RED;
+ Color c = null;
+ // if score is above .50, use the green color, otherwise the
red one
+ if (score > 0.50) {
+ c = new Color(gc.getRed(), gc.getGreen(), gc.getBlue(),
+ ((int) (score * 100) * 2) - 100);
+ } else {
+ c = new Color(rc.getRed(), rc.getGreen(), rc.getBlue(),
+ 100 - ((int) (score * 100) * 2));
+ }
+ return "style=\"background-color: rgba(" + c.getRed() + ","
+ + c.getGreen() + "," + c.getBlue() + ","
+ + ((double) c.getAlpha() / 100) + ")\"";
+ }
+
+ /**
+ * Given the annotator name, the method finds out the annotation set
which
+ * contains annotations for that annotator
+ *
+ * @param doc
+ * @param annotatorName
+ * @return
+ */
+ private String getSetName(Document doc, String annotatorName) {
+
+ // if the annotatorName is inside the
annotationSetNamesToAnnotators
+ // it means, this is not an anonymous mode
+ if (annotationSetNamesAsAnnotators != null
+ &&
annotationSetNamesAsAnnotators.contains(annotatorName)) {
+ return annotatorName;
+ }
+
+ // otherwise find out the annotator name from the document
features
+ String inputAS = null;
+ for (Object key : doc.getFeatures().keySet()) {
+ Object val = doc.getFeatures().get(key);
+ if (val.equals(annotatorName)) {
+ inputAS = key.toString().substring(12);
+ break;
+ }
+ }
+ return inputAS;
+ }
+
+ /**
+ * Creating a temporary annotation set
+ *
+ * @param doc
+ * @param inputAS
+ * @param outputAS
+ */
+ private void createAnnSet(Document doc, String inputAS, String
outputAS) {
+ // if both inputAS and outputAS are same, we just return
+ // no need to create a temporarity annotation set
+ if (inputAS.equals(outputAS))
+ return;
+
+ AnnotationSet inAS = inputAS == null || inputAS.trim().length()
== 0 ? doc
+ .getAnnotations() : doc.getAnnotations(inputAS);
+ inAS = inAS.get(new HashSet<String>(annotationTypes));
+ AnnotationSet outAS = doc.getAnnotations(outputAS);
+ for (Annotation a : inAS) {
+ outAS.add(a.getStartNode(), a.getEndNode(), a.getType(),
+ a.getFeatures());
+ }
+ }
+
+ /**
+ * A method that calculates IAA between the given annotation sets
+ *
+ * @param corpus
+ * @param keyAS
+ * @param responseAS
+ * @param filePrefix
+ * @throws ExecutionException
+ */
+ private void calculateIAA(Corpus corpus, String keyAS, String
responseAS,
+ String filePrefix) throws ExecutionException {
+ try {
+ assurancePR.reInit();
+ } catch (ResourceInstantiationException rie) {
+ throw new ExecutionException(rie);
+ }
+ // lets set the params on qualityAssurancePR
+ assurancePR.setAnnotationTypes(annotationTypes);
+ assurancePR
+ .setFeatureNames(featureNames == null ? new
ArrayList<String>()
+ : featureNames);
+ assurancePR.setMeasure(measure);
+ assurancePR.setOutputFolderUrl(outputFolderUrl);
+ // the remaining two params will be set later
+ assurancePR.setKeyASName(keyAS);
+ assurancePR.setResponseASName(responseAS);
+ controller.setCorpus(corpus);
+ controller.execute();
+ // QA PR produces two stats file, one for corpus and one for
+ // documents
+ // for QAForTeamware, we don't need corpus one
+ File corpusFile = new File(outputFolderUrl.getFile(),
+ "corpus-stats.html");
+ corpusFile.delete();
+ File documentFile = new File(outputFolderUrl.getFile(),
+ "document-stats.html");
+ File documentOutFile = new File(outputFolderUrl.getFile(),
filePrefix
+ + "-document-stats.html");
+ documentFile.renameTo(documentOutFile);
+ }
+
+ /**
+ * Method called when a PR is unloaded.
+ */
+ public void cleanup() {
+ super.cleanup();
+ Factory.deleteResource(assurancePR);
+ Factory.deleteResource(controller);
+ }
+
+ /**
+ * Annotation types for which the stats should be calculated
+ *
+ * @return
+ */
+ public List<String> getAnnotationTypes() {
+ return annotationTypes;
+ }
+
+ /**
+ * Annotation types for which the stats should be calculated
+ *
+ * @param annotationTypes
+ */
+ @RunTime
+ @CreoleParameter
+ public void setAnnotationTypes(List<String> annotationTypes) {
+ this.annotationTypes = annotationTypes;
+ }
+
+ /**
+ * Features names for which the stats should be calculated
+ *
+ * @return
+ */
+ public List<String> getFeatureNames() {
+ return featureNames;
+ }
+
+ /**
+ * Features names for which the stats should be calculated
+ *
+ * @param featureNames
+ */
+ @RunTime
+ @Optional
+ @CreoleParameter
+ public void setFeatureNames(List<String> featureNames) {
+ this.featureNames = featureNames;
+ }
+
+ /**
+ * Measure to use for stats calculation
+ *
+ * @return
+ */
+ public Measure getMeasure() {
+ return measure;
+ }
+
+ /**
+ * Measure to use for stats calculation
+ *
+ * @param measure
+ */
+ @RunTime
+ @CreoleParameter
+ public void setMeasure(Measure measure) {
+ this.measure = measure;
+ }
+
+ /**
+ * URL of the folder to store output files into
+ *
+ * @return
+ */
+ public URL getOutputFolderUrl() {
+ return outputFolderUrl;
+ }
+
+ /**
+ * URL of the folder to store output files into
+ *
+ * @param outputFolderUrl
+ */
+ @RunTime
+ @CreoleParameter(suffixes = "html")
+ public void setOutputFolderUrl(URL outputFolderUrl) {
+ this.outputFolderUrl = outputFolderUrl;
+ }
+
+ /**
+ * annotators to ignore when computing IAA
+ *
+ * @return
+ */
+ public List<String> getAnnotatorsToIgnore() {
+ return annotatorsToIgnore;
+ }
+
+ /**
+ * annotators to ingore when computing IAA. Please note that if there's
at
+ * least one value provided for the annotationSetNamesAsAnnotators, this
+ * parameter will have no effect.
+ *
+ * @param annotatorsToIgnore
+ */
+ @RunTime
+ @Optional
+ @CreoleParameter
+ public void setAnnotatorsToIgnore(List<String> annotatorsToIgnore) {
+ this.annotatorsToIgnore = annotatorsToIgnore;
+ }
+
+ public List<String> getAnnotationSetNamesAsAnnotators() {
+ return annotationSetNamesAsAnnotators;
+ }
+
+ /**
+ * If this parameter is provided, it is assumed that the anonymous mode
is
+ * not in place and therefore the names of annotation sets should be
+ * considered as author names. Also note that if this parameter is
provided
+ * the parameter annotatorsToIgnore is ignored entirely. In other
words, it
+ * has no effect.
+ *
+ * @param annotationSetNamesAsAnnotators
+ */
+ @RunTime
+ @Optional
+ @CreoleParameter
+ public void setAnnotationSetNamesAsAnnotators(
+ List<String> annotationSetNamesAsAnnotators) {
+ this.annotationSetNamesAsAnnotators =
annotationSetNamesAsAnnotators;
+ }
+
+ /**
+ * Storing individual results for each pair of annotators
+ *
+ * @author niraj
+ */
+ class Result {
+ // macro average
+ double macro = 0.0D;
+
+ // micro average
+ double micro = 0.0D;
+
+ // name of the file to link to
+ String documentFileName;
+ }
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Symantec Endpoint Protection 12 positioned as A LEADER in The Forrester
Wave(TM): Endpoint Security, Q1 2013 and "remains a good choice" in the
endpoint security space. For insight on selecting the right partner to
tackle endpoint security challenges, access the full report.
http://p.sf.net/sfu/symantec-dev2dev
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
