Revision: 17785
http://sourceforge.net/p/gate/code/17785
Author: ian_roberts
Date: 2014-04-09 14:48:36 +0000 (Wed, 09 Apr 2014)
Log Message:
-----------
Allow job creator to supply a "reason" why a gold-standard snippet has been
marked the way it has. This is displayed as feedback to the user if they get
the gold unit wrong.
Modified Paths:
--------------
gate/trunk/plugins/Crowd_Sourcing/src/gate/crowdsource/ne/EntityAnnotationJobBuilder.java
gate/trunk/plugins/Crowd_Sourcing/src/gate/crowdsource/rest/CrowdFlowerClient.java
Modified:
gate/trunk/plugins/Crowd_Sourcing/src/gate/crowdsource/ne/EntityAnnotationJobBuilder.java
===================================================================
---
gate/trunk/plugins/Crowd_Sourcing/src/gate/crowdsource/ne/EntityAnnotationJobBuilder.java
2014-04-09 01:21:03 UTC (rev 17784)
+++
gate/trunk/plugins/Crowd_Sourcing/src/gate/crowdsource/ne/EntityAnnotationJobBuilder.java
2014-04-09 14:48:36 UTC (rev 17785)
@@ -54,6 +54,8 @@
private String goldFeatureName;
private String goldFeatureValue;
+
+ private String goldReasonFeatureName;
private String entityAnnotationType;
@@ -172,6 +174,18 @@
this.goldFeatureValue = goldFeatureValue;
}
+ public String getGoldReasonFeatureName() {
+ return goldReasonFeatureName;
+ }
+
+ @Optional
+ @RunTime
+ @CreoleParameter(comment = "Feature on gold snippet annotations explaining "
+
+ "why the snippet's entities are correct")
+ public void setGoldReasonFeatureName(String goldReasonFeatureName) {
+ this.goldReasonFeatureName = goldReasonFeatureName;
+ }
+
@Override
public Resource init() throws ResourceInstantiationException {
if(apiKey == null || "".equals(apiKey)) {
@@ -206,12 +220,17 @@
if(isInterrupted()) throw new ExecutionInterruptedException();
AnnotationSet snippetTokens = Utils.getContainedAnnotations(tokens,
snippet);
AnnotationSet goldAnnots = null;
+ String goldReason = null;
if(goldFeatureValue.equals(snippet.getFeatures().get(goldFeatureName))) {
goldAnnots = Utils.getContainedAnnotations(goldAS, snippet);
+ if(goldReasonFeatureName != null) {
+ Object goldReasonValue =
snippet.getFeatures().get(goldReasonFeatureName);
+ if(goldReasonValue != null) goldReason =
goldReasonValue.toString();
+ }
}
long unitId = crowdFlowerClient.createAnnotationUnit(
- jobId, getDocument(), snippetASName, snippet, snippetTokens,
goldAnnots);
+ jobId, getDocument(), snippetASName, snippet, snippetTokens,
goldAnnots, goldReason);
// store the unit ID - we use the entity annotation type as part of
this feature
// name so the same sentences can hold units for different annotation
types
// e.g. Person, Location, Organization
Modified:
gate/trunk/plugins/Crowd_Sourcing/src/gate/crowdsource/rest/CrowdFlowerClient.java
===================================================================
---
gate/trunk/plugins/Crowd_Sourcing/src/gate/crowdsource/rest/CrowdFlowerClient.java
2014-04-09 01:21:03 UTC (rev 17784)
+++
gate/trunk/plugins/Crowd_Sourcing/src/gate/crowdsource/rest/CrowdFlowerClient.java
2014-04-09 14:48:36 UTC (rev 17785)
@@ -375,10 +375,15 @@
* includes the case where an empty annotation set is
* provided, as this represents a gold snippet where the
* correct answer is that this snippet contains no entities.
+ * @param goldReason for a gold-standard unit, the <em>reason</em>
+ * why the annotations should be considered correct. This
+ * will be shown to users as feedback if they get the gold
+ * unit wrong. Ignored for non-gold units.
* @return the ID of the newly-created unit.
*/
public long createAnnotationUnit(long jobId, Document doc, String asName,
- Annotation snippet, AnnotationSet tokens, AnnotationSet
correctAnnotations) {
+ Annotation snippet, AnnotationSet tokens, AnnotationSet
correctAnnotations,
+ String goldReason) {
String documentId = String.valueOf(doc.getLRPersistenceId());
int formDataSize = 6; // docId + asName + annId
List<Annotation> tokensList = Utils.inDocumentOrder(tokens);
@@ -401,6 +406,9 @@
} else {
formDataSize += 2*answerGold.size(); // answer=N for each token
}
+ if(goldReason != null) {
+ formDataSize += 2; // answer_gold_reason or noentities_gold_reason
+ }
}
String[] formData = new String[formDataSize];
@@ -421,13 +429,22 @@
if(answerGold.size() == 0) {
formData[i++] = "unit[data][noentities_gold]";
formData[i++] = "1";
+ if(goldReason != null) {
+ formData[i++] = "unit[data][noentities_gold_reason]";
+ formData[i++] = goldReason;
+ }
+ } else {
+ Integer[] goldArray = answerGold.toArray(new
Integer[answerGold.size()]);
+ Arrays.sort(goldArray);
+ for(Integer tokIndex : goldArray) {
+ formData[i++] = "unit[data][answer_gold][]";
+ formData[i++] = String.valueOf(tokIndex);
+ }
+ if(goldReason != null) {
+ formData[i++] = "unit[data][answer_gold_reason]";
+ formData[i++] = goldReason;
+ }
}
- Integer[] goldArray = answerGold.toArray(new Integer[answerGold.size()]);
- Arrays.sort(goldArray);
- for(Integer tokIndex : goldArray) {
- formData[i++] = "unit[data][answer_gold][]";
- formData[i++] = String.valueOf(tokIndex);
- }
}
try {
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Put Bad Developers to Shame
Dominate Development with Jenkins Continuous Integration
Continuously Automate Build, Test & Deployment
Start a new project now. Try Jenkins in the cloud.
http://p.sf.net/sfu/13600_Cloudbees
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
