Revision: 16760
http://sourceforge.net/p/gate/code/16760
Author: markagreenwood
Date: 2013-08-01 11:59:15 +0000 (Thu, 01 Aug 2013)
Log Message:
-----------
added the new MediaWiki corpus populater which uses the new ResourceHelper
class to provide a new corpus populate menu item
Modified Paths:
--------------
gate/trunk/plugins/Format_MediaWiki/src/gate/corpora/MediaWikiXMLDocumentFormat.java
Added Paths:
-----------
gate/trunk/plugins/Format_MediaWiki/src/gate/corpora/MediaWikiPopulater.java
Added:
gate/trunk/plugins/Format_MediaWiki/src/gate/corpora/MediaWikiPopulater.java
===================================================================
---
gate/trunk/plugins/Format_MediaWiki/src/gate/corpora/MediaWikiPopulater.java
(rev 0)
+++
gate/trunk/plugins/Format_MediaWiki/src/gate/corpora/MediaWikiPopulater.java
2013-08-01 11:59:15 UTC (rev 16760)
@@ -0,0 +1,190 @@
+/*
+ * MediaWikiPopulater.java
+ *
+ * Copyright (c) 2012-2013, The University of Sheffield. See the file
COPYRIGHT.txt
+ * in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt
+ *
+ * This file is part of GATE (see http://gate.ac.uk/), and is free software,
+ * licenced under the GNU Library General Public License, Version 2, June 1991
+ * (in the distribution as file licence.html, and also available at
+ * http://gate.ac.uk/gate/licence.html).
+ *
+ * Mark A. Greenwood, 01/08/2013
+ */
+
+package gate.corpora;
+
+import gate.Corpus;
+import gate.Document;
+import gate.Factory;
+import gate.FeatureMap;
+import gate.Resource;
+import gate.creole.metadata.AutoInstance;
+import gate.creole.metadata.CreoleResource;
+import gate.gui.MainFrame;
+import gate.gui.ResourceHelper;
+import gate.util.ExtensionFileFilter;
+import info.bliki.wiki.dump.IArticleFilter;
+import info.bliki.wiki.dump.Siteinfo;
+import info.bliki.wiki.dump.WikiArticle;
+import info.bliki.wiki.dump.WikiXMLParser;
+import info.bliki.wiki.model.WikiModel;
+
+import java.awt.event.ActionEvent;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import javax.swing.AbstractAction;
+import javax.swing.Action;
+import javax.swing.JFileChooser;
+
+import org.xml.sax.SAXException;
+
+@CreoleResource(name = "MediaWiki Corpus Populater", tool = true,
autoinstances = @AutoInstance)
+public class MediaWikiPopulater extends ResourceHelper {
+
+ /**
+ * so that we don't end up with a document littered with unparsed
+ * "magic words" we we need a custom model that we can use to filter them out
+ */
+ private final static WikiModel model = new WikiModel("${image}", "${title}")
{
+ @Override
+ public String getRawWikiContent(String namespace, String articleName,
+ Map<String, String> templateParameters) {
+ String rawContent =
+ super.getRawWikiContent(namespace, articleName, templateParameters);
+
+ if(rawContent == null) {
+ // if we return 'null' then the magic variables end up in the doc with
+ // full markup which isn't really what we want, so we just return the
+ // empty string instead to remove them entirely from the document
+ return "";
+ } else {
+ return rawContent;
+ }
+ }
+ };
+
+ @Override
+ protected List<Action> buildActions(final Resource resource) {
+ List<Action> actions = new ArrayList<Action>();
+
+ if(!(resource instanceof Corpus)) return actions;
+
+ actions.add(new AbstractAction("Populate from MediaWiki XML Dump") {
+
+ @Override
+ public void actionPerformed(ActionEvent e) {
+
+ // configure the file chooser ready for use
+ final JFileChooser filer = MainFrame.getFileChooser();
+ filer.setFileSelectionMode(JFileChooser.FILES_ONLY);
+ filer.setDialogTitle("Select a MediaWiki XML Dump File");
+ filer.resetChoosableFileFilters();
+ filer.setAcceptAllFileFilterUsed(false);
+ ExtensionFileFilter filter =
+ new ExtensionFileFilter("MediaWiki XML Dump Files (*.xml)", "xml");
+ filer.addChoosableFileFilter(filter);
+ filer.setFileFilter(filter);
+
+ // if no file was selected then just stop
+ if(filer.showOpenDialog(MainFrame.getInstance()) !=
JFileChooser.APPROVE_OPTION)
+ return;
+
+ // we want to run the population in a separate thread so we don't lock
+ // up the GUI
+ Thread thread =
+ new Thread(Thread.currentThread().getThreadGroup(),
+ "MediaWiki XML Dump Corpus Populater") {
+ public void run() {
+ try {
+ populateCorpus((Corpus)resource, filer.getSelectedFile()
+ .toURI().toURL());
+ } catch(MalformedURLException e) {
+ // this really should not be possible so just dump the
+ // exception and quit
+ e.printStackTrace();
+ }
+ }
+ };
+ thread.setPriority(Thread.MIN_PRIORITY);
+ thread.start();
+ }
+ });
+
+ return actions;
+ }
+
+ public static void populateCorpus(final Corpus corpus, URL xml) {
+ try {
+ // get the model ready for parsing
+ model.setUp();
+
+ // the parser needs an InputStream so lets build one up from the original
+ // document content
+ InputStream in = xml.openStream();
+
+ // create a parser to load the XML document
+ WikiXMLParser parser = new WikiXMLParser(in, new IArticleFilter() {
+
+ @Override
+ public void process(WikiArticle article, Siteinfo site)
+ throws SAXException {
+
+ try {
+ // extract the page content and convert it to HTML
+ // copy relevant metadata onto the document
+ FeatureMap features = Factory.newFeatureMap();
+ features.put("mediawiki.title", article.getTitle());
+ features.put("mediawiki.timestamp", article.getTimeStamp());
+ features.put("mediawiki.id", article.getId());
+ features.put("mediawiki.revision", article.getRevisionId());
+ features.put("mediawiki.sitename", site.getSitename());
+ features.put("mediawiki.base", site.getBase());
+
+ FeatureMap params = Factory.newFeatureMap();
+ params.put(Document.DOCUMENT_STRING_CONTENT_PARAMETER_NAME,
+ article.getText());
+ params.put(Document.DOCUMENT_MIME_TYPE_PARAMETER_NAME,
+ "text/x-mediawiki");
+
+ Document doc =
+ (Document)Factory.createResource("gate.corpora.DocumentImpl",
+ params, features, article.getTitle());
+
+ corpus.add(doc);
+
+ if(corpus.getLRPersistenceId() != null) {
+ // persistent corpus -> unload the document
+ corpus.unloadDocument(doc);
+ Factory.deleteResource(doc);
+ }
+ } catch(Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ // now we are all set let's parse the MediaWiki XML file
+ parser.parse();
+
+ if(corpus.getDataStore() != null) {
+ // if this corpus is in a datastore make sure we sync it back
+ corpus.getDataStore().sync(corpus);
+ }
+
+ } catch(Exception e) {
+ // oh dear, something went wrong and it's unlikely there is anything we
+ // can do about it so let's just throw our hands in the air and pass the
+ // responsibility up the stack and hope someone else will deal with it!
+ throw new RuntimeException(e);
+ } finally {
+ // signal that, at least for now, we have finished with the model
+ model.tearDown();
+ }
+ }
+}
Modified:
gate/trunk/plugins/Format_MediaWiki/src/gate/corpora/MediaWikiXMLDocumentFormat.java
===================================================================
---
gate/trunk/plugins/Format_MediaWiki/src/gate/corpora/MediaWikiXMLDocumentFormat.java
2013-08-01 11:57:45 UTC (rev 16759)
+++
gate/trunk/plugins/Format_MediaWiki/src/gate/corpora/MediaWikiXMLDocumentFormat.java
2013-08-01 11:59:15 UTC (rev 16760)
@@ -1,14 +1,14 @@
/*
* MediaWikiXMLDocumentFormat.java
- *
+ *
* Copyright (c) 2012, The University of Sheffield. See the file COPYRIGHT.txt
* in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt
- *
+ *
* This file is part of GATE (see http://gate.ac.uk/), and is free software,
* licenced under the GNU Library General Public License, Version 2, June 1991
* (in the distribution as file licence.html, and also available at
* http://gate.ac.uk/gate/licence.html).
- *
+ *
* Mark A. Greenwood, 31/10/2012
*/
@@ -37,96 +37,104 @@
import org.xml.sax.SAXException;
/**
- * A document format for parsing MediaWiki XML dump files. The format
- * extracts the MediaWiki markup from the XML file which is then converted
- * into HTML which in turn is passed to the standard HTML document format
- * for final parsing into a GATE document. The format is activated by
- * specifying the text/xml+mediawiki MIME type.
+ * A document format for parsing MediaWiki XML dump files. The format extracts
+ * the MediaWiki markup from the XML file which is then converted into HTML
+ * which in turn is passed to the standard HTML document format for final
+ * parsing into a GATE document. The format is activated by specifying the
+ * text/xml+mediawiki MIME type.
*
+ * @deprecated don't use this document format directly as it only loads the
last
+ * page from the dump file. Use the corpus populater instead.
* @author Mark A. Greenwood
*/
-@CreoleResource(name = "MediaWiki XML Document Format", isPrivate = true,
- autoinstances = {@AutoInstance(hidden = true)})
+@Deprecated
+@CreoleResource(name = "MediaWiki XML Document Format", isPrivate = true,
autoinstances = {@AutoInstance(hidden = true)})
public class MediaWikiXMLDocumentFormat extends NekoHtmlDocumentFormat {
-
+
/**
- * so that we don't end up with a document littered with unparsed "magic
words" we
- * we need a custom model that we can use to filter them out
+ * so that we don't end up with a document littered with unparsed
+ * "magic words" we we need a custom model that we can use to filter them out
*/
private final WikiModel model = new WikiModel("${image}", "${title}") {
@Override
public String getRawWikiContent(String namespace, String articleName,
- Map<String, String> templateParameters) {
- String rawContent = super.getRawWikiContent(namespace, articleName,
templateParameters);
+ Map<String, String> templateParameters) {
+ String rawContent =
+ super.getRawWikiContent(namespace, articleName, templateParameters);
- if (rawContent == null){
- //if we return 'null' then the magic variables end up in the doc with
- //full markup which isn't really what we want, so we just return the
- //empty string instead to remove them entirely from the document
- return "";
- }
- else {
- return rawContent;
- }
+ if(rawContent == null) {
+ // if we return 'null' then the magic variables end up in the doc with
+ // full markup which isn't really what we want, so we just return the
+ // empty string instead to remove them entirely from the document
+ return "";
+ } else {
+ return rawContent;
}
+ }
};
-
+
@Override
public Boolean supportsRepositioning() {
return false;
}
-
+
@Override
public Resource init() throws ResourceInstantiationException {
-
+
// create the MIME type object
- MimeType mime = new MimeType("text","xml+mediawiki");
-
+ MimeType mime = new MimeType("text", "xml+mediawiki");
+
// Register the class handler for this mime type
- mimeString2ClassHandlerMap.put(mime.getType()+ "/" + mime.getSubtype(),
this);
-
+ mimeString2ClassHandlerMap.put(mime.getType() + "/" + mime.getSubtype(),
+ this);
+
// Register the mime type with mine string
mimeString2mimeTypeMap.put(mime.getType() + "/" + mime.getSubtype(), mime);
-
+
// Set the mimeType for this language resource
setMimeType(mime);
-
+
return this;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see gate.corpora.TextualDocumentFormat#cleanup()
*/
@Override
public void cleanup() {
super.cleanup();
-
+
MimeType mime = getMimeType();
-
- mimeString2ClassHandlerMap.remove(mime.getType()+ "/" + mime.getSubtype());
+
+ mimeString2ClassHandlerMap.remove(mime.getType() + "/" +
mime.getSubtype());
mimeString2mimeTypeMap.remove(mime.getType() + "/" + mime.getSubtype());
}
@Override
public void unpackMarkup(final Document doc) throws DocumentFormatException {
-
+
try {
-
+
// get the model ready for parsing
model.setUp();
-
- // the parser needs an InputStream so lets build one up from the
original document content
- InputStream in = new ReaderInputStream(new
StringReader(doc.getContent().toString()));
-
+
+ // the parser needs an InputStream so lets build one up from the original
+ // document content
+ InputStream in =
+ new ReaderInputStream(new StringReader(doc.getContent().toString()));
+
// create a parser to load the XML document
WikiXMLParser parser = new WikiXMLParser(in, new IArticleFilter() {
-
+
@Override
- public void process(WikiArticle article, Siteinfo site) throws
SAXException {
-
+ public void process(WikiArticle article, Siteinfo site)
+ throws SAXException {
+
// extract the page content and convert it to HTML
String htmlText = model.render(article.getText());
-
+
// copy relevant metadata onto the document
FeatureMap features = doc.getFeatures();
features.put("mediawiki.title", article.getTitle());
@@ -135,38 +143,39 @@
features.put("mediawiki.revision", article.getRevisionId());
features.put("mediawiki.sitename", site.getSitename());
features.put("mediawiki.base", site.getBase());
-
+
// use the HTML to update the document content
doc.setContent(new DocumentContentImpl(htmlText));
-
+
}
});
-
+
// now we are all set let's parse the MediaWiki XML file
parser.parse();
-
+
} catch(Exception e) {
- // oh dear, something went wrong and it's unlikely there is anything we
can
- // do about it so let's just throw our hands in the air and pass the
responsibility
- // up the stack and hope someone else will deal with it!
+ // oh dear, something went wrong and it's unlikely there is anything we
+ // can
+ // do about it so let's just throw our hands in the air and pass the
+ // responsibility
+ // up the stack and hope someone else will deal with it!
throw new DocumentFormatException(e);
- }
- finally {
+ } finally {
// signal that, at least for now, we have finished with the model
model.tearDown();
}
-
+
// we need to nullify the source URL otherwise the HTML doc format we
// are about to use will re-parse the original document which will
// undo everything we have just done
URL url = doc.getSourceUrl();
doc.setSourceUrl(null);
-
+
// now let the HTML unpacker also do its job
super.unpackMarkup(doc);
-
+
// now we can put the URL back as it won't mess anything up
doc.setSourceUrl(url);
}
-
+
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Get your SQL database under version control now!
Version control is standard for application code, but databases havent
caught up. So what steps can you take to put your SQL databases under
version control? Why should you start doing it? Read more to find out.
http://pubads.g.doubleclick.net/gampad/clk?id=49501711&iu=/4140/ostg.clktrk
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
