Tjones has uploaded a new change for review.
https://gerrit.wikimedia.org/r/298479
Change subject: Re-skin the TextCat Demo
......................................................................
Re-skin the TextCat Demo
Re-skin the TextCat Demo according to a version done by Jan. Merge
his changes (branched from an earlier version).
Change-Id: I6d31fb4a14dfea8ef7083bbe74c19831007b163f
---
M index.html
M textcatdemo.css
M textcatdemo.js
3 files changed, 767 insertions(+), 539 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/textcat-demo
refs/changes/79/298479/1
diff --git a/index.html b/index.html
index 65382c0..9503d78 100644
--- a/index.html
+++ b/index.html
@@ -6,7 +6,6 @@
<meta name="description" content="Live demo of modernized TextCat libraries
for language detection, from the Wikimedia Foundation Discovery department." />
<meta name="viewport" content="width=device-width, initial-scale=1.0,
minimum-scale=1.0">
<link rel="stylesheet" type="text/css" href="textcatdemo.css">
-<script src="textcatdemo.js"></script>
<!--
This program is free software; you can redistribute it and/or modify
@@ -26,10 +25,19 @@
</head>
<body>
-<div class="container">
+<div id="infoPop">
+ <div onclick="closeInfo()">
+ <img src="close.png" class=icon alt="Close info box." title="Close info
box.">
+ </div>
+ <div id="infoPopText">
+ <span class="dimText">Additional Information...</span>
+ </div>
+</div>
-<div id="demoPop" class="modalPop">
- <div class="dismissModal" onclick="hideDemo(1)"><img src="close.png"
class=icon alt="Close demo." title="Close Demo."></div>
+<div class="container" id="container">
+
+<div id="demoPop">
+ <div class="righty" onclick="closeDemo(1)"><img src="close.png"
class=icon alt="Close demo." title="Close Demo."></div>
<div id="demoContainer">
<div id="demoTitle">
Demo Title
@@ -38,22 +46,28 @@
Demo info.
</div>
<br class='clearIt'>
- <div class="demoContinue" id="demoContinue"
onclick="demoNext();">Continue ➤</div>
+ <div id="demoContinue"
onclick="demoNext();">Continue ➤</div>
<div class="details">Use “Continue” or the right arrow key to
move through the demo.</div>
<br class='clearIt'>
</div>
</div>
-<div id="infoPop" class="modalPop">
- <div class="dismissModal" onclick="closeInfo()"><img src="close.png"
class=icon alt="Close info box." title="Close info box."></div>
- <div id="infoPopText">
- <span class="dimText">Additional Information...</span>
- </div>
-</div>
-
-<div id="TextCatHeaderContainer" class="subContainer zeroth">
+<div id="TextCatHeaderContainer" class="subContainer">
<div id="TextCatHeader" class="lefty">
- <h1>TextCat Demo</h1>
+ <pre class='asciiCat'>
+\ /\
+ ) ( ')
+( / )
+ \(__)|</pre>
+ <h1 class='headline'>
+ <span class=info onclick="showInfo(0)">
+ <img src='info.png' style='height:1em' alt="Get more
info." title="Get more info.">
+ </span>
+ <span class='headline-main'>
+ TextCat Demo</span>
+ <span class='subhead'>
+ Language Detection Library</span>
+ </h1>
</div>
<div class="pickADemo righty">
@@ -67,24 +81,28 @@
</select>
<span class=info onclick="showInfo(1)"><img src='info.png'
class=icon alt="Get more info." title="Get more info."></span>
</div>
-
-
<br class='clearIt'>
-
- <p>
- <b>TextCat</b> is a language detection library based on <a
class=external href='https://en.wikipedia.org/wiki/N-gram'
target=_blank>n-gram</a> text categorization. This is a demo of the <a
class=external href='https://www.mediawiki.org/wiki/TextCat'
target=_blank>modernized TextCat libraries</a>, ported from Perl to PHP by the
Wikimedia Foundation Discovery department. It provides a look into how TextCat
works—and sometimes doesn’t work—and it’s also just fun to
play with. Select a demo to the right, or enter a query below to try out the
language detection.
- </p>
-
</div>
-<div class="subContainer odd">
- <div class="info righty" onclick="showInfo(2)"><img src='info.png'
class=icon alt="Get more info." title="Get more info."></div>
- <div class="LMContainer">
- <div class="LM" onclick="LMDirSelect(this)"
id="LMDirQuery">Query Lan­guage Models</div>
- <div class="LM dimText" onclick="LMDirSelect(this)"
id="LMDir">Wiki-Text Lan­guage Models</div>
+<div class="subContainer">
+ <br class='clearIt'>
+ <div>
+ <h3 class='subtitle'>
+ Language Models
+ <div class="info" onclick="showInfo(2)"><img src='info.png'
style='height:1em' alt="Get more info." title="Get more info."></div>
+ </h3>
+ <div class="LMContainer">
+ <div class="LM" onclick="LMDirSelect(this)"
id="LMDirQuery">
+ Query Lan­guage Models
+ </div>
+ <div class="LM" onclick="LMDirSelect(this)" id="LMDir">
+ Wiki-Text Lan­guage Models
+ </div>
+ </div>
</div>
+
+ <p class="details">Select the languages TextCat should consider.</p>
<div class="LMList">
- <div class="LMSelectDeselect lefty"><div class="dismiss"
onclick="LMAll()"><img src="add.png" class=icon alt="Select all languages."
title="Select all languages."></div><div class="dismiss"
onclick="LMClear()"><img src="close.png" class=icon alt="Deselect all
languages." title="Deselect all languages."></div></div>
<span class="LMPick" onclick="LMToggle(this)"
id="Afrikaans">Afrikaans</span>
<span class="LMPick" onclick="LMToggle(this)"
id="Albanian">Albanian</span>
<span class="LMPick" onclick="LMToggle(this)"
id="Arabic">Arabic</span>
@@ -155,41 +173,62 @@
<span class="LMPick" onclick="LMToggle(this)"
id="Vietnamese">Vietnamese</span>
<span class="LMPick" onclick="LMToggle(this)"
id="Welsh">Welsh</span>
<span class="LMPick" onclick="LMToggle(this)"
id="WesternPunjabi">Western Punjabi</span>
- <br class='clearIt'>
<div class="moreLess" id="moreLess"
onclick="extraLangToggle(this);">more...</div>
- <br class='clearIt'>
- <div class="details">Select the languages to be considered when
identifying the query below.</div>
+ <div class="selectAllNoneLM">
+ (
+ <span class="unselectLM" onclick="LMClear()">
+ Clear all
+ </span>
+ ·
+ <span class="selectLM" onclick="LMAll()">
+ Select all
+ </span>
+ )
+ </div>
+
</div>
+
<br class='clearIt'>
</div>
-<div class="subContainer even">
- <div class="info righty" onclick="showInfo(3)"><img src='info.png'
class=icon alt="Get more info." title="Get more info."></div>
- <div>
- Query<br>
- <textarea rows="3" class="embiggen" onkeypress="return
catchReturn(this);" id="queryTA"></textarea>
- </div>
- <div class="details">Type your query and then <Enter> or
<Return> to submit your query for language detection. Results will be
shown below.</div>
+
+<div class="subContainer">
+ <h3 class='subtitle'>Query
+ <div class="info" onclick="showInfo(3)"><img src='info.png'
class=icon alt="Get more info." title="Get more info."></div>
+ </h3>
+ <form id="queryForm">
+ <textarea placeholder="what language is this?"
class="largeInput" class="embiggen" onkeypress="return catchReturn(this);"
id="queryTA"></textarea>
+ <button type="submit" class="primaryButton">Guess
Language</button>
+ </form>
+</div>
+
+<div class="subContainer">
+ <br class='clearIt'>
+ <h3 class='subtitle'>
+ Textcats Results
+ <div class="info" onclick="showInfo(4)"><img src='info.png'
style='height:1em' alt="Get more info." title="Get more info."></div>
+ </h3>
+
+ <p id="IDresults" class="embiggen">
+ <span class="dimText">Language ID results...</span>
+ </p>
+ <p class="details">Click on the name of a language above to view search
results below.</p>
<br class='clearIt'>
</div>
-<div class="subContainer odd">
- <div class="info righty" onclick="showInfo(4)"><img src='info.png'
class=icon alt="Get more info." title="Get more info."></div>
- <div id="IDresults" class="embiggen">
- <span class="dimText">Language ID results...</span>
- </div>
- <div class="details">Click on the name of a language to view search
results below.</div>
- <br class='clearIt'>
-</div>
-
-<div class="subContainer even">
-<div>Wikipedia search results</div>
- <div class="info righty" onclick="showInfo(5)"><img src='info.png'
class=icon alt="Get more info." title="Get more info."></div>
+<div class="subContainer">
+ <h3 class='subtitle'>
+ Wikipedia Search Results
+ <div class="info" onclick="showInfo(5)"><img src='info.png'
style='height:1em' alt="Get more info." title="Get more info."></div>
+ </h3>
<iframe src="blank.html" class="embiggen" name="iframeWiki"></iframe>
<br class='clearIt'>
</div>
</div>
+
+<script src="textcatdemo.js"></script>
+
</body>
</html>
\ No newline at end of file
diff --git a/textcatdemo.css b/textcatdemo.css
index a9b43bf..a7bd468 100644
--- a/textcatdemo.css
+++ b/textcatdemo.css
@@ -1,287 +1,474 @@
body {
- background-color: #f9f9f9;
- margin: 0;
- font-family: sans-serif;
+ margin: 0;
+ font-family: sans-serif;
+ background-color: #F1F1F1;
+ color: #252525;
}
-h1 {
- padding: 0;
- margin: 0;
- font-variant: small-caps;
+#TextCatHeaderContainer {
+ margin-bottom: 1rem;
}
-textarea {
- margin-top: .5em;
- font-size: 1em;
+#TextCatHeader.hidden {
+ display: none;
+}
+
+.asciiCat {
+ display: inline-block;
+ width: 70px;
+ margin: 0;
+ vertical-align: bottom;
+}
+
+.headline {
+ padding: 0;
+ margin: 0;
+ vertical-align: bottom;
+ display: inline-block;
+ font-size: 2rem;
+}
+
+.headline-main {
+ float: left;
+}
+
+.headline .subhead {
+ font-size: 1rem;
+ color: grey;
+ float: left;
+ clear: left;
+ font-weight: 400;
+}
+
+.subtitle {
+ margin-bottom: .8em;
+ font-size: 1.1rem;
+ margin-top: 0;
+ text-transform: uppercase;
+ font-weight: 200;
+ letter-spacing: 1px;
}
iframe {
- margin: .5em auto;
- height: 400px;
- border: 1px solid #ccc;
- display: block;
+ margin: .5em auto;
+ height: 400px;
+ border: 1px solid #ccc;
+ display: block;
/* iOS hack */
- width: 1px;
- min-width: 80%;
+ width: 1px;
+ min-width: 100%;
/* iOS hack */
}
.lefty {
- float: left;
+ float: left;
}
.righty {
- float: right;
+ float: right;
}
.external::after {
- content: url('external.png');
- padding-left: .2em;
+ content: url('external.png');
+ padding-left: .2em;
}
.embiggen {
- width: 95%;
+ width: 95%;
}
.container {
- margin: 1em;
- padding: 0;
- border: 1px solid #bdf;
- background-color: #fff;
+ padding: 2rem;
+ width: 70%;
+ box-sizing: border-box;
+ margin: 0 auto;
+ float: none;
+ box-shadow: 0 1px 20px -5px rgba(0,0,0,0.5);
+ background-color: #fff;
}
.subContainer {
- margin: .5em;
- padding: 1em;
+ margin: 0;
+ padding: 0;
}
.info {
- font-size: 1rem;
- cursor: help;
- vertical-align: middle;
- font-variant: normal;
- margin-left: .5rem;
+ font-size: 1rem;
+ cursor: help;
+ vertical-align: middle;
+ font-variant: normal;
+ margin-left: .5rem;
+ display: inline-block;
}
.icon {
- height: 1.2em;
+ height: 1em;
}
-.infoIconDiv {
- float: left;
- font-size: 150%;
- font-weight: 700;
- margin: .5rem .25rem .25rem;
+.selectAllNoneLM {
+ display: inline-block;
+ font-size: .75rem;
}
-.dismiss,.dismissModal {
- cursor: pointer;
- vertical-align: middle;
+.unselectLM,.selectLM {
+ display: inline-block;
+ cursor: pointer;
+ font-variant: normal;
}
-.dismissModal {
- float: right;
- margin: .25em .25em 0 .1em;
+.unselectLM:hover,.selectLM:hover {
+ text-decoration: underline;
+}
+
+.unselectLM {
+ color: #af1f1d;
+}
+
+.selectLM {
+ color: #00af89;
}
.LM {
- float: left;
- margin: -1px -1px 0 0;
- padding: .5em;
- border: 1px solid #a3bfb1;
- cursor: pointer;
- background-color: #cef2e0;
+ margin-bottom: .5em;
+ cursor: pointer;
+ font-size: .9em;
+ border: 1px solid #CECDCD;
+ padding: .5em;
+ float: left;
+ width: 50%;
+ box-sizing: border-box;
+ text-align: center;
+ transition: background-color .1s ease-in;
+}
+
+.LM:hover {
+ background-color: #F0F8FF;
+}
+
+.LM.active {
+ background-color: #5E9ECE;
+ color: #fff;
+}
+
+.LMdetails {
+ font-size: small;
+ margin-top: 0;
+ font-weight: 400;
+}
+
+.LMPick:hover {
+ background-color: #D6EDFF;
}
.LMPick {
- padding: .1em;
- cursor: pointer;
+ padding: 0 .4em;
+ cursor: pointer;
+ margin: .2em;
+ display: inline-block;
+ font-size: .8rem;
+ border-radius: 2px;
+ border: 1px solid #CECDCD;
+ transition: background-color .1s ease-in;
+}
+
+.LMPick.active {
+ color: #fff;
+ background-color: #5E9ECE;
+ border-color: #418BC3;
}
.LMList {
- clear: left;
- margin-left: 2em;
- padding-top: .5em;
- line-height: 150%;
+ clear: left;
+ line-height: 150%;
}
.LMContainer {
- display: flex;
+ display: flex;
+ margin-bottom: 1em;
+ margin-top: 1em;
}
-#LMDirQuery {
- font-weight: 700;
+.LMContainer:before,.LMContainer:after {
+ content: "";
+ display: table;
}
-.LMSelectDeselect {
- margin-left: -1.5em;
+.LMContainer:after {
+ clear: both;
}
.dimText {
- color: #ccc;
+ opacity: .5;
+ font-size: .9rem;
}
.clearIt {
- clear: both;
- line-height: 0;
+ clear: both;
+ line-height: 0;
}
.details {
- color: #999;
- font-size: 90%;
- font-style: italic;
- line-height: 120%;
- float: left;
- clear: left;
- margin-bottom: -.5rem;
- margin-top: .4rem;
+ font-size: .9rem;
+ line-height: 120%;
+ float: left;
+ clear: left;
+ margin-top: 0;
+ margin-bottom: 1em;
+ padding-top: 0;
+ font-style: italic;
}
.moreLess {
- float: right;
- margin-bottom: -1rem;
- padding: 0 .5em 0 .6em;
- border: 1px solid #ccc;
- background-color: #f4f4f4;
- font-size: 75%;
- cursor: pointer;
- font-weight: 700;
-}
-
-.modalPop {
- position: fixed;
- top: 2em;
- right: 2em;
- left: 2em;
- font-size: 90%;
- display: none;
- line-height: 140%;
- overflow: scroll;
+ font-size: .75rem;
+ cursor: pointer;
+ font-weight: 800;
+ display: inline-block;
+ text-decoration: underline;
}
#infoPop {
- border: 3px solid #a3b0bf;
- background-color: #cedff2;
- max-height: 80%;
+ display: none;
+}
+
+#infoPop.active {
+ display: block;
+ overflow: scroll;
+ width: 30%;
+ float: right;
+ box-sizing: border-box;
+ padding: 1.5rem;
+ font-size: .9rem;
+ line-height: 1.4rem;
+}
+
+#infoPop.active + .container {
+ float: left;
}
#infoPopText {
- margin: 1em;
+ margin: 1em;
}
#demoPop {
- border: 3px solid #dd9;
- background-color: #ffd;
- max-height: 50%;
+ border: 3px solid #5E9ECE;
+ background-color: #fff;
+ max-height: 50%;
+ width: 68%;
+ margin: 0 auto;
+ position: fixed;
+ top: 1.5em;
+ right: 2em;
+ left: 2em;
+ font-size: .9rem;
+ display: none;
+ line-height: 140%;
+ overflow: scroll;
+}
+
+#demoPop.active {
+ display: block;
}
#demoTitle {
- font-weight: 700;
+ font-weight: 700;
}
#demoContainer {
- margin: 1em;
+ margin: 1em;
}
-.demoContinue {
- float: right;
- padding: 0 .5em 0 .6em;
- margin-top: -.75rem;
- border: 1px solid #ccc;
- background-color: #f4f4f4;
- font-size: 75%;
- cursor: pointer;
- font-weight: 700;
+#demoContinue {
+ float: right;
+ padding: 0 .5em 0 .6em;
+ margin-top: -.75rem;
+ border: 1px solid #ccc;
+ background-color: #f4f4f4;
+ font-size: 75%;
+ cursor: pointer;
+ font-weight: 700;
}
.pickADemo {
- margin-top: .5em;
+ margin-top: .5em;
}
.langRes {
- float: left;
- margin-right: 1em;
- margin-bottom: .5em;
+ margin-right: 1em;
+ margin-bottom: .5em;
+}
+
+.largeInput {
+ width: 100%;
+ padding: .7em;
+ box-sizing: border-box;
+ height: 5em;
+ border: 1px solid #CECDCD;
+ vertical-align: middle;
+ font-family: sans-serif;
+ font-size: .9rem;
+}
+
+.primaryButton {
+ border: none;
+ background-color: #5E9ECE;
+ color: #FFF;
+ font-weight: 400;
+ display: inline-block;
+ float: right;
+ cursor: pointer;
+ font-size: .9rem;
+ margin: .5em 0 1em;
+ padding: .5em;
}
#IDresults {
- min-height: 2em;
+ min-height: 2em;
}
-.odd,.even,.zeroth {
- border: 1px solid #ccc;
-}
+@media (min-width:676px) and (max-width: 800px) {
+ #infoPop.active {
+ width: 40%;
+ }
-.zeroth {
- background-color: #f9f9f9;
-}
+ .container {
+ width: 80%;
+ }
-.odd {
- background-color: #f5fffa;
-}
+ .container.withPopup {
+ margin-left: -20%;
+ }
-.even {
- background-color: #f5faff;
+ #demoPop {
+ width: 76%;
+ }
}
@media (min-width:450px) and (max-width: 675px) {
- h1 {
- font-size: 150%;
- }
+ .headline {
+ font-size: 1.5rem;
+ }
- .embiggen {
- width: 90%;
- }
+ .embiggen {
+ width: 90%;
+ }
- .pickADemo {
- margin-top: .1em;
- }
+ .pickADemo {
+ margin-top: .1em;
+ }
+
+ #infoPop.active {
+ width: 50%;
+ }
+
+ .container {
+ width: 90%;
+ }
+
+ .container.withPopup {
+ margin-left: -40%;
+ }
+
+ #demoPop {
+ width: 86%;
+ }
}
@media (min-width:301px) and (max-width: 449px) {
- h1 {
- font-size: 120%;
- }
+ .headline {
+ font-size: 1.2rem;
+ }
- .embiggen {
- width: 85%;
- }
+ .embiggen {
+ width: 85%;
+ }
- .pickADemo {
- margin-top: 0;
- }
+ .pickADemo {
+ margin-top: 0;
+ }
+
+ #infoPop.active {
+ width: 70%;
+ }
+
+ .container {
+ width: 100%;
+ }
+
+ .container.withPopup {
+ margin-left: -70%;
+ }
+
+ #demoPop {
+ width: 96%;
+ right: 0;
+ left: 0;
+ }
+
+ .asciiCat {
+ font-size: .7rem;
+ width: 45px;
+ vertical-align: 75%;
+ }
}
@media (max-width: 300px) {
- body {
- font-size: 80%;
- }
+ body {
+ font-size: 80%;
+ }
- h1 {
- font-size: 120%;
- }
+ .headline {
+ font-size: 1.2rem;
+ }
- .embiggen {
- width: 100%;
- clear: right;
- }
+ .headline .subhead {
+ font-size: 1rem;
+ }
- .pickADemo {
- margin-top: .1em;
- }
+ .embiggen {
+ width: 100%;
+ clear: right;
+ }
- .demoPicker {
- width: 8em;
- }
+ .pickADemo {
+ margin-top: .1em;
+ }
- .LMContainer {
- width: 90%;
- display: block;
- clear: right;
- }
+ .demoPicker {
+ width: 8em;
+ }
- .LM {
- float: none;
- width: 100%;
- }
+ .LMContainer {
+ width: 90%;
+ display: block;
+ clear: right;
+ }
+
+ .LM {
+ float: none;
+ width: 100%;
+ }
+
+ #infoPop.active {
+ width: 85%;
+ }
+
+ .container {
+ width: 100%;
+ }
+
+ .container.withPopup {
+ margin-left: -85%;
+ }
+
+ #demoPop {
+ width: 96%;
+ right: 0;
+ left: 0;
+ }
+
+ .asciiCat {
+ font-size: .7rem;
+ width: 45px;
+ vertical-align: 0;
+ }
}
\ No newline at end of file
diff --git a/textcatdemo.js b/textcatdemo.js
index 2b46869..d77a8ce 100644
--- a/textcatdemo.js
+++ b/textcatdemo.js
@@ -1,24 +1,24 @@
var initLangs = ['English', 'French', 'German', 'Italian', 'Spanish'];
var extraLangs = ['Afrikaans', 'Albanian', 'Armenian', 'Basque', 'Belarusian',
- 'Bengali', 'Bosnian', 'Breton', 'Bulgarian', 'Burmese', 'Cantonese',
- 'Catalan', 'Croatian', 'Czech', 'Danish', 'Dutch', 'Esperanto',
- 'Estonian', 'Finnish', 'Georgian', 'Gujarati', 'Hindi', 'Hungarian',
- 'Icelandic', 'Indonesian', 'Irish', 'Javanese', 'Kannada', 'Latin',
- 'Latvian', 'Lithuanian', 'Malay', 'Malayalam', 'Marathi', 'Norwegian',
- 'Oriya', 'Persian', 'Polish', 'Portuguese', 'Romanian', 'Scots',
- 'Serbian', 'SerboCroatian', 'Slovak', 'Slovenian', 'Sundanese',
- 'Swedish', 'Tagalog', 'Tamil', 'Telugu', 'Turkish', 'Turkmen',
- 'Ukrainian', 'Urdu', 'Vietnamese', 'Welsh', 'WesternPunjabi'
+ 'Bengali', 'Bosnian', 'Breton', 'Bulgarian', 'Burmese', 'Cantonese',
+ 'Catalan', 'Croatian', 'Czech', 'Danish', 'Dutch', 'Esperanto',
+ 'Estonian', 'Finnish', 'Georgian', 'Gujarati', 'Hindi', 'Hungarian',
+ 'Icelandic', 'Indonesian', 'Irish', 'Javanese', 'Kannada', 'Latin',
+ 'Latvian', 'Lithuanian', 'Malay', 'Malayalam', 'Marathi', 'Norwegian',
+ 'Oriya', 'Persian', 'Polish', 'Portuguese', 'Romanian', 'Scots',
+ 'Serbian', 'SerboCroatian', 'Slovak', 'Slovenian', 'Sundanese',
+ 'Swedish', 'Tagalog', 'Tamil', 'Telugu', 'Turkish', 'Turkmen',
+ 'Ukrainian', 'Urdu', 'Vietnamese', 'Welsh', 'WesternPunjabi'
];
var wikiLangs = ['Afrikaans', 'Albanian', 'Basque', 'Belarusian', 'Bosnian',
- 'Breton', 'Burmese', 'Cantonese', 'Catalan', 'Croatian', 'Danish',
- 'Esperanto', 'Estonian', 'Finnish', 'Gujarati', 'Hungarian',
- 'Icelandic', 'Irish', 'Javanese', 'Kannada', 'Latin', 'Latvian',
- 'Lithuanian', 'Malay', 'Malayalam', 'Marathi', 'Norwegian', 'Oriya',
- 'Romanian', 'Scots', 'Serbian', 'SerboCroatian', 'Slovak', 'Slovenian',
- 'Sundanese', 'Tagalog', 'Turkmen', 'Urdu', 'Welsh', 'WesternPunjabi'
+ 'Breton', 'Burmese', 'Cantonese', 'Catalan', 'Croatian', 'Danish',
+ 'Esperanto', 'Estonian', 'Finnish', 'Gujarati', 'Hungarian',
+ 'Icelandic', 'Irish', 'Javanese', 'Kannada', 'Latin', 'Latvian',
+ 'Lithuanian', 'Malay', 'Malayalam', 'Marathi', 'Norwegian', 'Oriya',
+ 'Romanian', 'Scots', 'Serbian', 'SerboCroatian', 'Slovak', 'Slovenian',
+ 'Sundanese', 'Tagalog', 'Turkmen', 'Urdu', 'Welsh', 'WesternPunjabi'
];
var activeLangs = {};
@@ -32,320 +32,321 @@
var demoNum = -1;
var demoStep = -1;
+var queryForm = document.getElementById('queryForm')
+
function setInitLangs() {
- initializeLangs(initLangs, 0, 0);
+ initializeLangs(initLangs, 0, 0);
}
function initializeLangs(theLangs, theDir, theMore) {
- activeLangs = {};
+ activeLangs = {};
- showWikiLangs = 1 - theDir;
- LMDirSelect(document.querySelectorAll(".LM")[theDir]);
+ showWikiLangs = 1 - theDir;
+ LMDirSelect(document.querySelectorAll(".LM")[theDir]);
- showExtraLangs = 1 - theMore;
- extraLangToggle(document.querySelector("#moreLess"));
+ showExtraLangs = 1 - theMore;
+ extraLangToggle(document.querySelector("#moreLess"));
- for (var i = 0; i < theLangs.length; i++) {
- var lang = document.querySelector("#" + theLangs[i]);
- LMToggle(lang);
- }
- displayLangs();
+ for (var i = 0; i < theLangs.length; i++) {
+ var lang = document.querySelector("#" + theLangs[i]);
+ LMToggle(lang);
+ }
+ displayLangs();
}
document.addEventListener("DOMContentLoaded", function(event) {
- setInitLangs();
+ setInitLangs();
});
document.addEventListener("keydown", catchKeyDownEvent, false);
+queryForm.addEventListener("submit", function(event) {
+ event.preventDefault();
+ submitQuery();
+});
+
function LMClear() {
- var langs = document.querySelectorAll(".LMPick");
- for (var i = 0; i < langs.length; i++) {
- langs[i].style.color = "black";
- langs[i].style.border = "";
- }
- activeLangs = {};
+ var langs = document.querySelectorAll(".LMPick");
+ for (var i = 0; i < langs.length; i++) {
+ langs[i].classList.remove('active');
+ }
+ activeLangs = {};
}
function LMAll() {
- var langs = document.querySelectorAll(".LMPick");
- for (var i = 0; i < langs.length; i++) {
- langs[i].style.color = "black";
- langs[i].style.border = "";
- }
- activeLangs = {};
- for (var i = 0; i < langs.length; i++) {
- LMToggle(langs[i]);
- }
+ var langs = document.querySelectorAll(".LMPick");
+ activeLangs = {};
+ for (var i = 0; i < langs.length; i++) {
+ LMToggle(langs[i]);
+ }
}
function LMToggleByName(name) {
- LMToggle(document.querySelector("#" + name));
+ LMToggle(document.querySelector("#" + name));
}
function LMToggle(item) {
- if (activeLangs[item.id] == 1) {
- item.style.color = "black";
- item.style.border = "";
- activeLangs[item.id] = 0;
- } else {
- item.style.color = "green";
- item.style.border = "1px solid #a3bfb1";
- activeLangs[item.id] = 1;
- }
+ if (activeLangs[item.id] == 1) {
+ item.classList.remove('active');
+ activeLangs[item.id] = 0;
+ } else {
+ item.classList.add('active');
+ activeLangs[item.id] = 1;
+ }
}
function LMDirSelect(item) {
- var tabs = document.querySelectorAll(".LM");
- showWikiLangs = 0;
- for (var i = 0; i < tabs.length; i++) {
- if (tabs[i] == item) {
- tabs[i].style.color = "#000";
- tabs[i].style.fontWeight = "bold";
- textCatDir = textCatDirs[i];
- showWikiLangs = i;
- } else {
- tabs[i].style.color = "#ccc";
- tabs[i].style.fontWeight = "normal";
- }
- }
- displayLangs();
+ var tabs = document.querySelectorAll(".LM");
+ showWikiLangs = 0;
+ for (var i = 0; i < tabs.length; i++) {
+ if (tabs[i] == item) {
+ tabs[i].classList.add('active')
+ textCatDir = textCatDirs[i];
+ showWikiLangs = i;
+ } else {
+ tabs[i].classList.remove('active')
+ }
+ }
+ displayLangs();
}
function extraLangToggle(item) {
- if (showExtraLangs == 1) {
- item.innerHTML = "more...";
- showExtraLangs = 0;
- } else {
- item.innerHTML = "less...";
- showExtraLangs = 1;
- }
- displayLangs();
+ if (showExtraLangs == 1) {
+ item.innerHTML = "more...";
+ showExtraLangs = 0;
+ } else {
+ item.innerHTML = "less...";
+ showExtraLangs = 1;
+ }
+ displayLangs();
}
function displayLangs() {
- for (var i = 0; i < extraLangs.length; i++) {
- var lang = document.querySelector("#" + extraLangs[i]);
- lang.style.display = showExtraLangs ? "inline" : "none";
- }
- for (var i = 0; i < wikiLangs.length; i++) {
- var lang = document.querySelector("#" + wikiLangs[i]);
- lang.style.display = showWikiLangs ? "inline" : "none";
- }
- if (!showExtraLangs) {
- for (var i = 0; i < extraLangs.length; i++) {
- var lang = document.querySelector("#" + extraLangs[i]);
- lang.style.display = "none";
- }
- }
+ for (var i = 0; i < extraLangs.length; i++) {
+ var lang = document.querySelector("#" + extraLangs[i]);
+ lang.style.display = showExtraLangs ? "inline-block" : "none";
+ }
+ for (var i = 0; i < wikiLangs.length; i++) {
+ var lang = document.querySelector("#" + wikiLangs[i]);
+ lang.style.display = showWikiLangs ? "inline-block" : "none";
+ }
+ if (!showExtraLangs) {
+ for (var i = 0; i < extraLangs.length; i++) {
+ var lang = document.querySelector("#" + extraLangs[i]);
+ lang.style.display = "none";
+ }
+ }
}
function catchReturn(ta) {
- if (window.event.keyCode == 13) {
- submitQuery();
- return false;
- }
- return true;
+ if (window.event && window.event.keyCode == 13) {
+ submitQuery();
+ return false;
+ }
+ return true;
}
function catchKeyDownEvent(e) {
- if (e.keyCode == 13) {
- submitQuery();
- return;
- }
- if (e.keyCode == 39) {
- if (document.getElementById("queryTA") != document.activeElement) {
- demoNext();
- }
- return;
- }
- if (e.keyCode == 27) {
- closeInfo();
- hideDemo(0);
- return;
- }
+ if (e.keyCode == 13) {
+ submitQuery();
+ return;
+ }
+ if (e.keyCode == 27) {
+ closeInfo();
+ closeDemo(0);
+ return;
+ }
+ if (document.getElementById("queryTA") != document.activeElement) {
+ if (e.keyCode == 39) {
+ demoNext();
+ }
+ return;
+ }
}
function langIDLink(lang, score, basescore, query) {
- //
https://en.m.wikipedia.org/w/index.php?title=Special:Search&fulltext=Search&search=qqq
- var relScore = score / basescore;
- return "<a target='iframeWiki' href='https://"; + lang +
- ".m.wikipedia.org/w/index.php?title=Special:Search&" +
- "fulltext=Search&uselang=en&search=" + encodeURI(query) + "'>" +
- code2lang[lang] + "</a> (" + relScore.toFixed(4) + ")";
+ //
https://en.m.wikipedia.org/w/index.php?title=Special:Search&fulltext=Search&search=qqq
+ var relScore = score / basescore;
+ return "<a target='iframeWiki' href='https://"; + lang +
+ ".m.wikipedia.org/w/index.php?title=Special:Search&" +
+ "fulltext=Search&uselang=en&search=" + encodeURI(query) + "'>" +
+ code2lang[lang] + "</a> (" + relScore.toFixed(4) + ")";
}
function submitQuery(query) {
- var ta = document.getElementById("queryTA");
- if (query !== undefined) {
- ta.value = query;
- }
- var idResults = document.getElementById("IDresults");
- ta.value = ta.value.replace(/^\s+/, "");
- ta.value = ta.value.replace(/\s+$/, "");
- var query = ta.value;
+ var ta = document.getElementById("queryTA");
+ if (typeof query === 'String' || typeof query === 'string') {
+ ta.value = query;
+ }
+ var idResults = document.getElementById("IDresults");
+ ta.value = ta.value.replace(/^\s+/, "");
+ ta.value = ta.value.replace(/\s+$/, "");
+ var query = ta.value;
- if (query != "") {
- idResults.innerHTML = "<span class='dimText'>...Thinking...</span>";
- var lang = "";
- for (var l in activeLangs) {
- if (activeLangs[l]) {
- if (document.getElementById(l).style.display != "none") {
- lang += (lang ? "," : "") + lang2code[l];
- }
- }
- }
- var msize = 3000;
- if (lang) {
- var xmlhttp = new XMLHttpRequest();
- xmlhttp.onreadystatechange = function() {
- if (xmlhttp.readyState == 4 && xmlhttp.status == 200) {
- var output = "";
- if (xmlhttp.responseText.match(/^Cannot/)) {
- output = xmlhttp.responseText;
- } else {
- var phpResults = JSON.parse(xmlhttp.responseText);
- var basescore = 1;
- for (var detectedLang in phpResults) {
- if (output == "") {
- basescore = phpResults[detectedLang];
- }
- output += "<div class='langRes'>" +
- langIDLink(detectedLang, phpResults[
- detectedLang], basescore, query) +
- "</div>";
- }
- }
- idResults.innerHTML = output;
- }
- };
- xmlhttp.open("GET", "textcatdemo.php?q=" + encodeURI(query) +
- "&lang=" + encodeURI(lang) + "&dir=" + encodeURI(
- textCatDir) + "&msize=" + encodeURI(msize), true);
- xmlhttp.send();
- return;
- }
- }
- clearIDresults();
+ if (query != "") {
+ idResults.innerHTML = "<span
class='dimText'>...Thinking...</span>";
+ var lang = "";
+ for (var l in activeLangs) {
+ if (activeLangs[l]) {
+ // Only consider languages currently on-screen.
+ // Selected langauges that are hidden should
not be used.
+ if (document.getElementById(l).style.display !=
"none") {
+ lang += (lang ? "," : "") +
lang2code[l];
+ }
+ }
+ }
+ var msize = 3000;
+ if (lang) {
+ var xmlhttp = new XMLHttpRequest();
+ xmlhttp.onreadystatechange = function() {
+ if (xmlhttp.readyState == 4 && xmlhttp.status
== 200) {
+ var output = "";
+ if
(xmlhttp.responseText.match(/^Cannot/)) {
+ output = xmlhttp.responseText;
+ } else {
+ var phpResults =
JSON.parse(xmlhttp.responseText);
+ var basescore = 1;
+ for (var detectedLang in
phpResults) {
+ if (output == "") {
+ basescore =
phpResults[detectedLang];
+ }
+ output += "<div
class='langRes'>" +
+
langIDLink(detectedLang, phpResults[
+
detectedLang], basescore, query) +
+ "</div>";
+ }
+ }
+ idResults.innerHTML = output;
+ }
+ };
+ xmlhttp.open("GET", "textcatdemo.php?q=" +
encodeURI(query) +
+ "&lang=" + encodeURI(lang) + "&dir=" +
encodeURI(
+ textCatDir) + "&msize=" +
encodeURI(msize), true);
+ xmlhttp.send();
+ return;
+ }
+ }
+ clearIDresults();
}
function clearIDresults() {
- document.getElementById("IDresults").innerHTML =
- '<span class="dimText">Language ID results...</span>';
+ document.getElementById("IDresults").innerHTML =
+ '<span class="dimText">Language ID results...</span>';
}
function closeInfo() {
- document.querySelector("#infoPop").style.display = "none";
- document.querySelector("#infoPopText").innerHTML =
- '<span class="dimText">Additional Information...</span>';
+ document.querySelector("#infoPop").classList.remove('active');
+ document.querySelector("#container").classList.remove('withPopup');
+ document.querySelector("#infoPopText").innerHTML =
+ '<span class="dimText">Additional Information...</span>';
}
function showInfo(msgNum) {
- document.querySelector("#infoPopText").innerHTML =
- "<div class='infoIconDiv'><img src='info.png' class=icon></div>" +
- infoMsg[msgNum];
- document.querySelector("#infoPop").style.display = "block";
- }
+ closeDemo(1);
+ document.querySelector("#infoPopText").innerHTML = infoMsg[msgNum];
+ document.querySelector("#infoPop").classList.add("active");
+ document.querySelector("#container").classList.add("withPopup");
+}
// Demos
demoScript = [
- // Europanto
- ["Europanto", [
- ["<a class=external href='https://en.wikipedia.org/wiki/Europanto'
target=_blank>Europanto</a> is a macaronic language (i.e., a mixture of
languages), made up of whatever European languages the speaker wishes to use.
Of course, it will be difficult for TextCat to handle since it is a mix of
languages and there is no single right answer. In this demo we’ll look at
how TextCat’s results depend on the languages selected for comparison.",
- ""],
- ["Let’s use the Query Language Models with the larger list of
languages, and activate Dutch, English, French, German, Italian, Portuguese,
and Spanish.",
-
"initializeLangs(['English','Spanish','French','German','Portuguese',
'Italian', 'Dutch'], 0, 1)"],
- ["After pasting the <a class=external
href='https://en.wikipedia.org/wiki/Europanto#Example' target=_blank>Europanto
Example text</a> from English Wikipedia into the Query box, submit it to
TextCat. The results should be German followed closely by Dutch, with English
and French in third and fourth place, and Italian coming in a distant fifth.",
- "submitQuery('Europanto want nicht informe aber amuse. Porqué
betanke aan die illuminante forza van eine laugh, sometimes man remarque things
dat anders el hadde noticed nicht. Laugh esse directe emanatione des
intelligence, Aristotele dixit.');"],
- ["Let’s disable German and re-submit to TextCat. Now English is
first with French a very close second, followed by Italian, as before. Spanish
is now fourth. When German was available, Spanish didn’t score well
enough to be reported.",
- "LMToggleByName('German'); LMToggleByName('Dutch');
submitQuery();"],
- ["Let’s disable English and French and try again. Now Italian
moves up to first place, with Spanish in second. Portuguese makes an
appearance. You can try the same query string using the Wiki-Text Language
Models to see how they behave differently from the Query Language Models.",
- "LMToggleByName('English'); LMToggleByName('French');
submitQuery();"],
- ["For use on a Wikipedia—say, to redirect queries in another language
to a different wiki—we need to tune the languages we want to consider based on
what other-language queries occur most frequently on that Wikipedia. We
don’t want to include languages that rarely occur or that get many more
errors than correct answers. You can read a lot more about this kind of <a
target=_blank class=external
href='https://www.mediawiki.org/wiki/User:TJones_(WMF)/Notes/TextCat_Optimization_for_frwiki_eswiki_itwiki_and_dewiki'>tuning
for French, Spanish, Italian, and German Wikipedias</a>.",
- ""],
- ]],
- // Names are Weird
- ["Names are Weird", [
- ["Names are particularly hard when it comes to language
identification, in part because they are often short, and in part because they
aren’t really in a particular language. The line between the <a class=external
href='https://en.wikipedia.org/wiki/Ethnolinguistics'
target=_blank>ethnolinguistic</a> source of a name and a given language can be
more or less blurry, depending on the writing system, the popularity of the
name around the world (<i><a class=external
href='https://en.wikipedia.org/wiki/Maria_(given_name)'
target=_blank>Maria</a>,</i> for example, is common across incredibly diverse
cultures), the ethnolinguistic distinctiveness of the name elements, where the
person was born, etc.",
- ""],
- ["Let’s look at four celebrity names of distinctive ethnolinguistic
decent; all of these celebrities were born in the United States. These names
were chosen in part for their relative fame, and so happen to have articles in
the languages TextCat thinks they are in. That’s not always the case for less
famous people, who may be notable in one culture/language, while not being
well-known in the language most associated with their names.",
- ""],
- ["We’ll use the Query Language Models with the larger list of
languages, and activate all of the languages, just to make things hard. (And a
bit slower—so be patient.)",
- "extraLangToggle(document.querySelector('#moreLess'));LMAll();"],
- ["Our first name is <a class=external
href='https://en.wikipedia.org/wiki/Carla_Gugino' target=_blank>Carla
Gugino</a>, an American actress of Italian descent, born in Florida. Her name
is very Italian.",
- "submitQuery('Carla Gugino');"],
- ["Our next name is <a class=external
href='https://en.wikipedia.org/wiki/Mehmet_Oz' target=_blank>Mehmet Öz</a>, the
full name of the Turkish-American surgeon and TV personality better known as
Dr. Oz. He was born in Ohio. His name is very Turkish, in particular his given
name, <i>Mehmet,</i> which is the Turkish form of <i>Muhammad.</i>",
- "submitQuery('Mehmet Öz');"],
- ["Next up is <a class=external
href='https://en.wikipedia.org/wiki/Jane_Kaczmarek' target=_blank>Jane
Kaczmarek</a>, an American actress of Polish descent, born in Wisconsin. While
<i>Jane</i> is a common name in English-speaking countries, <i>Kaczmarek</i>
has the very distinctive Polish <i>cz</i> in it.",
- "submitQuery('Jane Kaczmarek');"],
- ["<a class=external
href='https://en.wikipedia.org/wiki/Robert_Kardashian' target=_blank>Ռոբերտ
Քարդաշյան</a> is Robert Kardashian’s name in Armenian script. He was born in
California, but the Armenian version of his name, not surprisingly, is very
Armenian.",
- "submitQuery('Ռոբերտ Քարդաշյան');"],
- ["Names with multiple ethnolinguistic sources are most likely to
give... “interesting” results. You can try the names of other celebrities, with
all the languages, or a more restricted set. For example, <a class=external
href='https://en.wikipedia.org/wiki/C%C3%A9lia_%C5%A0a%C5%A1i%C4%87'
target=_blank>Célia Šašić</a> is a German footballer. She has a French given
name by birth and a Croatian surname by marriage. What’s the right “language”
in which to categorize her name?<br><br>Try the Query Language Models and the
Wiki-Text Language Models on her name before finishing the demo.",
- "clearIDresults(); document.getElementById('queryTA').value='Célia
Šašić';"],
- ]],
- // Unexpected Results
- ["Unexpected Results and Language Tuning", [
- ["Since TextCat’s language identification is based on statistical
models, there will always be the possibility of odd results, particularly on
shorter strings—where there isn’t much on which to gather statistics—and
strings that either happen not to be very typical of their language, or happen
to have characteristics typical of another language.",
- ""],
- ["When deciding which languages models to use on German Wikipedia, for
example, we looked to see what languages other than German were present in a
sample of poorly performing queries (i.e., those with fewer than 3 results).
The <a class=external
href='https://www.mediawiki.org/wiki/User:TJones_(WMF)/Notes/TextCat_Optimization_for_frwiki_eswiki_itwiki_and_dewiki#German_Results'
target=_blank>languages found</a>, by volume, were English, Italian, Spanish,
French, Chinese, Polish, Vietnamese, Turkish, Swedish, and Dutch. There was
also some Latin, but we don’t have a model for that. Let’s activate the rest of
them.",
- "initializeLangs(['English', 'Italian', 'Spanish', 'French',
'Chinese', 'Polish', 'Vietnamese', 'Turkish', 'Swedish', 'Dutch'], 0, 1)"],
- ["Our first examples is <i>impstoffe adhaesive</i>, a typo for
<i>impfstoffe adhaesive,</i> which means “vaccines adhesive”. TextCat
recognizes it as English. Restoring the missing <i>f</i> gives is the more
distinctively German <i>pf</i> cluster, and German edges out English instead of
the other way around. <br><br>Mistakes will happen, and in our test set, there
were 124 out of 520 queries in English. 76/124 were correctly tagged as
English, with only 4 other non-English tagged incorrectly. So having English in
the mix does more good than not.",
- "submitQuery('impstoffe adhaesive');"],
- ["Medical and scientific terminology often get borrowed from language
to language. The word <i>aortotomie,</i> meaning “aortotomy”—an incision of the
aorta—is identified by TextCat as Italian. The source of <i>aorta</i> in
English is Latin, from earlier Greek, and it is spelled exactly the same in at
least a <a class=external href='https://en.wiktionary.org/wiki/aorta'
target=_blank>dozen languages</a>, and similarly in many others. While
<i>Aortotomie</i>, properly capitalized in German, is a German word, it’s also
the same in French. Scientific terms in isolation can be very
ambiguous.<br><br>Overall, there were only 8 out of 520 Italian queries in our
sample. 5/8 were identified as Italian, but 12 more were incorrectly identified
as Italian. Including Italian does more harm than good.",
- "submitQuery('aortotomie');"],
- ["Next, <i>personalunion</i>, meaning “staff union”, is identified as
Spanish. Overall, there were only 8 out of 520 Spanish queries in our sample.
7/8 were identified as Spanish (pretty good!), but 14 non-Spanish queries were
incorrectly identified as Spanish. Having Spanish does more harm than good.",
- "submitQuery('personalunion');"],
- ["Compound nouns are very Germanic, but
<i>organisationsdokumentation</i>, “organizational documentation”, was
identified as French. There were only 5 out of 520 French queries in our
sample, with 4 tagged correctly, but 13 non-French queries tagged incorrectly.
Including French does more harm than good.",
- "submitQuery('organisationsdokumentation');"],
- ["You can also click to try:<ul><li><i><a href=''
onclick=\"submitQuery('Biolumineszenz'); return false;\">Bioluminiszez</a>,</i>
typo for <i>Biolumineszenz,</i> “bioluminescence”, which is tagged as
Polish</li><li><i><a href='' onclick=\"submitQuery('humanmedezin'); return
false;\">humanmedezin</a>,</i> typo for <i>humanmedizin,</i> “human medicine”,
which is tagged as Turkish</li><li><i><a href=''
onclick=\"submitQuery('Auflösevermögen'); return
false;\">Auflösevermögen</a>,</i> “resolving power”, which is tagged as
Swedish, and</li><li><i><a href='' onclick=\"submitQuery('kammilentee'); return
false;\">kammilentee</a>,</i> typo for <i>kamillentee,</i> “Camomile tea”,
which is tagged as Dutch.</li></ul>Dutch and Swedish were both rare—only one
example in our sample—but both were very bad about incorrectly tagging queries
in German, with more than 30 each incorrectly tagged in our sample of only
520.",
- "submitQuery('');"],
- ["When tuning the choice of languages to include for detection for a
given wiki, it’s important to know how often they are likely to occur, and how
often they are likely to be tagged incorrectly. If Italian, Spanish, or French
were more common among queries on German Wikipedia, it would make sense to
tolerate a relatively small number of identification errors. Dutch or Swedish
would have to be much more common to offset the likelihood of errors. As it is,
though, including these languages generates significantly more incorrect
answers than correct, so it’s best to exclude them.",
- ""],
- ]],
- // Gibberish
- ["Gibberish: Garbage In, Garbage Out", [
- ["We get a surprising number of gibberish queries on the various
Wikipedias. For example, <i>fdshkfldshflshdkflsr.</i> These pose a minor
problem for language identification. Obviously they aren’t in any particular
language, but they may be identified as being in some language by TextCat,
because the statistics will always have a least bad match.",
- ""],
- ["Enabling all the smaller set of Query Language Models, we can submit
<i>fdshkfldshflshdkflsr</i> for identification and see that it is scores
similarly for all of German, English, French, Italian, and Spanish—that is, all
the languages available that use the Latin alphabet; there really isn’t much to
distinguish it among them other than slight statistical variations.",
- "LMAll();submitQuery('fdshkfldshflshdkflsr');"],
- ["If we enable more languages TextCat cannot even guess at what
language <i>fdshkfldshflshdkflsr</i> is in, as the scores across all the
languages in the Latin alphabet are too similar, and—as configured here—TextCat
deems any result as too ambiguous if more than five languages are within 5% of
each other.",
-
"extraLangToggle(document.querySelector('#moreLess'));submitQuery('fdshkfldshflshdkflsr');"],
- ["If we enable five models for languages that don’t use the Latin
alphabet, the scores on our gibberish (or any other string of Latin characters)
are all 1.0000. Every <a class=external
href='https://en.wikipedia.org/wiki/N-gram' target=_blank>n-gram</a> in the
string received the unknown n-gram penalty score in each language, so they are
all the same. Upon closer inspection, note that the results are sorted in
reverse alphabetical order by language code (the code for Chinese is “zh”, for
example), rather than numerically, because that’s all there is to sort on.",
- "LMClear();initializeLangs(['Arabic', 'Armenian', 'Bengali',
'Bulgarian', 'Chinese'], 0, 1); submitQuery('fdshkfldshflshdkflsr');"],
- ["In an actual Wikipedia search scenario, a gibberish query is
unlikely to get a match on the wiki it’s on. If language detection returns
another language, it’s also unlikely to get any results on that other wiki,
which just wastes CPU cycles. A potential future project is gibberish
detection, which would allow us to note this kind of query and allow us to
treat them differently—not expending too much extra effort to get results, or
ignoring them in certain data analyses.<br><br>But of course we’d still search
for them! Old school sci-fi fans would recognize that <a class=external
href='https://en.wikipedia.org/wiki/Electronic_Labyrinth:_THX_1138_4EB'
target=_blank>THX 1138 4EB</a> is not gibberish, and elite H4x0rs would
certainly note that <a class=external
href='https://en.wikipedia.org/wiki/10100111001' target=_blank>10100111001</a>
is meaningful.",
- ""],
- ]],
- // Chinese Challenges
- ["Chinese Can Be Harder Than It Would Seem", [
- ["It seems like Chinese should be easy to identify, and when it occurs
by itself, it usually is. But when it is mixed with other languages, or even
just with text in the Latin alphabet, weird things can happen.",
- "extraLangToggle(document.querySelector('#moreLess'));
LMClear();"],
- ["The current Chinese language model was built like all the other
language models, and so it includes <a class=external
href='https://en.wikipedia.org/wiki/N-gram' target=_blank>n-grams</a> of size
from 1 to 5. When many languages share a writing system, like the Latin
alphabet, the distribution of characters and the co-occurence of characters are
very important in distinguishing languages. Although Chinese characters
<i>are</i> used in other writing systems, they are distinctly Chinese when they
occur alone.",
- ""],
- ["Another important issue is that Chinese writing uses many more
unique characters than most writing systems, so not all of them are present in
our models. Our default model size is 3000 n-grams, and at the moment it’s
important to use the same model size across language models so the scoring is
normalizes. Even a 5000 n-gram model has fewer than 2300 distinct characters in
it, so the rarer Chinese characters are not even represented. Also note that at
the level of individual characters, common single Latin letters are much more
frequent in their language models (such as <i>e, a, i, o, n, r, t,</i> and
<i>s</i> in English) than any individual Chinese character is in the Chinese
language model.",
- ""],
- ["As a result, we can get some unexpected results when Chinese and
Latin characters are mixed, especially if the Chinese characters are rare or
few in number. We have a fairly frequent pattern of queries that consist of a
snippet of Chinese text followed by something that looks like a serial number
or other arbitrary string of letters and numbers. Sometimes the letters happen
to be very characteristic of a particular language that uses the Latin
alphabet, and it overwhelms the Chinese statistics.",
- ""],
- ["Let’s activate a number of Chinese along with a number of European
languages. Try any of these (see if you can guess the language before you
click):<ul style='margin-bottom:0'><li><a href=''
onclick=\"submitQuery('的科学研究17lho4232'); return
false;\">的科学研究17lho4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17ein4232'); return
false;\">的科学研究17ein4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17eau4232'); return
false;\">的科学研究17eau4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17van4232'); return
false;\">的科学研究17van4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17ight4232'); return
false;\">的科学研究17ight4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17ione4232'); return
false;\">的科学研究17ione4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17los4232'); return
false;\">的科学研究17los4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17osz4232'); return
false;\">的科学研究17osz4232</a></li></ul>",
- "initializeLangs(['Chinese', 'Portuguese', 'German', 'French',
'Dutch', 'English', 'Italian', 'Spanish', 'Polish'], 0, 1);"],
- ["A future project may include doing some sort of Chinese-specific
language modeling to improve coverage of and statistics for Chinese
characters.",
- ""],
- ]],
+ // Europanto
+ ["Europanto", [
+ ["<a class=external
href='https://en.wikipedia.org/wiki/Europanto' target=_blank>Europanto</a> is a
macaronic language (i.e., a mixture of languages), made up of whatever European
languages the speaker wishes to use. Of course, it will be difficult for
TextCat to handle since it is a mix of languages and there is no single right
answer. In this demo we’ll look at how TextCat’s results depend on
the languages selected for comparison.",
+ ""],
+ ["Let’s use the Query Language Models with the larger
list of languages, and activate Dutch, English, French, German, Italian,
Portuguese, and Spanish.",
+
"initializeLangs(['English','Spanish','French','German','Portuguese',
'Italian', 'Dutch'], 0, 1)"],
+ ["After pasting the <a class=external
href='https://en.wikipedia.org/wiki/Europanto#Example' target=_blank>Europanto
Example text</a> from English Wikipedia into the Query box, submit it to
TextCat. The results should be German followed closely by Dutch, with English
and French in third and fourth place, and Italian coming in a distant fifth.",
+ "submitQuery('Europanto want nicht informe aber amuse.
Porqué betanke aan die illuminante forza van eine laugh, sometimes man remarque
things dat anders el hadde noticed nicht. Laugh esse directe emanatione des
intelligence, Aristotele dixit.');"],
+ ["Let’s disable German and Dutch and re-submit to
TextCat. Now English is first with French a very close second, followed by
Italian, as before. Spanish is now fourth. When German and Dutch were
available, Spanish didn’t score well enough to be reported.",
+ "LMToggleByName('German'); LMToggleByName('Dutch');
submitQuery();"],
+ ["Let’s disable English and French and try again. Now
Italian moves up to first place, with Spanish in second. Portuguese makes an
appearance. You can try the same query string using the Wiki-Text Language
Models to see how they behave differently from the Query Language Models.",
+ "LMToggleByName('English'); LMToggleByName('French');
submitQuery();"],
+ ["For use on a Wikipedia—say, to redirect queries in another
language to a different wiki—we need to tune the languages we want to consider
based on what other-language queries occur most frequently on that Wikipedia.
We don’t want to include languages that rarely occur or that get many
more errors than correct answers. You can read a lot more about this kind of <a
target=_blank class=external
href='https://www.mediawiki.org/wiki/User:TJones_(WMF)/Notes/TextCat_Optimization_for_frwiki_eswiki_itwiki_and_dewiki'>tuning
for French, Spanish, Italian, and German Wikipedias</a>.",
+ ""],
+ ]],
+ // Names are Weird
+ ["Names are Weird", [
+ ["Names are particularly hard when it comes to language
identification, in part because they are often short, and in part because they
aren’t really in a particular language. The line between the <a class=external
href='https://en.wikipedia.org/wiki/Ethnolinguistics'
target=_blank>ethnolinguistic</a> source of a name and a given language can be
more or less blurry, depending on the writing system, the popularity of the
name around the world (<i><a class=external
href='https://en.wikipedia.org/wiki/Maria_(given_name)'
target=_blank>Maria</a>,</i> for example, is common across incredibly diverse
cultures), the ethnolinguistic distinctiveness of the name elements, where the
person was born, etc.",
+ ""],
+ ["Let’s look at four celebrity names of distinctive
ethnolinguistic decent; all of these celebrities were born in the United
States. These names were chosen in part for their relative fame, and so happen
to have articles in the languages TextCat thinks they are in. That’s not always
the case for less famous people, who may be notable in one culture/language,
while not being well-known in the language most associated with their names.",
+ ""],
+ ["We’ll use the Query Language Models with the larger
list of languages, and activate all of the languages, just to make things hard.
(And a bit slower—so be patient.)",
+
"extraLangToggle(document.querySelector('#moreLess'));LMAll();"],
+ ["Our first name is <a class=external
href='https://en.wikipedia.org/wiki/Carla_Gugino' target=_blank>Carla
Gugino</a>, an American actress of Italian descent, born in Florida. Her name
is very Italian.",
+ "submitQuery('Carla Gugino');"],
+ ["Our next name is <a class=external
href='https://en.wikipedia.org/wiki/Mehmet_Oz' target=_blank>Mehmet Öz</a>, the
full name of the Turkish-American surgeon and TV personality better known as
Dr. Oz. He was born in Ohio. His name is very Turkish, in particular his given
name, <i>Mehmet,</i> which is the Turkish form of <i>Muhammad.</i>",
+ "submitQuery('Mehmet Öz');"],
+ ["Next up is <a class=external
href='https://en.wikipedia.org/wiki/Jane_Kaczmarek' target=_blank>Jane
Kaczmarek</a>, an American actress of Polish descent, born in Wisconsin. While
<i>Jane</i> is a common name in English-speaking countries, <i>Kaczmarek</i>
has the very distinctive Polish <i>cz</i> in it.",
+ "submitQuery('Jane Kaczmarek');"],
+ ["<a class=external
href='https://en.wikipedia.org/wiki/Robert_Kardashian' target=_blank>Ռոբերտ
Քարդաշյան</a> is Robert Kardashian’s name in Armenian script. He was born in
California, but the Armenian version of his name, not surprisingly, is very
Armenian.",
+ "submitQuery('Ռոբերտ Քարդաշյան');"],
+ ["Names with multiple ethnolinguistic sources are most likely
to give... “interesting” results. You can try the names of other celebrities,
with all the languages, or a more restricted set. For example, <a
class=external
href='https://en.wikipedia.org/wiki/C%C3%A9lia_%C5%A0a%C5%A1i%C4%87'
target=_blank>Célia Šašić</a> is a German footballer. She has a French given
name by birth and a Croatian surname by marriage. What’s the right “language”
in which to categorize her name?<br><br>Try the Query Language Models and the
Wiki-Text Language Models on her name before finishing the demo.",
+ "clearIDresults();
document.getElementById('queryTA').value='Célia Šašić';"],
+ ]],
+ // Unexpected Results
+ ["Unexpected Results and Language Tuning", [
+ ["Since TextCat’s language identification is based on
statistical models, there will always be the possibility of odd results,
particularly on shorter strings—where there isn’t much on which to gather
statistics—and strings that either happen not to be very typical of their
language, or happen to have characteristics typical of another language.",
+ ""],
+ ["When deciding which languages models to use on German
Wikipedia, for example, we looked to see what languages other than German were
present in a sample of poorly performing queries (i.e., those with fewer than 3
results). The <a class=external
href='https://www.mediawiki.org/wiki/User:TJones_(WMF)/Notes/TextCat_Optimization_for_frwiki_eswiki_itwiki_and_dewiki#German_Results'
target=_blank>languages found</a>, by volume, were English, Italian, Spanish,
French, Chinese, Polish, Vietnamese, Turkish, Swedish, and Dutch. There was
also some Latin, but we don’t have a model for that. Let’s activate the rest of
them.",
+ "initializeLangs(['English', 'Italian', 'Spanish',
'French', 'Chinese', 'Polish', 'Vietnamese', 'Turkish', 'Swedish', 'Dutch'], 0,
1)"],
+ ["Our first examples is <i>impstoffe adhaesive</i>, a typo for
<i>impfstoffe adhaesive,</i> which means “vaccines adhesive”. TextCat
recognizes it as English. Restoring the missing <i>f</i> gives is the more
distinctively German <i>pf</i> cluster, and German edges out English instead of
the other way around. <br><br>Mistakes will happen, and in our test set, there
were 124 out of 520 queries in English. 76/124 were correctly tagged as
English, with only 4 other non-English tagged incorrectly. So having English in
the mix does more good than not.",
+ "submitQuery('impstoffe adhaesive');"],
+ ["Medical and scientific terminology often get borrowed from
language to language. The word <i>aortotomie,</i> meaning “aortotomy”—an
incision of the aorta—is identified by TextCat as Italian. The source of
<i>aorta</i> in English is Latin, from earlier Greek, and it is spelled exactly
the same in at least a <a class=external
href='https://en.wiktionary.org/wiki/aorta' target=_blank>dozen languages</a>,
and similarly in many others. While <i>Aortotomie</i>, properly capitalized in
German, is a German word, it’s also the same in French. Scientific terms in
isolation can be very ambiguous.<br><br>Overall, there were only 8 out of 520
Italian queries in our sample. 5/8 were identified as Italian, but 12 more were
incorrectly identified as Italian. Including Italian does more harm than good.",
+ "submitQuery('aortotomie');"],
+ ["Next, <i>personalunion</i>, meaning “staff union”, is
identified as Spanish. Overall, there were only 8 out of 520 Spanish queries in
our sample. 7/8 were identified as Spanish (pretty good!), but 14 non-Spanish
queries were incorrectly identified as Spanish. Having Spanish does more harm
than good.",
+ "submitQuery('personalunion');"],
+ ["Compound nouns are very Germanic, but
<i>organisationsdokumentation</i>, “organizational documentation”, was
identified as French. There were only 5 out of 520 French queries in our
sample, with 4 tagged correctly, but 13 non-French queries tagged incorrectly.
Including French does more harm than good.",
+ "submitQuery('organisationsdokumentation');"],
+ ["You can also click to try:<ul><li><i><a href=''
onclick=\"submitQuery('Biolumineszenz'); return false;\">Bioluminiszez</a>,</i>
typo for <i>Biolumineszenz,</i> “bioluminescence”, which is tagged as
Polish</li><li><i><a href='' onclick=\"submitQuery('humanmedezin'); return
false;\">humanmedezin</a>,</i> typo for <i>humanmedizin,</i> “human medicine”,
which is tagged as Turkish</li><li><i><a href=''
onclick=\"submitQuery('Auflösevermögen'); return
false;\">Auflösevermögen</a>,</i> “resolving power”, which is tagged as
Swedish, and</li><li><i><a href='' onclick=\"submitQuery('kammilentee'); return
false;\">kammilentee</a>,</i> typo for <i>kamillentee,</i> “Camomile tea”,
which is tagged as Dutch.</li></ul>Dutch and Swedish were both rare—only one
example in our sample—but both were very bad about incorrectly tagging queries
in German, with more than 30 each incorrectly tagged in our sample of only
520.",
+ "submitQuery('');"],
+ ["When tuning the choice of languages to include for detection
for a given wiki, it’s important to know how often they are likely to occur,
and how often they are likely to be tagged incorrectly. If Italian, Spanish, or
French were more common among queries on German Wikipedia, it would make sense
to tolerate a relatively small number of identification errors. Dutch or
Swedish would have to be much more common to offset the likelihood of errors.
As it is, though, including these languages generates significantly more
incorrect answers than correct, so it’s best to exclude them.",
+ ""],
+ ]],
+ // Gibberish
+ ["Gibberish: Garbage In, Garbage Out", [
+ ["We get a surprising number of gibberish queries on the
various Wikipedias. For example, <i>fdshkfldshflshdkflsr.</i> These pose a
minor problem for language identification. Obviously they aren’t in any
particular language, but they may be identified as being in some language by
TextCat, because the statistics will always have a least bad match.",
+ ""],
+ ["Enabling all the smaller set of Query Language Models, we can
submit <i>fdshkfldshflshdkflsr</i> for identification and see that it is scores
similarly for all of German, English, French, Italian, and Spanish—that is, all
the languages available that use the Latin alphabet; there really isn’t much to
distinguish it among them other than slight statistical variations.",
+ "LMAll();submitQuery('fdshkfldshflshdkflsr');"],
+ ["If we enable more languages TextCat cannot even guess at what
language <i>fdshkfldshflshdkflsr</i> is in, as the scores across all the
languages in the Latin alphabet are too similar, and—as configured here—TextCat
deems any result as too ambiguous if more than five languages are within 5% of
each other.",
+
"extraLangToggle(document.querySelector('#moreLess'));submitQuery('fdshkfldshflshdkflsr');"],
+ ["If we enable five models for languages that don’t use the
Latin alphabet, the scores on our gibberish (or any other string of Latin
characters) are all 1.0000. Every <a class=external
href='https://en.wikipedia.org/wiki/N-gram' target=_blank>n-gram</a> in the
string received the unknown n-gram penalty score in each language, so they are
all the same. Upon closer inspection, note that the results are sorted in
reverse alphabetical order by language code (the code for Chinese is “zh”, for
example), rather than numerically, because that’s all there is to sort on.",
+ "LMClear();initializeLangs(['Arabic', 'Armenian',
'Bengali', 'Bulgarian', 'Chinese'], 0, 1);
submitQuery('fdshkfldshflshdkflsr');"],
+ ["In an actual Wikipedia search scenario, a gibberish query is
unlikely to get a match on the wiki it’s on. If language detection returns
another language, it’s also unlikely to get any results on that other wiki,
which just wastes CPU cycles. A potential future project is gibberish
detection, which would allow us to note this kind of query and allow us to
treat them differently—not expending too much extra effort to get results, or
ignoring them in certain data analyses.<br><br>But of course we’d still search
for them! Old school sci-fi fans would recognize that <a class=external
href='https://en.wikipedia.org/wiki/Electronic_Labyrinth:_THX_1138_4EB'
target=_blank>THX 1138 4EB</a> is not gibberish, and elite H4x0rs would
certainly note that <a class=external
href='https://en.wikipedia.org/wiki/10100111001' target=_blank>10100111001</a>
is meaningful.",
+ ""],
+ ]],
+ // Chinese Challenges
+ ["Chinese Can Be Harder Than It Would Seem", [
+ ["It seems like Chinese should be easy to identify, and when it
occurs by itself, it usually is. But when it is mixed with other languages, or
even just with text in the Latin alphabet, weird things can happen.",
+ "extraLangToggle(document.querySelector('#moreLess'));
LMClear();"],
+ ["The current Chinese language model was built like all the
other language models, and so it includes <a class=external
href='https://en.wikipedia.org/wiki/N-gram' target=_blank>n-grams</a> of size
from 1 to 5. When many languages share a writing system, like the Latin
alphabet, the distribution of characters and the co-occurence of characters are
very important in distinguishing languages. Although Chinese characters
<i>are</i> used in other writing systems, they are distinctly Chinese when they
occur alone.",
+ ""],
+ ["Another important issue is that Chinese writing uses many
more unique characters than most writing systems, so not all of them are
present in our models. Our default model size is 3000 n-grams, and at the
moment it’s important to use the same model size across language models so the
scoring is normalizes. Even a 5000 n-gram model has fewer than 2300 distinct
characters in it, so the rarer Chinese characters are not even represented.
Also note that at the level of individual characters, common single Latin
letters are much more frequent in their language models (such as <i>e, a, i, o,
n, r, t,</i> and <i>s</i> in English) than any individual Chinese character is
in the Chinese language model.",
+ ""],
+ ["As a result, we can get some unexpected results when Chinese
and Latin characters are mixed, especially if the Chinese characters are rare
or few in number. We have a fairly frequent pattern of queries that consist of
a snippet of Chinese text followed by something that looks like a serial number
or other arbitrary string of letters and numbers. Sometimes the letters happen
to be very characteristic of a particular language that uses the Latin
alphabet, and it overwhelms the Chinese statistics.",
+ ""],
+ ["Let’s activate a number of Chinese along with a number of
European languages. Try any of these (see if you can guess the language before
you click):<ul style='margin-bottom:0'><li><a href=''
onclick=\"submitQuery('的科学研究17lho4232'); return
false;\">的科学研究17lho4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17ein4232'); return
false;\">的科学研究17ein4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17eau4232'); return
false;\">的科学研究17eau4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17van4232'); return
false;\">的科学研究17van4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17ight4232'); return
false;\">的科学研究17ight4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17ione4232'); return
false;\">的科学研究17ione4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17los4232'); return
false;\">的科学研究17los4232</a></li><li><a href=''
onclick=\"submitQuery('的科学研究17osz4232'); return
false;\">的科学研究17osz4232</a></li></ul>",
+ "initializeLangs(['Chinese', 'Portuguese', 'German',
'French', 'Dutch', 'English', 'Italian', 'Spanish', 'Polish'], 0, 1);"],
+ ["A future project may include doing some sort of
Chinese-specific language modeling to improve coverage of and statistics for
Chinese characters.",
+ ""],
+ ]],
];
// Info Boxes
var infoMsg = [
- // TextCat - moved to not be in an info box
- "",
- // demos
- "These <b>demos</b> provide step-by-step walkthroughs of some of the
features of TextCat, demonstrate how some of the settings affect
TextCat’s behavior, and show some interesting ways it can fail.",
- //language models
- "<p>We have two sets of <b>language models</b> available to TextCat. In
general, these language models are frequency lists of <a class=external
href='https://en.wikipedia.org/wiki/N-gram' target=_blank>n-grams</a> (in this
case, substrings of 1 to 5 characters) found in example text. The frequency
rank of n-grams extracted from new text to be identified is compared to these
known models, and the closest matches are reported by TextCat.</p><p><b>Click
on the languages you want TextCat to consider when identifying your
query.</b></p><p>The <i>Query Language Models</i> are built on real query log
data from Wikipedia in various languages. The <a class=external
href='https://www.mediawiki.org/wiki/User:TJones_(WMF)/Notes/Language_Detection_with_TextCat#Query_Data_Collection'
target=_blank>query data is very messy</a>, but better represents the
idiosyncrasies of queries—shorter strings, more question words, fewer function
words, frequent absence of diacritics, etc. There are fewer models, because
they require more manual effort to create. The selection is based largely on
query volume on the respective Wikipedias. These models generally work better
on shorter strings in a less formal style of the sort that show up in the query
logs.</p><p>The <i>Wiki-Text Language Models</i> are built on text extracted
from Wikipedias in various languages. The data is generally cleaner, the
writing more formal, and the models much less effort to generate. Many more
models are available. The selection is based in part on sizes of Wikipedias,
and the list of languages supported by the original version of TextCat. These
models are probably better for more general use.</p>",
- // queries
- "Enter a <b>query</b> and hit return to send it off to TextCat, which will
consider only the languages you’ve selected above, using the model type
you’ve selected. Language identification and relative scores will be
displayed below.",
- //results
- "<p>The scores used by TextCat and shown in the <b>language identification
results</b> are technically <i>costs,</i> so the least costly model to match is
the one TextCat likes best.</p><p>Since the cost of matching a model depends on
the length of the string being matched—every <a class=external
href='https://en.wikipedia.org/wiki/N-gram' target=_blank>n-gram</a> has a
cost, so longer strings have bigger costs, even if they are good matches—the
raw numbers don’t mean much.</p><p>Here we show the costs relative to the
lowest cost, which always has a score of 1.0000. TextCat only considers
alternatives that have a score that is close to the best score (we’ve set
it for within 5%, so the maximum score that could be shown is 1.0500).</p><p>If
too many languages are detected (here set for more than five), TextCat says it
can’t determine the language the text is in.</p>",
- //Wikipedia search results
- "<p>If you click on the name of one of the languages detected by TextCat,
your query will be sent to the appropriate Wikipedia and displayed.</p><p>Keep
in mind that Wikipedia has a maximum query length of 300 characters, so you
might be able to do language detection on your query/text, but not be able to
search for it on Wikipedia.</p><p>Since it’s unlikely that anyone will be
able to read all the languages available in this demo, the <b>Wikipedia search
results</b> are configured to use English as the user interface language for
all results shown. The mobile version of the results is also used to keep
things simple and streamlined.</p>"
+ // TextCat
+ "<b>TextCat</b> is a language detection library based on <a
class=external href='https://en.wikipedia.org/wiki/N-gram'
target=_blank>n-gram</a> text categorization. This is a demo of the <a
class=external href='https://www.mediawiki.org/wiki/TextCat'
target=_blank>modernized TextCat libraries</a>, ported from Perl to PHP by the
Wikimedia Foundation Discovery department. It provides a look into how TextCat
works—and sometimes doesn’t work—and it’s also just fun to
play with.",
+ // demos
+ "These <b>demos</b> provide step-by-step walkthroughs of some of the
features of TextCat, demonstrate how some of the settings affect
TextCat’s behavior, and show some interesting ways it can fail.",
+ //language models
+ "<p>We have two sets of <b>language models</b> available to TextCat. In
general, these language models are frequency lists of <a class=external
href='https://en.wikipedia.org/wiki/N-gram' target=_blank>n-grams</a> (in this
case, substrings of 1 to 5 characters) found in example text. The frequency
rank of n-grams extracted from new text to be identified is compared to these
known models, and the closest matches are reported by TextCat.</p><p><b>Click
on the languages you want TextCat to consider when identifying your
query.</b></p><p>The <i>Query Language Models</i> are built on real query log
data from Wikipedia in various languages. The <a class=external
href='https://www.mediawiki.org/wiki/User:TJones_(WMF)/Notes/Language_Detection_with_TextCat#Query_Data_Collection'
target=_blank>query data is very messy</a>, but better represents the
idiosyncrasies of queries—shorter strings, more question words, fewer function
words, frequent absence of diacritics, etc. There are fewer models, because
they require more manual effort to create. The selection is based largely on
query volume on the respective Wikipedias. These models generally work better
on shorter strings in a less formal style of the sort that show up in the query
logs.</p><p>The <i>Wiki-Text Language Models</i> are built on text extracted
from Wikipedias in various languages. The data is generally cleaner, the
writing more formal, and the models much less effort to generate. Many more
models are available. The selection is based in part on sizes of Wikipedias,
and the list of languages supported by the original version of TextCat. These
models are probably better for more general use.</p>",
+ // queries
+ "Enter a <b>query</b> and hit return to send it off to TextCat, which
will consider only the languages you’ve selected, using the model type
you’ve selected. Language identification and relative scores will be
displayed below.",
+ //results
+ "<p>The scores used by TextCat and shown in the <b>language
identification results</b> are technically <i>costs,</i> so the least costly
model to match is the one TextCat likes best.</p><p>Since the cost of matching
a model depends on the length of the string being matched—every <a
class=external href='https://en.wikipedia.org/wiki/N-gram'
target=_blank>n-gram</a> has a cost, so longer strings have bigger costs, even
if they are good matches—the raw numbers don’t mean much.</p><p>Here we
show the costs relative to the lowest cost, which always has a score of 1.0000.
TextCat only considers alternatives that have a score that is close to the best
score (we’ve set it for within 5%, so the maximum score that could be
shown is 1.0500).</p><p>If too many languages are detected (here set for more
than five), TextCat says it can’t determine the language the text is
in.</p>",
+ //Wikipedia search results
+ "<p>If you click on the name of one of the languages detected by
TextCat, your query will be sent to the appropriate Wikipedia and
displayed.</p><p>Keep in mind that Wikipedia has a maximum query length of 300
characters, so you might be able to do language detection on your query/text,
but not be able to search for it on Wikipedia.</p><p>Since it’s unlikely
that anyone will be able to read all the languages available in this demo, the
<b>Wikipedia search results</b> are configured to use English as the user
interface language for all results shown. The mobile version of the results is
also used to keep things simple and streamlined.</p>"
];
var lang2code = {};
@@ -423,7 +424,7 @@
// create inverse index
var code2lang = {};
for (var lang in lang2code) {
- code2lang[lang2code[lang]] = lang;
+ code2lang[lang2code[lang]] = lang;
}
// add alternate forms without spaces or dashes
@@ -432,66 +433,67 @@
// Demos
function startDemo(num) {
- hideDemo(1);
+ closeInfo();
+ closeDemo(1);
- if (num == -1) {
- return;
- }
- demoNum = num;
- demoStep = 0;
- document.querySelector("#demoPop").style.display = "block";
- document.querySelector("#TextCatHeader").style.display = "none";
- demoNext();
+ if (num == -1) {
+ return;
+ }
+ demoNum = num;
+ demoStep = 0;
+ document.querySelector("#demoPop").classList.add("active");
+ document.querySelector("#TextCatHeader").classList.add("hidden");
+ demoNext();
}
-function hideDemo(forceIt) {
- if (!forceIt && demoNum < 0) {
- return;
- }
- demoStep = -1;
- demoNum = -1;
- demoText.innerHTML = "";
+function closeDemo(forceIt) {
+ if (!forceIt && demoNum < 0) {
+ return;
+ }
+ demoStep = -1;
+ demoNum = -1;
+ demoText.innerHTML = "";
- window.scrollTo(0, 0);
- document.getElementById("TextCatHeader").style.display = "block";
- document.getElementById("TextCatHeaderContainer").style.paddingTop =
- '1em';
- document.getElementById("demoPicker").selectedIndex = 0;
- document.getElementById("demoPop").style.display = "none";
- document.getElementById("demoContinue").innerHTML = "Continue ➤";
- clearIDresults();
+ window.scrollTo(0, 0);
- LMClear();
- LMDirSelect(document.querySelector("#LMDirQuery"));
- document.getElementById('queryTA').value = "";
- setInitLangs();
+ document.querySelector("#TextCatHeader").classList.remove("hidden");
+ document.getElementById("TextCatHeaderContainer").style.paddingTop =
+ '0';
+ document.getElementById("demoPicker").selectedIndex = 0;
+ document.getElementById("demoPop").classList.remove("active");
+ document.getElementById("demoContinue").innerHTML = "Continue ➤";
+ clearIDresults();
+
+ LMClear();
+ LMDirSelect(document.querySelector("#LMDirQuery"));
+ document.getElementById('queryTA').value = "";
+ setInitLangs();
}
function demoNext() {
- if (demoNum < 0) {
- return;
- }
+ if (demoNum < 0) {
+ return;
+ }
- var demoPop = document.querySelector("#demoPop");
- var demoText = document.querySelector("#demoText");
- var demoTitle = document.querySelector("#demoTitle");
- demoTitle.innerHTML = demoScript[demoNum][0];
- var demoSteps = demoScript[demoNum][1];
- if (demoStep < demoSteps.length) {
- demoText.innerHTML = demoSteps[demoStep][0];
- demoPop.scrollTop = 0;
- if (demoSteps[demoStep][1] != "") {
- eval(demoSteps[demoStep][1]);
- }
- if (demoStep == demoSteps.length - 1) {
- document.querySelector("#demoContinue").innerHTML = "End.";
- }
- document.querySelector("#TextCatHeaderContainer").style.paddingTop =
- document.querySelector("#demoPop").offsetHeight + 8 + "px";
- } else if (demoStep >= demoSteps.length) {
- hideDemo(1);
- return;
- }
- demoStep += 1;
-}
-
+ var demoPop = document.querySelector("#demoPop");
+ var demoText = document.querySelector("#demoText");
+ var demoTitle = document.querySelector("#demoTitle");
+ demoTitle.innerHTML = demoScript[demoNum][0];
+ var demoSteps = demoScript[demoNum][1];
+ if (demoStep < demoSteps.length) {
+ demoText.innerHTML = demoSteps[demoStep][0];
+ demoPop.scrollTop = 0;
+ if (demoSteps[demoStep][1] != "") {
+ eval(demoSteps[demoStep][1]);
+ }
+ if (demoStep == demoSteps.length - 1) {
+ document.querySelector("#demoContinue").innerHTML =
"End.";
+ }
+
document.querySelector("#TextCatHeaderContainer").style.paddingTop =
+ document.querySelector("#demoPop").offsetHeight + "px";
+ } else {
+ closeDemo(1);
+ return;
+ }
+ demoStep += 1;
+}
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/298479
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6d31fb4a14dfea8ef7083bbe74c19831007b163f
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/textcat-demo
Gerrit-Branch: master
Gerrit-Owner: Tjones <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
