Ram has uploaded a new change for review.
https://gerrit.wikimedia.org/r/61033
Change subject: Dump entire global config to a file.
......................................................................
Dump entire global config to a file.
This patch dumps the entire singleton GlobalConfiguration object to the file
/var/tmp/global.txt. Most of the functionality of Lucene search is driven by
this singleton object so this dump will be useful while we are stabilizing
search. The file is around 10MB and the dump is controlled by the DBG variable
in GlobalConfiguration which can be set to 'false' when we deem lsearchd to
be adequately stable. I decided to make a separate file for this output since
I didn't want it to be clobbered by the rolling file appenders.
Since this dump is done just once during initialization, there should be no
performance impact thereafter.
Change-Id: I60b1be9441b3652514c76e5160750a1e2dcc4ff0
---
M src/org/wikimedia/lsearch/config/GlobalConfiguration.java
M src/org/wikimedia/lsearch/config/IndexId.java
A src/org/wikimedia/lsearch/util/Dump.java
3 files changed, 208 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/debs/lucene-search-2
refs/changes/33/61033/1
diff --git a/src/org/wikimedia/lsearch/config/GlobalConfiguration.java
b/src/org/wikimedia/lsearch/config/GlobalConfiguration.java
index 316d341..dad4524 100644
--- a/src/org/wikimedia/lsearch/config/GlobalConfiguration.java
+++ b/src/org/wikimedia/lsearch/config/GlobalConfiguration.java
@@ -5,9 +5,12 @@
package org.wikimedia.lsearch.config;
import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.io.PrintWriter;
import java.net.Inet4Address;
import java.net.InetAddress;
import java.net.MalformedURLException;
@@ -32,6 +35,10 @@
import org.wikimedia.lsearch.util.Localization;
import org.wikimedia.lsearch.util.PHPParser;
import org.wikimedia.lsearch.util.StringUtils;
+
+// for debugging
+import static org.wikimedia.lsearch.util.Dump.dumpIterable;
+import static org.wikimedia.lsearch.util.Dump.dumpMap;
/**
* Read and parse the global configuration file, is also used
@@ -330,6 +337,40 @@
return true;
}
+ private final boolean DBG = true;
+ private void dump( PrintWriter out ) {
+ dumpMap( "database", database, 0, out );
+ dumpMap( "databaseHostGroup", databaseHostGroup, 0, out );
+ dumpMap( "searchGroup", searchGroup, 0, out );
+ dumpMap( "search", search, 0, out );
+ dumpMap( "searchWildcard", searchWildcard, 0, out );
+ dumpIterable( "searchOrphans", searchOrphans, 0, out );
+ dumpMap( "index", index, 0, out );
+ dumpMap( "indexWildcard", indexWildcard, 0, out );
+ dumpMap( "indexLocation", indexLocation, 0, out );
+ dumpMap( "indexRsyncPath", indexRsyncPath, 0, out );
+ dumpMap( "namespacePrefix", namespacePrefix, 0, out );
+ dumpMap( "oaiRepo", oaiRepo, 0, out );
+ dumpMap( "wgLanguageCode", wgLanguageCode, 0, out );
+ dumpMap( "wgServer", wgServer, 0, out );
+ dumpMap( "wgDefaultSearch", wgDefaultSearch, 0, out );
+ dumpMap( "suffixIwMap", suffixIwMap, 0, out );
+ dumpMap( "wgContentNamespaces", wgContentNamespaces, 0, out );
+ dumpMap( "wgNamespacesWithSubpages", wgNamespacesWithSubpages,
0, out );
+
+ out.format( "indexPath = %s%nnamespacePrefixAll =
%s%ncommonsWiki = %s%n",
+ indexPath, namespacePrefixAll, commonsWiki );
+ out.format( "indexIdPool = { %d%n", indexIdPool.size() );
+ int i = 0;
+ ArrayList<String> keys = new ArrayList<String>(
indexIdPool.keySet() );
+ Collections.sort( keys );
+ for ( String key : keys ) {
+ out.format( "[IndexId %d] %s =>%n", ++i, key );
+ indexIdPool.get( key ).dump( out );
+ }
+ out.println( "}" );
+ } // dump
+
/**
* Read a config file from a given URL
*
@@ -338,13 +379,18 @@
*/
public void readFromURL(URL url, String indexpath, String nullH )
throws IOException{
this.nullHost = nullH;
+
BufferedReader in;
try {
in = new BufferedReader(
new InputStreamReader(
url.openStream()));
read(in,indexpath);
-
+ if ( DBG ) {
+ PrintWriter out = new PrintWriter( new
BufferedWriter( new FileWriter( "/var/tmp/global.txt" ) ) );
+ dump( out );
+ out.close();
+ }
} catch (IOException e) {
System.out.println("I/O Error in opening or reading
global config at url "+url);
throw e;
@@ -577,6 +623,7 @@
wgContentNamespaces = parser.getContentNamespaces(text);
Localization.readDBLocalizations(text);
} catch (Exception e) {
+ e.printStackTrace();
System.out.println("Error reading
InitialiseSettings.php from url "+initset+" : "+e.getMessage());
}
}
diff --git a/src/org/wikimedia/lsearch/config/IndexId.java
b/src/org/wikimedia/lsearch/config/IndexId.java
index ed315bb..3a74197 100644
--- a/src/org/wikimedia/lsearch/config/IndexId.java
+++ b/src/org/wikimedia/lsearch/config/IndexId.java
@@ -2,6 +2,8 @@
import java.io.File;
import java.io.IOException;
+import java.io.PrintWriter;
+
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Hashtable;
@@ -13,6 +15,10 @@
import org.wikimedia.lsearch.analyzers.FilterFactory;
import org.wikimedia.lsearch.search.NamespaceFilter;
import org.wikimedia.lsearch.search.SearcherCache;
+
+// for debugging
+import static org.wikimedia.lsearch.util.Dump.dumpIterable;
+import static org.wikimedia.lsearch.util.Dump.dumpMap;
/**
* Encapsulated an index ID in form db.part, e.g. entest.mainpart.
@@ -35,7 +41,7 @@
/** Where the index is */
protected String indexHost;
/** Path to index on remote machine */
- protected String indexRsyncPath;
+ protected String indexRsyncPath;
/** Type (single, mainsplit, split) in string repesentation */
protected String typeString;
@@ -386,7 +392,59 @@
} */
}
-
+
+ private void dumpArray( String name, String v[], PrintWriter out ) {
// dump small array; for debugging
+ if ( null == v ) return;
+
+ out.format( "%s = [", name );
+ for ( String s : v ) {
+ out.format( "%s, ", s );
+ }
+ out.format( "]%n" );
+ } // dumpArray
+
+ public void dump( PrintWriter out ) { // for debugging
+ dumpMap( "typeParams", typeParams, 0, out );
+ dumpMap( "nssplitMap", nssplitMap, 0, out );
+ dumpMap( "params", params, 0, out );
+ dumpMap( "suffixIwMap", suffixIwMap, 0, out );
+ dumpMap( "suffixToDbname", suffixToDbname, 0, out );
+
+ dumpIterable( "searchHosts", searchHosts, 0, out );
+ dumpIterable( "mySearchHosts", mySearchHosts, 0, out );
+ dumpIterable( "namespaceSet", namespaceSet, 0, out );
+
+ dumpArray( "splitParts", splitParts, out );
+ dumpArray( "subParts", subParts, out );
+
+ out.format( "dbrole = %s%n" + "indexHost = %s%n"
+ "indexRsyncPath = %s%n" +
+ "typeString = %s%n" + "mySearch = %s%n"
+ "dbname = %s%n" +
+ "part = %s%n" + "subpart = %s%n"
+ "splitFactor = %d%n" +
+ "myIndex = %s%n" + "subpartNum = %d%n"
+ "subFactor = %d%n" +
+ "furtherSubdivided = %s%n" + "highlight = %s%n"
+ "type = %s%n" +
+ "partNum = %d%n" + "indexPath = %s%n"
+ "searchPath = %s%n" +
+ "snapshotPath = %s%n" + "updatePath = %s%n"
+ "importPath = %s%n" +
+ "statusPath = %s%n" + "tempPath = %s%n"
+ "titlesIndex = %s%n" +
+ "titlesSuffix = %s%n" + "reducedDbname = %s%n"
+ "OAIRepository = %s%n" +
+ "rsyncSnapshotPath = %s%n" + "langCode = %s%n"
+ "exactCase = %s%n" +
+ "defaultNs = %s%n" + "nsWithSubpages =
%s%n" + "contentNamespaces = %s%n" +
+ "disabled = %s%n" +
+ "%n",
+ dbrole, indexHost, indexRsyncPath,
+ typeString, mySearch, dbname,
+ part, subpart, splitFactor,
+ myIndex, subpartNum, subFactor,
+ furtherSubdivided, highlight, type,
+ partNum, indexPath, searchPath,
+ snapshotPath, updatePath, importPath,
+ statusPath, tempPath, titlesIndex,
+ titlesSuffix, reducedDbname, OAIRepository,
+ rsyncSnapshotPath, langCode, exactCase,
+ defaultNs, nsWithSubpages,
contentNamespaces,
+ disabled
+ );
+ } // dump
+
/** If this is logical name referring to a set of split indexes */
public boolean isLogical(){
return (part == null && type != IndexType.SINGLE) ||
(subpart==null && furtherSubdivided);
diff --git a/src/org/wikimedia/lsearch/util/Dump.java
b/src/org/wikimedia/lsearch/util/Dump.java
new file mode 100644
index 0000000..47101cc
--- /dev/null
+++ b/src/org/wikimedia/lsearch/util/Dump.java
@@ -0,0 +1,100 @@
+package org.wikimedia.lsearch.util;
+
+import java.io.PrintWriter;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.Set;
+import java.lang.reflect.Method;
+
+public class Dump {
+ private static void indent( int level, PrintWriter out ) {
+ for ( int i = 0; i < level; ++i ) {
+ out.format( " " );
+ }
+ } // indent
+
+ public static void dumpIterable( String name, Iterable a, int level,
PrintWriter out ) {
+ if ( null == a ) return;
+
+ indent( level, out );
+ final int size = (a instanceof ArrayList) ?
((ArrayList)a).size()
+ : (a instanceof Set) ? ((Set)a).size()
+ : -1;
+
+ if ( 0 == size ) {
+ out.format( "%s = []%n", name );
+ return;
+ }
+ out.format( "%s = [ %d%n", name, size );
+
+ final int nLevel = 1 + level;
+ Iterator iter = a.iterator();
+ for ( int i = 0; iter.hasNext(); ++i ) {
+ indent( nLevel, out );
+ out.format( "%d: %s%n", i, iter.next().toString() );
+ }
+ indent( level, out );
+ out.println( "]" );
+ } // dumpIterable
+
+ public static void dumpMap( String name, Map h, int level, PrintWriter
out ) {
+ if ( null == h ) return;
+
+ indent( level, out );
+ int size = h.size();
+ if ( 0 == size ) {
+ out.format( "%s = {}%n", name );
+ return;
+ }
+ out.format( "%s = { %d%n", name, size );
+
+ final int nLevel = 1 + level;
+ ArrayList keys = new ArrayList( h.keySet() );
+ if ( keys.get( 0 ) instanceof Pattern ) {
+ Collections.sort( keys, new Comparator() {
+ public int compare( Object o1, Object
o2 ) {
+ return o1.toString().compareTo(
o2.toString() );
+ } } );
+ } else {
+ Collections.sort( keys );
+ }
+ for ( Object key : keys ) {
+ String skey = key.toString();
+ Object val = h.get( key );
+ if ( val instanceof String || val instanceof Boolean ||
val instanceof Integer ||
+ val instanceof Float || val instanceof Double ) {
// primitive types
+ indent( nLevel, out );
+ out.format( "%s = %s%n", skey, val );
+ continue;
+ }
+ if ( val instanceof Map ) {
+ dumpMap( skey, (Map)val, nLevel, out );
+ continue;
+ }
+ if ( val instanceof Iterable ) {
+ dumpIterable( skey, (Iterable)val, nLevel, out
);
+ continue;
+ }
+
+ // convert to string if possible
+ Method m = null;
+ try { m = val.getClass().getMethod( "toString" ); }
catch (Exception e) {}
+ if ( null != m ) {
+ indent( nLevel, out );
+ out.format( "%s = %s%n", skey, val );
+ continue;
+ }
+
+ // just print class name
+ out.format( "<%s>%n", val.getClass().getName() );
+ }
+ indent( level, out );
+ out.println( "}" );
+ } // dumpMap
+
+}
--
To view, visit https://gerrit.wikimedia.org/r/61033
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I60b1be9441b3652514c76e5160750a1e2dcc4ff0
Gerrit-PatchSet: 1
Gerrit-Project: operations/debs/lucene-search-2
Gerrit-Branch: master
Gerrit-Owner: Ram <r...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
