MaxSem has uploaded a new change for review.
https://gerrit.wikimedia.org/r/133886
Change subject: Kill GeoData Solr, decom servers
......................................................................
Kill GeoData Solr, decom servers
Change-Id: I3b82b7c5f884df04c9c74c6fb7ab5952c11f2b6b
---
M manifests/misc/maintenance.pp
M manifests/role/solr.pp
M manifests/site.pp
M modules/install-server/files/autoinstall/netboot.cfg
M modules/install-server/files/dhcpd/linux-host-entries.ttyS1-115200
D modules/solr/files/schema-geodata.xml
D templates/misc/clear-killlist.erb
D templates/misc/update-geodata.erb
8 files changed, 12 insertions(+), 400 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/86/133886/1
diff --git a/manifests/misc/maintenance.pp b/manifests/misc/maintenance.pp
index d7fee56..1fc0b2e 100644
--- a/manifests/misc/maintenance.pp
+++ b/manifests/misc/maintenance.pp
@@ -292,11 +292,19 @@
class misc::maintenance::geodata( $enabled = false ) {
file {
'/usr/local/bin/update-geodata':
- ensure => present,
+ ensure => $enabled ?{
+ true => present,
+ false => absent,
+ default => absent
+ };
content => template( 'misc/update-geodata.erb' ),
mode => '0555';
'/usr/local/bin/clear-killlist':
- ensure => present,
+ ensure => $enabled ?{
+ true => present,
+ false => absent,
+ default => absent
+ };
content => template( 'misc/clear-killlist.erb' ),
mode => '0555';
}
diff --git a/manifests/role/solr.pp b/manifests/role/solr.pp
index f149227..3749c68 100644
--- a/manifests/role/solr.pp
+++ b/manifests/role/solr.pp
@@ -45,22 +45,3 @@
max_heap => "1G",
}
}
-
-class role::solr::geodata($replication_master = 'solr1001.eqiad.wmnet',
$max_heap = undef) {
- system::role { "solr-geodata": description => "Solr server for GeoData"
}
-
- include standard
-
- class { "role::solr":
- schema => "puppet:///modules/solr/schema-geodata.xml",
- replication_master => $replication_master,
- max_heap => $max_heap,
- }
-}
-
-class role::solr::geodata::labs {
- class { "role::solr::geodata":
- replication_master => undef,
- max_heap => "256M",
- }
-}
\ No newline at end of file
diff --git a/manifests/site.pp b/manifests/site.pp
index 71e96e0..e3e88c6 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -2494,7 +2494,7 @@
enabled => true,
}
class { 'misc::maintenance::geodata':
- enabled => true,
+ enabled => false,
}
class { 'misc::maintenance::update_flaggedrev_stats':
enabled => true,
@@ -2765,11 +2765,6 @@
class { 'lvs::realserver':
realserver_ips => ['10.2.2.28'],
}
-}
-
-node /^solr100[1-3]\.eqiad\.wmnet/ {
- include standard
- include role::solr::geodata
}
node 'ytterbium.wikimedia.org' {
diff --git a/modules/install-server/files/autoinstall/netboot.cfg
b/modules/install-server/files/autoinstall/netboot.cfg
index c08cb3d..7d28b5c 100755
--- a/modules/install-server/files/autoinstall/netboot.cfg
+++ b/modules/install-server/files/autoinstall/netboot.cfg
@@ -76,7 +76,7 @@
snapshot[1-4]|snapshot100[1-4]) echo partman/snapshot.cfg ;; \
stat1002) echo partman/lvm-noraid-large.a.cfg ;; \
argon|bast4001|cerium|copper|iridium|neon|praseodymium|ruthenium|ssl[1-3]0[0-9][0-9]|ssl[0-9]|titanium|ytterbium|xenon|zirconium)
echo partman/raid1-lvm.cfg ;; \
- solr100[1-3]) echo partman/lvm.cfg ;; \
+ ) echo partman/lvm.cfg ;; \
virt[5-9]|virt1[0-1]) echo partman/virt-raid10-cisco.cfg ;; \
virt100[1-9]) echo partman/virt-raid10-cisco.cfg ;; \
labnet1001) echo partman/lvm.cfg ;; \
diff --git a/modules/install-server/files/dhcpd/linux-host-entries.ttyS1-115200
b/modules/install-server/files/dhcpd/linux-host-entries.ttyS1-115200
index 64dcfb2..fa23607 100644
--- a/modules/install-server/files/dhcpd/linux-host-entries.ttyS1-115200
+++ b/modules/install-server/files/dhcpd/linux-host-entries.ttyS1-115200
@@ -3055,21 +3055,6 @@
fixed-address sodium.wikimedia.org;
}
-host solr1001 {
- hardware ethernet 90:B1:1C:23:23:BF;
- fixed-address solr1001.eqiad.wmnet;
-}
-
-host solr1002 {
- hardware ethernet 90:B1:1C:1D:E7:E5;
- fixed-address solr1002.eqiad.wmnet;
-}
-
-host solr1003 {
- hardware ethernet 90:B1:1C:23:36:1D;
- fixed-address solr1003.eqiad.wmnet;
-}
-
host ssl1001 {
hardware ethernet 78:2b:cb:09:10:91;
fixed-address ssl1001.wikimedia.org;
diff --git a/modules/solr/files/schema-geodata.xml
b/modules/solr/files/schema-geodata.xml
deleted file mode 100644
index f221817..0000000
--- a/modules/solr/files/schema-geodata.xml
+++ /dev/null
@@ -1,332 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- This is the Solr schema file. This file should be named "schema.xml" and
- should be in the conf directory under the solr home
- (i.e. ./solr/conf/schema.xml by default)
- or located where the classloader for the Solr webapp can find it.
-
- This example schema is the recommended starting point for users.
- It should be kept correct and concise, usable out-of-the-box.
-
- For more information, on how to customize this file, please see
- http://wiki.apache.org/solr/SchemaXml
-
- PERFORMANCE NOTE: this schema includes many optional features and should not
- be used for benchmarking. To improve performance one could
- - set stored="false" for all fields possible (esp large fields) when you
- only need to search on the field but don't need to return the original
- value.
- - set indexed="false" if you don't need to search on the field, but only
- return the field as a result of searching on other indexed fields.
- - remove all unneeded copyField statements
- - for best index size and searching performance, set "index" to false
- for all general text fields, use copyField to copy them to the
- catchall "text" field, and use that for searching.
- - For maximum indexing performance, use the StreamingUpdateSolrServer
- java client.
- - Remember to run the JVM in server mode, and use a higher logging level
- that avoids logging every request
--->
-
-<schema name="geodata" version="1.5">
-
- <fields>
- <!-- Valid attributes for fields:
- name: mandatory - the name for the field
- type: mandatory - the name of a field type from the
- <types> fieldType section
- indexed: true if this field should be indexed (searchable or sortable)
- stored: true if this field should be retrievable
- multiValued: true if this field may contain multiple values per document
- omitNorms: (expert) set to true to omit the norms associated with
- this field (this disables length normalization and index-time
- boosting for the field, and saves some memory). Only full-text
- fields or fields that need an index-time boost need norms.
- Norms are omitted for primitive (non-analyzed) types by default.
- termVectors: [false] set to true to store the term vector for a
- given field.
- When using MoreLikeThis, fields used for similarity should be
- stored for best performance.
- termPositions: Store position information with the term vector.
- This will increase storage costs.
- termOffsets: Store offset information with the term vector. This
- will increase storage costs.
- required: The field is required. It will throw an error if the
- value does not exist
- default: a value that should be used if no value is specified
- when adding a document.
- -->
-
- <!-- field names should consist of alphanumeric or underscore characters
only and
- not start with a digit. This is not currently strictly enforced,
- but other field names will not have first class support from all
components
- and back compatibility is not guaranteed. Names with both leading and
- trailing underscores (e.g. _version_) are reserved.
- -->
-
- <field name="_version_" type="long" indexed="true" stored="true"/>
-
- <field name="id" type="string" indexed="true" stored="true" required="true"
/>
- <field name="wiki" type="string" indexed="true" stored="false"
required="true" omitNorms="true"/>
- <field name="coord" type="location" indexed="true" stored="true"
required="true"/>
- <field name="page_id" type="tlong" indexed="true" stored="true"
required="true"/>
- <field name="globe" type="string" indexed="true" stored="true"
omitNorms="true" required="true"/>
- <field name="primary" type="boolean" indexed="true" stored="true"
required="true"/>
- <field name="dim" type="float" indexed="true" stored="true"
required="false"/>
- <field name="type" type="string" indexed="false" stored="true"
omitNorms="true"/>
- <field name="name" type="string" indexed="false" stored="true"
omitNorms="true"/>
- <field name="country" type="string" indexed="true" stored="true"
omitNorms="true"/>
- <field name="region" type="string" indexed="true" stored="true"
omitNorms="true"/>
-
- <!-- Dynamic field definitions allow using convention over configuration
- for fields via the specification of patterns to match field names.
- EXAMPLE: name="*_i" will match any field ending in _i (like myid_i,
z_i)
- RESTRICTION: the glob-like pattern in the name attribute must have
- a "*" only at the start or the end. -->
-
- <!-- Type used to index the lat and lon components for the "location"
FieldType -->
- <dynamicField name="*_coordinate" type="tdouble" indexed="true"
stored="false" />
-
- <!-- Uncommenting the following will create a "timestamp" field using
- a default value of "NOW" to indicate when each document was indexed.
- -->
- <!--
- <field name="timestamp" type="date" indexed="true" stored="true"
default="NOW" multiValued="false"/>
- -->
-
- <!-- uncomment the following to ignore any fields that don't already match
an existing
- field name or dynamic field, rather than reporting them as an error.
- alternately, change the type="ignored" to some other type e.g. "text"
if you want
- unknown fields indexed and/or stored by default -->
- <!--dynamicField name="*" type="ignored" multiValued="true" /-->
-
- </fields>
-
- <!-- Field to use to determine and enforce document uniqueness.
- Unless this field is marked with required="false", it will be a required
field
- -->
- <uniqueKey>id</uniqueKey>
-
- <!-- DEPRECATED: The defaultOperator (AND|OR) is consulted by various query
parsers
- when parsing a query string to determine if a clause of the query should be
marked as
- required or optional, assuming the clause isn't already marked by some
operator.
- The default is OR, which is generally assumed so it is not a good idea to
change it
- globally here. The "q.op" request parameter takes precedence over this.
- <solrQueryParser defaultOperator="OR"/> -->
-
-
- <types>
- <!-- field type definitions. The "name" attribute is
- just a label to be used by field definitions. The "class"
- attribute and any other attributes determine the real
- behavior of the fieldType.
- Class names starting with "solr" refer to java classes in a
- standard package such as org.apache.solr.analysis
- -->
-
- <!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
- <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
-
- <!-- boolean type: "true" or "false" -->
- <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
-
- <!-- sortMissingLast and sortMissingFirst attributes are optional
attributes are
- currently supported on types that are sorted internally as strings
- and on numeric types.
- This includes "string","boolean", and, as of 3.5 (and 4.x),
- int, float, long, date, double, including the "Trie" variants.
- - If sortMissingLast="true", then a sort on this field will cause
documents
- without the field to come after documents with the field,
- regardless of the requested sort order (asc or desc).
- - If sortMissingFirst="true", then a sort on this field will cause
documents
- without the field to come before documents with the field,
- regardless of the requested sort order.
- - If sortMissingLast="false" and sortMissingFirst="false" (the default),
- then default lucene sorting will be used which places docs without the
- field first in an ascending sort and last in a descending sort.
- -->
-
- <!--
- Default numeric field types. For faster range queries, consider the
tint/tfloat/tlong/tdouble types.
- -->
- <fieldType name="int" class="solr.TrieIntField" precisionStep="0"
positionIncrementGap="0"/>
- <fieldType name="float" class="solr.TrieFloatField" precisionStep="0"
positionIncrementGap="0"/>
- <fieldType name="long" class="solr.TrieLongField" precisionStep="0"
positionIncrementGap="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0"
positionIncrementGap="0"/>
-
- <!--
- Numeric field types that index each value at various levels of precision
- to accelerate range queries when the number of values between the range
- endpoints is large. See the javadoc for NumericRangeQuery for internal
- implementation details.
-
- Smaller precisionStep values (specified in bits) will lead to more tokens
- indexed per value, slightly larger index size, and faster range queries.
- A precisionStep of 0 disables indexing at different precision levels.
- -->
- <fieldType name="tint" class="solr.TrieIntField" precisionStep="8"
positionIncrementGap="0"/>
- <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8"
positionIncrementGap="0"/>
- <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8"
positionIncrementGap="0"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8"
positionIncrementGap="0"/>
-
- <!-- The format for this date field is of the form 1995-12-31T23:59:59Z,
and
- is a more restricted form of the canonical representation of dateTime
- http://www.w3.org/TR/xmlschema-2/#dateTime
- The trailing "Z" designates UTC time and is mandatory.
- Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
- All other components are mandatory.
-
- Expressions can also be used to denote calculations that should be
- performed relative to "NOW" to determine the value, ie...
-
- NOW/HOUR
- ... Round to the start of the current hour
- NOW-1DAY
- ... Exactly 1 day prior to now
- NOW/DAY+6MONTHS+3DAYS
- ... 6 months and 3 days in the future from the start of
- the current day
-
- Consult the DateField javadocs for more information.
-
- Note: For faster range queries, consider the tdate type
- -->
- <fieldType name="date" class="solr.TrieDateField" precisionStep="0"
positionIncrementGap="0"/>
-
- <!-- A Trie based date field for faster date range queries and date
faceting. -->
- <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6"
positionIncrementGap="0"/>
-
-
- <!--Binary data type. The data should be sent/retrieved in as Base64
encoded Strings -->
- <fieldtype name="binary" class="solr.BinaryField"/>
-
- <!--
- Note:
- These should only be used for compatibility with existing indexes
(created with lucene or older Solr versions).
- Use Trie based fields instead. As of Solr 3.5 and 4.x, Trie based fields
support sortMissingFirst/Last
-
- Plain numeric field types that store and index the text
- value verbatim (and hence don't correctly support range queries, since
the
- lexicographic ordering isn't equal to the numeric ordering)
- -->
- <fieldType name="pint" class="solr.IntField"/>
- <fieldType name="plong" class="solr.LongField"/>
- <fieldType name="pfloat" class="solr.FloatField"/>
- <fieldType name="pdouble" class="solr.DoubleField"/>
- <fieldType name="pdate" class="solr.DateField" sortMissingLast="true"/>
-
- <!-- The "RandomSortField" is not used to store or search any
- data. You can declare fields of this type it in your schema
- to generate pseudo-random orderings of your docs for sorting
- or function purposes. The ordering is generated based on the field
- name and the version of the index. As long as the index version
- remains unchanged, and the same field name is reused,
- the ordering of the docs will be consistent.
- If you want different psuedo-random orderings of documents,
- for the same version of the index, use a dynamicField and
- change the field name in the request.
- -->
- <fieldType name="random" class="solr.RandomSortField" indexed="true" />
-
- <!-- solr.TextField allows the specification of custom text analyzers
- specified as a tokenizer and a list of token filters. Different
- analyzers may be specified for indexing and querying.
-
- The optional positionIncrementGap puts space between multiple fields
of
- this type on the same document, with the purpose of preventing false
phrase
- matching across fields.
-
- For more info on customizing your analyzer chain, please see
- http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
- -->
-
- <!-- One can also specify an existing Analyzer class that has a
- default constructor via the class attribute on the analyzer element.
- Example:
- <fieldType name="text_greek" class="solr.TextField">
- <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
- </fieldType>
- -->
-
- <!-- A text field that only splits on whitespace for exact matching of
words -->
- <fieldType name="text_ws" class="solr.TextField"
positionIncrementGap="100">
- <analyzer>
- <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- </analyzer>
- </fieldType>
-
-
-
- <!-- charFilter + WhitespaceTokenizer -->
- <!--
- <fieldType name="text_char_norm" class="solr.TextField"
positionIncrementGap="100" >
- <analyzer>
- <charFilter class="solr.MappingCharFilterFactory"
mapping="mapping-ISOLatin1Accent.txt"/>
- <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- </analyzer>
- </fieldType>
- -->
-
- <!-- lowercases the entire field value, keeping it as a single token. -->
- <fieldType name="lowercase" class="solr.TextField"
positionIncrementGap="100">
- <analyzer>
- <tokenizer class="solr.KeywordTokenizerFactory"/>
- <filter class="solr.LowerCaseFilterFactory" />
- </analyzer>
- </fieldType>
-
- <fieldType name="text_path" class="solr.TextField"
positionIncrementGap="100">
- <analyzer>
- <tokenizer class="solr.PathHierarchyTokenizerFactory"/>
- </analyzer>
- </fieldType>
-
-
- <!-- since fields of this type are by default not stored or indexed,
- any data added to them will be ignored outright. -->
- <fieldtype name="ignored" stored="false" indexed="false"
multiValued="true" class="solr.StrField" />
-
- <!-- This point type indexes the coordinates as separate fields (subFields)
- If subFieldType is defined, it references a type, and a dynamic field
- definition is created matching *___<typename>. Alternately, if
- subFieldSuffix is defined, that is used to create the subFields.
- Example: if subFieldType="double", then the coordinates would be
- indexed in fields myloc_0___double,myloc_1___double.
- Example: if subFieldSuffix="_d" then the coordinates would be indexed
- in fields myloc_0_d,myloc_1_d
- The subFields are an implementation detail of the fieldType, and end
- users normally should not need to know about them.
- -->
- <fieldType name="point" class="solr.PointType" dimension="2"
subFieldSuffix="_d"/>
-
- <!-- A specialized field for geospatial search. If indexed, this fieldType
must not be multivalued. -->
- <fieldType name="location" class="solr.LatLonType"
subFieldSuffix="_coordinate"/>
-
- <!--
- A Geohash is a compact representation of a latitude longitude pair in a
single field.
- See http://wiki.apache.org/solr/SpatialSearch
- -->
- <fieldtype name="geohash" class="solr.GeoHashField"/>
-
- </types>
-
-
-
-</schema>
diff --git a/templates/misc/clear-killlist.erb
b/templates/misc/clear-killlist.erb
deleted file mode 100644
index afcdc59..0000000
--- a/templates/misc/clear-killlist.erb
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-DAYS=3
-
-<% if @realm == 'production' -%>
-/usr/local/bin/foreachwikiindblist /usr/local/apache/common/wikipedia.dblist
extensions/GeoData/solrupdate.php --clear-killlist $DAYS --noindex
-/usr/local/bin/foreachwikiindblist /usr/local/apache/common/special.dblist
extensions/GeoData/solrupdate.php --clear-killlist $DAYS --noindex
-/usr/local/bin/foreachwikiindblist /usr/local/apache/common/wikivoyage.dblist
extensions/GeoData/solrupdate.php --clear-killlist $DAYS --noindex
-<% else -%>
-/usr/local/bin/foreachwikiindblist /usr/local/apache/common/all-labs.dblist
extensions/GeoData/solrupdate.php --clear-killlist $DAYS --noindex
-<% end -%>
diff --git a/templates/misc/update-geodata.erb
b/templates/misc/update-geodata.erb
deleted file mode 100644
index f69e3f3..0000000
--- a/templates/misc/update-geodata.erb
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-# Allow disabling this script temporarily
-if [ -e /tmp/disable-update-geodata ] ; then
- exit
-fi
-
-<% if @realm == 'production' -%>
-/usr/local/bin/foreachwikiindblist /usr/local/apache/common/wikipedia.dblist
extensions/GeoData/solrupdate.php
-/usr/local/bin/foreachwikiindblist /usr/local/apache/common/special.dblist
extensions/GeoData/solrupdate.php
-/usr/local/bin/foreachwikiindblist /usr/local/apache/common/wikivoyage.dblist
extensions/GeoData/solrupdate.php
-<% else -%>
-/usr/local/bin/foreachwikiindblist /usr/local/apache/common/all-labs.dblist
extensions/GeoData/solrupdate.php
-<% end -%>
--
To view, visit https://gerrit.wikimedia.org/r/133886
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3b82b7c5f884df04c9c74c6fb7ab5952c11f2b6b
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: MaxSem <maxsem.w...@gmail.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
