Filippo Giunchedi has uploaded a new change for review.
https://gerrit.wikimedia.org/r/184620
Change subject: lsearchd: remove lucene role and class
......................................................................
lsearchd: remove lucene role and class
Bug: T86150
Change-Id: I0054c6c026b8c01e885e464d00a44177f731e83e
---
D files/lucene/lucene.jobs.sh
D files/lucene/nooptimize.dblist
D hieradata/hosts/searchidx1001.yaml
M hieradata/regex.yaml
D manifests/role/lucene.pp
D manifests/search.pp
M manifests/site.pp
D templates/lucene/lsearch-global-2.1.conf.erb
D templates/lucene/lsearch.conf.erb
D templates/lucene/lsearch.log4j.erb
D templates/lucene/lucene.jobs.conf.erb
11 files changed, 0 insertions(+), 1,085 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/20/184620/1
diff --git a/files/lucene/lucene.jobs.sh b/files/lucene/lucene.jobs.sh
deleted file mode 100755
index aa2917c..0000000
--- a/files/lucene/lucene.jobs.sh
+++ /dev/null
@@ -1,145 +0,0 @@
-#!/bin/bash
-
-base=/a/search
-confs="$base/conf"
-import="$base/indexes/import" # here the importer builds the index
-index="$base/indexes/index" # here indexer keeps it's copy
-dumps="$base/dumps"
-ls2="$base/lucene-search"
-
-MWinstall="/srv/mediawiki"
-dblist="$confs/all.dblist"
-
-JAVA_OPTS_IMPORTER='-Xms128m -Xmx2000m'
-JAVA_OPTS_PREFIXINDEXBUILDER='-Xmx4000m'
-JAVA_OPTS_RELATEDBUILDER='-Xmx4000m'
-JAVA_OPTS_SUGGESTBUILDER='-Xmx8000m'
-
-# load configuration file maintained by puppet
-if [ -s "$confs/lucene.jobs.conf" ]; then
- . "$confs/lucene.jobs.conf"
-fi
-
-# detect realm (production or labs)
-WMF_REALM='production'
-if [ -f /etc/wikimedia-realm ]
-then
- WMF_REALM=`cat /etc/wikimedia-realm`
-fi
-# Per realm override
-case "$WMF_REALM" in
- 'labs')
- dblist="$confs/all-labs.dblist"
- ;;
-esac
-
-pvtlist="$confs/private.dblist"
-
-function build-new {
- cd $ls2
- rm -f $base/indexes/status/$1
- java -cp LuceneSearch.jar org.wikimedia.lsearch.oai.IncrementalUpdater
$1
- java -cp LuceneSearch.jar org.wikimedia.lsearch.util.Snapshot -p
${1}.links
- java -cp LuceneSearch.jar org.wikimedia.lsearch.related.RelatedBuilder
$1
- java -cp LuceneSearch.jar org.wikimedia.lsearch.oai.IncrementalUpdater
$1
- java -cp LuceneSearch.jar org.wikimedia.lsearch.util.Snapshot -p $1
- java -cp LuceneSearch.jar org.wikimedia.lsearch.util.Snapshot -p ${1}.hl
- java -cp LuceneSearch.jar org.wikimedia.lsearch.util.Snapshot -pre -p
${1}.spell.pre
- java -cp LuceneSearch.jar org.wikimedia.lsearch.spell.SuggestBuilder -s
$1
- java -cp LuceneSearch.jar org.wikimedia.lsearch.util.Snapshot -p
${1}.spell
- java -cp LuceneSearch.jar
org.wikimedia.lsearch.prefix.PrefixIndexBuilder $1
-}
-
-function import-file {
- echo "Importing $2 ..."
- # Syntax: import-file <xmldump> <dbname>
- cd $ls2 &&
- java $JAVA_OPTS_IMPORTER -cp $ls2/LuceneSearch.jar
org.wikimedia.lsearch.importer.BuildAll $1 $2
-}
-
-function import-db {
- dbname="$1"
- dumpfile="$dumps/dump-$dbname.xml"
- timestamp=`date -u +%Y-%m-%d`
-
- # not going to compute this param as it's broken in dumpBackup.php
- #slave=`php $MWinstall/multiversion/MWScript.php getSlaveServer.php
$dbname`
-
- echo "Dumping $dbname..."
- php $MWinstall/multiversion/MWScript.php dumpBackup.php $dbname
--current > $dumpfile &&
- import-file $dumpfile $dbname &&
- (
- if [ -e $import ]; then
- echo "Imported $dbname"
- else
- echo "Failed $dbname"
- fi
- )
-}
-
-function import-private {
- # Import all dbs in the cluster
- for dbname in `<$confs/private.dblist`;do
- import-db $dbname >> $base/log/log-private 2>&1
- done
-
- for dbname in `<$confs/fishbowl.dblist`;do
- import-db $dbname >> $base/log/log-private 2>&1
- done
-}
-
-function indexer-cron {
- cd $ls2 &&
- java $JAVA_OPTS_SUGGESTBUILDER -cp LuceneSearch.jar
org.wikimedia.lsearch.spell.SuggestBuilder -l >> $base/log/log-spell 2>&1 &
-
- cd $ls2 &&
- java $JAVA_OPTS_RELATEDBUILDER -cp LuceneSearch.jar
org.wikimedia.lsearch.related.RelatedBuilder -l >> $base/log/log-related 2>&1 &
-}
-
-function build-prefix {
- cd $ls2 &&
- java -cp LuceneSearch.jar org.wikimedia.lsearch.util.Snapshot -pre -p
*.prefix.pre >> $base/log/log-prefix 2>&1 &&
- java $JAVA_OPTS_PREFIXINDEXBUILDER -cp LuceneSearch.jar
org.wikimedia.lsearch.prefix.PrefixIndexBuilder -l -s >> $base/log/log-prefix
2>&1 &
-}
-
-function inc-updater-start {
- echo "Starting incremental updater ..."
-
- if [ -n "$2" ]; then
- timestamp="-dt $2"
- fi
-
- while true; do
- cd $ls2 &&
- java -cp $ls2/LuceneSearch.jar
-Djava.rmi.server.hostname=$HOSTNAME
org.wikimedia.lsearch.oai.IncrementalUpdater -n -f $dblist -ef $pvtlist -e
dewikisource $timestamp -nof /a/search/conf/nooptimize.dblist
- sleep 15m
- done >> $base/log/log-all 2>&1 &
-}
-
-if [ -z "$1" ] ; then
- echo "$0: Requires an argument"
- exit 42
-fi
-
-if [ "$1" = "snapshot" ] ; then
- curl http://localhost:8321/snapshot >> $base/log/log-snapshot 2>&1
-elif [ "$1" = "snapshot-precursors" ] ; then
- curl "http://localhost:8321/snapshotPrecursors?p=*.spell.pre"; >>
$base/log/log-snapshot-pre 2>&1
-elif [ "$1" = "indexer-cron" ] ; then
- indexer-cron
-elif [ "$1" = "import-private" ] ; then
- import-private
-elif [ "$1" = "import-broken" ] ; then
- import-db dewikisource >> $base/log/log-dewikisource 2>&1 &
-elif [ "$1" = "build-prefix" ] ; then
- build-prefix
-elif [ "$1" = "inc-updater-start" ] ; then
- inc-updater-start
-elif [ "$1" = "build-new" ] && [ "$2" ] ; then
- build-new $2
-elif [ "$1" = "import-db" ] && [ "$2" ] ; then
- import-db $2
-else
- echo "$0: argument not recognized"
- exit 1
-fi
diff --git a/files/lucene/nooptimize.dblist b/files/lucene/nooptimize.dblist
deleted file mode 100644
index b748426..0000000
--- a/files/lucene/nooptimize.dblist
+++ /dev/null
@@ -1,13 +0,0 @@
-enwiki
-dewiki
-frwiki
-itwiki
-ptwiki
-jawiki
-plwiki
-nlwiki
-ruwiki
-svwiki
-zhwiki
-eswiki
-commonswiki
diff --git a/hieradata/hosts/searchidx1001.yaml
b/hieradata/hosts/searchidx1001.yaml
deleted file mode 100644
index 465b455..0000000
--- a/hieradata/hosts/searchidx1001.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-nagios_group: lucene
-cluster: search
diff --git a/hieradata/regex.yaml b/hieradata/regex.yaml
index 010b90e..139cd0d 100644
--- a/hieradata/regex.yaml
+++ b/hieradata/regex.yaml
@@ -43,11 +43,6 @@
__regex: !ruby/regexp /stat100[123].eqiad.wmnet/
mainrole: statistics
-lsearchd:
- __regex: !ruby/regexp /^search10[0-2][0-9]\.eqiad\.wmnet$/
- cluster: search
- nagios_group: lucene
-
rcs_eqiad:
__regex: !ruby/regexp /^rcs100[0-9]\.eqiad\.wmnet/
cluster: rcstream
diff --git a/manifests/role/lucene.pp b/manifests/role/lucene.pp
deleted file mode 100644
index 6ddfcc5..0000000
--- a/manifests/role/lucene.pp
+++ /dev/null
@@ -1,221 +0,0 @@
-# vim: set noet :
-
-class role::lucene {
- class configuration {
- $nodes = {
- 'production' => {
- 'eqiad' => {
- 'front_ends' => {
- # enwiki
- 'pool1' => {
- 'search1001' => ['enwiki.nspart1.sub1',
'enwiki.nspart1.sub2', 'enwiki.spell'],
- 'search1002' => ['enwiki.nspart1.sub1',
'enwiki.nspart1.sub2'],
- 'search1003' => ['enwiki.nspart1.sub1',
'enwiki.nspart1.sub2', 'enwiki.nspart2*'],
- 'search1004' => ['enwiki.nspart1.sub1.hl',
'enwiki.nspart1.sub2.hl'],
- 'search1005' => ['enwiki.nspart1.sub1.hl',
'enwiki.nspart1.sub2.hl'],
- 'search1006' => ['enwiki.nspart2*',
'enwiki.spell'],
- },
- # de,fr,jawiki
- 'pool2' => {
- 'search1007' => ['frwiki.nspart1',
'frwiki.nspart2', 'jawiki.nspart1', 'jawiki.nspart2', 'dewiki.nspart1',
'dewiki.nspart2'],
- 'search1008' => ['frwiki.nspart1',
'frwiki.nspart2', 'jawiki.nspart1', 'jawiki.nspart2', 'dewiki.nspart1',
'dewiki.nspart2'],
- 'search1009' => ['dewiki.nspart1.hl',
'dewiki.nspart2.hl', 'frwiki.nspart1.hl', 'frwiki.nspart2.hl', 'frwiki.spell',
'dewiki.spell'],
- 'search1010' => ['dewiki.nspart1.hl',
'dewiki.nspart2.hl', 'frwiki.nspart1.hl', 'frwiki.nspart2.hl', 'frwiki.spell',
'dewiki.spell'],
- },
- # it,nl,ru,sv,pl,pt,es,zhwiki
- 'pool3' => {
- 'search1011' => ['eswiki itwiki.nspart1',
'ruwiki.nspart1', 'nlwiki.nspart1',
- 'svwiki.nspart1', 'plwiki.nspart1',
'ptwiki.nspart1', 'zhwiki.nspart1', 'eswiki.hl'],
- 'search1012' => ['eswiki itwiki.nspart1',
'ruwiki.nspart1', 'nlwiki.nspart1',
- 'svwiki.nspart1', 'plwiki.nspart1',
'ptwiki.nspart1', 'zhwiki.nspart1', 'eswiki.hl'],
- 'search1013' => ['itwiki.nspart1.hl',
'itwiki.nspart2.hl', 'nlwiki.nspart1.hl', 'nlwiki.nspart2.hl',
'ruwiki.nspart1.hl', 'ruwiki.nspart2.hl',
- 'itwiki.spell', 'nlwiki.spell',
'ruwiki.spell', 'svwiki.spell', 'plwiki.spell', 'ptwiki.spell', 'eswiki.spell'],
- 'search1014' => ['itwiki.nspart1.hl',
'itwiki.nspart2.hl', 'nlwiki.nspart1.hl', 'nlwiki.nspart2.hl',
'ruwiki.nspart1.hl', 'ruwiki.nspart2.hl',
- 'itwiki.spell', 'nlwiki.spell',
'ruwiki.spell', 'svwiki.spell', 'plwiki.spell', 'ptwiki.spell', 'eswiki.spell'],
- 'search1023' => ['svwiki.nspart1.hl',
'svwiki.nspart2.hl', 'plwiki.nspart1.hl', 'plwiki.nspart2.hl',
'ptwiki.nspart1.hl', 'ptwiki.nspart2.hl',
- 'itwiki.nspart2', 'nlwiki.nspart2',
'ruwiki.nspart2', 'svwiki.nspart2', ' plwiki.nspart2', 'ptwiki.nspart2',
'zhwiki.nspart2'],
- 'search1024' => ['svwiki.nspart1.hl',
'svwiki.nspart2.hl', 'plwiki.nspart1.hl', 'plwiki.nspart2.hl',
'ptwiki.nspart1.hl', 'ptwiki.nspart2.hl',
- 'itwiki.nspart2', 'nlwiki.nspart2',
'ruwiki.nspart2', 'svwiki.nspart2', ' plwiki.nspart2', 'ptwiki.nspart2',
'zhwiki.nspart2'],
- },
- # everything else
- 'pool4' => {
- 'search1015' => ['*?'],
- 'search1016' => ['*?'],
- 'search1019' => ['commonswiki.nspart1',
'commonswiki.nspart1.hl', 'commonswiki.nspart2', 'commonswiki.nspart2.hl',
- 'wikidatawiki', 'metawiki', 'enwiktionary',
-
'(?!(enwiki.|dewiki.|frwiki.|itwiki.|nlwiki.|ruwiki.|svwiki.|plwiki.|eswiki.|ptwiki.))*.spell'],
- 'search1020' => ['commonswiki.nspart1',
'commonswiki.nspart1.hl', 'commonswiki.nspart2', 'commonswiki.nspart2.hl',
- 'wikidatawiki', 'metawiki', 'enwiktionary',
-
'(?!(enwiki.|dewiki.|frwiki.|itwiki.|nlwiki.|ruwiki.|svwiki.|plwiki.|eswiki.|ptwiki.))*.spell'],
- 'search1021' =>
['(?!(enwiki.|dewiki.|frwiki.|itwiki.|nlwiki.|ruwiki.|svwiki.|plwiki.|eswiki.|ptwiki.|jawiki.|zhwiki.))*.hl'],
- 'search1022' =>
['(?!(enwiki.|dewiki.|frwiki.|itwiki.|nlwiki.|ruwiki.|svwiki.|plwiki.|eswiki.|ptwiki.|jawiki.|zhwiki.))*.hl'],
- },
- # prefix hosts for all pools
- 'prefix' => {
- 'search1017' => ['*.prefix'],
- 'search1018' => ['*.prefix'],
- },
- # assigned to fake host to disable them
- 'disabled' => {
- 'search1000x' => ['*tspart1', '*tspart2',
'en-titles*', 'de-titles*', 'ja-titles*', 'it-titles*',
- 'sv-titles*', 'pl-titles*', 'pt-titles*',
'es-titles*', 'zh-titles*', 'nl-titles*', 'ru-titles*', 'fr-titles*',
- 'commonswiki.spell', 'commonswiki.nspart1.hl',
'commonswiki.nspart1', 'commonswiki.nspart2.hl', 'commonswiki.nspart2',
- '*.related', 'jawiki.nspart1.hl',
'jawiki.nspart2.hl', 'zhwiki.nspart1.hl', 'zhwiki.nspart2.hl'],
- }
- },
- 'indexers' => {
- 'searchidx1001' => ['*']
- }
- }
- }
- }
-
- # hash for lsearch-global configuration template
- $lsearch_global = {
- 'production' => {
- 'alldblist' => '/a/search/conf/all.dblist',
- 'initialisesettings' =>
'/a/search/conf/InitialiseSettings.php',
- },
- 'labs' => {
- 'alldblist' => '/a/search/conf/all-labs.dblist',
- 'initialisesettings' => '/a/search/conf/mw-beta-context.php',
- }
- }
-
- # lucene.jobs.sh tweaking
- # java_heap_size_initial => -Xms
- # java_heap_size_maximum => -Xmx
- $lucene_jobs = {
- 'production' => {
- 'importer' => {
- 'java_heap_size_initial' => '128m',
- 'java_heap_size_maximum' => '2000m',
- },
- 'prefixindexbuilder' => {
- 'java_heap_size_initial' => '',
- 'java_heap_size_maximum' => '4000m',
- },
- 'relatedbuilder' => {
- 'java_heap_size_initial' => '',
- 'java_heap_size_maximum' => '4000m',
- },
- 'suggestbuilder' => {
- 'java_heap_size_initial' => '',
- 'java_heap_size_maximum' => '8000m',
- },
- },
- 'labs' => {
- 'importer' => {
- 'java_heap_size_initial' => '128m',
- 'java_heap_size_maximum' => '2000m',
- },
- 'prefixindexbuilder' => {
- 'java_heap_size_initial' => '',
- 'java_heap_size_maximum' => '4000m',
- },
- 'relatedbuilder' => {
- 'java_heap_size_initial' => '',
- 'java_heap_size_maximum' => '4000m',
- },
- 'suggestbuilder' => {
- 'java_heap_size_initial' => '',
- 'java_heap_size_maximum' => '8000m',
- },
- },
- }
- }
-
- class beta {
- mount { '/a':
- ensure => mounted,
- device => '/dev/vdb',
- fstype => 'auto',
- options => 'defaults,nobootwait,comment=cloudconfig',
- }
- }
-
- class indexer {
- system::role { 'role::lucene::indexer':
- description => 'Lucene search indexer',
- }
-
- if $::realm == 'labs' {
- require role::lucene::beta
- }
-
- # Include packages needed for MW maintenance scripts
- include standard
- include mediawiki
- include mediawiki::php
-
- # dependency for wikimedia-task-appserver
- service { 'apache':
- ensure => stopped,
- name => 'apache2',
- enable => false,
- }
-
- class { 'lucene::server':
- indexer => true,
- udplogging => false,
- }
- }
-
- class front_end {
- class common($search_pool) {
- system::role { 'role::lucene::front-end':
- description => 'Front end lucene search server',
- }
-
- include standard
-
- $updloggingpool = $search_pool ? {
- 'search_prefix' => false,
- default => true
- }
-
- class { 'lucene::server':
- udplogging => $updloggingpool,
- }
- }
- class pool1 {
- class { 'role::lucene::front_end::common':
- search_pool => 'search_pool1',
- }
- }
- class pool2 {
- class { 'role::lucene::front_end::common':
- search_pool => 'search_pool2',
- }
- }
- class pool3 {
- class { 'role::lucene::front_end::common':
- search_pool => 'search_pool3',
- }
- }
- class pool4 {
- class { 'role::lucene::front_end::common':
- search_pool => 'search_pool4',
- }
- }
- class pool5 {
- class { 'role::lucene::front_end::common':
- search_pool => 'search_pool5',
- }
- }
- # Search frontend for the beta cluster
- class poolbeta {
- if $::realm == 'labs' {
- require role::lucene::beta
- }
- class { 'role::lucene::front_end::common':
- search_pool => 'search_poolbeta',
- }
- }
- class prefix {
- class { 'role::lucene::front_end::common':
- search_pool => 'search_prefix',
- }
- }
- }
-}
diff --git a/manifests/search.pp b/manifests/search.pp
deleted file mode 100644
index 63cdcba..0000000
--- a/manifests/search.pp
+++ /dev/null
@@ -1,245 +0,0 @@
-# search.pp
-# vim: set noet :
-
-# Virtual resource for the monitoring server
-@monitoring::group { 'lucene': description => 'eqiad search servers' }
-
-class lucene {
-
- class server($indexer=false, $udplogging=true) {
- Class['lucene::packages'] -> Class['lucene::server']
-
- require role::lucene::configuration
-
- include passwords::lucene
- $lucene_oai_pass = $passwords::lucene::oai_pass
-
- include lucene::packages,
- lucene::service
-
- if $indexer == true {
- include lucene::indexer
- }
-
- file { '/a/search/conf/lsearch-global-2.1.conf':
- ensure => 'present',
- require => File['/a/search/conf'],
- owner => 'lsearch',
- group => 'lsearch',
- mode => '0444',
- content => template('lucene/lsearch-global-2.1.conf.erb'),
- }
-
- file { '/etc/lsearch.conf':
- ensure => 'present',
- owner => 'lsearch',
- group => 'lsearch',
- mode => '0444',
- content => template('lucene/lsearch.conf.erb'),
- }
-
- file { '/a/search/conf/lsearch.log4j':
- ensure => 'present',
- require => File['/a/search/conf'],
- owner => 'lsearch',
- group => 'lsearch',
- mode => '0444',
- content => template('lucene/lsearch.log4j.erb'),
- }
-
- file { [ '/a/search',
- '/a/search/indexes',
- '/a/search/log',
- '/a/search/conf',
- '/a/search/dumps'
- ]:
- ensure => 'directory',
- owner => 'lsearch',
- group => 'lsearch',
- mode => '0775',
- }
- if $lucene::server::indexer == true {
- file { '/etc/logrotate.d/lucene-indexer':
- ensure => 'present',
- owner => 'root',
- group => 'root',
- mode => '0444',
- source => 'puppet:///files/logrotate/search-indexer',
- }
-
- file { '/a/search/conf/lucene.jobs.conf':
- ensure => 'present',
- owner => 'root',
- group => 'root',
- mode => '0444',
- content => template('lucene/lucene.jobs.conf.erb'),
- }
- }
-
- # Conf for sync-conf-from-common cronjob
- if $::realm == 'production' {
- $sync_conf_all_dblist =
'tin.eqiad.wmnet::common/*.dblist'
- $sync_conf_initialisesettings =
'tin.eqiad.wmnet::common/wmf-config/InitialiseSettings.php'
- $sync_conf_initialisesettings_labs = ''
- $sync_conf_messages =
'tin.eqiad.wmnet::common/php/languages/messages'
- } else {
- $sync_conf_all_dblist = '/srv/mediawiki/*.dblist'
- $sync_conf_initialisesettings =
'/srv/mediawiki/wmf-config/InitialiseSettings.php'
- $sync_conf_initialisesettings_labs =
'/srv/mediawiki/wmf-config/InitialiseSettings-labs.php'
- $sync_conf_messages =
'/srv/mediawiki/php-master/languages/messages'
- }
- cron { 'sync-conf-from-common':
- ## to occassionally poll for mediawiki configs
- ensure => 'present',
- require => File['/a/search/conf'],
- command => "rsync -a --delete --exclude=**/.svn/lock --no-perms
${sync_conf_all_dblist} /a/search/conf/ && rsync -a --delete
--exclude=**/.svn/lock --no-perms ${sync_conf_initialisesettings}
${sync_conf_initialisesettings_labs} /a/search/conf/ && rsync -a --delete
--exclude=**/.svn/lock --no-perms ${sync_conf_messages} /a/search/conf/",
- user => 'lsearch',
- minute => '15',
- }
-
- cron { 'delete-old-logs':
- ## this is to compliment log4j's log rotation.
- ## we want to use log4j's logrotate ability,
- ## as it's easier on the system,
- ## but log4j does not yet have "delete old logs" capability :/
- ensure => 'present',
- command =>'find /a/search/log/log.* -type f -mtime +3 -exec rm -f
{} \;',
- user => 'lsearch',
- hour => '0',
- minute => '0',
- }
- }
-
- class packages {
- package { ['oracle-j2sdk1.6', 'libudp2log-log4j-java']:
- ensure => 'latest',
- }
- package { 'liblog4j1.2-java':
- ensure => 'latest',
- require => Package['oracle-j2sdk1.6'],
- }
- package { 'lucene-search-2':
- # Present instead of latest for controlled upgrade
- ensure => 'present',
- require => Package['oracle-j2sdk1.6'],
- }
- }
-
- class service {
- service { 'lucene-search-2':
- ensure => 'running',
- require => [ File['/etc/lsearch.conf'],
- File['/a/search/conf/lsearch-global-2.1.conf'],
- File['/a/search/indexes'],
- File['/a/search/log']
- ],
- }
-
- if $lucene::server::indexer == false {
- monitoring::service { 'lucene':
- description => 'Lucene',
- check_command => 'check_lucene',
- retries => '6'
- }
-
- }
- }
-
- class users {
-
- group { 'lsearch':
- ensure => present,
- name => 'lsearch',
- system => true,
- }
-
- user { 'lsearch':
- name => 'lsearch',
- gid => 'lsearch',
- managehome => true,
- system => true,
- home => '/var/lib/search',
- }
- }
-
- class indexer {
-
- include rsync::server
- rsync::server::module { 'search':
- path => '/a/search/indexes',
- read_only => 'yes',
- comment => 'Lucene Search 2 index data',
- }
-
- file { '/a/search/conf/nooptimize.dblist':
- owner => 'lsearch',
- group => 'lsearch',
- mode => '0444',
- source => 'puppet:///files/lucene/nooptimize.dblist',
- }
-
- file { '/a/search/lucene.jobs.sh':
- owner => 'lsearch',
- group => 'lsearch',
- mode => '0755',
- source => 'puppet:///files/lucene/lucene.jobs.sh',
- }
-
- cron { 'snapshot':
- ensure => 'present',
- require => File['/a/search/lucene.jobs.sh'],
- command => '/a/search/lucene.jobs.sh snapshot',
- user => 'lsearch',
- hour => '4',
- minute => '30',
- }
-
- cron { 'snapshot-precursors':
- ensure => 'present',
- require => File['/a/search/lucene.jobs.sh'],
- command => '/a/search/lucene.jobs.sh snapshot-precursors',
- user => 'lsearch',
- weekday => '5',
- hour => '9',
- minute => '30',
- }
-
- cron { 'indexer-cron':
- ensure => 'present',
- require => File['/a/search/lucene.jobs.sh'],
- command => '/a/search/lucene.jobs.sh indexer-cron',
- user => 'lsearch',
- weekday => '6',
- hour => '0',
- minute => '0',
- }
-
- cron { 'import-private':
- ensure => 'present',
- require => File['/a/search/lucene.jobs.sh'],
- command => '/a/search/lucene.jobs.sh import-private',
- user => 'lsearch',
- hour => '2',
- minute => '0',
- }
-
- cron { 'import-broken':
- ensure => 'present',
- require => File['/a/search/lucene.jobs.sh'],
- command => '/a/search/lucene.jobs.sh import-broken',
- user => 'lsearch',
- hour => '3',
- minute => '0',
- }
-
- cron { 'build-prefix':
- ensure => 'present',
- require => File['/a/search/lucene.jobs.sh'],
- command => '/a/search/lucene.jobs.sh build-prefix',
- user => 'lsearch',
- hour => '9',
- minute => '25',
- }
- }
-}
-
diff --git a/manifests/site.pp b/manifests/site.pp
index f2bc999..0aa1424 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -12,7 +12,6 @@
import 'nfs.pp'
import 'role/*.pp'
import 'role/analytics/*.pp'
-import 'search.pp'
import 'swift.pp'
# Include stages last
@@ -2348,107 +2347,6 @@
include ldap::role::server::corp
include ldap::role::client::corp
class { 'admin': groups => ['oit'] }
-}
-
-node /^search100[0-6]\.eqiad\.wmnet/ {
- if $::hostname =~ /^search100(1|2)$/ {
- $ganglia_aggregator = true
- }
-
- class { 'admin':
- groups => [
- 'deployment',
- 'search-roots',
- ],
- }
- include role::lucene::front_end::pool1
-}
-
-node /^search10(0[7-9]|10)\.eqiad\.wmnet/ {
- class { 'admin':
- groups => [
- 'deployment',
- 'search-roots',
- ],
- }
- include role::lucene::front_end::pool2
-}
-
-node /^search101[1-4]\.eqiad\.wmnet/ {
- class { 'admin':
- groups => [
- 'deployment',
- 'search-roots',
- ],
- }
- include role::lucene::front_end::pool3
-}
-
-node /^search101[56]\.eqiad\.wmnet/ {
- class { 'admin':
- groups => [
- 'deployment',
- 'search-roots',
- ],
- }
- include role::lucene::front_end::pool4
-}
-
-node /^search10(19|20)\.eqiad\.wmnet/ {
- class { 'admin':
- groups => [
- 'deployment',
- 'search-roots',
- ],
- }
- include role::lucene::front_end::pool5
-}
-
-node /^search101[78]\.eqiad\.wmnet/ {
- class { 'admin':
- groups => [
- 'deployment',
- 'search-roots',
- ],
- }
- include role::lucene::front_end::prefix
-}
-
-node /^search10(19|2[0-2])\.eqiad\.wmnet/ {
- class { 'admin':
- groups => [
- 'deployment',
- 'search-roots',
- ],
- }
- include role::lucene::front_end::pool4
-}
-
-node /^search102[3-4]\.eqiad\.wmnet/ {
- class { 'admin':
- groups => [
- 'deployment',
- 'search-roots',
- ],
- }
- include role::lucene::front_end::pool3
-}
-
-node /^searchidx100[0-2]\.eqiad\.wmnet/ {
- class { 'admin':
- groups => [
- 'deployment',
- 'search-roots',
- ],
- }
- mount { '/srv/mediawiki':
- ensure => present,
- fstype => 'none',
- options => 'bind',
- device => '/a/bind-mount-mediawiki',
- before => Exec['fetch_mediawiki']
- }
- include role::lucene::indexer
}
node 'sodium.wikimedia.org' {
diff --git a/templates/lucene/lsearch-global-2.1.conf.erb
b/templates/lucene/lsearch-global-2.1.conf.erb
deleted file mode 100644
index 7f05809..0000000
--- a/templates/lucene/lsearch-global-2.1.conf.erb
+++ /dev/null
@@ -1,148 +0,0 @@
-####################################################################
-### THIS FILE IS MANAGED BY PUPPET ###
-### puppet:///templates/search/lsearch-global-2.1.conf.eqiad.erb ###
-####################################################################
-<% # Shortcuts for convenience
-active_nodes = scope.lookupvar('role::lucene::configuration::nodes')
-lsearch_global = scope.lookupvar('role::lucene::configuration::lsearch_global')
-realm = @realm
-site = @site
--%>
-
-# Logical structure, maps different roles to certain db
-[Database]
-{file://<%= lsearch_global[realm]['alldblist'] -%>} : (single,true,20,1000)
(prefix) (spell,10,3)
-<% if realm == 'production' -%>
-enwiki : (nssplit,2)
-enwiki : (nspart1,[0],true,20,500,2)
-enwiki : (nspart2,[],true,20,500)
-enwiki : (spell,40,10) (warmup,500)
-mediawikiwiki, metawiki, commonswiki, strategywiki : (language,en)
-commonswiki : (nssplit,2) (nspart1,[6]) (nspart2,[])
-dewiki, frwiki : (spell,20,5)
-dewiki, frwiki, itwiki, ptwiki, jawiki, plwiki, nlwiki, ruwiki, svwiki, zhwiki
: (nssplit,2) (nspart1,[0,2,4,12,14]) (nspart2,[]) (warmup,0)
-
-<% end -%>
-<% if realm == 'production' -%>
-[Database-Group]
-<all> : (titles_by_suffix,2) (tspart1,[ wiki|w ]) (tspart2,[ wiktionary|wikt,
wikibooks|b, wikinews|n, wikiquote|q, wikisource|s, wikiversity|v])
-sv-titles: (titles_by_suffix,2) (tspart1,[ svwiki|w ]) (tspart2,[
svwiktionary|wikt, svwikibooks|b, svwikinews|n, svwikiquote|q,
svwikisource|src])
-mw-titles: (titles_by_suffix,1) (tspart1, [ mediawikiwiki|mw, metawiki|meta ])
-<% end -%>
-
-# Search hosts layout
-[Search-Group]
-
-<% active_nodes[realm][site]['front_ends'].keys.sort.each do |pool| -%>
-# search <%= pool %>
-<% active_nodes[realm][site]['front_ends'][pool].sort.each do |node, indexes|
-%>
-<%= node %>: <%= indexes.join(" ") %>
-<% end -%>
-
-<% end -%>
-
-# Indexers
-[Index]
-<% active_nodes[realm][site]['indexers'].each do |node, indexes| -%>
-<%= node %>: <%= indexes.join(" ") %>
-<% end -%>
-
-# Rsync path where indexes are on hosts, after default value put
-# hosts where the location differs
-# Syntax: host : <path>
-[Index-Path]
-<default> : /search
-
-<% if @realm == 'production' -%>
-[OAI]
-simplewiki : http://simple.wikipedia.org/w/index.php
-rswikimedia : http://rs.wikimedia.org/w/index.php
-ilwikimedia : http://il.wikimedia.org/w/index.php
-nzwikimedia : http://nz.wikimedia.org/w/index.php
-sewikimedia : http://se.wikimedia.org/w/index.php
-alswiki : http://als.wikipedia.org/w/index.php
-alswikibooks : http://als.wikibooks.org/w/index.php
-alswikiquote : http://als.wikibooks.org/w/index.php
-alswiktionary : http://als.wiktionary.org/w/index.php
-chwikimedia : http://www.wikimedia.ch/w/index.php
-crhwiki : http://chr.wikipedia.org/w/index.php
-roa_rupwiki : http://roa-rup.wikipedia.org/w/index.php
-roa_rupwiktionary : http://roa-rup.wiktionary.org/w/index.php
-be_x_oldwiki : http://be-x-old.wikipedia.org/w/index.php
-uawikimedia : http://ua.wikimedia.org/w/index.php
-ukwikimedia : http://uk.wikimedia.org/w/index.php
-brwikimedia : http://br.wikimedia.org/w/index.php
-dkwikimedia : http://dk.wikimedia.org/w/index.php
-trwikimedia : http://tr.wikimedia.org/w/index.php
-arwikimedia : http://ar.wikimedia.org/w/index.php
-mxwikimedia : http://mx.wikimedia.org/w/index.php
-commonswiki: http://commons.wikimedia.org/w/index.php
-<% end -%>
-
-[Namespace-Boost]
-commonswiki : (0, 1) (6, 4)
-<default> : (0, 1) (1, 0.0005) (2, 0.005) (3, 0.001) (4, 0.01), (6, 0.02), (8,
0.005), (10, 0.0005), (12, 0.01), (14, 0.02)
-
-# Global properies
-[Properties]
-# suffixes to database name, the rest is assumed to be language code
-Database.suffix=wiki wiktionary wikiquote wikibooks wikisource wikinews
wikiversity wikimedia wikivoyage
-
-# Allow only up to 500 results per page
-Search.maxlimit=501
-
-# Age scaling based on last edit, default is no scaling
-# Below are suffixes (or whole names) with various scaling strength
-AgeScaling.strong=wikinews
-AgeScaling.medium=mediawikiwiki metawiki
-#AgeScaling.weak=wiki
-
-# Use additional per-article ranking data, more suitable for non-encyclopedias
-AdditionalRank.suffix=mediawikiwiki metawiki
-
-# suffix for databases that should also have exact-case index built
-# note: this will also turn off stemming!
-ExactCase.suffix=wiktionary jbowiki
-
-# wmf-style init file, attempt to read OAI and lang info from it
-# for sample see http://noc.wikimedia.org/conf/InitialiseSettings.php.html
-WMF.InitialiseSettings=file://<%= lsearch_global[realm]['initialisesettings']
-%>
-
-# Where common images are
-Commons.wiki=commonswiki.nspart1
-
-# Set a high RMI read timeout to avoid breaking indexing
-Search.rmiReadTimeout=7200
-
-
-# Syntax: <prefix_name> : <coma separated list of namespaces>
-# <all> is a special keyword meaning all namespaces
-# E.g. all_talk : 1,3,5,7,9,11,13,15
-[Namespace-Prefix]
-all : <all>
-[0] : 0
-[1] : 1
-[2] : 2
-[3] : 3
-[4] : 4
-[5] : 5
-[6] : 6
-[7] : 7
-[8] : 8
-[9] : 9
-[10] : 10
-[11] : 11
-[12] : 12
-[13] : 13
-[14] : 14
-[15] : 15
-[100] : 100
-[101] : 101
-[104] : 104
-[105] : 105
-[106] : 106
-[0,6,12,14,100,106]: 0,6,12,14,100,106
-[0,100,104] : 0,100,104
-[0,2,4,12,14] : 0,2,4,12,14
-[0,14] : 0,14
-[4,12] : 4,12
diff --git a/templates/lucene/lsearch.conf.erb
b/templates/lucene/lsearch.conf.erb
deleted file mode 100644
index f69c95c..0000000
--- a/templates/lucene/lsearch.conf.erb
+++ /dev/null
@@ -1,122 +0,0 @@
-######################################################
-##### THIS FILE IS MANAGED BY PUPPET ####
-##### puppet:///templates/search/lsearch.conf.erb ####
-######################################################
-
-# By default, will check /etc/mwsearch.conf
-
-################################################
-# Global configuration
-################################################
-
-# URL to global configuration, this is the shared main config file, it can
-# be on a NFS partition or available somewhere on the network or pushed out
-# by puppet or another conf management tool
-MWConfig.global=file:///a/search/conf/lsearch-global-2.1.conf
-
-# Local path to root directory of indexes
-Indexes.path=/a/search/indexes
-
-# Path to rsync
-Rsync.path=/usr/bin/rsync
-
-# Extra params for rsync
-#Rsync.params=--bwlimit=8192
-
-################################################
-# Search node related configuration
-################################################
-
-# Port of http daemon, if different from default 8123
-# Search.port=8000
-
-# non-existent host used to disable indexes/wikis
-Search.nullHost=search1000x
-
-# In minutes, how frequently will the index host be checked for updates
-Search.updateinterval=0.5
-
-# In seconds, delay after which the update will be fetched
-# used to scatter the updates around the hour
-Search.updatedelay=0
-
-# In seconds, how frequently the dead search nodes should be checked
-Search.checkinterval=30
-
-# Disable wordnet aliases
-Search.disablewordnet=<% if scope.lookupvar('lucene::server::indexer') == true
then -%>true<% else -%>false<% end -%>
-
-################################################
-# Indexer related configuration
-################################################
-
-# In minutes, how frequently is a clean snapshot of index created
-# 2880 = two days
-Index.snapshotinterval=2880
-
-# Daemon type (http is started by default)
-#Index.daemon=xmlrpc
-
-# Port of daemon (default is 8321)
-#Index.port=8080
-
-# Maximal queue size after which index is being updated
-Index.maxqueuecount=5000
-
-# Maximal time an update can remain in queue before being processed (in
seconds)
-Index.maxqueuetimeout=120
-
-# Keep only the most recent update
-# Search.deleteOldUpdates=true
-
-<% if scope.lookupvar('lucene::server::indexer') == true then -%>
-Index.delsnapshots=true
-<% end -%>
-
-################################################
-# Log, ganglia, localization
-################################################
-
-SearcherPool.size=6
-
-# URL to message files, {0} is replaced with language code, i.e. En
-Localization.url=file:///a/search/conf/messages
-
-<% if scope.lookupvar('lucene::server::indexer') == true then -%>
-# Pattern for OAI repo. {0} is replaced with dbname, {1} with language
-#OAI.repo=http://{1}.wikipedia.org/wiki/Special:OAIRepository
-OAI.username=lsearch2
-OAI.password=<%= @lucene_oai_pass %>
-<% end -%>
-# Max queue size on remote indexer after which we wait a bit
-OAI.maxqueue=5000
-
-# Number of docs to buffer before sending to inc updater
-OAI.bufferdocs=500
-
-# UDP Logger config
-# currently set to send logs to oxygen.w.o:51234
-UDPLogger.port=51234
-UDPLogger.host=208.80.154.15
-
-# RecentUpdateDaemon udp and tcp ports
-#RecentUpdateDaemon.udp=8111
-#RecentUpdateDaemon.tcp=8112
-# Hot spare
-#RecentUpdateDaemon.hostspareHost=vega
-#RecentUpdateDaemon.hostspareUdpPort=8111
-#RecentUpdateDaemon.hostspareTcpPort=8112
-
-# Log configuration
-Logging.logconfig=/a/search/conf/lsearch.log4j
-
-# Set debug to true to diagnose problems with log4j configuration
-Logging.debug=false
-
-# Turn this on to broadcast status to a Ganglia reporting system.
-# Requires that 'gmetric' be in the PATH and runnable. You can
-# override the default UDP broadcast port and interface if required.
-Ganglia.report=true
-#Ganglia.port=8649
-#Ganglia.interface=eth0
-
diff --git a/templates/lucene/lsearch.log4j.erb
b/templates/lucene/lsearch.log4j.erb
deleted file mode 100644
index 55d6f39..0000000
--- a/templates/lucene/lsearch.log4j.erb
+++ /dev/null
@@ -1,43 +0,0 @@
-# Set root logger level and its only appender to A1.
-log4j.rootLogger=INFO, A1<% if @udplogging == true then -%>, udp2log<% end -%>
-
-# A1 is set to be a ConsoleAppender.
-#log4j.appender.A1=org.apache.log4j.ConsoleAppender
-
-# Append to log file
-log4j.appender.A1=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.A1.File=/a/search/log/log
-log4j.appender.A1.Append=true
-log4j.appender.A1.Threshold=WARN
-log4j.appender.A1.MaxBackupIndex=6
-
-# A1 uses PatternLayout.
-log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c %x - %m%n
-
-# Log progress messages from the incremental indexer
-log4j.logger.org.wikimedia.lsearch.oai.IncrementalUpdater=INFO
-
-<% if @udplogging == true then -%>
-# udp2log log4j appender.
-# Logs to Oxygen's udp2log-lucene instance on port 51234
-log4j.appender.udp2log=org.wikimedia.log4j.Udp2LogAppender
-log4j.appender.udp2log.remoteHost=208.80.154.15 # oxygen
-log4j.appender.udp2log.remotePort=51234
-log4j.appender.udp2log.layout=org.apache.log4j.PatternLayout
-log4j.appender.udp2log.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss'Z'}\t%m%n
-log4j.appender.udp2log.errorhandler.logger.appender-ref=stdout
-<% end -%>
-
-#log4j.logger.org.wikimedia.lsearch.search.SearchEngine=WARN
-#log4j.logger.org.wikimedia.lsearch.frontend.HttpHandler=WARN
-#log4j.logger.org.wikimedia.lsearch.search.WikiSearcher=ERROR
-#log4j.logger.org.wikimedia.lsearch.index.WikiIndexModifier=DEBUG
-#log4j.logger.org.wikimedia.lsearch.spell.Suggest=DEBUG
-#log4j.logger.org.wikimedia.lsearch.interoperability.RMIMessengerClient=DEBUG
-#log4j.logger.org.wikimedia.lsearch.ranks.Links=DEBUG
-#log4j.logger.org.wikimedia.lsearch.prefix.PrefixIndexBuilder=DEBUG
-#log4j.logger.org.wikimedia.lsearch.spell.CleanIndexWriter=DEBUG
-#log4j.logger.org.wikimedia.lsearch.spell.api.TitleNgramIndexer=DEBUG
-#log4j.logger.org.wikimedia.lsearch.search.NetworkStatusThread=DEBUG
-#log4j.logger.org.wikimedia.lsearch.search.SearcherCache=DEBUG
diff --git a/templates/lucene/lucene.jobs.conf.erb
b/templates/lucene/lucene.jobs.conf.erb
deleted file mode 100644
index 952620e..0000000
--- a/templates/lucene/lucene.jobs.conf.erb
+++ /dev/null
@@ -1,39 +0,0 @@
-#######################################################################
-# WARNING: this file is managed by Puppet
-# puppet:///templates/search/lucene.jobs.conf.erb
-#
-# Configuration variables comes from role::lucene::configuration
-#######################################################################
-
-# Configuration script for lucene.jobs.sh runner. It is sourced inside
-# the shell script and let you override the inner settings.
-<%
-
-# Fetch configuration hash from puppet scope
-full_conf = scope.lookupvar('role::lucene::configuration::lucene_jobs')
-realm = @realm
-jobs_config = full_conf[realm]
-
-# Helper to convert the puppet hash to bash variables having
-# the -Xms and -Xmx settings.
-def java_opts(name, conf_hash)
- opts = ''
-
- heap_init = conf_hash.fetch( 'java_heap_size_initial' )
- heap_max = conf_hash.fetch( 'java_heap_size_maximum' )
-
- # Dont forget to add a space between options =)
- opts += "-Xms#{heap_init} " if heap_init and ! heap_init.empty?
- opts += "-Xms#{heap_max} " if heap_max and ! heap_max.empty?
-
- if opts.empty?
- return "##{name}=''" # Variable commented out
- else
- return "#{name}='#{opts}'"
- end
-end
--%>
-<%= java_opts( 'JAVA_OPTS_IMPORTER', jobs_config['importer']) %>
-<%= java_opts( 'JAVA_OPTS_PREFIXINDEXBUILDER',
jobs_config['prefixindexbuilder']) %>
-<%= java_opts( 'JAVA_OPTS_RELATEDBUILDER', jobs_config['relatedbuilder']) %>
-<%= java_opts( 'JAVA_OPTS_SUGGESTBUILDER', jobs_config['suggestbuilder']) %>
--
To view, visit https://gerrit.wikimedia.org/r/184620
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0054c6c026b8c01e885e464d00a44177f731e83e
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Filippo Giunchedi <fgiunch...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
