[MediaWiki-commits] [Gerrit] Fix edit conflicts after calls to save_claim() - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/268877

Change subject: Fix edit conflicts after calls to save_claim()
..

Fix edit conflicts after calls to save_claim()

Unlike DataSite.addClaim(), DataSite.save_claim() was not updating
the latest revision ID of the item of the saved claim. As a result,
any submission to Wikidata after modifying a claim, such as
Claim.changeRank(), would result in an edit conflict.

Now DataSite.save_claim() updates the latest revision ID of the
item of the saved claim.

Change-Id: I2079228b50abca2e171bbc63e6bdd581532cd703
---
M pywikibot/site.py
1 file changed, 1 insertion(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/77/268877/1

diff --git a/pywikibot/site.py b/pywikibot/site.py
index fb92abe..39ac5cb 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -7071,6 +7071,7 @@
 
 req = self._simple_request(**params)
 data = req.submit()
+claim.on_item.latest_revision_id = data['pageinfo']['lastrevid']
 return data
 
 @must_be(group='user')

-- 
To view, visit https://gerrit.wikimedia.org/r/268877
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2079228b50abca2e171bbc63e6bdd581532cd703
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Gallaecio 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] replaceCategoryLinks() no longer normalizes unmodified categ... - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/246459

Change subject: replaceCategoryLinks() no longer normalizes unmodified category 
links
..

replaceCategoryLinks() no longer normalizes unmodified category links

replaceCategoryLinks() no longer normalizes unmodified category links. For
example, if a template is used in an existing category link that is not
replaced, this function no longer expands the template. However,
note that replaceCategoryLinks() still normalizes the namespace (e.g.
localizes it if it is in English in a non-English wiki).

Users can optionally pass Category.aslink() the text that they want the
cateogory link to contain, as opposed to the normalized name of the
category. This feature is used by replaceCategoryLinks().

Create textlib.normalize_page_name(), refactoring existing code.

Move html2unicode() from page.py to textlib.py.

Change-Id: I23851646319e5d8b49e9528be908bb885c446ea5
---
M pywikibot/__init__.py
M pywikibot/page.py
M pywikibot/textlib.py
3 files changed, 167 insertions(+), 127 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/59/246459/1

diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py
index ba3aae3..eeb0ccc 100644
--- a/pywikibot/__init__.py
+++ b/pywikibot/__init__.py
@@ -84,14 +84,14 @@
 'getCategoryLinks', 'categoryFormat', 'replaceCategoryLinks',
 'removeCategoryLinks', 'removeCategoryLinksAndSeparator',
 'replaceCategoryInPlace', 'compileLinkR', 'extract_templates_and_params',
-'TimeStripper',
+'TimeStripper', 'html2unicode',
 )
 
 __all__ = (
 'config', 'ui', 'UnicodeMixin', 'translate',
 'Page', 'FilePage', 'Category', 'Link', 'User',
 'ItemPage', 'PropertyPage', 'Claim',
-'html2unicode', 'url2unicode', 'unicode2html',
+'url2unicode', 'unicode2html',
 'stdout', 'output', 'warning', 'error', 'critical', 'debug',
 'exception', 'input_choice', 'input', 'input_yn', 'inputChoice',
 'handle_args', 'handleArgs', 'showHelp', 'ui', 'log',
@@ -660,7 +660,7 @@
 PropertyPage,
 Claim,
 )
-from pywikibot.page import html2unicode, url2unicode, unicode2html
+from pywikibot.page import url2unicode, unicode2html
 
 
 link_regex = re.compile(r'\[\[(?P[^\]|[<>{}]*)(\|.*?)?\]\]')
diff --git a/pywikibot/page.py b/pywikibot/page.py
index cc2c1b5..ec62261 100644
--- a/pywikibot/page.py
+++ b/pywikibot/page.py
@@ -26,11 +26,6 @@
 import re
 import sys
 
-try:
-import unicodedata2 as unicodedata
-except ImportError:
-import unicodedata
-
 from collections import defaultdict, namedtuple
 from warnings import warn
 
@@ -2336,7 +2331,7 @@
  % title)
 
 @deprecated_args(forceInterwiki=None, textlink=None, noInterwiki=None)
-def aslink(self, sortKey=None):
+def aslink(self, sortKey=None, name_text=None):
 """Return a link to place a page in this Category.
 
 Use this only to generate a "true" category link, not for interwikis
@@ -2348,11 +2343,14 @@
 
 """
 key = sortKey or self.sortKey
+title_without_section = self.title(withSection=False)
+if name_text:
+title_without_section = '%s:%s' % (
+self.site.namespace(self.namespace()), name_text)
 if key is not None:
-titleWithSortKey = '%s|%s' % (self.title(withSection=False),
-  key)
+titleWithSortKey = '%s|%s' % (title_without_section, key)
 else:
-titleWithSortKey = self.title(withSection=False)
+titleWithSortKey = title_without_section
 return '[[%s]]' % titleWithSortKey
 
 @deprecated_args(startFrom=None, cacheResults=None)
@@ -4719,41 +4717,7 @@
 encodings = [self._source.encoding()] + list(self._source.encodings())
 
 self._text = url2unicode(self._text, encodings=encodings)
-
-# Clean up the name, it can come from anywhere.
-# Convert HTML entities to unicode
-t = html2unicode(self._text)
-
-# Normalize unicode string to a NFC (composed) format to allow
-# proper string comparisons to strings output from MediaWiki API.
-# Due to Python issue 10254, this is not possible on Python 2.6.6
-# if the string contains combining characters.  See T102461.
-if (PYTHON_VERSION == (2, 6, 6) and
-unicodedata.__name__ != 'unicodedata2' and
-any(unicodedata.combining(c) for c in t)):
-raise UnicodeError(
-'Link(%r, %s): combining characters detected, which are '
-'not supported by Pywikibot on Python 2.6.6. See '
-'https://phabricator.wikimedia.org/T102461'
-% (t, self._source))
-t = unicodedata.normalize('NFC', t)
-
-# This code was adapted from Title.php : secureAndSplit()
-#
-if 

[MediaWiki-commits] [Gerrit] Respect title inclusion exceptions from the user fixes file - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/229074

Change subject: Respect title inclusion exceptions from the user fixes file
..

Respect title inclusion exceptions from the user fixes file

Change-Id: I3c0ee5f6cd1ed870d4e5aadfb1f03bff69aaccf7
---
M scripts/replace.py
1 file changed, 21 insertions(+), 10 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/74/229074/1

diff --git a/scripts/replace.py b/scripts/replace.py
index 61faf7b..d06b7f4 100755
--- a/scripts/replace.py
+++ b/scripts/replace.py
@@ -279,10 +279,11 @@
 A replacement entry for ReplacementList.
 
 def __init__(self, old, new, fix_set, edit_summary=None,
- default_summary=True):
+ default_summary=True, name=None):
 super(ReplacementListEntry, self).__init__(old, new, edit_summary,
default_summary)
 self.fix_set = fix_set
+self.name = name
 
 @property
 def case_insensitive(self):
@@ -480,18 +481,20 @@
 self.summary = summary
 self.changed_pages = 0
 
-def isTitleExcepted(self, title):
+def isTitleExcepted(self, title, exceptions=None):
 
 Return True iff one of the exceptions applies for the given title.
 
 @rtype: bool
 
-if title in self.exceptions:
-for exc in self.exceptions['title']:
+if exceptions is None:
+exceptions = self.exceptions
+if title in exceptions:
+for exc in exceptions['title']:
 if exc.search(title):
 return True
-if require-title in self.exceptions:
-for req in self.exceptions['require-title']:
+if require-title in exceptions:
+for req in exceptions['require-title']:
 if not req.search(title):
 return True
 return False
@@ -508,7 +511,7 @@
 return True
 return False
 
-def apply_replacements(self, original_text, applied):
+def apply_replacements(self, original_text, applied, page=None):
 
 Apply all replacements to the given text.
 
@@ -521,6 +524,12 @@
 for replacement in self.replacements:
 if self.sleep is not None:
 time.sleep(self.sleep)
+if page is not None and self.isTitleExcepted(
+page.title(), replacement.exceptions):
+pywikibot.output(uSkipping replacement '{}' on {} because 
+uthe title is on the exceptions list..format(
+replacement.name, page.title(asLink=True)))
+continue
 old_text = new_text
 new_text = textlib.replaceExcept(
 new_text, replacement.old_regex, replacement.new,
@@ -598,7 +607,8 @@
 last_text = None
 while new_text != last_text:
 last_text = new_text
-new_text = self.apply_replacements(last_text, applied)
+new_text = self.apply_replacements(last_text, applied,
+   page)
 if not self.recursive:
 break
 if new_text == original_text:
@@ -871,9 +881,9 @@
 'description of the\nchanges your bot will make:')
 
 # Perform one of the predefined actions.
-for fix in fixes_set:
+for fix_name in fixes_set:
 try:
-fix = fixes.fixes[fix]
+fix = fixes.fixes[fix_name]
 except KeyError:
 pywikibot.output(u'Available predefined fixes are: %s'
  % ', '.join(fixes.fixes.keys()))
@@ -907,6 +917,7 @@
 new=replacement[1],
 fix_set=replacement_set,
 edit_summary=summary,
+name=fix_name,
 ))
 
 # Set the regular expression flags

-- 
To view, visit https://gerrit.wikimedia.org/r/229074
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I3c0ee5f6cd1ed870d4e5aadfb1f03bff69aaccf7
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Gallaecio adriyeticha...@gmail.com

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] replace.py: Added support for replacing in page titles - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/182444

Change subject: replace.py: Added support for replacing in page titles
..

replace.py: Added support for replacing in page titles

Change-Id: I27f929dbfd08efda559000f6d281b40f5d7e4098
---
M scripts/replace.py
1 file changed, 186 insertions(+), 89 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/44/182444/1

diff --git a/scripts/replace.py b/scripts/replace.py
index a8fc8cd..0d527a6 100755
--- a/scripts/replace.py
+++ b/scripts/replace.py
@@ -70,6 +70,13 @@
   (or no replacements are defined via -fix or the arguments)
   it'll ask for additional replacements at start.
 
+-scope:XYZWorks only on the specified scope. XYZ may be any of the
+  following:
+  * content (default): Search and replace on page content.
+  * titles: Search and replace on page titles.
+  * all: Search and replace on both page content and page
+titles.
+
 fixes-help;
 
 -always   Don't prompt you for each replacement
@@ -388,7 +395,8 @@
 
 def __init__(self, generator, replacements, exceptions={},
  acceptall=False, allowoverlap=False, recursive=False,
- addedCat=None, sleep=None, summary='', site=None):
+ addedCat=None, sleep=None, summary='', site=None,
+ scopes=[content]):
 
 Constructor.
 
@@ -405,6 +413,9 @@
  replaced.
 * addedCat - If set to a value, add this category to every page
  touched.
+* scope- Determines the scopes where the replace robot
+ works. Possible scopes are page content
+ (content) and page names (names).
 
 Structure of the exceptions dictionary:
 This dictionary can have these keys:
@@ -441,6 +452,8 @@
 self.sleep = sleep
 self.summary = summary
 self.changed_pages = 0
+self.moved_pages = 0
+self.scopes = scopes
 
 def isTitleExcepted(self, title):
 
@@ -528,6 +541,153 @@
 ))
 return u'; '.join(summary_messages)
 
+def work_on_page_content(self, page):
+applied = set()
+original_text = page.get(get_redirect=True)
+new_text = original_text
+while True:
+if self.isTextExcepted(new_text):
+pywikibot.output(u'Skipping content of %s because it contains '
+ u'text that is on the exceptions list.'
+ % page.title(asLink=True))
+break
+last_text = None
+while new_text != last_text:
+last_text = new_text
+new_text = self.apply_replacements(last_text, applied)
+if not self.recursive:
+break
+if new_text == original_text:
+pywikibot.output(u'No changes were necessary in content of %s'
+% page.title(asLink=True))
+break
+if hasattr(self, addedCat):
+cats = page.categories(nofollow_redirects=True)
+if self.addedCat not in cats:
+cats.append(self.addedCat)
+new_text = textlib.replaceCategoryLinks(new_text,
+cats,
+site=page.site)
+# Show the title of the page we're working on.
+# Highlight the title in purple.
+pywikibot.output(u\n\n \03{lightpurple}%s\03{default} 
+% page.title())
+pywikibot.showDiff(original_text, new_text)
+if self.acceptall:
+break
+choice = pywikibot.input_choice(
+u'Do you want to accept these changes?',
+[('Yes', 'y'), ('No', 'n'), ('Edit', 'e'),
+('open in Browser', 'b'), ('all', 'a')],
+default='N')
+if choice == 'e':
+editor = editarticle.TextEditor()
+as_edited = editor.edit(original_text)
+# if user didn't press Cancel
+if as_edited and as_edited != new_text:
+new_text = as_edited
+continue
+if choice == 'b':
+webbrowser.open(http://%s%s; % (
+page.site.hostname(),
+page.site.nice_get_address(page.title(asUrl=True))
+))
+i18n.input('pywikibot-enter-finished-browser')
+try:
+original_text = page.get(get_redirect=True, force=True)

[MediaWiki-commits] [Gerrit] login: Use the site object’s username if defined - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/176482

Change subject: login: Use the site object’s username if defined
..

login: Use the site object’s username if defined

Change-Id: Idac9c21f6b4d703bb464306cac82b7e61d2c31d8
---
M pywikibot/login.py
1 file changed, 2 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/82/176482/1

diff --git a/pywikibot/login.py b/pywikibot/login.py
index db7be8f..9178db2 100644
--- a/pywikibot/login.py
+++ b/pywikibot/login.py
@@ -59,6 +59,8 @@
 self.site = pywikibot.Site()
 if user:
 self.username = user
+elif self.site.username():
+self.username = self.site.username()
 elif sysop:
 try:
 family_sysopnames = config.sysopnames[self.site.family.name]

-- 
To view, visit https://gerrit.wikimedia.org/r/176482
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Idac9c21f6b4d703bb464306cac82b7e61d2c31d8
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Gallaecio adriyeticha...@gmail.com

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] scripts.catall: Make script documentation more clear - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/176485

Change subject: scripts.catall: Make script documentation more clear
..

scripts.catall: Make script documentation more clear

Fixes T60549

Change-Id: I9c980da9df2f3c83d6e27147831aa398ed4f4d3b
---
M scripts/catall.py
1 file changed, 7 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/85/176485/1

diff --git a/scripts/catall.py b/scripts/catall.py
index dbd1705..bd4f938 100755
--- a/scripts/catall.py
+++ b/scripts/catall.py
@@ -1,6 +1,12 @@
 # -*- coding: utf-8 -*-
 
-Add or change categories on a number of pages.
+For each page in the target wiki, this script lets you do either of the
+following:
+
+* If a page contains no categories, you can specify a list of categories to add
+  to the page.
+* If a page already contains one or more categories, you can specify a new list
+  of categories to replace the current list of categories of the page.
 
 Usage:
 catall.py [start]

-- 
To view, visit https://gerrit.wikimedia.org/r/176485
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I9c980da9df2f3c83d6e27147831aa398ed4f4d3b
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Gallaecio adriyeticha...@gmail.com

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] Use batch limit from the API instead of a hardcoded value - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/154336

Change subject: Use batch limit from the API instead of a hardcoded value
..

Use batch limit from the API instead of a hardcoded value

Change-Id: I5aef183069abecc91c0c353eeb7676d2ec7b8849
---
M pywikibot/pagegenerators.py
M pywikibot/site.py
M scripts/casechecker.py
M scripts/interwiki.py
M scripts/makecat.py
M scripts/reflinks.py
M scripts/welcome.py
7 files changed, 71 insertions(+), 22 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/36/154336/1

diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index 0220107..f8ec1be 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -981,30 +981,40 @@
 
 @deprecate_arg(pageNumber, step)
 @deprecate_arg(lookahead, None)
-def PreloadingGenerator(generator, step=50):
-Yield preloaded pages taken from another generator.
+def PreloadingGenerator(generator, step=None):
+Yield preloaded pages taken from another generator.
+
+If you do not specify a number of pages to query, use the maximum
+number that the API allows.
+
+
 
 # pages may be on more than one site, for example if an interwiki
 # generator is used, so use a separate preloader for each site
 sites = {}
+steps = {}
 # build a list of pages for each site found in the iterator
 for page in generator:
 site = page.site
 sites.setdefault(site, []).append(page)
-if len(sites[site]) = step:
+if site not in steps:
+steps[site] = step
+if steps[site] is None:
+steps[site] = site.getquerylimit()
+if len(sites[site]) = steps[site]:
 # if this site is at the step, process it
 group = sites[site]
 sites[site] = []
-for i in site.preloadpages(group, step):
+for i in site.preloadpages(group, steps[site]):
 yield i
 for site in sites:
 if sites[site]:
 # process any leftover sites that never reached the step
-for i in site.preloadpages(sites[site], step):
+for i in site.preloadpages(sites[site], steps[site]):
 yield i
 
 
-def PreloadingItemGenerator(generator, step=50):
+def PreloadingItemGenerator(generator, step=None):
 
 Yield preloaded pages taken from another generator.
 
@@ -1014,19 +1024,24 @@
 @param step: how many pages to preload at once
 
 sites = {}
+steps = {}
 for page in generator:
 site = page.site
 sites.setdefault(site, []).append(page)
-if len(sites[site]) = step:
+if site not in steps:
+steps[site] = step
+if steps[site] is None:
+steps[site] = site.getquerylimit()
+if len(sites[site]) = steps[site]:
 # if this site is at the step, process it
 group = sites[site]
 sites[site] = []
-for i in site.preloaditempages(group, step):
+for i in site.preloaditempages(group, steps[site]):
 yield i
 for site in sites:
 if sites[site]:
 # process any leftover sites that never reached the step
-for i in site.preloaditempages(sites[site], step):
+for i in site.preloaditempages(sites[site], steps[site]):
 yield i
 
 
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 63fca60..011efef 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -1852,9 +1852,26 @@
 page._redirtarget = target
 return page._redirtarget
 
-def preloadpages(self, pagelist, groupsize=50, templates=False,
+def getquerylimit(self):
+if not hasattr(self, __querylimit):
+request = 
api.CachedRequest(expiry=pywikibot.config.API_config_expiry, site=self,
+action=paraminfo, 
querymodules=info)
+data = request.submit()
+assert paraminfo in data
+assert querymodules in data[paraminfo]
+assert len(data[paraminfo][querymodules]) = 1
+assert parameters in data[paraminfo][querymodules][0]
+assert len(data[paraminfo][querymodules][0][parameters]) = 1
+assert limit in 
data[paraminfo][querymodules][0][parameters][0]
+self.__querylimit = 
int(data[paraminfo][querymodules][0][parameters][0][limit])
+return self.__querylimit
+
+def preloadpages(self, pagelist, groupsize=None, templates=False,
  langlinks=False):
 Return a generator to a list of preloaded pages.
+
+If you do not specify a number of pages to query, use the maximum
+number that the API allows.
 
 Note that [at least in current implementation] pages may be iterated
 in a different order than in the 

[MediaWiki-commits] [Gerrit] Add support for generating lists of pages matching the searc... - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/153562

Change subject: Add support for generating lists of pages matching the search 
string
..

Add support for generating lists of pages matching the search string

This support was part of the old PyWikiBot, this patch simply restores that 
feature.

Change-Id: I3fc2dcfb4847ca0b529bbed5687adec423c04c87
---
M scripts/replace.py
1 file changed, 93 insertions(+), 19 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/62/153562/1

diff --git a/scripts/replace.py b/scripts/replace.py
index 3ddc0ec..77810ad 100755
--- a/scripts/replace.py
+++ b/scripts/replace.py
@@ -33,6 +33,17 @@
   before the one specified (may also be given as
   -xmlstart:Article).
 
+-save Saves the titles of the articles to a file instead of
+  modifying the articles. This way you may collect titles to
+  work on in automatic mode, and process them later with
+  -file. Opens the file for append, if exists.
+  If you insert the contents of the file into a wikipage, it
+  will appear as a numbered list, and may be used with -links.
+  Argument may also be given as -save:filename.
+
+-savenew  Just like -save, except that overwrites the existing file.
+  Argument may also be given as -savenew:filename.
+
 -addcat:cat_name  Adds cat_name category to every altered page.
 
 -excepttitle:XYZ  Skip pages with titles that contain XYZ. If the -regex
@@ -107,6 +118,14 @@
 
 python replace.py -page:John_Doe -fix:isbn
 
+Let's suppose, you want to change color to colour manually, but gathering
+the articles is too slow, so you want to save the list while you are sleeping.
+You have Windows, so python is not necessary. Use this:
+
+replace.py -xml -save:color.txt color colour -always
+
+You may use color.txt later with -file or -links, if you upload it to the wiki.
+
 This command will change 'referer' to 'referrer', but not in pages which
 talk about HTTP, where the typo has become part of the standard:
 
@@ -123,6 +142,7 @@
 __version__ = '$Id$'
 #
 
+import codecs
 import re
 import time
 import pywikibot
@@ -225,7 +245,8 @@
 
 def __init__(self, generator, replacements, exceptions={},
  acceptall=False, allowoverlap=False, recursive=False,
- addedCat=None, sleep=None, summary='', site=None):
+ addedCat=None, sleep=None, summary='', site=None,
+ articles=None):
 
 Arguments:
 * generator- A generator that yields Page objects.
@@ -240,6 +261,9 @@
  replaced.
 * addedCat - If set to a value, add this category to every page
  touched.
+* articles - An open file to save the page titles. If None,
+ we work on our wikisite immediately (default).
+ Corresponds to titlefile variable of main().
 
 Structure of the exceptions dictionary:
 This dictionary can have these keys:
@@ -275,6 +299,7 @@
cat_ns + ':' + addedCat)
 self.sleep = sleep
 self.summary = summary
+self.articles = articles
 
 def isTitleExcepted(self, title):
 
@@ -335,7 +360,7 @@
 try:
 # Load the page's text from the wiki
 original_text = page.get(get_redirect=True)
-if not page.canBeEdited():
+if not (self.articles or page.canBeEdited()):
 pywikibot.output(uYou can't edit page %s
  % page.title(asLink=True))
 continue
@@ -403,25 +428,37 @@
 if choice == 'a':
 self.acceptall = True
 if choice == 'y':
-page.put_async(new_text, self.summary)
+if not self.articles:
+page.put_async(new_text, self.summary)
+else:
+self.articles.write(u'#%s\n'
+% (page.title(asLink=True,
+  textlink=True)))
+self.articles.flush()
 # choice must be 'N'
 break
 if self.acceptall and new_text != original_text:
-try:
-page.put(new_text, self.summary)
-except pywikibot.EditConflict:
-pywikibot.output(u'Skipping %s because of edit conflict'
- % (page.title(),))
-except pywikibot.SpamfilterError as e:
-

[MediaWiki-commits] [Gerrit] Ignore warnings due to family codenames with dots - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/153563

Change subject: Ignore warnings due to family codenames with dots
..

Ignore warnings due to family codenames with dots

PyWikiBot works perfectly when family names have dots in them, however it still
shows a warning. This patch avoids that warning.

Change-Id: I18ab14a490186782c68bdddf6e7fb41bdaef4775
---
M pywikibot/site.py
1 file changed, 5 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/63/153563/1

diff --git a/pywikibot/site.py b/pywikibot/site.py
index 16415fb..b8f2504 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -108,7 +108,11 @@
 return _families[fam]
 
 try:
-myfamily = imp.load_source(fam, config.family_files[fam])
+# Ignore warnings due to dots in family names.
+import warnings
+with warnings.catch_warnings():
+warnings.simplefilter(ignore)
+myfamily = imp.load_source(fam, config.family_files[fam])
 except (ImportError, KeyError):
 if fatal:
 pywikibot.error(u\

-- 
To view, visit https://gerrit.wikimedia.org/r/153563
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I18ab14a490186782c68bdddf6e7fb41bdaef4775
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Gallaecio adriyeticha...@gmail.com

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] Allow defining entries in the password file that are specifi... - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/127453

Change subject: Allow defining entries in the password file that are specific 
to a site but not to a language on that site
..

Allow defining entries in the password file that are specific to a site but not 
to a language on that site

Change-Id: I1390e0cc8ff72ee165bf115ef3c9d69e7d2f35af
---
M pywikibot/login.py
1 file changed, 10 insertions(+), 5 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/53/127453/1

diff --git a/pywikibot/login.py b/pywikibot/login.py
index aead4be..48e6bfb 100644
--- a/pywikibot/login.py
+++ b/pywikibot/login.py
@@ -142,21 +142,26 @@
 
 (umy_username, umy_default_password)
 (umy_sysop_user, umy_sysop_password)
-(uen, uwikipedia, umy_en_user, umy_en_pass)
+(uwikipedia, umy_wikipedia_user, umy_wikipedia_pass)
+(uen, uwikipedia, umy_en_wikipedia_user, umy_en_wikipedia_pass)
 
 password_f = open(config.password_file)
 for line in password_f:
 if not line.strip():
 continue
 entry = eval(line.decode('utf-8'))
-if len(entry) == 2:# for default userinfo
-if entry[0] == self.username:
-self.password = entry[1]
-elif len(entry) == 4:  # for userinfo included code and family
+if len(entry) == 4: # for userinfo included code and family
 if entry[0] == self.site.code and \
entry[1] == self.site.family.name and \
entry[2] == self.username:
 self.password = entry[3]
+elif len(entry) == 3:   # for userinfo included family
+if entry[0] == self.site.family.name and \
+   entry[1] == self.username:
+self.password = entry[2]
+elif len(entry) == 2:   # for default userinfo
+if entry[0] == self.username:
+self.password = entry[1]
 password_f.close()
 
 def login(self, retry=False):

-- 
To view, visit https://gerrit.wikimedia.org/r/127453
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I1390e0cc8ff72ee165bf115ef3c9d69e7d2f35af
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Gallaecio adriyeticha...@gmail.com

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] Fix replace.py ignoring the -lang parameter - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/127454

Change subject: Fix replace.py ignoring the -lang parameter
..

Fix replace.py ignoring the -lang parameter

Since the generator is initialized before the -lang parameter reaches the 
global pywikibot parameter handling, it sometimes may happen that you ask 
replace.py for a specific language of a site using the -lang parameter, and 
replace.py would ignore this and search on the default (e.g. English) version 
of the site.

This is now fixed by initializing the generator only after pywikibot has 
handled all global parameters.

Change-Id: I61c2987505eff17bf53994b084283ac3ca601aa4
---
M scripts/replace.py
1 file changed, 17 insertions(+), 6 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/54/127454/1

diff --git a/scripts/replace.py b/scripts/replace.py
index ad9c11b..16a427d 100755
--- a/scripts/replace.py
+++ b/scripts/replace.py
@@ -441,6 +441,7 @@
 # summary message
 summary_commandline = False
 edit_summary = u
+unhandledArgs = []
 # Array which will collect commandline parameters.
 # First element is original text, second element is replacement text.
 commandline_replacements = []
@@ -478,18 +479,13 @@
 allowoverlap = False
 # Do not recurse replacement
 recursive = False
-# This factory is responsible for processing command line arguments
-# that are also used by other scripts and that determine on which pages
-# to work on.
-genFactory = pagegenerators.GeneratorFactory()
 # Between a regex and another (using -fix) sleep some time (not to waste
 # too much CPU
 sleep = None
 
 # Read commandline parameters.
+unhandledArgs = []
 for arg in pywikibot.handleArgs(*args):
-if genFactory.handleArg(arg):
-continue
 if arg == '-regex':
 regex = True
 elif arg.startswith('-xmlstart'):
@@ -543,8 +539,23 @@
 elif arg.startswith('-allowoverlap'):
 allowoverlap = True
 else:
+unhandledArgs.append(arg)
+
+# This factory is responsible for processing command line arguments
+# that are also used by other scripts and that determine on which pages
+# to work on.
+# Note: You must create the generator factory after you parse other
+# parameters, so that the generator factory uses the right site if 
specified
+# with the ‘-family’ and ‘-lang’ command-line parameters.
+genFactory = pagegenerators.GeneratorFactory()
+for arg in unhandledArgs:
+if genFactory.handleArg(arg):
+continue
+else:
 commandline_replacements.append(arg)
+
 pywikibot.Site().login()
+genFactory.site = pywikibot.Site()
 gen = genFactory.getCombinedGenerator()
 if (len(commandline_replacements) % 2):
 raise pywikibot.Error('require even number of replacements.')

-- 
To view, visit https://gerrit.wikimedia.org/r/127454
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I61c2987505eff17bf53994b084283ac3ca601aa4
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Gallaecio adriyeticha...@gmail.com

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] Allow defining a step value for page generators in the user ... - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/127455

Change subject: Allow defining a step value for page generators in the user 
config file
..

Allow defining a step value for page generators in the user config file

Add a new option to the user settings that is used as the default value for the 
step parameter of the PreloadingGenerator function in pagegenerators.py, where 
so far there was a hard-coded 50.

Change-Id: I84e6510df478e580f45405be60901b96020b34b3
---
M pywikibot/config2.py
M pywikibot/pagegenerators.py
2 files changed, 4 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/55/127455/1

diff --git a/pywikibot/config2.py b/pywikibot/config2.py
index 3f6342c..508a46f 100644
--- a/pywikibot/config2.py
+++ b/pywikibot/config2.py
@@ -412,6 +412,9 @@
 # Minimum time to wait before resubmitting a failed API request.
 retry_wait = 5
 
+# Maximum of pages which can be retrieved in a row by a page generator.
+page_group_limit = 50
+
 ## TABLE CONVERSION BOT SETTINGS ##
 
 # will split long paragraphs for better reading the source.
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index b33d3a3..82f1ea3 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -861,7 +861,7 @@
 
 @deprecate_arg(pageNumber, step)
 @deprecate_arg(lookahead, None)
-def PreloadingGenerator(generator, step=50):
+def PreloadingGenerator(generator, step=config.page_group_limit):
 Yield preloaded pages taken from another generator.
 
 # pages may be on more than one site, for example if an interwiki

-- 
To view, visit https://gerrit.wikimedia.org/r/127455
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I84e6510df478e580f45405be60901b96020b34b3
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Gallaecio adriyeticha...@gmail.com

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] Make it possible to define a custom families folder - change (pywikibot/core)

[Gallaecio (Code Review)]

Gallaecio has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/124636

Change subject: Make it possible to define a custom families folder
..

Make it possible to define a custom families folder

Allow to define a custom family folder with a global function from the 
user-config.py file.
Add try-except structures that prevent issues when you try to use the API from 
login.py on
a wiki that requires you to first log in to use any part of the API.

Change-Id: I631ecc551900cefb85d45b860dc45e187c0a7395
---
M pywikibot/config2.py
M pywikibot/site.py
2 files changed, 47 insertions(+), 27 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core 
refs/changes/36/124636/1

diff --git a/pywikibot/config2.py b/pywikibot/config2.py
index 3f6342c..3b11a96 100644
--- a/pywikibot/config2.py
+++ b/pywikibot/config2.py
@@ -105,9 +105,6 @@
 #  relevant summary for bot edits
 default_edit_summary = u'Wikipedia python library v.2'
 
-
-# Get the names of all known families, and initialize
-# with empty dictionaries
 def _get_base_dir():
 Return the directory in which user-specific information is stored.
 
@@ -162,14 +159,26 @@
 return base_dir
 
 _base_dir = _get_base_dir()
-# families/ is a subdirectory of the directory in which config.py is found
-for _filename in os.listdir(
-os.path.join(os.path.dirname(__file__), 'families')):
-if _filename.endswith(_family.py):
-familyName = _filename[:-len(_family.py)]
-usernames[familyName] = {}
-sysopnames[familyName] = {}
-disambiguation_comment[familyName] = {}
+
+
+familyFiles = {}
+
+def registerFamilyFile(familyName, filePath):
+# NOTE: Although it is not currently used, the new implementation will 
probably need to know the exact location of each family file.
+usernames[familyName] = {}
+sysopnames[familyName] = {}
+disambiguation_comment[familyName] = {}
+familyFiles[familyName] = filePath
+
+def registerFamiliesFolder(folderPath):
+for fileName in os.listdir(folderPath):
+if fileName.endswith(_family.py):
+familyName = fileName[:-len(_family.py)]
+registerFamilyFile(familyName, os.path.join(folderPath, fileName))
+
+# Get the names of all known families, and initialize with empty dictionaries.
+# ‘families/’ is a subdirectory of the directory in which config2.py is found.
+registerFamiliesFolder(os.path.join(os.path.dirname(__file__), 'families'))
 
 # Set to True to override the {{bots}} exclusion protocol (at your own risk!)
 ignore_bot_templates = False
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 141f0e9..77d6036 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -16,6 +16,7 @@
 except ImportError:
 from md5 import md5
 import datetime
+import imp
 import itertools
 import os
 import re
@@ -94,14 +95,18 @@
 sys.path.append(config.datafilepath('families'))
 myfamily = __import__(%s_family % fam)
 except ImportError:
-if fatal:
-pywikibot.error(u\
+# Next, see if user has registered a custom family file.
+try:
+myfamily = imp.load_source(name, config.familyFiles[fam])
+except (ImportError, KeyError):
+if fatal:
+pywikibot.error(u\
 Error importing the %s family. This probably means the family
 does not exist. Also check your configuration file.
-% fam, exc_info=True)
-sys.exit(1)
-else:
-raise Error(Family %s does not exist % fam)
+% fam, exc_info=True)
+sys.exit(1)
+else:
+raise Error(Family %s does not exist % fam)
 return myfamily.Family()
 
 
@@ -881,10 +886,13 @@
 self._loginstatus = LoginStatus.IN_PROGRESS
 if hasattr(self, _userinfo):
 del self._userinfo
-self.getuserinfo()
-if self.userinfo['name'] == self._username[sysop] and \
-   self.logged_in(sysop):
-return
+try:
+self.getuserinfo()
+if self.userinfo['name'] == self._username[sysop] and \
+self.logged_in(sysop):
+return
+except pywikibot.data.api.APIError: # May occur if you are not logged 
in (no API read permissions).
+pass
 loginMan = api.LoginManager(site=self, sysop=sysop,
 user=self._username[sysop])
 if loginMan.login(retry=True):
@@ -1331,12 +1339,15 @@
 version numbers and any other text contained in the version.
 
 
-if force:
-self._getsiteinfo(force=True)# drop/expire cache and reload
-versionstring = self.siteinfo['generator']
-m = re.match(r^MediaWiki ([0-9]+)\.([0-9]+)(.*)$, versionstring)
-