Murfel has uploaded a new change for review.
https://gerrit.wikimedia.org/r/179149
Change subject: Added batch upload support
......................................................................
Added batch upload support
T57088
Change-Id: I13d469b3ddcac78b02543efe53868a8672966ec0
---
M scripts/upload.py
A tests/data/images/1rightarrow.png
A tests/data/images/MP_sounds.png
A tests/uploadbot_tests.py
4 files changed, 109 insertions(+), 30 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/49/179149/1
diff --git a/scripts/upload.py b/scripts/upload.py
index 91e1976..fa83a12 100755
--- a/scripts/upload.py
+++ b/scripts/upload.py
@@ -21,14 +21,14 @@
'M': Megabytes (1000000 B)
'Ki': Kibibytes (1024 B)
'Mi': Mebibytes (1024x1024 B)
- The suffixes are case insensitive.
+ The suffixes are case insenstive.
-If any other arguments are given, the first is the URL or filename to upload,
-and the rest is a proposed description to go with the upload. If none of these
-are given, the user is asked for the file or URL to upload. The bot will then
-upload the image to the wiki.
+If any other arguments are given, the first is either directory, filename or
URL
+to upload, and the rest is a proposed description to go with the upload. If
none
+of these are given, the user is asked for the directory, file or URL to upload.
+The bot will then upload the image to the wiki.
-The script will ask for the location of an image, if not given as a parameter,
+The script will ask for the location of an image(s), if not given as a
parameter,
and for a description.
"""
#
@@ -76,7 +76,13 @@
selectively ignore specific warnings.
"""
- self.url = url
+ if os.path.isdir(url):
+ url = os.path.abspath(url)
+ self.urlList = [os.path.join(url, fileUrl)
+ for fileUrl in os.listdir(url)
+ if os.path.isfile(os.path.join(url, fileUrl))]
+ else:
+ self.urlList = [os.path.abspath(url)]
self.urlEncoding = urlEncoding
self.description = description
self.useFilename = useFilename
@@ -90,16 +96,18 @@
'commons')
else:
self.targetSite = targetSite or pywikibot.Site()
- self.targetSite.forceLogin()
+ self.targetSite.login()
self.uploadByUrl = uploadByUrl
def urlOK(self):
- """Return True if self.url is an URL or an existing local file."""
- return "://" in self.url or os.path.exists(self.url)
+ """Return True if self.urlList[0] is a URL or an existing local
file."""
+ if self.urlList:
+ return "://" in self.urlList[0] or os.path.exists(self.urlList[0])
+ return False
- def read_file_content(self):
+ def read_file_content(self, imageUrl):
"""Return name of temp file in which remote file is saved."""
- pywikibot.output(u'Reading file %s' % self.url)
+ pywikibot.output(u'Reading file %s' % imageUrl)
resume = False
rlen = 0
_contents = None
@@ -112,7 +120,7 @@
pywikibot.output(u"Resume download...")
uo.addheader('Range', 'bytes=%s-' % rlen)
- infile = uo.open(self.url)
+ infile = uo.open(imageUrl)
if 'text/html' in infile.info().getheader('Content-Type'):
pywikibot.output(u"Couldn't download the image: "
@@ -155,10 +163,10 @@
t.close()
return tempname
- def process_filename(self):
- """Return base filename portion of self.url."""
+ def process_filename(self, imageUrl):
+ """Return base filename portion of imageUrl."""
# Isolate the pure name
- filename = self.url
+ filename = imageUrl
# Filename may be either a local file path or a URL
if "://" in filename:
# extract the path portion of the URL
@@ -247,34 +255,34 @@
else:
return warn_code in self.ignoreWarning
- def upload_image(self, debug=False):
- """Upload the image at self.url to the target wiki.
+ def upload_image(self, imageUrl, debug=False):
+ """Upload the image at imageUrl to the target wiki.
Return the filename that was used to upload the image.
If the upload fails, ask the user whether to try again or not.
If the user chooses not to retry, return null.
"""
- filename = self.process_filename()
+ filename = self.process_filename(imageUrl)
site = self.targetSite
imagepage = pywikibot.FilePage(site, filename) # normalizes filename
imagepage.text = self.description
- pywikibot.output(u'Uploading file to %s via API....' % site)
+ pywikibot.output(u'Uploading file to %s via API...' % site)
try:
apiIgnoreWarnings = False
if self.ignoreWarning is True:
apiIgnoreWarnings = True
if self.uploadByUrl:
- site.upload(imagepage, source_url=self.url,
+ site.upload(imagepage, source_url=imageUrl,
ignore_warnings=apiIgnoreWarnings)
else:
- if "://" in self.url:
- temp = self.read_file_content()
+ if "://" in imageUrl:
+ temp = self.read_file_content(imageUrl)
else:
- temp = self.url
+ temp = imageUrl
site.upload(imagepage, source_filename=temp,
ignore_warnings=apiIgnoreWarnings,
chunk_size=self.chunk_size)
@@ -307,7 +315,7 @@
else:
# No warning, upload complete.
- pywikibot.output(u"Upload successful.")
+ pywikibot.output(u"Upload of %s successful." % filename)
return filename # data['filename']
def run(self):
@@ -327,12 +335,22 @@
return
while not self.urlOK():
- if not self.url:
- pywikibot.output(u'No input filename given')
+ if not self.urlList:
+ pywikibot.output(u'No input filename given.')
else:
pywikibot.output(u'Invalid input filename given. Try again.')
- self.url = pywikibot.input(u'File or URL where image is now:')
- return self.upload_image()
+ url = pywikibot.input(u'Directory, file or URL where images are
now:')
+ if os.path.isdir(url):
+ url = os.path.abspath(url)
+ self.urlList = [os.path.join(url, fileUrl)
+ for fileUrl in os.listdir(url)
+ if os.path.isfile(os.path.join(url, fileUrl))]
+ else:
+ self.urlList = [os.path.abspath(url)]
+
+ for imageUrl in self.urlList:
+ self.upload_image(imageUrl)
+ return "Upload completed"
def main(*args):
@@ -404,7 +422,7 @@
else:
description.append(arg)
description = u' '.join(description)
- bot = UploadRobot(url, description=description, useFilename=useFilename,
+ bot = UploadRobot(url=url, description=description,
useFilename=useFilename,
keepFilename=keepFilename,
verifyDescription=verifyDescription,
aborts=aborts, ignoreWarning=ignorewarn,
diff --git a/tests/data/images/1rightarrow.png
b/tests/data/images/1rightarrow.png
new file mode 100644
index 0000000..fbce893
--- /dev/null
+++ b/tests/data/images/1rightarrow.png
Binary files differ
diff --git a/tests/data/images/MP_sounds.png b/tests/data/images/MP_sounds.png
new file mode 100644
index 0000000..4bf78d5
--- /dev/null
+++ b/tests/data/images/MP_sounds.png
Binary files differ
diff --git a/tests/uploadbot_tests.py b/tests/uploadbot_tests.py
new file mode 100644
index 0000000..d4d5048
--- /dev/null
+++ b/tests/uploadbot_tests.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+"""
+UploadRobot test.
+
+These tests write to the wiki.
+"""
+#
+# (C) Pywikibot team, 2014
+#
+# Distributed under the terms of the MIT license.
+#
+
+import os
+
+from scripts import upload
+from tests import _data_dir
+from tests.aspects import unittest, TestCase
+
+
+class TestUploadbot(TestCase):
+
+ """Test cases for upload."""
+
+ write = True
+
+ family = 'wikipedia'
+ code = 'test'
+
+ def test_png_batch(self):
+ """Test uploading a batch of pngs using upload.py."""
+ bot = upload.UploadRobot(url=os.path.join(_data_dir, 'images'),
+ description="pywikibot upload.py script test",
+ useFilename=None, keepFilename=True,
+ verifyDescription=True, aborts=set(),
+ ignoreWarning=True, chunk_size=0)
+ bot.run()
+
+ def test_png(self):
+ """Test uploading a png using upload.py."""
+ bot = upload.UploadRobot(url=os.path.join(_data_dir, "MP_sounds.png"),
+ description="pywikibot upload.py script test",
+ useFilename=None, keepFilename=True,
+ verifyDescription=True, aborts=set(),
+ ignoreWarning=True, chunk_size=0)
+ bot.run()
+
+ def test_png_url(self):
+ """Test uploading a png from url using upload.py."""
+ bot =
upload.UploadRobot(url='https://upload.wikimedia.org/wikipedia/commons/f/fc/MP_sounds.png',
+ description="pywikibot upload.py script test",
+ useFilename=None, keepFilename=True,
+ verifyDescription=True, aborts=set(),
+ ignoreWarning=True, chunk_size=0)
+ bot.run()
+
+
+if __name__ == '__main__':
+ try:
+ unittest.main()
+ except SystemExit:
+ pass
--
To view, visit https://gerrit.wikimedia.org/r/179149
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I13d469b3ddcac78b02543efe53868a8672966ec0
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Murfel <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
