jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/356089 )
Change subject: Ensure sql transactions are committed
......................................................................
Ensure sql transactions are committed
Our upgrade in db seems to now allow for transactions. As a result
the transactions need to be committed in order for the results to
actually get stored back in the database.
@Note:
The commons database connection have not been closed in the same
way since these are never used for writing.
Change-Id: I6b60422303ec0c0679530539df881e210d04e351
---
M erfgoedbot/add_coord_to_articles.py
M erfgoedbot/add_object_location_monuments.py
M erfgoedbot/categorize_images.py
M erfgoedbot/database_connection.py
M erfgoedbot/database_statistics.py
M erfgoedbot/images_of_monuments_without_id.py
M erfgoedbot/missing_commonscat_links.py
M erfgoedbot/populate_image_table.py
M erfgoedbot/top_streets.py
M erfgoedbot/unused_monument_images.py
M erfgoedbot/update_database.py
M erfgoedbot/update_id_dump.py
M tests/test_populate_image_table.py
13 files changed, 81 insertions(+), 12 deletions(-)
Approvals:
Jean-Frédéric: Looks good to me, approved
jenkins-bot: Verified
diff --git a/erfgoedbot/add_coord_to_articles.py
b/erfgoedbot/add_coord_to_articles.py
index 3e75b55..38f136c 100644
--- a/erfgoedbot/add_coord_to_articles.py
+++ b/erfgoedbot/add_coord_to_articles.py
@@ -20,7 +20,11 @@
import pywikibot
import monuments_config as mconfig
-from database_connection import connect_to_monuments_database,
connect_to_commons_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database,
+ connect_to_commons_database
+)
# coordinate templates for different language wikipedias
wikiData = {
@@ -305,6 +309,8 @@
pywikibot.output(u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
processCountry(countrycode, lang, countryconfig,
wikiData.get(lang), connMon, cursorMon)
+ close_database_connection(connMon, cursorMon)
+
if __name__ == "__main__":
try:
diff --git a/erfgoedbot/add_object_location_monuments.py
b/erfgoedbot/add_object_location_monuments.py
index b61d421..d8b2f67 100644
--- a/erfgoedbot/add_object_location_monuments.py
+++ b/erfgoedbot/add_object_location_monuments.py
@@ -9,7 +9,11 @@
from pywikibot import pagegenerators
import monuments_config as mconfig
-from database_connection import connect_to_monuments_database,
connect_to_commons_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database,
+ connect_to_commons_database
+)
def locateCountry(countrycode, lang, countryconfig, conn, cursor, conn2,
cursor2):
@@ -234,6 +238,8 @@
locateCountry(
countrycode, lang, countryconfig, conn, cursor, conn2,
cursor2)
+ close_database_connection(conn, cursor)
+
if __name__ == "__main__":
try:
diff --git a/erfgoedbot/categorize_images.py b/erfgoedbot/categorize_images.py
index 9aa5de1..ff44476 100644
--- a/erfgoedbot/categorize_images.py
+++ b/erfgoedbot/categorize_images.py
@@ -25,7 +25,10 @@
from pywikibot import textlib
import monuments_config as mconfig
-from database_connection import connect_to_monuments_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database
+)
_logger = "categorize_images"
@@ -536,6 +539,8 @@
outputStatistics(statistics)
+ close_database_connection(conn, cursor)
+
if __name__ == "__main__":
main()
diff --git a/erfgoedbot/database_connection.py
b/erfgoedbot/database_connection.py
index ca290e9..741b061 100644
--- a/erfgoedbot/database_connection.py
+++ b/erfgoedbot/database_connection.py
@@ -60,3 +60,9 @@
use_unicode=True, charset='latin1')
cursor = conn.cursor()
return (conn, cursor)
+
+
+def close_database_connection(conn, cursor):
+ """Close the cursor and commit the current transactions."""
+ conn.commit()
+ cursor.close()
diff --git a/erfgoedbot/database_statistics.py
b/erfgoedbot/database_statistics.py
index bd3b694..b13b55c 100755
--- a/erfgoedbot/database_statistics.py
+++ b/erfgoedbot/database_statistics.py
@@ -7,7 +7,10 @@
'''
import pywikibot
-from database_connection import connect_to_monuments_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database
+)
def getCount(query, cursor):
@@ -321,6 +324,7 @@
country, language, conn, cursor)
outputStatistics(statistics)
+ close_database_connection(conn, cursor)
if __name__ == "__main__":
diff --git a/erfgoedbot/images_of_monuments_without_id.py
b/erfgoedbot/images_of_monuments_without_id.py
index dfaa6fe..a4842b3 100644
--- a/erfgoedbot/images_of_monuments_without_id.py
+++ b/erfgoedbot/images_of_monuments_without_id.py
@@ -16,7 +16,11 @@
import pywikibot
import monuments_config as mconfig
-from database_connection import connect_to_monuments_database,
connect_to_commons_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database,
+ connect_to_commons_database
+)
def processCountry(countrycode, lang, countryconfig, conn, cursor, conn2,
cursor2):
@@ -215,6 +219,8 @@
processCountry(
countrycode, lang, countryconfig, conn, cursor, conn2, cursor2)
+ close_database_connection(conn, cursor)
+
if __name__ == "__main__":
try:
diff --git a/erfgoedbot/missing_commonscat_links.py
b/erfgoedbot/missing_commonscat_links.py
index a7982d1..7b2acf8 100644
--- a/erfgoedbot/missing_commonscat_links.py
+++ b/erfgoedbot/missing_commonscat_links.py
@@ -16,7 +16,11 @@
import pywikibot
import monuments_config as mconfig
-from database_connection import connect_to_monuments_database,
connect_to_commons_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database,
+ connect_to_commons_database
+)
_logger = "missing_commonscat"
@@ -226,6 +230,8 @@
countrycode, lang, countryconfig, conn, cursor, conn2, cursor2)
makeStatistics(mconfig, totals)
+ close_database_connection(conn, cursor)
+
if __name__ == "__main__":
main()
diff --git a/erfgoedbot/populate_image_table.py
b/erfgoedbot/populate_image_table.py
index 9396f4a..26f76f5 100644
--- a/erfgoedbot/populate_image_table.py
+++ b/erfgoedbot/populate_image_table.py
@@ -32,7 +32,11 @@
import pywikibot
import monuments_config as mconfig
-from database_connection import connect_to_monuments_database,
connect_to_commons_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database,
+ connect_to_commons_database
+)
class CannotNormalizeException(Exception):
@@ -112,7 +116,8 @@
tracked_photos += 1
updateImage(countrycode, monumentId, name, image_has_geolocation,
conn, cursor)
- cursor.close()
+
+ close_database_connection(conn, cursor)
return (len(photos), tracked_photos)
diff --git a/erfgoedbot/top_streets.py b/erfgoedbot/top_streets.py
index c9d9452..155372e 100644
--- a/erfgoedbot/top_streets.py
+++ b/erfgoedbot/top_streets.py
@@ -10,7 +10,10 @@
import pywikibot
-from database_connection import connect_to_monuments_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database
+)
def getAddresses(countrycode, lang, municipality, conn, cursor):
@@ -105,6 +108,8 @@
else:
print u'Usage'
+ close_database_connection(conn, cursor)
+
if __name__ == "__main__":
try:
diff --git a/erfgoedbot/unused_monument_images.py
b/erfgoedbot/unused_monument_images.py
index 29dec72..6f4e636 100644
--- a/erfgoedbot/unused_monument_images.py
+++ b/erfgoedbot/unused_monument_images.py
@@ -16,7 +16,11 @@
import pywikibot
import monuments_config as mconfig
-from database_connection import connect_to_monuments_database,
connect_to_commons_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database,
+ connect_to_commons_database
+)
_logger = "unused_images"
@@ -223,6 +227,8 @@
countrycode, lang, countryconfig, conn, cursor, conn2, cursor2)
makeStatistics(mconfig, totals)
+ close_database_connection(conn, cursor)
+
if __name__ == "__main__":
main()
diff --git a/erfgoedbot/update_database.py b/erfgoedbot/update_database.py
index 75381c3..d77407f 100755
--- a/erfgoedbot/update_database.py
+++ b/erfgoedbot/update_database.py
@@ -32,7 +32,10 @@
check_integer,
check_lat_with_lon
)
-from database_connection import connect_to_monuments_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database
+)
_logger = "update_database"
@@ -443,6 +446,8 @@
u"%s in lang %s\n%s" % (countrycode, lang, str(e)))
continue
+ close_database_connection(conn, cursor)
+
if __name__ == "__main__":
main()
diff --git a/erfgoedbot/update_id_dump.py b/erfgoedbot/update_id_dump.py
index f6f16f5..d49378f 100755
--- a/erfgoedbot/update_id_dump.py
+++ b/erfgoedbot/update_id_dump.py
@@ -19,7 +19,10 @@
from converters import (
extract_elements_from_template_param
)
-from database_connection import connect_to_monuments_database
+from database_connection import (
+ close_database_connection,
+ connect_to_monuments_database
+)
def updateMonument(countrycode, lang, identifier, source, countryconfig, conn,
cursor):
@@ -148,6 +151,8 @@
u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
processCountry(countrycode, lang, countryconfig, conn, cursor)
+ close_database_connection(conn, cursor)
+
if __name__ == "__main__":
try:
diff --git a/tests/test_populate_image_table.py
b/tests/test_populate_image_table.py
index 23f7c72..496811b 100644
--- a/tests/test_populate_image_table.py
+++ b/tests/test_populate_image_table.py
@@ -70,9 +70,13 @@
mock_connect_to_monuments_database = patcher_3.start()
mock_connect_to_monuments_database.return_value = (None,
self.mock_cursor_monuments)
+ patcher_4 =
mock.patch('erfgoedbot.populate_image_table.close_database_connection')
+ mock_close_database_connection = patcher_4.start()
+
self.addCleanup(patcher.stop)
self.addCleanup(patcher_2.stop)
self.addCleanup(patcher_3.stop)
+ self.addCleanup(patcher_4.stop)
@mock.patch('erfgoedbot.populate_image_table.has_geolocation',
autospec=True)
@mock.patch('erfgoedbot.populate_image_table.updateImage', autospec=True)
--
To view, visit https://gerrit.wikimedia.org/r/356089
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I6b60422303ec0c0679530539df881e210d04e351
Gerrit-PatchSet: 2
Gerrit-Project: labs/tools/heritage
Gerrit-Branch: wikidata
Gerrit-Owner: Lokal Profil <[email protected]>
Gerrit-Reviewer: Jean-Frédéric <[email protected]>
Gerrit-Reviewer: Lokal Profil <[email protected]>
Gerrit-Reviewer: Multichill <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
