Mforns has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/362103 )
Change subject: [WIP] Fix timestamp infinite loop in EL purging script (1)
......................................................................
[WIP] Fix timestamp infinite loop in EL purging script (1)
Change-Id: Ib4ab6edb0c7be445e24724b756564f8c1eb2fd41
---
M modules/role/files/mariadb/eventlogging_cleaner.py
1 file changed, 17 insertions(+), 10 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/03/362103/1
diff --git a/modules/role/files/mariadb/eventlogging_cleaner.py
b/modules/role/files/mariadb/eventlogging_cleaner.py
index 2de0160..dcdbe62 100644
--- a/modules/role/files/mariadb/eventlogging_cleaner.py
+++ b/modules/role/files/mariadb/eventlogging_cleaner.py
@@ -215,7 +215,7 @@
result = self.database.execute(command, params,
dry_run=self.dry_run)
time.sleep(self.sleep_between_batches)
- def _get_uuids_and_last_ts(self, table, start_ts):
+ def _get_uuid_info(self, table, start_ts, already_processed):
"""
Return the first <batch_size> uuids of the events between start_ts
and self.end. Also return the timestamp of the last of those events.
@@ -227,7 +227,8 @@
"""
command = (
"SELECT timestamp, uuid from {} WHERE timestamp >= %(start_ts)s "
- "AND timestamp < %(end_ts)s ORDER BY timestamp LIMIT
%(batch_size)s"
+ "AND timestamp < %(end_ts)s ORDER BY timestamp, uuid "
+ "LIMIT %(batch_size)s OFFSET %(already_processed)s"
.format(table)
)
params = {
@@ -235,13 +236,17 @@
'end_ts': self.end,
'batch_size': self.batch_size,
}
- result = self.database.execute(command, params, self.dry_run)
- if result['rows']:
- uuids = [x[1] for x in result['rows']]
- last_ts = result['rows'][-1][0]
- return (uuids, last_ts)
+ results = self.database.execute(command, params, self.dry_run)['rows']
+ if results:
+ uuids = [x[1] for x in results]
+ last_ts = results[-1][0]
+ if start_ts == last_ts:
+ uuids_with_last_ts = already_processed + len(results)
+ else:
+ uuids_with_last_ts = sum(x[0] == last_ts for x in results)
+ return (uuids, last_ts, uuids_with_last_ts)
else:
- return ([], None)
+ return ([], None, 0)
def sanitize(self, table):
"""
@@ -265,7 +270,8 @@
fields_to_purge = filter(lambda f: f not in fields_to_keep, fields)
values_string = ','.join([field + ' = NULL' for field in
fields_to_purge])
- uuids_current_batch, last_ts = self._get_uuids_and_last_ts(table,
self.start)
+ uuid_info = self._get_uuid_info(table, self.start, 0)
+ uuids_current_batch, last_ts, uuids_with_last_ts = uuid_info
command_template = (
"UPDATE {0} "
"SET {1} "
@@ -300,7 +306,8 @@
# to sanitize.
uuids_current_batch = []
else:
- uuids_current_batch, last_ts =
self._get_uuids_and_last_ts(table, last_ts)
+ uuid_info = self._get_uuid_info(table, last_ts,
uuids_with_last_ts)
+ uuids_current_batch, last_ts, uuids_with_last_ts = uuid_info
time.sleep(self.sleep_between_batches)
--
To view, visit https://gerrit.wikimedia.org/r/362103
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib4ab6edb0c7be445e24724b756564f8c1eb2fd41
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mforns <mfo...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
