Merlijn van Deen has submitted this change and it was merged.
Change subject: Rename some date model keys to be clearer
......................................................................
Rename some date model keys to be clearer
- Remove 'replacewith', since we will replace everything with only
SQL 'NULL'
- 'condition' in the columns becomes 'null_if' since replacewith
is no longer a thing, and 'null_if' makes it clearer that we
are nulling columns if this is true, rather than false.
- 'where' in the table defintion becomes 'include_row_if', which
makes it extremely clear what exactly it does.
Change-Id: I892f0ba4e6fa2db25696f1b7922f1e7c44630466
---
M auditor/models.py
M auditor/reports/viewdiffs.py
M greylisted.yaml
3 files changed, 103 insertions(+), 109 deletions(-)
Approvals:
Merlijn van Deen: Verified; Looks good to me, approved
diff --git a/auditor/models.py b/auditor/models.py
index af4131c..5d5cd92 100644
--- a/auditor/models.py
+++ b/auditor/models.py
@@ -24,7 +24,7 @@
"""
A column that is potentially conditionally or unconditionally nulled
"""
- def __init__(self, name, whitelisted=True, condition=None,
replacewith=None):
+ def __init__(self, name, whitelisted=True, null_if=None):
"""
A column in a table that can be whitelisted or conditionally nulled
@@ -32,23 +32,22 @@
:param name: Name of this column
:param whitelisted: True if the column is whitelisted
- :param condition: Conditions under which this column is replaced.
- Specified as a string that is valid SQL expression
- :param replacewith: What to replace the column's value with if
condition is matched
- If set to None with whitelisted = False, implies
SQL None
+ :param null_if: Conditions under which this column is nulled.
+ Specified as a string that is valid SQL expression.
+ If set to None & whitelisted=False, then nulled all
the time
+
:return:
"""
self.name = name
self.whitelisted = whitelisted
- self.condition = condition
- self.replacewith = replacewith
+ self.null_if = null_if
class Table(object):
- def __init__(self, name, columns={}, where=None, table_name=None):
+ def __init__(self, name, columns={}, include_row_if=None, table_name=None):
self.name = name
self.columns = columns
- self.where = where
+ self.include_row_if = include_row_if
self.table_name = table_name if table_name else name
def add_column(self, column):
@@ -60,8 +59,8 @@
Convert Table to a minimal dict from which it can be reconstituted
"""
tabledict = {}
- if self.where:
- tabledict['where'] = self.where
+ if self.include_row_if:
+ tabledict['include_row_if'] = self.include_row_if
if all([c.whitelisted for c in self.columns.values()]):
# Everything is whitelisted!
tabledict['columns'] = [c.name for c in self.columns]
@@ -70,10 +69,8 @@
for c in self.columns.values():
columndict = {}
columndict['whitelisted'] = c.whitelisted
- if c.condition:
- columndict['condition'] = c.condition
- if c.replacewith:
- columndict['replacewith'] = c.replacewith
+ if c.null_if:
+ columndict['null_if'] = c.null_if
tabledict['columns'][c.name] = columndict
if self.table_name != self.name:
tabledict['table_name'] = self.table_name
@@ -81,7 +78,7 @@
@classmethod
def from_dict(cls, tablename, tabledata):
- table = cls(tablename, {}, tabledata.get('where', None),
tabledata.get('table_name', None))
+ table = cls(tablename, {}, tabledata.get('include_row_if', None),
tabledata.get('table_name', None))
if isinstance(tabledata['columns'], list):
# Whitelisted table
for colname in tabledata['columns']:
@@ -89,10 +86,7 @@
else:
# Greylisted table!
for colname, coldata in tabledata['columns'].items():
- col = Column(colname,
- whitelisted=coldata.get('whitelisted'),
- condition=coldata.get('condition'),
- replacewith=coldata.get('replacewith'))
+ col = Column(colname, coldata.get('whitelisted'),
coldata.get('null_if'))
table.add_column(col)
return table
diff --git a/auditor/reports/viewdiffs.py b/auditor/reports/viewdiffs.py
index e9e91d0..6bd800c 100644
--- a/auditor/reports/viewdiffs.py
+++ b/auditor/reports/viewdiffs.py
@@ -33,11 +33,11 @@
def diff_columns(expected, actual):
- return _diff(expected, actual, ('name', 'whitelisted', 'condition',
'replacewith'))
+ return _diff(expected, actual, ('name', 'whitelisted', 'null_if'))
def diff_tables(expected, actual):
- table_diff = _diff(expected, actual, ('name', 'where', 'table_name'))
+ table_diff = _diff(expected, actual, ('name', 'include_row_if',
'table_name'))
missing_cols, extra_cols = diff_iters(expected.columns, actual.columns)
common_columns = common_iters(actual.columns, expected.columns)
column_diffs = {}
@@ -87,11 +87,11 @@
# Where cond is a valid SQL relational expression, column-name, wikiname,
tablename are strings
identifier = alphanums + "_"
-if_definition = "if(" + SkipTo(",")("condition") + "," + Word(identifier) +
"," + Word(identifier) + ")"
+if_definition = "if(" + SkipTo(",")("null_if") + "," + Word(identifier) + ","
+ Word(identifier) + ")"
column_definition = (if_definition ^ Word(identifier)('expression')) + "AS" +
Word(identifier)("name") + Optional(",")
sql_definition = "select" + OneOrMore(column_definition)("columns") + \
"from" + Word(identifier) + "." +
Word(identifier)("tablename") + \
- Optional("where" + SkipTo(StringEnd())("where")) + StringEnd()
+ Optional("where" + SkipTo(StringEnd())("include_row_if")) +
StringEnd()
def _table_from_definer(sql, viewname):
@@ -99,11 +99,11 @@
Build a Table object given a cleaned up SQL statement that defines the view
"""
res = sql_definition.parseString(sql)
- table = Table(viewname, {}, res.where if res.where else None,
res.tablename)
+ table = Table(viewname, {}, res.include_row_if if res.include_row_if else
None, res.tablename)
for tokens, start, end in column_definition.scanString(sql):
table.add_column(Column(tokens.name,
- whitelisted=tokens.condition == '' and
tokens.expression != 'NULL',
- condition=tokens.condition if tokens.condition
else None))
+ whitelisted=tokens.null_if == '' and
tokens.expression != 'NULL',
+ null_if=tokens.null_if if tokens.null_if else
None))
return table
diff --git a/greylisted.yaml b/greylisted.yaml
index 8b0af0b..e0c04d4 100644
--- a/greylisted.yaml
+++ b/greylisted.yaml
@@ -3,7 +3,7 @@
af_actions:
whitelisted: true
af_comments:
- condition: af_hidden
+ null_if: af_hidden
whitelisted: false
af_deleted:
whitelisted: true
@@ -20,7 +20,7 @@
af_id:
whitelisted: true
af_pattern:
- condition: af_hidden
+ null_if: af_hidden
whitelisted: false
af_public_comments:
whitelisted: true
@@ -153,7 +153,7 @@
ar_id:
whitelisted: true
ar_len:
- condition: (ar_deleted & 1)
+ null_if: (ar_deleted & 1)
whitelisted: false
ar_minor_edit:
whitelisted: true
@@ -166,22 +166,22 @@
ar_rev_id:
whitelisted: true
ar_sha1:
- condition: (ar_deleted & 1)
+ null_if: (ar_deleted & 1)
whitelisted: false
ar_text:
whitelisted: false
ar_text_id:
- condition: (ar_deleted & 1)
+ null_if: (ar_deleted & 1)
whitelisted: false
ar_timestamp:
whitelisted: true
ar_title:
whitelisted: true
ar_user:
- condition: (ar_deleted & 4)
+ null_if: (ar_deleted & 4)
whitelisted: false
ar_user_text:
- condition: (ar_deleted & 4)
+ null_if: (ar_deleted & 4)
whitelisted: false
archive_userindex:
table_name: archive
@@ -195,7 +195,7 @@
ar_id:
whitelisted: true
ar_len:
- condition: (ar_deleted & 1)
+ null_if: (ar_deleted & 1)
whitelisted: false
ar_minor_edit:
whitelisted: true
@@ -208,12 +208,12 @@
ar_rev_id:
whitelisted: true
ar_sha1:
- condition: (ar_deleted & 1)
+ null_if: (ar_deleted & 1)
whitelisted: false
ar_text:
whitelisted: false
ar_text_id:
- condition: (ar_deleted & 1)
+ null_if: (ar_deleted & 1)
whitelisted: false
ar_timestamp:
whitelisted: true
@@ -223,13 +223,13 @@
whitelisted: true
ar_user_text:
whitelisted: true
- where: ((ar_deleted & 4) = 0)
+ include_row_if: ((ar_deleted & 4) = 0)
filearchive:
columns:
fa_archive_name:
whitelisted: true
fa_bits:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_deleted:
whitelisted: true
@@ -240,32 +240,32 @@
fa_deleted_user:
whitelisted: true
fa_description:
- condition: (fa_deleted & 2)
+ null_if: (fa_deleted & 2)
whitelisted: false
fa_height:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_id:
whitelisted: true
fa_major_mime:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_media_type:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_metadata:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_minor_mime:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_name:
whitelisted: true
fa_sha1:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_size:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_storage_group:
whitelisted: true
@@ -274,13 +274,13 @@
fa_timestamp:
whitelisted: true
fa_user:
- condition: (fa_deleted & 4)
+ null_if: (fa_deleted & 4)
whitelisted: false
fa_user_text:
- condition: (fa_deleted & 4)
+ null_if: (fa_deleted & 4)
whitelisted: false
fa_width:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
filearchive_userindex:
table_name: filearchive
@@ -288,7 +288,7 @@
fa_archive_name:
whitelisted: true
fa_bits:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_deleted:
whitelisted: true
@@ -299,32 +299,32 @@
fa_deleted_user:
whitelisted: true
fa_description:
- condition: (fa_deleted & 2)
+ null_if: (fa_deleted & 2)
whitelisted: false
fa_height:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_id:
whitelisted: true
fa_major_mime:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_media_type:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_metadata:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_minor_mime:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_name:
whitelisted: true
fa_sha1:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_size:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
fa_storage_group:
whitelisted: true
@@ -337,13 +337,13 @@
fa_user_text:
whitelisted: true
fa_width:
- condition: (fa_deleted & 1)
+ null_if: (fa_deleted & 1)
whitelisted: false
- where: ((fa_deleted & 4) = 0)
+ include_row_if: ((fa_deleted & 4) = 0)
ipblocks:
columns:
ipb_address:
- condition: (ipb_auto <> 0)
+ null_if: (ipb_auto <> 0)
whitelisted: false
ipb_allow_usertalk:
whitelisted: true
@@ -370,10 +370,10 @@
ipb_parent_block_id:
whitelisted: true
ipb_range_end:
- condition: (ipb_auto <> 0)
+ null_if: (ipb_auto <> 0)
whitelisted: false
ipb_range_start:
- condition: (ipb_auto <> 0)
+ null_if: (ipb_auto <> 0)
whitelisted: false
ipb_reason:
whitelisted: true
@@ -381,7 +381,7 @@
whitelisted: true
ipb_user:
whitelisted: true
- where: (ipb_deleted = 0)
+ include_row_if: (ipb_deleted = 0)
ipblocks_ipindex:
table_name: ipblocks
columns:
@@ -421,49 +421,49 @@
whitelisted: true
ipb_user:
whitelisted: true
- where: ((ipb_deleted = 0) and (ipb_auto = 0))
+ include_row_if: ((ipb_deleted = 0) and (ipb_auto = 0))
logging:
columns:
log_action:
- condition: (log_deleted & 1)
+ null_if: (log_deleted & 1)
whitelisted: false
log_comment:
- condition: (log_deleted & 2)
+ null_if: (log_deleted & 2)
whitelisted: false
log_deleted:
whitelisted: true
log_id:
whitelisted: true
log_namespace:
- condition: (log_deleted & 1)
+ null_if: (log_deleted & 1)
whitelisted: false
log_page:
- condition: (log_deleted & 1)
+ null_if: (log_deleted & 1)
whitelisted: false
log_params:
- condition: log_deleted
+ null_if: log_deleted
whitelisted: false
log_timestamp:
whitelisted: true
log_title:
- condition: (log_deleted & 1)
+ null_if: (log_deleted & 1)
whitelisted: false
log_type:
whitelisted: true
log_user:
- condition: (log_deleted & 4)
+ null_if: (log_deleted & 4)
whitelisted: false
log_user_text:
- condition: (log_deleted & 4)
+ null_if: (log_deleted & 4)
whitelisted: false
- where: (log_type <> 'suppress')
+ include_row_if: (log_type <> 'suppress')
logging_logindex:
table_name: logging
columns:
log_action:
whitelisted: true
log_comment:
- condition: (log_deleted & 2)
+ null_if: (log_deleted & 2)
whitelisted: false
log_deleted:
whitelisted: true
@@ -475,7 +475,7 @@
whitelisted: true
log_params:
whitelisted: false
- condition: log_deleted
+ null_if: log_deleted
log_timestamp:
whitelisted: true
log_title:
@@ -483,38 +483,38 @@
log_type:
whitelisted: true
log_user:
- condition: (log_deleted & 4)
+ null_if: (log_deleted & 4)
whitelisted: false
log_user_text:
- condition: (log_deleted & 4)
+ null_if: (log_deleted & 4)
whitelisted: false
- where: (((log_deleted & 1) = 0) and (log_type <> 'suppress'))
+ include_row_if: (((log_deleted & 1) = 0) and (log_type <> 'suppress'))
logging_userindex:
table_name: logging
columns:
log_action:
- condition: (log_deleted & 1)
+ null_if: (log_deleted & 1)
whitelisted: false
log_comment:
- condition: (log_deleted & 2)
+ null_if: (log_deleted & 2)
whitelisted: false
log_deleted:
whitelisted: true
log_id:
whitelisted: true
log_namespace:
- condition: (log_deleted & 1)
+ null_if: (log_deleted & 1)
whitelisted: false
log_page:
- condition: (log_deleted & 1)
+ null_if: (log_deleted & 1)
whitelisted: false
log_params:
- condition: log_deleted
+ null_if: log_deleted
whitelisted: false
log_timestamp:
whitelisted: true
log_title:
- condition: (log_deleted & 1)
+ null_if: (log_deleted & 1)
whitelisted: false
log_type:
whitelisted: true
@@ -522,7 +522,7 @@
whitelisted: true
log_user_text:
whitelisted: true
- where: (((log_deleted & 4) = 0) and (log_type <> 'suppress'))
+ include_row_if: (((log_deleted & 4) = 0) and (log_type <> 'suppress'))
mark_as_helpful:
columns:
mah_id:
@@ -556,7 +556,7 @@
oi_deleted:
whitelisted: true
oi_description:
- condition: (oi_deleted & 2)
+ null_if: (oi_deleted & 2)
whitelisted: false
oi_height:
whitelisted: true
@@ -577,10 +577,10 @@
oi_timestamp:
whitelisted: true
oi_user:
- condition: (oi_deleted & 4)
+ null_if: (oi_deleted & 4)
whitelisted: false
oi_user_text:
- condition: (oi_deleted & 4)
+ null_if: (oi_deleted & 4)
whitelisted: false
oi_width:
whitelisted: true
@@ -594,7 +594,7 @@
oi_deleted:
whitelisted: true
oi_description:
- condition: (oi_deleted & 2)
+ null_if: (oi_deleted & 2)
whitelisted: false
oi_height:
whitelisted: true
@@ -620,13 +620,13 @@
whitelisted: true
oi_width:
whitelisted: true
- where: ((oi_deleted & 4) = 0)
+ include_row_if: ((oi_deleted & 4) = 0)
recentchanges:
columns:
rc_bot:
whitelisted: true
rc_comment:
- condition: (rc_deleted & 2)
+ null_if: (rc_deleted & 2)
whitelisted: false
rc_cur_id:
whitelisted: true
@@ -671,10 +671,10 @@
rc_type:
whitelisted: true
rc_user:
- condition: (rc_deleted & 4)
+ null_if: (rc_deleted & 4)
whitelisted: false
rc_user_text:
- condition: (rc_deleted & 4)
+ null_if: (rc_deleted & 4)
whitelisted: false
recentchanges_userindex:
table_name: recentchanges
@@ -682,7 +682,7 @@
rc_bot:
whitelisted: true
rc_comment:
- condition: (rc_deleted & 2)
+ null_if: (rc_deleted & 2)
whitelisted: false
rc_cur_id:
whitelisted: true
@@ -730,11 +730,11 @@
whitelisted: true
rc_user_text:
whitelisted: true
- where: ((rc_deleted & 4) = 0)
+ include_row_if: ((rc_deleted & 4) = 0)
revision:
columns:
rev_comment:
- condition: (rev_deleted & 2)
+ null_if: (rev_deleted & 2)
whitelisted: false
rev_content_format:
whitelisted: true
@@ -745,7 +745,7 @@
rev_id:
whitelisted: true
rev_len:
- condition: (rev_deleted & 1)
+ null_if: (rev_deleted & 1)
whitelisted: false
rev_minor_edit:
whitelisted: true
@@ -754,24 +754,24 @@
rev_parent_id:
whitelisted: true
rev_sha1:
- condition: (rev_deleted & 1)
+ null_if: (rev_deleted & 1)
whitelisted: false
rev_text_id:
- condition: (rev_deleted & 1)
+ null_if: (rev_deleted & 1)
whitelisted: false
rev_timestamp:
whitelisted: true
rev_user:
- condition: (rev_deleted & 4)
+ null_if: (rev_deleted & 4)
whitelisted: false
rev_user_text:
- condition: (rev_deleted & 4)
+ null_if: (rev_deleted & 4)
whitelisted: false
revision_userindex:
table_name: revision
columns:
rev_comment:
- condition: (rev_deleted & 2)
+ null_if: (rev_deleted & 2)
whitelisted: false
rev_content_format:
whitelisted: true
@@ -782,7 +782,7 @@
rev_id:
whitelisted: true
rev_len:
- condition: (rev_deleted & 1)
+ null_if: (rev_deleted & 1)
whitelisted: false
rev_minor_edit:
whitelisted: true
@@ -791,10 +791,10 @@
rev_parent_id:
whitelisted: true
rev_sha1:
- condition: (rev_deleted & 1)
+ null_if: (rev_deleted & 1)
whitelisted: false
rev_text_id:
- condition: (rev_deleted & 1)
+ null_if: (rev_deleted & 1)
whitelisted: false
rev_timestamp:
whitelisted: true
@@ -802,7 +802,7 @@
whitelisted: true
rev_user_text:
whitelisted: true
- where: ((rev_deleted & 4) = 0)
+ include_row_if: ((rev_deleted & 4) = 0)
user:
columns:
user_editcount:
@@ -867,4 +867,4 @@
whitelisted: true
up_value:
whitelisted: true
- where: (up_property in
('disablemail','fancysig','gender','language','nickname','skin','timecorrection','variant'))
+ include_row_if: (up_property in
('disablemail','fancysig','gender','language','nickname','skin','timecorrection','variant'))
--
To view, visit https://gerrit.wikimedia.org/r/184131
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I892f0ba4e6fa2db25696f1b7922f1e7c44630466
Gerrit-PatchSet: 4
Gerrit-Project: operations/software/labsdb-auditor
Gerrit-Branch: master
Gerrit-Owner: Yuvipanda <yuvipa...@gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhall...@arctus.nl>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
