This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new e341681 YETUS-1121. rewrite shelldocs (#219)
e341681 is described below
commit e341681e706690ee13e63b80a86153ed12bb960f
Author: Allen Wittenauer <[email protected]>
AuthorDate: Tue Sep 28 07:52:03 2021 -0700
YETUS-1121. rewrite shelldocs (#219)
---
shelldocs/src/main/python/shelldocs.py | 590 +++++++++++++++------------------
1 file changed, 265 insertions(+), 325 deletions(-)
diff --git a/shelldocs/src/main/python/shelldocs.py
b/shelldocs/src/main/python/shelldocs.py
index 3660e3b..7248125 100755
--- a/shelldocs/src/main/python/shelldocs.py
+++ b/shelldocs/src/main/python/shelldocs.py
@@ -17,10 +17,13 @@
""" process bash scripts and generate documentation from them """
# Do this immediately to prevent compiled forms
-import sys
+
+import logging
import os
+import pathlib
import re
-import errno
+import sys
+
from argparse import ArgumentParser
sys.dont_write_bytecode = True
@@ -45,54 +48,30 @@ ASFLICENSE = '''
-->
'''
-FUNCTIONRE = re.compile(r"^(\w+) *\(\) *{")
-
-def docstrip(key, dstr):
- '''remove extra spaces from shelldoc phrase'''
- dstr = re.sub("^## @%s " % key, "", dstr)
- dstr = dstr.lstrip()
- dstr = dstr.rstrip()
- return dstr
-
-
-def toc(tlist):
- '''build a table of contents'''
- tocout = []
- header = ()
- for i in tlist:
- if header != i.getinter():
- header = i.getinter()
- line = " * %s\n" % (i.headerbuild())
- tocout.append(line)
- line = " * [%s](#%s)\n" % (i.getname().replace("_",
- r"\_"), i.getname())
- tocout.append(line)
- return tocout
-
-
-class ShellFunction: # pylint: disable=too-many-public-methods,
too-many-instance-attributes
+class ShellFunction: # pylint: disable=too-many-instance-attributes
"""a shell function"""
- def __init__(self, filename):
+ def __init__(self, filename='Unknown'):
'''Initializer'''
- self.name = None
- self.audience = None
- self.stability = None
- self.replacetext = None
- self.replaceb = False
- self.returnt = None
- self.desc = None
- self.params = None
+ self.audience = ''
+ self.description = []
self.filename = filename
self.linenum = 0
+ self.name = ''
+ self.params = []
+ self.replacebool = False
+ self.replacerawtext = ''
+ self.replacetext = 'Not Replaceable'
+ self.returnt = []
+ self.stability = ''
def __lt__(self, other):
'''comparison'''
if self.audience == other.audience:
if self.stability == other.stability:
- if self.replaceb == other.replaceb:
+ if self.replacebool == other.replacebool:
return self.name < other.name
- if self.replaceb:
+ if self.replacebool:
return True
else:
if self.stability == "Stable":
@@ -102,320 +81,272 @@ class ShellFunction: # pylint:
disable=too-many-public-methods, too-many-instan
return True
return False
- def reset(self):
- '''empties current function'''
- self.name = None
- self.audience = None
- self.stability = None
- self.replacetext = None
- self.replaceb = False
- self.returnt = None
- self.desc = None
- self.params = None
- self.linenum = 0
- self.filename = None
-
- def getfilename(self):
- '''get the name of the function'''
- if self.filename is None:
- return "undefined"
- return self.filename
-
- def setname(self, text):
- '''set the name of the function'''
- if FUNCTIONRE.match(text):
- definition = FUNCTIONRE.match(text).groups()[0]
- else:
- definition = text.split()[1]
- self.name = definition.replace("(", "").replace(")", "")
-
- def getname(self):
- '''get the name of the function'''
- if self.name is None:
- return "None"
- return self.name
-
- def setlinenum(self, linenum):
- '''set the line number of the function'''
- self.linenum = linenum
-
- def getlinenum(self):
- '''get the line number of the function'''
- return self.linenum
-
- def setaudience(self, text):
- '''set the audience of the function'''
- self.audience = docstrip("audience", text)
- self.audience = self.audience.capitalize()
-
- def getaudience(self):
- '''get the audience of the function'''
- if self.audience is None:
- return "None"
- return self.audience
-
- def setstability(self, text):
- '''set the stability of the function'''
- self.stability = docstrip("stability", text)
- self.stability = self.stability.capitalize()
-
- def getstability(self):
- '''get the stability of the function'''
- if self.stability is None:
- return "None"
- return self.stability
-
- def setreplace(self, text):
- '''set the replacement state'''
- self.replacetext = docstrip("replaceable", text)
- if self.replacetext.capitalize() == "Yes":
- self.replaceb = True
-
- def getreplace(self):
- '''get the replacement state'''
- if self.replaceb:
- return "Yes"
- return "No"
-
- def getreplacetext(self):
- '''get the replacement state text'''
- return self.replacetext
-
- def getinter(self):
- '''get the function state'''
- return self.getaudience(), self.getstability(), self.getreplace()
-
- def addreturn(self, text):
- '''add a return state'''
- if self.returnt is None:
- self.returnt = []
- self.returnt.append(docstrip("return", text))
-
- def getreturn(self):
- '''get the complete return state'''
- if self.returnt is None:
- return "Nothing"
- return "\n\n".join(self.returnt)
-
- def adddesc(self, text):
- '''add to the description'''
- if self.desc is None:
- self.desc = []
- self.desc.append(docstrip("description", text))
-
- def getdesc(self):
- '''get the description'''
- if self.desc is None:
- return "None"
- return " ".join(self.desc)
-
- def addparam(self, text):
- '''add a parameter'''
- if self.params is None:
- self.params = []
- self.params.append(docstrip("param", text))
-
- def getparams(self):
- '''get all of the parameters'''
- if self.params is None:
- return ""
- return " ".join(self.params)
-
- def getusage(self):
- '''get the usage string'''
- line = "%s %s" % (self.name, self.getparams())
- return line.rstrip()
-
- def headerbuild(self):
+ def header(self):
'''get the header for this function'''
- if self.getreplace() == "Yes":
- replacetext = "Replaceable"
- else:
- replacetext = "Not Replaceable"
- line = "%s/%s/%s" % (self.getaudience(), self.getstability(),
- replacetext)
- return line
+ return f"{self.audience}/{self.stability}/{self.replacetext}"
def getdocpage(self):
'''get the built document page for this function'''
- line = "### `%s`\n\n"\
- "* Synopsis\n\n"\
- "```\n%s\n"\
- "```\n\n" \
- "* Description\n\n" \
- "%s\n\n" \
- "* Returns\n\n" \
- "%s\n\n" \
- "| Classification | Level |\n" \
- "| :--- | :--- |\n" \
- "| Audience | %s |\n" \
- "| Stability | %s |\n" \
- "| Replaceable | %s |\n\n" \
- % (self.getname(),
- self.getusage(),
- self.getdesc(),
- self.getreturn(),
- self.getaudience(),
- self.getstability(),
- self.getreplace())
- return line
+ params = " ".join(self.params)
+ usage = f"{self.name} {params}"
+ description = "\n".join(self.description)
+ if not self.returnt:
+ returntext = 'Nothing'
+ else:
+ returntext = "\n".join(self.returnt)
+ return (f"### `{self.name}`\n\n"
+ "* Synopsis\n\n"
+ f"```\n{usage}\n"
+ "```\n\n"
+ "* Description\n\n"
+ f"{description}\n\n"
+ "* Returns\n\n"
+ f"{returntext}\n\n"
+ "| Classification | Level |\n"
+ "| :--- | :--- |\n"
+ f"| Audience | {self.audience} |\n"
+ f"| Stability | {self.stability} |\n"
+ f"| Replaceable | {self.replacebool} |\n\n")
+
+ def isprivateandnotreplaceable(self):
+ ''' is this function Private and not replaceable? '''
+ return self.audience == "Private" and not self.replacebool
def lint(self):
'''Lint this function'''
- getfuncs = {
- "audience": self.getaudience,
- "stability": self.getstability,
- "replaceable": self.getreplacetext,
- }
validvalues = {
"audience": ("Public", "Private"),
"stability": ("Stable", "Evolving"),
- "replaceable": ("yes", "no"),
+ "replacerawtext": ("yes", "no"),
}
- messages = []
- for attr in ("audience", "stability", "replaceable"):
- value = getfuncs[attr]()
- if value == "None" and attr != 'replaceable':
- messages.append("%s:%u: ERROR: function %s has no @%s" %
- (self.getfilename(), self.getlinenum(),
- self.getname(), attr.lower()))
- elif value not in validvalues[attr]:
- validvalue = "|".join(v.lower() for v in validvalues[attr])
- messages.append(
- "%s:%u: ERROR: function %s has invalid value (%s) for @%s
(%s)"
- % (self.getfilename(), self.getlinenum(), self.getname(),
- value.lower(), attr.lower(), validvalue))
- return "\n".join(messages)
+ for attribute in validvalues:
+ value = getattr(self, attribute)
+ if (not value or value == ''):
+ logging.error("%s:%u:ERROR: function %s has no @%s",
+ self.filename, self.linenum, self.name,
+ attribute.lower().replace('rawtext', 'able'))
+ elif value not in validvalues[attribute]:
+ validvalue = "|".join(v.lower()
+ for v in validvalues[attribute])
+ logging.error(
+ "%s:%d:ERROR: function %s has invalid value (%s) for @%s
(%s)",
+ self.filename, self.linenum, self.name, value.lower(),
+ attribute.lower().replace('rawtext', 'able'), validvalue)
def __str__(self):
'''Generate a string for this function'''
- line = "{%s %s %s %s}" \
- % (self.getname(),
- self.getaudience(),
- self.getstability(),
- self.getreplace())
- return line
+ return f"{{{self.name} {self.audience} {self.stability}
{self.replacebool}}}"
-def marked_as_ignored(file_path):
- """Checks for the presence of the marker(SHELLDOC-IGNORE) to ignore the
file.
+class ProcessFile:
+ ''' shell file processor '''
- Marker needs to be in a line of its own and can not
- be an inline comment.
+ FUNCTIONRE = re.compile(r"^(\w+) *\(\) *{")
- A leading '#' and white-spaces(leading or trailing)
- are trimmed before checking equality.
+ def __init__(self, filename=None, skipsuperprivate=False):
+ self.filename = filename
+ self.functions = []
+ self.skipsuperprivate = skipsuperprivate
- Comparison is case sensitive and the comment must be in
- UPPERCASE.
- """
- with open(file_path) as input_file:
- for line in input_file:
- if line.startswith("#") and line[1:].strip() == "SHELLDOC-IGNORE":
- return True
- return False
+ def isignored(self):
+ """Checks for the presence of the marker(SHELLDOC-IGNORE) to ignore
the file.
+ Marker needs to be in a line of its own and can not
+ be an inline comment.
-def process_file(filename, skipprnorep):
- """ stuff all of the functions into an array """
- allfuncs = []
- try:
- with open(filename, "r") as shellcode:
- # if the file contains a comment containing
- # only "SHELLDOC-IGNORE" then skip that file
- if marked_as_ignored(filename):
- return None
- funcdef = ShellFunction(filename)
- linenum = 0
- for line in shellcode:
- linenum = linenum + 1
- if line.startswith('## @description'):
- funcdef.adddesc(line)
- elif line.startswith('## @audience'):
- funcdef.setaudience(line)
- elif line.startswith('## @stability'):
- funcdef.setstability(line)
- elif line.startswith('## @replaceable'):
- funcdef.setreplace(line)
- elif line.startswith('## @param'):
- funcdef.addparam(line)
- elif line.startswith('## @return'):
- funcdef.addreturn(line)
- elif line.startswith('function') or FUNCTIONRE.match(line):
- funcdef.setname(line)
- funcdef.setlinenum(linenum)
- if skipprnorep and \
- funcdef.getaudience() == "Private" and \
- funcdef.getreplace() == "No":
- pass
- else:
- allfuncs.append(funcdef)
- funcdef = ShellFunction(filename)
- except IOError as err:
- print("ERROR: Failed to read from file: %s. Skipping." % err.filename,
- file=sys.stderr)
- return None
- return allfuncs
+ A leading '#' and white-spaces(leading or trailing)
+ are trimmed before checking equality.
+
+ Comparison is case sensitive and the comment must be in
+ UPPERCASE.
+ """
+ with open(self.filename) as input_file:
+ for line in input_file:
+ if line.startswith(
+ "#") and line[1:].strip() == "SHELLDOC-IGNORE":
+ return True
+ return False
+
+ def _docstrip(self, key, dstr): #pylint: disable=no-self-use
+ '''remove extra spaces from shelldoc phrase'''
+ dstr = re.sub(f"^## @{key} ", "", dstr)
+ dstr = dstr.strip()
+ return dstr
+
+ def _process_description(self, funcdef, text=None):
+ if not text:
+ funcdef.description = []
+ return
+ funcdef.description.append(self._docstrip('description', text))
+
+ def _process_audience(self, funcdef, text=None):
+ '''set the audience of the function'''
+ if not text:
+ return
+ funcdef.audience = self._docstrip('audience', text)
+ funcdef.audience = funcdef.audience.capitalize()
+
+ def _process_stability(self, funcdef, text=None):
+ '''set the stability of the function'''
+ if not text:
+ return
+ funcdef.stability = self._docstrip('stability', text)
+ funcdef.stability = funcdef.stability.capitalize()
+
+ def _process_replaceable(self, funcdef, text=None):
+ '''set the replacement state'''
+ if not text:
+ return
+ funcdef.replacerawtext = self._docstrip("replaceable", text)
+ if funcdef.replacerawtext in ['yes', 'Yes', 'true', 'True']:
+ funcdef.replacebool = True
+ else:
+ funcdef.replacebool = False
+ if funcdef.replacebool:
+ funcdef.replacetext = 'Replaceable'
+ else:
+ funcdef.replacetext = 'Not Replaceable'
+
+ def _process_param(self, funcdef, text=None):
+ '''add a parameter'''
+ if not text:
+ funcdef.params = []
+ return
+ funcdef.params.append(self._docstrip('param', text))
+
+ def _process_return(self, funcdef, text=None):
+ '''add a return value'''
+ if not text:
+ funcdef.returnt = []
+ return
+ funcdef.returnt.append(self._docstrip('return', text))
+
+ def _process_function(self, funcdef, text=None, linenum=1): # pylint:
disable=no-self-use
+ '''set the name of the function'''
+ if ProcessFile.FUNCTIONRE.match(text):
+ definition = ProcessFile.FUNCTIONRE.match(text).groups()[0]
+ else:
+ definition = text.split()[1]
+ funcdef.name = definition.replace("(", "").replace(")", "")
+ funcdef.linenum = linenum
+
+ def process_file(self):
+ """ stuff all of the functions into an array """
+ self.functions = []
+
+ mapping = {
+ '## @description': '_process_description',
+ '## @audience': '_process_audience',
+ '## @stability': '_process_stability',
+ '## @replaceable': '_process_replaceable',
+ '## @param': '_process_param',
+ '## @return': '_process_return',
+ }
+
+ if self.isignored():
+ return
+
+ try:
+ with open(self.filename, "r") as shellcode:
+ # if the file contains a comment containing
+ # only "SHELLDOC-IGNORE" then skip that file
+
+ funcdef = ShellFunction(self.filename)
+ linenum = 0
+ for line in shellcode:
+ linenum = linenum + 1
+ for text, method in mapping.items():
+ if line.startswith(text):
+ getattr(self, method)(funcdef, text=line)
+
+ if line.startswith(
+ 'function') or ProcessFile.FUNCTIONRE.match(line):
+ self._process_function(funcdef,
+ text=line,
+ linenum=linenum)
+
+ if self.skipsuperprivate and
funcdef.isprivateandnotreplaceable(
+ ):
+ pass
+ else:
+ self.functions.append(funcdef)
+ funcdef = ShellFunction(self.filename)
+
+ except OSError as err:
+ logging.error("ERROR: Failed to read from file: %s. Skipping.",
+ err.filename)
+ self.functions = []
+
+
+class MarkdownReport:
+ ''' generate a markdown report '''
+ def __init__(self, functions, filename=None):
+ self.filename = filename
+ self.filepath = pathlib.Path(self.filename)
+ if functions:
+ self.functions = sorted(functions)
+ else:
+ self.functions = None
+
+ def write_tableofcontents(self, fhout):
+ '''build a table of contents'''
+ header = None
+ for function in self.functions:
+ if header != function.header():
+ header = function.header()
+ fhout.write(f" * {header}\n")
+ markdownsafename = function.name.replace("_", r"\_")
+ fhout.write(f" * [{markdownsafename}](#{function.name})\n")
+
+ def write_output(self):
+ """ write the markdown file """
+
+ self.filepath.parent.mkdir(parents=True, exist_ok=True)
+
+ with open(self.filename, "w", encoding='utf-8') as outfile:
+ outfile.write(ASFLICENSE)
+ self.write_tableofcontents(outfile)
+ outfile.write("\n------\n\n")
+
+ header = []
+ for function in self.functions:
+ if header != function.header():
+ header = function.header()
+ outfile.write(f"## {header}\n")
+ outfile.write(function.getdocpage())
def process_input(inputlist, skipprnorep):
""" take the input and loop around it """
+ def call_process_file(filename, skipsuperprivate):
+ ''' handle building a ProcessFile '''
+ fileprocessor = ProcessFile(filename=filename,
+ skipsuperprivate=skipsuperprivate)
+ fileprocessor.process_file()
+ return fileprocessor.functions
+
allfuncs = []
- for filename in inputlist: #pylint: disable=too-many-nested-blocks
- if os.path.isdir(filename):
- for root, dirs, files in os.walk(filename): #pylint:
disable=unused-variable
- for fname in files:
+ for inputname in inputlist:
+ if pathlib.Path(inputname).is_dir():
+ for dirpath, dirnames, filenames in os.walk(inputname): #pylint:
disable=unused-variable
+ for fname in filenames:
if fname.endswith('sh'):
- newfuncs = process_file(filename=os.path.join(
- root, fname),
- skipprnorep=skipprnorep)
- if newfuncs:
- allfuncs = allfuncs + newfuncs
+ allfuncs = allfuncs + call_process_file(
+ filename=pathlib.Path(dirpath).joinpath(fname),
+ skipsuperprivate=skipprnorep)
else:
- newfuncs = process_file(filename=filename, skipprnorep=skipprnorep)
- if newfuncs:
- allfuncs = allfuncs + newfuncs
-
+ allfuncs = allfuncs + call_process_file(
+ filename=inputname, skipsuperprivate=skipprnorep)
if allfuncs is None:
- print("ERROR: no functions found.", file=sys.stderr)
+ logging.error("ERROR: no functions found.")
sys.exit(1)
allfuncs = sorted(allfuncs)
return allfuncs
-def write_output(filename, functions):
- """ write the markdown file """
- try:
- directory = os.path.dirname(filename)
- if directory:
- if not os.path.exists(directory):
- os.makedirs(directory)
- except OSError as exc:
- if exc.errno == errno.EEXIST and os.path.isdir(directory):
- pass
- else:
- print("Unable to create output directory %s: %u, %s" % \
- (directory, exc.errno, exc.strerror))
- sys.exit(1)
-
- with open(filename, "w") as outfile:
- outfile.write(ASFLICENSE)
- for line in toc(functions):
- outfile.write(line)
- outfile.write("\n------\n\n")
-
- header = []
- for funcs in functions:
- if header != funcs.getinter():
- header = funcs.getinter()
- line = "## %s\n" % (funcs.headerbuild())
- outfile.write(line)
- outfile.write(funcs.getdocpage())
-
-
-def main():
- '''main entry point'''
+def process_arguments():
+ ''' deal with parameters '''
parser = ArgumentParser(
prog='shelldocs',
epilog="You can mark a file to be ignored by shelldocs by adding"
@@ -454,8 +385,8 @@ def main():
options = parser.parse_args()
if options.release_version:
- with open(os.path.join(os.path.dirname(__file__), "../VERSION"),
- 'r') as ver_file:
+ verfile = pathlib.Path(__file__).parent.joinpath('VERSION')
+ with open(verfile, encoding='utf-8') as ver_file:
print(ver_file.read())
sys.exit(0)
@@ -465,16 +396,25 @@ def main():
parser.error(
"At least one of output file and lint mode needs to be specified")
+ return options
+
+
+def main():
+ '''main entry point'''
+
+ logging.basicConfig(format='%(message)s')
+
+ options = process_arguments()
+
allfuncs = process_input(options.infile, options.skipprnorep)
if options.lint:
for funcs in allfuncs:
- message = funcs.lint()
- if message:
- print(message)
+ funcs.lint()
- if options.outfile is not None:
- write_output(options.outfile, allfuncs)
+ if options.outfile:
+ mdreport = MarkdownReport(allfuncs, filename=options.outfile)
+ mdreport.write_output()
if __name__ == "__main__":
