This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new 89f15d8 YETUS-452. Remove python2; rewrite python bits for python3
(#201)
89f15d8 is described below
commit 89f15d8c998575535bba1f4b1a3bc312cb155c9f
Author: Allen Wittenauer <[email protected]>
AuthorDate: Mon Nov 16 23:24:55 2020 -0800
YETUS-452. Remove python2; rewrite python bits for python3 (#201)
Signed-off-by: Akira Ajisaka <[email protected]>
---
.circleci/config.yml | 3 +-
.cirrus.yml | 3 +-
.github/workflows/action-test.yml | 4 +-
.github/workflows/yetus.yml | 1 -
.gitlab-ci.yml | 3 +-
.semaphore/semaphore-build.sh | 3 +-
.travis.yml | 3 +-
Jenkinsfile | 5 +-
asf-site-src/source/contribute/website.html.md | 4 +-
.../in-progress/precommit/plugins/pylint.html.md | 3 +-
.../in-progress/releasedocmaker.html.md | 4 +-
.../documentation/in-progress/shelldocs.html.md | 2 +-
precommit/src/main/python/jenkins-admin.py | 191 ++++----
.../src/main/shell/test-patch-docker/Dockerfile | 35 +-
releasedocmaker/src/main/python/releasedocmaker.py | 3 +-
.../src/main/python/releasedocmaker/__init__.py | 538 +++++++++++----------
.../src/main/python/releasedocmaker/utils.py | 61 ++-
shelldocs/src/main/python/shelldocs.py | 464 +++++++++++++++++-
shelldocs/src/main/python/shelldocs/__init__.py | 472 ------------------
19 files changed, 881 insertions(+), 921 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 1dc6731..a291f46 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -40,12 +40,11 @@ jobs:
--plugins=all
--java-home=/usr/lib/jvm/java-8-openjdk-amd64
--patch-dir=/tmp/yetus-out
- --pylint=pylint2
--html-report-file=/tmp/yetus-out/report.html
--console-report-file=/tmp/yetus-out/console.txt
--brief-report-file=/tmp/yetus-out/brief.txt
--bugcomments=briefreport,htmlout,junit
- --tests-filter=checkstyle,javadoc,rubocop,test4tests
+ --tests-filter=checkstyle,test4tests
--junit-report-xml=/tmp/yetus-out/junit-report.xml
- store_test_results:
diff --git a/.cirrus.yml b/.cirrus.yml
index fe84b31..e54fc9b 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -28,8 +28,7 @@ yetus_task:
--java-home=/usr/lib/jvm/java-8-openjdk-amd64
--junit-report-xml=/tmp/yetus-out/junit.xml
--plugins=all
- --pylint=pylint2
- --tests-filter=checkstyle,javadoc,rubocop,test4tests
+ --tests-filter=checkstyle,test4tests
always:
junit_artifacts:
path: "yetus-out/junit.xml"
diff --git a/.github/workflows/action-test.yml
b/.github/workflows/action-test.yml
index 17861f3..262458e 100644
--- a/.github/workflows/action-test.yml
+++ b/.github/workflows/action-test.yml
@@ -41,9 +41,7 @@ jobs:
patchdir: ./out
buildtool: maven
githubtoken: ${{ secrets.GITHUB_TOKEN }}
- pylint: pylint2
- pip: pip2
- testsfilter: checkstyle,javadoc,test4tests
+ testsfilter: checkstyle,test4tests
- name: Artifact output
if: ${{ always() }}
uses: actions/upload-artifact@v2
diff --git a/.github/workflows/yetus.yml b/.github/workflows/yetus.yml
index 8a6fbea..8a6b908 100644
--- a/.github/workflows/yetus.yml
+++ b/.github/workflows/yetus.yml
@@ -52,7 +52,6 @@ jobs:
--plugins=all
--proclimit=2000
--project=yetus
- --pylint=pylint2
--tests-filter=checkstyle,javadoc,rubocop,test4tests
- name: Artifact output
if: ${{ always() }}
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 91b71a1..bec0520 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,13 +23,12 @@ buretoolbox-job:
precommit/src/main/shell/test-patch.sh
--patch-dir=/tmp/yetus-out
--plugins=all
- --pylint=pylint2
--java-home=/usr/lib/jvm/java-8-openjdk-amd64
--html-report-file=/tmp/yetus-out/report.html
--console-report-file=/tmp/yetus-out/console.txt
--brief-report-file=/tmp/yetus-out/brief.txt
--bugcomments=briefreport,htmlout,gitlab,junit
- --tests-filter=checkstyle,javadoc,rubocop,test4tests
+ --tests-filter=checkstyle,test4tests
--junit-report-xml=/tmp/yetus-out/junit-report.xml
artifacts:
diff --git a/.semaphore/semaphore-build.sh b/.semaphore/semaphore-build.sh
index e861676..879c661 100755
--- a/.semaphore/semaphore-build.sh
+++ b/.semaphore/semaphore-build.sh
@@ -22,8 +22,7 @@ PRECOMMITDIR=precommit/src/main/shell
--mvn-custom-repos \
--mvn-custom-repos-dir=/tmp/yetus-m2 \
--patch-dir=/tmp/yetus-out \
- --tests-filter=checkstyle,javadoc,rubocop,test4tests \
+ --tests-filter=checkstyle,test4tests \
--docker \
- --pylint=pylint2 \
--dockerfile="${PRECOMMITDIR}/test-patch-docker/Dockerfile" \
--docker-cache-from=apache/yetus-base:main,ubuntu:focal
diff --git a/.travis.yml b/.travis.yml
index 8afd242..28fab1f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,10 +32,9 @@ script:
--patch-dir=/tmp/yetus-out
--java-home=/usr/lib/jvm/java-8-openjdk-amd64
--plugins=all
- --pylint=pylint2
--docker-cache-from=apache/yetus:main
--html-report-file=/tmp/yetus-out/report.html
--console-report-file=/tmp/yetus-out/console.txt
--brief-report-file=/tmp/yetus-out/brief.txt
--bugcomments=briefreport,htmlout
- --tests-filter=checkstyle,javadoc,rubocop,test4tests
+ --tests-filter=checkstyle,test4tests
diff --git a/Jenkinsfile b/Jenkinsfile
index 02b7059..9acb826 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -101,9 +101,6 @@ pipeline {
YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
YETUS_ARGS+=(--jira-user="${JIRA_USER}")
- # pylint settings
- YETUS_ARGS+=('--pylint=pylint2')
-
# auto-kill any surefire stragglers during unit test runs
YETUS_ARGS+=(--reapermode=report)
@@ -122,7 +119,7 @@ pipeline {
# don't let these tests cause -1s because we aren't really
paying that
# much attention to them
-
YETUS_ARGS+=("--tests-filter=checkstyle,javadoc,rubocop,test4tests")
+ YETUS_ARGS+=("--tests-filter=checkstyle,test4tests")
if [[ "${USE_DEBUG_FLAG}" == true ]]; then
YETUS_ARGS+=("--debug")
diff --git a/asf-site-src/source/contribute/website.html.md
b/asf-site-src/source/contribute/website.html.md
index bca25ed..1edf8d4 100644
--- a/asf-site-src/source/contribute/website.html.md
+++ b/asf-site-src/source/contribute/website.html.md
@@ -37,7 +37,7 @@ by reading [Middleman's excellent
documentation](https://middlemanapp.com/basics
NOTE: You MUST have run `mvn install` at least once prior to running `mvn
site`.
-The following steps assume you have a working ruby 2.3+ environment setup:
+The following steps assume you have a working ruby 2.7+ environment setup:
```bash
$ sudo gem install bundler
@@ -45,7 +45,7 @@ $ cd asf-site-src
$ bundle install
```
-and a working Python 2.7 environment for
[releasedocmaker](/documentation/in-progress/releasedocmaker/).
+and a working Python 3.8 environment for
[releasedocmaker](/documentation/in-progress/releasedocmaker/).
## Make changes in asf-site-src/source
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/plugins/pylint.html.md
b/asf-site-src/source/documentation/in-progress/precommit/plugins/pylint.html.md
index 1195d73..44b8b6c 100644
---
a/asf-site-src/source/documentation/in-progress/precommit/plugins/pylint.html.md
+++
b/asf-site-src/source/documentation/in-progress/precommit/plugins/pylint.html.md
@@ -45,8 +45,7 @@ None
# Docker Notes
-The default Apache Yetus image comes with `pip2`/`pylint2` for Python 2.x
support and `pip3`/`pylint3` for Python 3.x support.
-The `pip` and `pylint` point to the Python 3.x versions.
+None
# Developer Notes
diff --git
a/asf-site-src/source/documentation/in-progress/releasedocmaker.html.md
b/asf-site-src/source/documentation/in-progress/releasedocmaker.html.md
index c946c3b..f399e86 100644
--- a/asf-site-src/source/documentation/in-progress/releasedocmaker.html.md
+++ b/asf-site-src/source/documentation/in-progress/releasedocmaker.html.md
@@ -53,9 +53,9 @@ In order to solve these problems, `releasedocmaker` was
written to automatically
# Requirements
-* Python 2.7 with dateutil extension
+* Python 3.8 with dateutil extension
-dateutil may be installed via pip: `pip2 install python-dateutil`
+dateutil may be installed via pip: `pip3 install python-dateutil`
# Basic Usage
diff --git a/asf-site-src/source/documentation/in-progress/shelldocs.html.md
b/asf-site-src/source/documentation/in-progress/shelldocs.html.md
index 90038ea..4cd0e6a 100644
--- a/asf-site-src/source/documentation/in-progress/shelldocs.html.md
+++ b/asf-site-src/source/documentation/in-progress/shelldocs.html.md
@@ -41,7 +41,7 @@ Some projects have complex shell functions that act as APIs.
`shelldocs` provide
# Requirements
-* Python 2.7
+* Python 3.8
# Function Annotations
diff --git a/precommit/src/main/python/jenkins-admin.py
b/precommit/src/main/python/jenkins-admin.py
index 0786cf4..18679fd 100755
--- a/precommit/src/main/python/jenkins-admin.py
+++ b/precommit/src/main/python/jenkins-admin.py
@@ -1,5 +1,5 @@
-#!/usr/bin/env python2
-#pylint: disable=invalid-name
+#!/usr/bin/env python3
+# pylint: disable=invalid-name
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,48 +18,37 @@
# specific language governing permissions and limitations
# under the License.
#
-
""" Process patch file attachments from JIRA using a query """
-from optparse import OptionParser
+from argparse import ArgumentParser
from tempfile import NamedTemporaryFile
from xml.etree import ElementTree
-import base64
-import httplib
import os
import re
import sys
-import urllib2
+import requests
+
def http_get(resource, ignore_error=False, username=None, password=None):
""" get the contents of a URL """
- request = urllib2.Request(resource)
- if username and password:
- base64string = base64.b64encode('%s:%s' % (username, password))
- request.add_header("Authorization", "Basic %s" % base64string)
+
try:
- response = urllib2.urlopen(request)
- except urllib2.HTTPError, http_err:
- code = http_err.code
- print '%s returns HTTP error %d: %s' \
- % (resource, code, http_err.reason)
- if ignore_error:
- return ''
- else:
- print 'Aborting.'
- sys.exit(1)
- except urllib2.URLError, url_err:
- print 'Error contacting %s: %s' % (resource, url_err.reason)
- if ignore_error:
- return ''
+ if username and password:
+ response = requests.get(resource, auth=(username, password))
else:
- raise url_err
- except httplib.BadStatusLine, err:
+ response = requests.get(resource)
+ response.raise_for_status()
+ except requests.exceptions.HTTPError as http_err:
+ errstr = str(http_err)
+ print(
+ f'%{resource} returns HTTP error %{response.status_code}:
%{errstr}\n'
+ )
if ignore_error:
return ''
- else:
- raise err
- return response.read()
+ print('Aborting.')
+ sys.exit(1)
+ return response.text
+
def parse_jira_data(filename):
""" returns a map of (project, issue) => attachment id """
@@ -88,9 +77,10 @@ def parse_jira_data(filename):
result[jiraissue] = attachmentids[-1]
return result
-def main(): #pylint: disable=too-many-branches, too-many-statements,
too-many-locals
+
+def main(): #pylint: disable=too-many-branches, too-many-statements,
too-many-locals
""" main program """
- parser = OptionParser(prog='jenkins-admin')
+ parser = ArgumentParser(prog='jenkins-admin')
if os.getenv('JENKINS_URL'):
parser.set_defaults(jenkinsurl=os.getenv('JENKINS_URL'))
if os.getenv('JOB_NAME'):
@@ -99,40 +89,61 @@ def main(): #pylint: disable=too-many-branches,
too-many-statements, too-many-lo
parser.set_defaults(jenkinsJobName='PreCommit-Admin')
parser.set_defaults(jenkinsJobTemplate='PreCommit-{project}')
- parser.add_option('--initialize', action='store_true',
- dest='jenkinsInit',
- help='Start a new patch_tested.txt file')
- parser.add_option('--jenkins-jobname', type='string',
- dest='jenkinsJobName',
- help='PreCommit-Admin JobName', metavar='JOB_NAME')
- parser.add_option('--jenkins-project-template', type='string',
- dest='jenkinsJobTemplate',
- help='Template for project jobs',
- metavar='TEMPLATE')
- parser.add_option('--jenkins-token', type='string',
- dest='jenkinsToken', help='Jenkins Token',
- metavar='TOKEN')
- parser.add_option('--jenkins-url', type='string', dest='jenkinsurl'
- , help='Jenkins base URL', metavar='URL')
- parser.add_option(
+ parser.add_argument('--initialize',
+ action='store_true',
+ dest='jenkinsInit',
+ help='Start a new patch_tested.txt file')
+ parser.add_argument('--jenkins-jobname',
+ type=str,
+ dest='jenkinsJobName',
+ help='PreCommit-Admin JobName',
+ metavar='JOB_NAME')
+ parser.add_argument('--jenkins-project-template',
+ type=str,
+ dest='jenkinsJobTemplate',
+ help='Template for project jobs',
+ metavar='TEMPLATE')
+ parser.add_argument('--jenkins-token',
+ type=str,
+ dest='jenkinsToken',
+ help='Jenkins Token',
+ metavar='TOKEN')
+ parser.add_argument('--jenkins-url',
+ type=str,
+ dest='jenkinsurl',
+ help='Jenkins base URL',
+ metavar='URL')
+ parser.add_argument(
'--jenkins-url-override',
- type='string',
+ type=str,
dest='jenkinsurloverrides',
action='append',
help='Project specific Jenkins base URL',
metavar='PROJECT=URL',
- )
- parser.add_option('--jira-filter', type='string', dest='jiraFilter',
- help='JIRA filter URL', metavar='URL')
- parser.add_option('--jira-user', type='string', dest='jiraUser',
- help='JIRA username')
- parser.add_option('--jira-password', type='string', dest='jiraPassword',
- help='JIRA password')
- parser.add_option('--live', dest='live', action='store_true',
- help='Submit Job to jenkins')
- parser.add_option('--max-history', dest='history', type='int',
- help='Maximum history to store', default=5000)
- parser.add_option(
+ )
+ parser.add_argument('--jira-filter',
+ type=str,
+ dest='jiraFilter',
+ help='JIRA filter URL',
+ metavar='URL')
+ parser.add_argument('--jira-user',
+ type=str,
+ dest='jiraUser',
+ help='JIRA username')
+ parser.add_argument('--jira-password',
+ type=str,
+ dest='jiraPassword',
+ help='JIRA password')
+ parser.add_argument('--live',
+ dest='live',
+ action='store_true',
+ help='Submit Job to jenkins')
+ parser.add_argument('--max-history',
+ dest='history',
+ type=int,
+ help='Maximum history to store',
+ default=5000)
+ parser.add_argument(
'-V',
'--version',
dest='release_version',
@@ -140,14 +151,13 @@ def main(): #pylint: disable=too-many-branches,
too-many-statements, too-many-lo
default=False,
help="display version information for jenkins-admin and exit.")
- (options, args) = parser.parse_args() #pylint: disable=unused-variable
+ options = parser.parse_args()
# Handle the version string right away and exit
if options.release_version:
- with open(
- os.path.join(
- os.path.dirname(__file__), "../../VERSION"), 'r') as ver_file:
- print ver_file.read()
+ with open(os.path.join(os.path.dirname(__file__), "../../VERSION"),
+ 'r') as ver_file:
+ print(ver_file.read())
sys.exit(0)
token_frag = ''
@@ -158,36 +168,39 @@ def main(): #pylint: disable=too-many-branches,
too-many-statements, too-many-lo
if not options.jiraFilter:
parser.error('ERROR: --jira-filter is a required argument.')
if not options.jenkinsurl:
- parser.error('ERROR: --jenkins-url or the JENKINS_URL environment
variable is required.'
- )
+ parser.error(
+ 'ERROR: --jenkins-url or the JENKINS_URL environment variable is
required.'
+ )
if options.history < 0:
- parser.error('ERROR: --max-history must be 0 or a positive integer.'
- )
+ parser.error('ERROR: --max-history must be 0 or a positive integer.')
jenkinsurloverrides = {}
if options.jenkinsurloverrides:
for override in options.jenkinsurloverrides:
if '=' not in override:
- parser.error('Invalid Jenkins Url Override: '
- + override)
+ parser.error('Invalid Jenkins Url Override: ' + override)
(project, url) = override.split('=', 1)
jenkinsurloverrides[project.upper()] = url
tempfile = NamedTemporaryFile(delete=False)
try:
jobloghistory = None
if not options.jenkinsInit:
- jobloghistory = http_get(options.jenkinsurl
- +
'/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt'
- % options.jenkinsJobName, True)
+ jobloghistory = http_get(
+ options.jenkinsurl +
+ '/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt' %
+ options.jenkinsJobName, True)
# if we don't have a successful build available try the last build
if not jobloghistory:
- jobloghistory = http_get(options.jenkinsurl
- +
'/job/%s/lastCompletedBuild/artifact/patch_tested.txt'
- % options.jenkinsJobName)
+ jobloghistory = http_get(
+ options.jenkinsurl +
+ '/job/%s/lastCompletedBuild/artifact/patch_tested.txt' %
+ options.jenkinsJobName)
jobloghistory = jobloghistory.strip().split('\n')
if 'TESTED ISSUES' not in jobloghistory[0]:
- print 'Downloaded patch_tested.txt control file may be
corrupted. Failing.'
+ print(
+ 'Downloaded patch_tested.txt control file may be
corrupted. Failing.'
+ )
sys.exit(1)
# we are either going to write a new one or rewrite the old one
@@ -204,12 +217,13 @@ def main(): #pylint: disable=too-many-branches,
too-many-statements, too-many-lo
else:
joblog.write('TESTED ISSUES\n')
joblog.flush()
- rssdata = http_get(options.jiraFilter, False, options.jiraUser,
options.jiraPassword)
- tempfile.write(rssdata)
+ rssdata = http_get(options.jiraFilter, False, options.jiraUser,
+ options.jiraPassword)
+ tempfile.write(rssdata.encode('utf-8'))
tempfile.flush()
- for (key, attachment) in parse_jira_data(tempfile.name).items():
+ for (key, attachment) in list(parse_jira_data(tempfile.name).items()):
(project, issue) = key
- if jenkinsurloverrides.has_key(project):
+ if jenkinsurloverrides.get(project):
url = jenkinsurloverrides[project]
else:
url = options.jenkinsurl
@@ -223,28 +237,29 @@ def main(): #pylint: disable=too-many-branches,
too-many-statements, too-many-lo
'project': project,
'issue': issue,
'attachment': attachment,
- }
+ }
jenkinsurl = jenkinsurltemplate.format(**url_args)
# submit job
jobname = '%s-%s,%s' % (project, issue, attachment)
if not jobloghistory or jobname not in jobloghistory:
- print jobname + ' has not been processed, submitting'
+ print(jobname + ' has not been processed, submitting')
joblog.write(jobname + '\n')
joblog.flush()
if options.live:
http_get(jenkinsurl, True)
else:
- print 'GET ' + jenkinsurl
+ print('GET ' + jenkinsurl)
else:
- print jobname + ' has been processed, ignoring'
+ print(jobname + ' has been processed, ignoring')
joblog.close()
finally:
if options.live:
os.remove(tempfile.name)
else:
- print 'JIRA Data is located: ' + tempfile.name
+ print('JIRA Data is located: ' + tempfile.name)
+
if __name__ == '__main__':
main()
diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile
b/precommit/src/main/shell/test-patch-docker/Dockerfile
index d874af2..ae0ccb8 100644
--- a/precommit/src/main/shell/test-patch-docker/Dockerfile
+++ b/precommit/src/main/shell/test-patch-docker/Dockerfile
@@ -324,41 +324,8 @@ RUN apt-get -q update && apt-get -q install
--no-install-recommends -y \
yamllint==1.24.2 \
&& rm -rf /root/.cache \
&& mv /usr/local/bin/pylint /usr/local/bin/pylint3
-
-######
-# Install python, pylint2, and yamllint
-######
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
- python2 \
- python-backports.functools-lru-cache \
- python-bcrypt \
- python-cffi \
- python-cryptography \
- python-dateutil \
- python-dev \
- python-enum34 \
- python-pkg-resources \
- python-setuptools \
- python-singledispatch \
- python-six \
- python-yaml \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/* \
- && curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py \
- && python2 /tmp/get-pip.py \
- && rm /tmp/get-pip.py /usr/local/bin/pip \
- && pip2 install -v \
- astroid==1.6.5 \
- configparser==4.0.2 \
- isort==4.3.21 \
- pylint==1.9.2 \
- && rm -rf /root/.cache \
- && mv /usr/local/bin/pylint /usr/local/bin/pylint2
-
-#####
-# all of the world's python2 code stopped working, right?
-#####
RUN ln -s /usr/local/bin/pylint3 /usr/local/bin/pylint
+RUN ln -s /usr/local/bin/pip3 /usr/local/bin/pip
####
# Install ruby and associated bits
diff --git a/releasedocmaker/src/main/python/releasedocmaker.py
b/releasedocmaker/src/main/python/releasedocmaker.py
index f4527c7..154abe9 100755
--- a/releasedocmaker/src/main/python/releasedocmaker.py
+++ b/releasedocmaker/src/main/python/releasedocmaker.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -15,7 +15,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
""" wrapper to launch releasedocmaker from the CLI """
import sys
diff --git a/releasedocmaker/src/main/python/releasedocmaker/__init__.py
b/releasedocmaker/src/main/python/releasedocmaker/__init__.py
index 16fa7ed..e73ff8b 100755
--- a/releasedocmaker/src/main/python/releasedocmaker/__init__.py
+++ b/releasedocmaker/src/main/python/releasedocmaker/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,28 +14,29 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
""" Generate releasenotes based upon JIRA """
# pylint: disable=too-many-lines
-from __future__ import print_function
-import sys
-from glob import glob
-from optparse import OptionParser
-from time import gmtime, strftime, sleep
-from distutils.version import LooseVersion
import errno
+import http.client
+import json
import os
import re
import shutil
-import urllib
-import urllib2
-import httplib
-import json
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+from pprint import pprint
+from glob import glob
+from argparse import ArgumentParser
+from time import gmtime, strftime, sleep
+
sys.dont_write_bytecode = True
-# pylint: disable=wrong-import-position,relative-import
-from utils import get_jira, to_unicode, sanitize_text, processrelnote, Outputs
+# pylint: disable=wrong-import-position
+from .utils import get_jira, to_unicode, sanitize_text, processrelnote, Outputs
# pylint: enable=wrong-import-position
try:
@@ -45,6 +46,27 @@ except ImportError:
"You can install it using:\n\t pip install python-dateutil")
sys.exit(1)
+# These are done in order of preference as to which one seems to be
+# more up-to-date at any given point in time. And yes, it is
+# ironic that packaging is usually the last one to be
+# correct.
+
+try:
+ from pip._vendor.packaging.version import LegacyVersion as PythonVersion
+except ImportError:
+ try:
+ from setuptools._vendor.packaging.version import LegacyVersion as
PythonVersion
+ except ImportError:
+ try:
+ from pkg_resources._vendor.packaging.version import LegacyVersion
as PythonVersion
+ except ImportError:
+ try:
+ from packaging.version import LegacyVersion as PythonVersion
+ except ImportError:
+ print(
+ "This script requires a packaging module to be installed.")
+ sys.exit(1)
+
RELEASE_VERSION = {}
JIRA_BASE_URL = "https://issues.apache.org/jira";
@@ -77,19 +99,20 @@ ASF_LICENSE = '''
-->
'''
+
def indexbuilder(title, asf_license, format_string):
"""Write an index file for later conversion using mvn site"""
versions = glob("[0-9]*.[0-9]*")
- versions.sort(key=LooseVersion, reverse=True)
+ versions = sorted(versions, reverse=True, key=PythonVersion)
with open("index" + EXTENSION, "w") as indexfile:
if asf_license is True:
indexfile.write(ASF_LICENSE)
for version in versions:
indexfile.write("* %s v%s\n" % (title, version))
for k in ("Changelog", "Release Notes"):
- indexfile.write(format_string %
- (k, version, k.upper().replace(" ", ""),
- version))
+ indexfile.write(
+ format_string %
+ (k, version, k.upper().replace(" ", ""), version))
def buildprettyindex(title, asf_license):
@@ -105,7 +128,7 @@ def buildindex(title, asf_license):
def buildreadme(title, asf_license):
"""Write an index file for Github using README.md"""
versions = glob("[0-9]*.[0-9]*")
- versions.sort(key=LooseVersion, reverse=True)
+ versions = sorted(versions, reverse=True, key=PythonVersion)
with open("README.md", "w") as indexfile:
if asf_license is True:
indexfile.write(ASF_LICENSE)
@@ -113,16 +136,15 @@ def buildreadme(title, asf_license):
indexfile.write("* %s v%s\n" % (title, version))
for k in ("Changelog", "Release Notes"):
indexfile.write(" * [%s](%s/%s.%s%s)\n" %
- (k, version, k.upper().replace(" ", ""),
- version, EXTENSION))
+ (k, version, k.upper().replace(
+ " ", ""), version, EXTENSION))
-class GetVersions(object): # pylint: disable=too-few-public-methods
+class GetVersions: # pylint: disable=too-few-public-methods
""" List of version strings """
-
def __init__(self, versions, projects):
self.newversions = []
- versions.sort(key=LooseVersion)
+ versions = sorted(versions, key=PythonVersion)
print("Looking for %s through %s" % (versions[0], versions[-1]))
newversions = set()
for project in projects:
@@ -130,18 +152,20 @@ class GetVersions(object): # pylint:
disable=too-few-public-methods
"/rest/api/2/project/%s/versions" % project.upper()
try:
resp = get_jira(url)
- except (urllib2.HTTPError, urllib2.URLError,
httplib.BadStatusLine):
+ except (urllib.error.HTTPError, urllib.error.URLError,
+ http.client.BadStatusLine):
sys.exit(1)
datum = json.loads(resp.read())
for data in datum:
- newversions.add(data['name'])
+ newversions.add(PythonVersion(data['name']))
newlist = list(newversions.copy())
- newlist.append(versions[0])
- newlist.append(versions[-1])
- newlist.sort(key=LooseVersion)
- start_index = newlist.index(versions[0])
- end_index = len(newlist) - 1 - newlist[::-1].index(versions[-1])
+ newlist.append(PythonVersion(versions[0]))
+ newlist.append(PythonVersion(versions[-1]))
+ newlist = sorted(newlist)
+ start_index = newlist.index(PythonVersion(versions[0]))
+ end_index = len(newlist) - 1 - newlist[::-1].index(
+ PythonVersion(versions[-1]))
for newversion in newlist[start_index + 1:end_index]:
if newversion in newversions:
print("Adding %s to the list" % newversion)
@@ -152,32 +176,8 @@ class GetVersions(object): # pylint:
disable=too-few-public-methods
return self.newversions
-class Version(object):
- """Represents a version number"""
-
- def __init__(self, data):
- self.mod = False
- self.data = data
- found = re.match(r'^((\d+)(\.\d+)*).*$', data)
- if found:
- self.parts = [int(p) for p in found.group(1).split('.')]
- else:
- self.parts = []
- # backfill version with zeros if missing parts
- self.parts.extend((0,) * (3 - len(self.parts)))
-
- def __str__(self):
- if self.mod:
- return '.'.join([str(p) for p in self.parts])
- return self.data
-
- def __cmp__(self, other):
- return cmp(self.parts, other.parts)
-
-
-class Jira(object):
+class Jira:
"""A single JIRA"""
-
def __init__(self, data, parent):
self.key = data['key']
self.fields = data['fields']
@@ -226,8 +226,8 @@ class Jira(object):
def get_components(self):
""" Get the component(s) """
if self.fields['components']:
- return ", ".join([comp['name'] for comp in
self.fields['components']
- ])
+ return ", ".join(
+ [comp['name'] for comp in self.fields['components']])
return ""
def get_summary(self):
@@ -258,26 +258,25 @@ class Jira(object):
ret = mid['key']
return to_unicode(ret)
- def __cmp__(self, other):
- result = 0
+ def __lt__(self, other):
if SORTTYPE == 'issueid':
# compare by issue name-number
selfsplit = self.get_id().split('-')
othersplit = other.get_id().split('-')
- result = cmp(selfsplit[0], othersplit[0])
- if result == 0:
- result = cmp(int(selfsplit[1]), int(othersplit[1]))
+ result = selfsplit[0] < othersplit[0]
+ if not result:
+ result = int(selfsplit[1]) < int(othersplit[1])
# dec is supported for backward compatibility
if SORTORDER in ['dec', 'desc']:
- result *= -1
+ result = not result
elif SORTTYPE == 'resolutiondate':
dts = dateutil.parser.parse(self.fields['resolutiondate'])
dto = dateutil.parser.parse(other.fields['resolutiondate'])
- result = cmp(dts, dto)
+ result = dts < dto
if SORTORDER == 'newer':
- result *= -1
+ result = not result
return result
@@ -317,16 +316,16 @@ class Jira(object):
return self.important
-class JiraIter(object):
+class JiraIter:
"""An Iterator of JIRAs"""
-
@staticmethod
def collect_fields():
"""send a query to JIRA and collect field-id map"""
try:
resp = get_jira(JIRA_BASE_URL + "/rest/api/2/field")
data = json.loads(resp.read())
- except (urllib2.HTTPError, urllib2.URLError, httplib.BadStatusLine,
ValueError):
+ except (urllib.error.HTTPError, urllib.error.URLError,
+ http.client.BadStatusLine, ValueError):
sys.exit(1)
field_id_map = {}
for part in data:
@@ -342,9 +341,11 @@ class JiraIter(object):
jql = "project in ('%s') and \
fixVersion in ('%s') and \
resolution = Fixed" % (pjs, ver)
- params = urllib.urlencode({'jql': jql,
- 'startAt': pos,
- 'maxResults': count})
+ params = urllib.parse.urlencode({
+ 'jql': jql,
+ 'startAt': pos,
+ 'maxResults': count
+ })
return JiraIter.load_jira(params, 0)
@staticmethod
@@ -352,12 +353,12 @@ class JiraIter(object):
"""send query to JIRA and collect with retries"""
try:
resp = get_jira(JIRA_BASE_URL + "/rest/api/2/search?%s" % params)
- except (urllib2.URLError, httplib.BadStatusLine) as err:
+ except (urllib.error.URLError, http.client.BadStatusLine) as err:
return JiraIter.retry_load(err, params, fail_count)
try:
data = json.loads(resp.read())
- except httplib.IncompleteRead as err:
+ except http.client.IncompleteRead as err:
return JiraIter.retry_load(err, params, fail_count)
return data
@@ -370,9 +371,8 @@ class JiraIter(object):
print("Connection failed %d times. Retrying." % (fail_count))
sleep(1)
return JiraIter.load_jira(params, fail_count)
- else:
- print("Connection failed %d times. Aborting." % (fail_count))
- sys.exit(1)
+ print("Connection failed %d times. Aborting." % (fail_count))
+ sys.exit(1)
@staticmethod
def collect_jiras(ver, projects):
@@ -384,7 +384,8 @@ class JiraIter(object):
while pos < end:
data = JiraIter.query_jira(ver, projects, pos)
if 'error_messages' in data:
- print("JIRA returns error message: %s" %
data['error_messages'])
+ print("JIRA returns error message: %s" %
+ data['error_messages'])
sys.exit(1)
pos = data['startAt'] + data['maxResults']
end = data['total']
@@ -409,19 +410,20 @@ class JiraIter(object):
def __iter__(self):
return self
- def next(self):
+ def __next__(self):
""" get next """
- data = self.iter.next()
+ data = next(self.iter)
j = Jira(data, self)
return j
-class Linter(object):
+class Linter:
"""Encapsulates lint-related functionality.
Maintains running lint statistics about JIRAs."""
- _valid_filters = ["incompatible", "important", "version", "component",
- "assignee"]
+ _valid_filters = [
+ "incompatible", "important", "version", "component", "assignee"
+ ]
def __init__(self, version, options):
self._warning_count = 0
@@ -429,8 +431,8 @@ class Linter(object):
self._lint_message = ""
self._version = version
- self._filters = dict(zip(self._valid_filters, [False] * len(
- self._valid_filters)))
+ self._filters = dict(
+ list(zip(self._valid_filters, [False] * len(self._valid_filters))))
self.enabled = False
self._parse_options(options)
@@ -439,12 +441,12 @@ class Linter(object):
def add_parser_options(parser):
"""Add Linter options to passed optparse parser."""
filter_string = ", ".join("'" + f + "'" for f in Linter._valid_filters)
- parser.add_option(
+ parser.add_argument(
"-n",
"--lint",
dest="lint",
action="append",
- type="string",
+ type=str,
help="Specify lint filters. Valid filters are " + filter_string +
". " + "'all' enables all lint filters. " +
"Multiple filters can be specified comma-delimited and " +
@@ -535,7 +537,8 @@ class Linter(object):
if not self.enabled:
return
if not jira.get_release_note():
- if self._filters["incompatible"] and
jira.get_incompatible_change():
+ if self._filters["incompatible"] and jira.get_incompatible_change(
+ ):
self._warning_count += 1
self._lint_message += "\nWARNING: incompatible change %s lacks
release notes." % \
(sanitize_text(jira.get_id()))
@@ -561,67 +564,64 @@ class Linter(object):
% (" and ".join(error_message), jira.get_id())
-def parse_args(): # pylint: disable=too-many-branches
+def parse_args(): # pylint: disable=too-many-branches
"""Parse command-line arguments with optparse."""
- usage = "usage: %prog [OPTIONS] " + \
- "--project PROJECT [--project PROJECT] " + \
- "--version VERSION [--version VERSION2 ...]"
- parser = OptionParser(
- usage=usage,
+ parser = ArgumentParser(
+ prog='releasedocmaker',
epilog=
- "Markdown-formatted CHANGELOG and RELEASENOTES files will be stored"
- " in a directory named after the highest version provided.")
- parser.add_option("--dirversions",
- dest="versiondirs",
- action="store_true",
- default=False,
- help="Put files in versioned directories")
- parser.add_option("--empty",
- dest="empty",
- action="store_true",
- default=False,
- help="Create empty files when no issues")
- parser.add_option("--extension",
- dest="extension",
- default=EXTENSION,
- type="string",
- help="Set the file extension of created Markdown files")
- parser.add_option("--fileversions",
- dest="versionfiles",
- action="store_true",
- default=False,
- help="Write files with embedded versions")
- parser.add_option("-i",
- "--index",
- dest="index",
- action="store_true",
- default=False,
- help="build an index file")
- parser.add_option("-l",
- "--license",
- dest="license",
- action="store_true",
- default=False,
- help="Add an ASF license")
- parser.add_option("-p",
- "--project",
- dest="projects",
- action="append",
- type="string",
- help="projects in JIRA to include in releasenotes",
- metavar="PROJECT")
- parser.add_option("--prettyindex",
- dest="prettyindex",
- action="store_true",
- default=False,
- help="build an index file with pretty URLs")
- parser.add_option("-r",
- "--range",
- dest="range",
- action="store_true",
- default=False,
- help="Given versions are a range")
- parser.add_option(
+ "--project and --version may be given multiple times.")
+ parser.add_argument("--dirversions",
+ dest="versiondirs",
+ action="store_true",
+ default=False,
+ help="Put files in versioned directories")
+ parser.add_argument("--empty",
+ dest="empty",
+ action="store_true",
+ default=False,
+ help="Create empty files when no issues")
+ parser.add_argument(
+ "--extension",
+ dest="extension",
+ default=EXTENSION,
+ type=str,
+ help="Set the file extension of created Markdown files")
+ parser.add_argument("--fileversions",
+ dest="versionfiles",
+ action="store_true",
+ default=False,
+ help="Write files with embedded versions")
+ parser.add_argument("-i",
+ "--index",
+ dest="index",
+ action="store_true",
+ default=False,
+ help="build an index file")
+ parser.add_argument("-l",
+ "--license",
+ dest="license",
+ action="store_true",
+ default=False,
+ help="Add an ASF license")
+ parser.add_argument("-p",
+ "--project",
+ dest="projects",
+ action="append",
+ type=str,
+ help="projects in JIRA to include in releasenotes",
+ metavar="PROJECT")
+ parser.add_argument("--prettyindex",
+ dest="prettyindex",
+ action="store_true",
+ default=False,
+ help="build an index file with pretty URLs")
+ parser.add_argument("-r",
+ "--range",
+ dest="range",
+ action="store_true",
+ default=False,
+ help="Given versions are a range")
+ parser.add_argument(
"--sortorder",
dest="sortorder",
metavar="TYPE",
@@ -629,67 +629,72 @@ def parse_args(): # pylint: disable=too-many-branches
# dec is supported for backward compatibility
choices=["asc", "dec", "desc", "newer", "older"],
help="Sorting order for sort type (default: %s)" % SORTORDER)
- parser.add_option("--sorttype",
- dest="sorttype",
- metavar="TYPE",
- default=SORTTYPE,
- choices=["resolutiondate", "issueid"],
- help="Sorting type for issues (default: %s)" % SORTTYPE)
- parser.add_option(
+ parser.add_argument("--sorttype",
+ dest="sorttype",
+ metavar="TYPE",
+ default=SORTTYPE,
+ choices=["resolutiondate", "issueid"],
+ help="Sorting type for issues (default: %s)" %
+ SORTTYPE)
+ parser.add_argument(
"-t",
"--projecttitle",
dest="title",
- type="string",
+ type=str,
help="Title to use for the project (default is Apache PROJECT)")
- parser.add_option("-u",
- "--usetoday",
- dest="usetoday",
- action="store_true",
- default=False,
- help="use current date for unreleased versions")
- parser.add_option("-v",
- "--version",
- dest="versions",
- action="append",
- type="string",
- help="versions in JIRA to include in releasenotes",
- metavar="VERSION")
- parser.add_option(
+ parser.add_argument("-u",
+ "--usetoday",
+ dest="usetoday",
+ action="store_true",
+ default=False,
+ help="use current date for unreleased versions")
+ parser.add_argument("-v",
+ "--version",
+ dest="versions",
+ action="append",
+ type=str,
+ help="versions in JIRA to include in releasenotes",
+ metavar="VERSION")
+ parser.add_argument(
"-V",
dest="release_version",
action="store_true",
default=False,
help="display version information for releasedocmaker and exit.")
- parser.add_option("-O",
- "--outputdir",
- dest="output_directory",
- action="append",
- type="string",
- help="specify output directory to put release docs to.")
- parser.add_option("-B",
- "--baseurl",
- dest="base_url",
- action="append",
- type="string",
- help="specify base URL of the JIRA instance.")
- parser.add_option(
+ parser.add_argument(
+ "-O",
+ "--outputdir",
+ dest="output_directory",
+ action="append",
+ type=str,
+ help="specify output directory to put release docs to.")
+ parser.add_argument("-B",
+ "--baseurl",
+ dest="base_url",
+ action="append",
+ type=str,
+ help="specify base URL of the JIRA instance.")
+ parser.add_argument(
"--retries",
dest="retries",
action="append",
- type="int",
+ type=int,
help="Specify how many times to retry connection for each URL.")
- parser.add_option(
+ parser.add_argument(
"--skip-credits",
dest="skip_credits",
action="store_true",
default=False,
- help="While creating release notes skip the 'reporter' and
'contributor' columns")
- parser.add_option("-X",
- "--incompatiblelabel",
- dest="incompatible_label",
- default="backward-incompatible",
- type="string",
- help="Specify the label to indicate backward
incompatibility.")
+ help=
+ "While creating release notes skip the 'reporter' and 'contributor'
columns"
+ )
+ parser.add_argument(
+ "-X",
+ "--incompatiblelabel",
+ dest="incompatible_label",
+ default="backward-incompatible",
+ type=str,
+ help="Specify the label to indicate backward incompatibility.")
Linter.add_parser_options(parser)
@@ -697,13 +702,12 @@ def parse_args(): # pylint: disable=too-many-branches
parser.print_help()
sys.exit(1)
- (options, _) = parser.parse_args()
+ options = parser.parse_args()
# Handle the version string right away and exit
if options.release_version:
- with open(
- os.path.join(
- os.path.dirname(__file__), "../VERSION"), 'r') as ver_file:
+ with open(os.path.join(os.path.dirname(__file__), "../VERSION"),
+ 'r') as ver_file:
print(ver_file.read())
sys.exit(0)
@@ -726,19 +730,21 @@ def parse_args(): # pylint: disable=too-many-branches
if options.range or len(options.versions) > 1:
if not options.versiondirs and not options.versionfiles:
- parser.error("Multiple versions require either --fileversions or
--dirversions")
+ parser.error(
+ "Multiple versions require either --fileversions or
--dirversions"
+ )
return options
-def main(): # pylint: disable=too-many-statements, too-many-branches,
too-many-locals
+def main(): # pylint: disable=too-many-statements, too-many-branches,
too-many-locals
""" hey, it's main """
- global JIRA_BASE_URL #pylint: disable=global-statement
- global BACKWARD_INCOMPATIBLE_LABEL #pylint: disable=global-statement
- global SORTTYPE #pylint: disable=global-statement
- global SORTORDER #pylint: disable=global-statement
- global NUM_RETRIES #pylint: disable=global-statement
- global EXTENSION #pylint: disable=global-statement
+ global JIRA_BASE_URL #pylint: disable=global-statement
+ global BACKWARD_INCOMPATIBLE_LABEL #pylint: disable=global-statement
+ global SORTTYPE #pylint: disable=global-statement
+ global SORTORDER #pylint: disable=global-statement
+ global NUM_RETRIES #pylint: disable=global-statement
+ global EXTENSION #pylint: disable=global-statement
options = parse_args()
@@ -753,7 +759,7 @@ def main(): # pylint: disable=too-many-statements,
too-many-branches, too-many-l
pass
else:
print("Unable to create output directory %s: %u, %s" % \
- (options.output_directory, exc.errno, exc.message))
+ (options.output_directory, exc.errno, exc.strerror))
sys.exit(1)
os.chdir(options.output_directory)
@@ -769,11 +775,10 @@ def main(): # pylint: disable=too-many-statements,
too-many-branches, too-many-l
projects = options.projects
if options.range is True:
- versions = [Version(v)
- for v in GetVersions(options.versions, projects).getlist()]
+ versions = GetVersions(options.versions, projects).getlist()
else:
- versions = [Version(v) for v in options.versions]
- versions.sort()
+ versions = [PythonVersion(v) for v in options.versions]
+ versions = sorted(versions)
SORTTYPE = options.sorttype
SORTORDER = options.sortorder
@@ -793,7 +798,8 @@ def main(): # pylint: disable=too-many-statements,
too-many-branches, too-many-l
linter = Linter(vstr, options)
jlist = sorted(JiraIter(vstr, projects))
if not jlist and not options.empty:
- print("There is no issue which has the specified version: %s" %
version)
+ print("There is no issue which has the specified version: %s" %
+ version)
continue
if vstr in RELEASE_VERSION:
@@ -807,57 +813,67 @@ def main(): # pylint: disable=too-many-statements,
too-many-branches, too-many-l
os.mkdir(vstr)
if options.versionfiles and options.versiondirs:
- reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s%(ext)s",
-
"%(ver)s/RELEASENOTES.%(key)s.%(ver)s%(ext)s", [],
- {"ver": version,
- "date": reldate,
- "title": title,
- "ext": EXTENSION})
+ reloutputs = Outputs(
+ "%(ver)s/RELEASENOTES.%(ver)s%(ext)s",
+ "%(ver)s/RELEASENOTES.%(key)s.%(ver)s%(ext)s", [], {
+ "ver": version,
+ "date": reldate,
+ "title": title,
+ "ext": EXTENSION
+ })
choutputs = Outputs("%(ver)s/CHANGELOG.%(ver)s%(ext)s",
"%(ver)s/CHANGELOG.%(key)s.%(ver)s%(ext)s", [],
- {"ver": version,
- "date": reldate,
- "title": title,
- "ext": EXTENSION})
+ {
+ "ver": version,
+ "date": reldate,
+ "title": title,
+ "ext": EXTENSION
+ })
elif options.versiondirs:
reloutputs = Outputs("%(ver)s/RELEASENOTES%(ext)s",
- "%(ver)s/RELEASENOTES.%(key)s%(ext)s", [],
- {"ver": version,
- "date": reldate,
- "title": title,
- "ext": EXTENSION})
+ "%(ver)s/RELEASENOTES.%(key)s%(ext)s", [], {
+ "ver": version,
+ "date": reldate,
+ "title": title,
+ "ext": EXTENSION
+ })
choutputs = Outputs("%(ver)s/CHANGELOG%(ext)s",
- "%(ver)s/CHANGELOG.%(key)s%(ext)s", [],
- {"ver": version,
- "date": reldate,
- "title": title,
- "ext": EXTENSION})
+ "%(ver)s/CHANGELOG.%(key)s%(ext)s", [], {
+ "ver": version,
+ "date": reldate,
+ "title": title,
+ "ext": EXTENSION
+ })
elif options.versionfiles:
reloutputs = Outputs("RELEASENOTES.%(ver)s%(ext)s",
- "RELEASENOTES.%(key)s.%(ver)s%(ext)s", [],
- {"ver": version,
- "date": reldate,
- "title": title,
- "ext": EXTENSION})
+ "RELEASENOTES.%(key)s.%(ver)s%(ext)s", [], {
+ "ver": version,
+ "date": reldate,
+ "title": title,
+ "ext": EXTENSION
+ })
choutputs = Outputs("CHANGELOG.%(ver)s%(ext)s",
- "CHANGELOG.%(key)s.%(ver)s%(ext)s", [],
- {"ver": version,
- "date": reldate,
- "title": title,
- "ext": EXTENSION})
+ "CHANGELOG.%(key)s.%(ver)s%(ext)s", [], {
+ "ver": version,
+ "date": reldate,
+ "title": title,
+ "ext": EXTENSION
+ })
else:
reloutputs = Outputs("RELEASENOTES%(ext)s",
- "RELEASENOTES.%(key)s%(ext)s", [],
- {"ver": version,
- "date": reldate,
- "title": title,
- "ext": EXTENSION})
- choutputs = Outputs("CHANGELOG%(ext)s",
- "CHANGELOG.%(key)s%(ext)s", [],
- {"ver": version,
- "date": reldate,
- "title": title,
- "ext": EXTENSION})
+ "RELEASENOTES.%(key)s%(ext)s", [], {
+ "ver": version,
+ "date": reldate,
+ "title": title,
+ "ext": EXTENSION
+ })
+ choutputs = Outputs("CHANGELOG%(ext)s", "CHANGELOG.%(key)s%(ext)s",
+ [], {
+ "ver": version,
+ "date": reldate,
+ "title": title,
+ "ext": EXTENSION
+ })
if options.license is True:
reloutputs.write_all(ASF_LICENSE)
@@ -915,8 +931,8 @@ def main(): # pylint: disable=too-many-statements,
too-many-branches, too-many-l
if not jira.get_release_note():
line = '\n**WARNING: No release note provided for this
change.**\n\n'
else:
- line = '\n%s\n\n' % (
- processrelnote(jira.get_release_note()))
+ line = '\n%s\n\n' % (processrelnote(
+ jira.get_release_note()))
reloutputs.write_key_raw(jira.get_project(), line)
linter.lint(jira)
@@ -945,25 +961,29 @@ def main(): # pylint: disable=too-many-statements,
too-many-branches, too-many-l
choutputs.write_all("### INCOMPATIBLE CHANGES:\n\n")
choutputs.write_all(change_header21)
choutputs.write_all(change_header22)
- choutputs.write_list(incompatlist, options.skip_credits,
JIRA_BASE_URL)
+ choutputs.write_list(incompatlist, options.skip_credits,
+ JIRA_BASE_URL)
if importantlist:
choutputs.write_all("\n\n### IMPORTANT ISSUES:\n\n")
choutputs.write_all(change_header21)
choutputs.write_all(change_header22)
- choutputs.write_list(importantlist, options.skip_credits,
JIRA_BASE_URL)
+ choutputs.write_list(importantlist, options.skip_credits,
+ JIRA_BASE_URL)
if newfeaturelist:
choutputs.write_all("\n\n### NEW FEATURES:\n\n")
choutputs.write_all(change_header21)
choutputs.write_all(change_header22)
- choutputs.write_list(newfeaturelist, options.skip_credits,
JIRA_BASE_URL)
+ choutputs.write_list(newfeaturelist, options.skip_credits,
+ JIRA_BASE_URL)
if improvementlist:
choutputs.write_all("\n\n### IMPROVEMENTS:\n\n")
choutputs.write_all(change_header21)
choutputs.write_all(change_header22)
- choutputs.write_list(improvementlist, options.skip_credits,
JIRA_BASE_URL)
+ choutputs.write_list(improvementlist, options.skip_credits,
+ JIRA_BASE_URL)
if buglist:
choutputs.write_all("\n\n### BUG FIXES:\n\n")
@@ -981,13 +1001,15 @@ def main(): # pylint: disable=too-many-statements,
too-many-branches, too-many-l
choutputs.write_all("\n\n### SUB-TASKS:\n\n")
choutputs.write_all(change_header21)
choutputs.write_all(change_header22)
- choutputs.write_list(subtasklist, options.skip_credits,
JIRA_BASE_URL)
+ choutputs.write_list(subtasklist, options.skip_credits,
+ JIRA_BASE_URL)
if tasklist or otherlist:
choutputs.write_all("\n\n### OTHER:\n\n")
choutputs.write_all(change_header21)
choutputs.write_all(change_header22)
- choutputs.write_list(otherlist, options.skip_credits,
JIRA_BASE_URL)
+ choutputs.write_list(otherlist, options.skip_credits,
+ JIRA_BASE_URL)
choutputs.write_list(tasklist, options.skip_credits, JIRA_BASE_URL)
choutputs.write_all("\n\n")
diff --git a/releasedocmaker/src/main/python/releasedocmaker/utils.py
b/releasedocmaker/src/main/python/releasedocmaker/utils.py
index 3964671..f3f93ae 100755
--- a/releasedocmaker/src/main/python/releasedocmaker/utils.py
+++ b/releasedocmaker/src/main/python/releasedocmaker/utils.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -15,21 +15,22 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
""" Utility methods used by releasedocmaker """
-from __future__ import print_function
import base64
import os
import re
-import urllib2
+import urllib.request
+import urllib.error
+import urllib.parse
import sys
import json
-import httplib
+import http.client
sys.dont_write_bytecode = True
NAME_PATTERN = re.compile(r' \([0-9]+\)')
+
def get_jira(jira_url):
""" Provide standard method for fetching content from apache jira and
handling of potential errors. Returns urllib2 response or
@@ -38,14 +39,15 @@ def get_jira(jira_url):
username = os.environ.get('RDM_JIRA_USERNAME')
password = os.environ.get('RDM_JIRA_PASSWORD')
- req = urllib2.Request(jira_url)
+ req = urllib.request.Request(jira_url)
if username and password:
- basicauth = base64.encodestring("%s:%s" % (username,
password)).replace('\n', '')
+ basicauth = base64.b64encode("%s:%s" % (username, password)).replace(
+ '\n', '')
req.add_header('Authorization', 'Basic %s' % basicauth)
try:
- response = urllib2.urlopen(req)
- except urllib2.HTTPError as http_err:
+ response = urllib.request.urlopen(req)
+ except urllib.error.HTTPError as http_err:
code = http_err.code
print("JIRA returns HTTP error %d: %s. Aborting." % \
(code, http_err.msg))
@@ -59,11 +61,11 @@ def get_jira(jira_url):
except ValueError:
print("FATAL: Could not parse json response from server.")
sys.exit(1)
- except urllib2.URLError as url_err:
+ except urllib.error.URLError as url_err:
print("Error contacting JIRA: %s\n" % jira_url)
print("Reason: %s" % url_err.reason)
raise url_err
- except httplib.BadStatusLine as err:
+ except http.client.BadStatusLine as err:
raise err
return response
@@ -78,24 +80,17 @@ def format_components(input_string):
ret = "."
return sanitize_markdown(re.sub(NAME_PATTERN, "", ret))
-def encode_utf8(input_string):
- """ Return the string encoded as UTF-8.
- This is necessary for handling markdown in Python. """
- return input_string.encode('utf-8')
-
def sanitize_markdown(input_string):
""" Sanitize Markdown input so it can be handled by Python.
The expectation is that the input is already valid Markdown,
so no additional escaping is required. """
- input_string = encode_utf8(input_string)
- input_string = input_string.replace("\r", "")
+ input_string = input_string.replace('\r', '')
input_string = input_string.rstrip()
return input_string
-
def sanitize_text(input_string):
""" Sanitize arbitrary text so it can be embedded in MultiMarkdown output.
@@ -122,7 +117,7 @@ def sanitize_text(input_string):
output_string = ""
for char in input_string:
out = char
- if char in escapes:
+ if escapes.get(char):
out = escapes[char]
output_string += out
@@ -144,12 +139,11 @@ def to_unicode(obj):
""" convert string to unicode """
if obj is None:
return ""
- return unicode(obj)
+ return str(obj)
-class Outputs(object):
+class Outputs:
"""Several different files to output to at the same time"""
-
def __init__(self, base_file_name, file_name_pattern, keys, params=None):
if params is None:
params = {}
@@ -175,12 +169,12 @@ class Outputs(object):
""" write everything without changes """
self.base.write(input_string)
if key in self.others:
- self.others[key].write(input_string)
+ self.others[key].write(input_string.decode("utf-8"))
def close(self):
""" close all the outputs """
self.base.close()
- for value in self.others.values():
+ for value in list(self.others.values()):
value.close()
def write_list(self, mylist, skip_credits, base_url):
@@ -192,13 +186,14 @@ class Outputs(object):
else:
line = '| [{id}]({base_url}/browse/{id}) | {summary} | ' \
'{priority} | {component} | {reporter} | {assignee} |\n'
- args = {'id': encode_utf8(jira.get_id()),
- 'base_url': base_url,
- 'summary': sanitize_text(jira.get_summary()),
- 'priority': sanitize_text(jira.get_priority()),
- 'component': format_components(jira.get_components()),
- 'reporter': sanitize_text(jira.get_reporter()),
- 'assignee': sanitize_text(jira.get_assignee())
- }
+ args = {
+ 'id': jira.get_id(),
+ 'base_url': base_url,
+ 'summary': sanitize_text(jira.get_summary()),
+ 'priority': sanitize_text(jira.get_priority()),
+ 'component': format_components(jira.get_components()),
+ 'reporter': sanitize_text(jira.get_reporter()),
+ 'assignee': sanitize_text(jira.get_assignee())
+ }
line = line.format(**args)
self.write_key_raw(jira.get_project(), line)
diff --git a/shelldocs/src/main/python/shelldocs.py
b/shelldocs/src/main/python/shelldocs.py
index f1c28fa..c953d1f 100755
--- a/shelldocs/src/main/python/shelldocs.py
+++ b/shelldocs/src/main/python/shelldocs.py
@@ -1,5 +1,32 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" process bash scripts and generate documentation from them """
+
+# Do this immediately to prevent compiled forms
+import sys
+import os
+import re
+import errno
+from argparse import ArgumentParser
+
+sys.dont_write_bytecode = True
+
+ASFLICENSE = '''
+<!---
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,13 +42,432 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+-->
+'''
-""" wrapper to run shelldocs from the CLI """
+FUNCTIONRE = re.compile(r"^(\w+) *\(\) *{")
-import sys
-sys.dont_write_bytecode = True
-# pylint: disable=wrong-import-position,import-self
-import shelldocs
-# pylint: enable=wrong-import-position
-# pylint: disable=no-member
-shelldocs.main()
+
+def docstrip(key, dstr):
+ '''remove extra spaces from shelldoc phrase'''
+ dstr = re.sub("^## @%s " % key, "", dstr)
+ dstr = dstr.lstrip()
+ dstr = dstr.rstrip()
+ return dstr
+
+
+def toc(tlist):
+ '''build a table of contents'''
+ tocout = []
+ header = ()
+ for i in tlist:
+ if header != i.getinter():
+ header = i.getinter()
+ line = " * %s\n" % (i.headerbuild())
+ tocout.append(line)
+ line = " * [%s](#%s)\n" % (i.getname().replace("_",
+ r"\_"), i.getname())
+ tocout.append(line)
+ return tocout
+
+
+class ShellFunction: # pylint: disable=too-many-public-methods,
too-many-instance-attributes
+ """a shell function"""
+ def __init__(self, filename):
+ '''Initializer'''
+ self.name = None
+ self.audience = None
+ self.stability = None
+ self.replaceb = None
+ self.returnt = None
+ self.desc = None
+ self.params = None
+ self.filename = filename
+ self.linenum = 0
+
+ def __lt__(self, other):
+ '''comparison'''
+ if self.audience == other.audience:
+ if self.stability == other.stability:
+ if self.replaceb == other.replaceb:
+ return self.name < other.name
+ if self.replaceb == "Yes":
+ return True
+ else:
+ if self.stability == "Stable":
+ return True
+ else:
+ if self.audience == "Public":
+ return True
+ return False
+
+ def reset(self):
+ '''empties current function'''
+ self.name = None
+ self.audience = None
+ self.stability = None
+ self.replaceb = None
+ self.returnt = None
+ self.desc = None
+ self.params = None
+ self.linenum = 0
+ self.filename = None
+
+ def getfilename(self):
+ '''get the name of the function'''
+ if self.filename is None:
+ return "undefined"
+ return self.filename
+
+ def setname(self, text):
+ '''set the name of the function'''
+ if FUNCTIONRE.match(text):
+ definition = FUNCTIONRE.match(text).groups()[0]
+ else:
+ definition = text.split()[1]
+ self.name = definition.replace("(", "").replace(")", "")
+
+ def getname(self):
+ '''get the name of the function'''
+ if self.name is None:
+ return "None"
+ return self.name
+
+ def setlinenum(self, linenum):
+ '''set the line number of the function'''
+ self.linenum = linenum
+
+ def getlinenum(self):
+ '''get the line number of the function'''
+ return self.linenum
+
+ def setaudience(self, text):
+ '''set the audience of the function'''
+ self.audience = docstrip("audience", text)
+ self.audience = self.audience.capitalize()
+
+ def getaudience(self):
+ '''get the audience of the function'''
+ if self.audience is None:
+ return "None"
+ return self.audience
+
+ def setstability(self, text):
+ '''set the stability of the function'''
+ self.stability = docstrip("stability", text)
+ self.stability = self.stability.capitalize()
+
+ def getstability(self):
+ '''get the stability of the function'''
+ if self.stability is None:
+ return "None"
+ return self.stability
+
+ def setreplace(self, text):
+ '''set the replacement state'''
+ self.replaceb = docstrip("replaceable", text)
+ self.replaceb = self.replaceb.capitalize()
+
+ def getreplace(self):
+ '''get the replacement state'''
+ if self.replaceb == "Yes":
+ return self.replaceb
+ return "No"
+
+ def getinter(self):
+ '''get the function state'''
+ return self.getaudience(), self.getstability(), self.getreplace()
+
+ def addreturn(self, text):
+ '''add a return state'''
+ if self.returnt is None:
+ self.returnt = []
+ self.returnt.append(docstrip("return", text))
+
+ def getreturn(self):
+ '''get the complete return state'''
+ if self.returnt is None:
+ return "Nothing"
+ return "\n\n".join(self.returnt)
+
+ def adddesc(self, text):
+ '''add to the description'''
+ if self.desc is None:
+ self.desc = []
+ self.desc.append(docstrip("description", text))
+
+ def getdesc(self):
+ '''get the description'''
+ if self.desc is None:
+ return "None"
+ return " ".join(self.desc)
+
+ def addparam(self, text):
+ '''add a parameter'''
+ if self.params is None:
+ self.params = []
+ self.params.append(docstrip("param", text))
+
+ def getparams(self):
+ '''get all of the parameters'''
+ if self.params is None:
+ return ""
+ return " ".join(self.params)
+
+ def getusage(self):
+ '''get the usage string'''
+ line = "%s %s" % (self.name, self.getparams())
+ return line.rstrip()
+
+ def headerbuild(self):
+ '''get the header for this function'''
+ if self.getreplace() == "Yes":
+ replacetext = "Replaceable"
+ else:
+ replacetext = "Not Replaceable"
+ line = "%s/%s/%s" % (self.getaudience(), self.getstability(),
+ replacetext)
+ return line
+
+ def getdocpage(self):
+ '''get the built document page for this function'''
+ line = "### `%s`\n\n"\
+ "* Synopsis\n\n"\
+ "```\n%s\n"\
+ "```\n\n" \
+ "* Description\n\n" \
+ "%s\n\n" \
+ "* Returns\n\n" \
+ "%s\n\n" \
+ "| Classification | Level |\n" \
+ "| :--- | :--- |\n" \
+ "| Audience | %s |\n" \
+ "| Stability | %s |\n" \
+ "| Replaceable | %s |\n\n" \
+ % (self.getname(),
+ self.getusage(),
+ self.getdesc(),
+ self.getreturn(),
+ self.getaudience(),
+ self.getstability(),
+ self.getreplace())
+ return line
+
+ def lint(self):
+ '''Lint this function'''
+ getfuncs = {
+ "audience": self.getaudience,
+ "stability": self.getstability,
+ "replaceable": self.getreplace,
+ }
+ validvalues = {
+ "audience": ("Public", "Private"),
+ "stability": ("Stable", "Evolving"),
+ "replaceable": ("Yes", "No"),
+ }
+ messages = []
+ for attr in ("audience", "stability", "replaceable"):
+ value = getfuncs[attr]()
+ if value == "None":
+ messages.append("%s:%u: ERROR: function %s has no @%s" %
+ (self.getfilename(), self.getlinenum(),
+ self.getname(), attr.lower()))
+ elif value not in validvalues[attr]:
+ validvalue = "|".join(v.lower() for v in validvalues[attr])
+ messages.append(
+ "%s:%u: ERROR: function %s has invalid value (%s) for @%s
(%s)"
+ % (self.getfilename(), self.getlinenum(), self.getname(),
+ value.lower(), attr.lower(), validvalue))
+ return "\n".join(messages)
+
+ def __str__(self):
+ '''Generate a string for this function'''
+ line = "{%s %s %s %s}" \
+ % (self.getname(),
+ self.getaudience(),
+ self.getstability(),
+ self.getreplace())
+ return line
+
+
+def marked_as_ignored(file_path):
+ """Checks for the presence of the marker(SHELLDOC-IGNORE) to ignore the
file.
+
+ Marker needs to be in a line of its own and can not
+ be an inline comment.
+
+ A leading '#' and white-spaces(leading or trailing)
+ are trimmed before checking equality.
+
+ Comparison is case sensitive and the comment must be in
+ UPPERCASE.
+ """
+ with open(file_path) as input_file:
+ for line in input_file:
+ if line.startswith("#") and line[1:].strip() == "SHELLDOC-IGNORE":
+ return True
+ return False
+
+
+def process_file(filename, skipprnorep):
+ """ stuff all of the functions into an array """
+ allfuncs = []
+ try:
+ with open(filename, "r") as shellcode:
+ # if the file contains a comment containing
+ # only "SHELLDOC-IGNORE" then skip that file
+ if marked_as_ignored(filename):
+ return None
+ funcdef = ShellFunction(filename)
+ linenum = 0
+ for line in shellcode:
+ linenum = linenum + 1
+ if line.startswith('## @description'):
+ funcdef.adddesc(line)
+ elif line.startswith('## @audience'):
+ funcdef.setaudience(line)
+ elif line.startswith('## @stability'):
+ funcdef.setstability(line)
+ elif line.startswith('## @replaceable'):
+ funcdef.setreplace(line)
+ elif line.startswith('## @param'):
+ funcdef.addparam(line)
+ elif line.startswith('## @return'):
+ funcdef.addreturn(line)
+ elif line.startswith('function') or FUNCTIONRE.match(line):
+ funcdef.setname(line)
+ funcdef.setlinenum(linenum)
+ if skipprnorep and \
+ funcdef.getaudience() == "Private" and \
+ funcdef.getreplace() == "No":
+ pass
+ else:
+ allfuncs.append(funcdef)
+ funcdef = ShellFunction(filename)
+ except IOError as err:
+ print("ERROR: Failed to read from file: %s. Skipping." % err.filename,
+ file=sys.stderr)
+ return None
+ return allfuncs
+
+
+def process_input(inputlist, skipprnorep):
+ """ take the input and loop around it """
+ allfuncs = []
+ for filename in inputlist: #pylint: disable=too-many-nested-blocks
+ if os.path.isdir(filename):
+ for root, dirs, files in os.walk(filename): #pylint:
disable=unused-variable
+ for fname in files:
+ if fname.endswith('sh'):
+ newfuncs = process_file(filename=os.path.join(
+ root, fname),
+ skipprnorep=skipprnorep)
+ if newfuncs:
+ allfuncs = allfuncs + newfuncs
+ else:
+ newfuncs = process_file(filename=filename, skipprnorep=skipprnorep)
+ if newfuncs:
+ allfuncs = allfuncs + newfuncs
+
+ if allfuncs is None:
+ print("ERROR: no functions found.", file=sys.stderr)
+ sys.exit(1)
+
+ allfuncs = sorted(allfuncs)
+ return allfuncs
+
+
+def write_output(filename, functions):
+ """ write the markdown file """
+ try:
+ directory = os.path.dirname(filename)
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+ except OSError as exc:
+ if exc.errno == errno.EEXIST and os.path.isdir(directory):
+ pass
+ else:
+ print("Unable to create output directory %s: %u, %s" % \
+ (directory, exc.errno, exc.strerror))
+ sys.exit(1)
+
+ with open(filename, "w") as outfile:
+ outfile.write(ASFLICENSE)
+ for line in toc(functions):
+ outfile.write(line)
+ outfile.write("\n------\n\n")
+
+ header = []
+ for funcs in functions:
+ if header != funcs.getinter():
+ header = funcs.getinter()
+ line = "## %s\n" % (funcs.headerbuild())
+ outfile.write(line)
+ outfile.write(funcs.getdocpage())
+
+
+def main():
+ '''main entry point'''
+ parser = ArgumentParser(
+ prog='shelldocs',
+ epilog="You can mark a file to be ignored by shelldocs by adding"
+ " 'SHELLDOC-IGNORE' as comment in its own line. "+
+ "--input may be given multiple times.")
+ parser.add_argument("-o",
+ "--output",
+ dest="outfile",
+ action="store",
+ type=str,
+ help="file to create",
+ metavar="OUTFILE")
+ parser.add_argument("-i",
+ "--input",
+ dest="infile",
+ action="append",
+ type=str,
+ help="file to read",
+ metavar="INFILE")
+ parser.add_argument("--skipprnorep",
+ dest="skipprnorep",
+ action="store_true",
+ help="Skip Private & Not Replaceable")
+ parser.add_argument("--lint",
+ dest="lint",
+ action="store_true",
+ help="Enable lint mode")
+ parser.add_argument(
+ "-V",
+ "--version",
+ dest="release_version",
+ action="store_true",
+ default=False,
+ help="display version information for shelldocs and exit.")
+
+ options = parser.parse_args()
+
+ if options.release_version:
+ with open(os.path.join(os.path.dirname(__file__), "../VERSION"),
+ 'r') as ver_file:
+ print(ver_file.read())
+ sys.exit(0)
+
+ if options.infile is None:
+ parser.error("At least one input file needs to be supplied")
+ elif options.outfile is None and options.lint is None:
+ parser.error(
+ "At least one of output file and lint mode needs to be specified")
+
+ allfuncs = process_input(options.infile, options.skipprnorep)
+
+ if options.lint:
+ for funcs in allfuncs:
+ message = funcs.lint()
+ if message:
+ print(message)
+
+ if options.outfile is not None:
+ write_output(options.outfile, allfuncs)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/shelldocs/src/main/python/shelldocs/__init__.py
b/shelldocs/src/main/python/shelldocs/__init__.py
deleted file mode 100755
index 29ed858..0000000
--- a/shelldocs/src/main/python/shelldocs/__init__.py
+++ /dev/null
@@ -1,472 +0,0 @@
-#!/usr/bin/env python2
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-""" process bash scripts and generate documentation from them """
-
-# Do this immediately to prevent compiled forms
-import sys
-import os
-import re
-import errno
-from optparse import OptionParser
-
-sys.dont_write_bytecode = True
-
-ASFLICENSE = '''
-<!---
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
--->
-'''
-
-FUNCTIONRE = re.compile(r"^(\w+) *\(\) *{")
-
-
-def docstrip(key, dstr):
- '''remove extra spaces from shelldoc phrase'''
- dstr = re.sub("^## @%s " % key, "", dstr)
- dstr = dstr.lstrip()
- dstr = dstr.rstrip()
- return dstr
-
-
-def toc(tlist):
- '''build a table of contents'''
- tocout = []
- header = ()
- for i in tlist:
- if header != i.getinter():
- header = i.getinter()
- line = " * %s\n" % (i.headerbuild())
- tocout.append(line)
- line = " * [%s](#%s)\n" % (i.getname().replace("_", r"\_"),
- i.getname())
- tocout.append(line)
- return tocout
-
-
-class ShellFunction(object): # pylint: disable=too-many-public-methods,
too-many-instance-attributes
- """a shell function"""
-
- def __init__(self, filename):
- '''Initializer'''
- self.name = None
- self.audience = None
- self.stability = None
- self.replaceb = None
- self.returnt = None
- self.desc = None
- self.params = None
- self.filename = filename
- self.linenum = 0
-
- def __cmp__(self, other):
- '''comparison'''
- if self.audience == other.audience:
- if self.stability == other.stability:
- if self.replaceb == other.replaceb:
- return cmp(self.name, other.name)
- else:
- if self.replaceb == "Yes":
- return -1
- else:
- if self.stability == "Stable":
- return -1
- else:
- if self.audience == "Public":
- return -1
- return 1
-
- def reset(self):
- '''empties current function'''
- self.name = None
- self.audience = None
- self.stability = None
- self.replaceb = None
- self.returnt = None
- self.desc = None
- self.params = None
- self.linenum = 0
- self.filename = None
-
- def getfilename(self):
- '''get the name of the function'''
- if self.filename is None:
- return "undefined"
- return self.filename
-
- def setname(self, text):
- '''set the name of the function'''
- if FUNCTIONRE.match(text):
- definition = FUNCTIONRE.match(text).groups()[0]
- else:
- definition = text.split()[1]
- self.name = definition.replace("(", "").replace(")", "")
-
- def getname(self):
- '''get the name of the function'''
- if self.name is None:
- return "None"
- return self.name
-
- def setlinenum(self, linenum):
- '''set the line number of the function'''
- self.linenum = linenum
-
- def getlinenum(self):
- '''get the line number of the function'''
- return self.linenum
-
- def setaudience(self, text):
- '''set the audience of the function'''
- self.audience = docstrip("audience", text)
- self.audience = self.audience.capitalize()
-
- def getaudience(self):
- '''get the audience of the function'''
- if self.audience is None:
- return "None"
- return self.audience
-
- def setstability(self, text):
- '''set the stability of the function'''
- self.stability = docstrip("stability", text)
- self.stability = self.stability.capitalize()
-
- def getstability(self):
- '''get the stability of the function'''
- if self.stability is None:
- return "None"
- return self.stability
-
- def setreplace(self, text):
- '''set the replacement state'''
- self.replaceb = docstrip("replaceable", text)
- self.replaceb = self.replaceb.capitalize()
-
- def getreplace(self):
- '''get the replacement state'''
- if self.replaceb == "Yes":
- return self.replaceb
- return "No"
-
- def getinter(self):
- '''get the function state'''
- return self.getaudience(), self.getstability(), self.getreplace()
-
- def addreturn(self, text):
- '''add a return state'''
- if self.returnt is None:
- self.returnt = []
- self.returnt.append(docstrip("return", text))
-
- def getreturn(self):
- '''get the complete return state'''
- if self.returnt is None:
- return "Nothing"
- return "\n\n".join(self.returnt)
-
- def adddesc(self, text):
- '''add to the description'''
- if self.desc is None:
- self.desc = []
- self.desc.append(docstrip("description", text))
-
- def getdesc(self):
- '''get the description'''
- if self.desc is None:
- return "None"
- return " ".join(self.desc)
-
- def addparam(self, text):
- '''add a parameter'''
- if self.params is None:
- self.params = []
- self.params.append(docstrip("param", text))
-
- def getparams(self):
- '''get all of the parameters'''
- if self.params is None:
- return ""
- return " ".join(self.params)
-
- def getusage(self):
- '''get the usage string'''
- line = "%s %s" % (self.name, self.getparams())
- return line.rstrip()
-
- def headerbuild(self):
- '''get the header for this function'''
- if self.getreplace() == "Yes":
- replacetext = "Replaceable"
- else:
- replacetext = "Not Replaceable"
- line = "%s/%s/%s" % (self.getaudience(), self.getstability(),
- replacetext)
- return line
-
- def getdocpage(self):
- '''get the built document page for this function'''
- line = "### `%s`\n\n"\
- "* Synopsis\n\n"\
- "```\n%s\n"\
- "```\n\n" \
- "* Description\n\n" \
- "%s\n\n" \
- "* Returns\n\n" \
- "%s\n\n" \
- "| Classification | Level |\n" \
- "| :--- | :--- |\n" \
- "| Audience | %s |\n" \
- "| Stability | %s |\n" \
- "| Replaceable | %s |\n\n" \
- % (self.getname(),
- self.getusage(),
- self.getdesc(),
- self.getreturn(),
- self.getaudience(),
- self.getstability(),
- self.getreplace())
- return line
-
- def lint(self):
- '''Lint this function'''
- getfuncs = {
- "audience": self.getaudience,
- "stability": self.getstability,
- "replaceable": self.getreplace,
- }
- validvalues = {
- "audience": ("Public", "Private"),
- "stability": ("Stable", "Evolving"),
- "replaceable": ("Yes", "No"),
- }
- messages = []
- for attr in ("audience", "stability", "replaceable"):
- value = getfuncs[attr]()
- if value == "None":
- messages.append("%s:%u: ERROR: function %s has no @%s" %
- (self.getfilename(), self.getlinenum(),
- self.getname(), attr.lower()))
- elif value not in validvalues[attr]:
- validvalue = "|".join(v.lower() for v in validvalues[attr])
- messages.append(
- "%s:%u: ERROR: function %s has invalid value (%s) for @%s
(%s)"
- % (self.getfilename(), self.getlinenum(), self.getname(),
- value.lower(), attr.lower(), validvalue))
- return "\n".join(messages)
-
- def __str__(self):
- '''Generate a string for this function'''
- line = "{%s %s %s %s}" \
- % (self.getname(),
- self.getaudience(),
- self.getstability(),
- self.getreplace())
- return line
-
-
-def marked_as_ignored(file_path):
- """Checks for the presence of the marker(SHELLDOC-IGNORE) to ignore the
file.
-
- Marker needs to be in a line of its own and can not
- be an inline comment.
-
- A leading '#' and white-spaces(leading or trailing)
- are trimmed before checking equality.
-
- Comparison is case sensitive and the comment must be in
- UPPERCASE.
- """
- with open(file_path) as input_file:
- for line in input_file:
- if line.startswith("#") and line[1:].strip() == "SHELLDOC-IGNORE":
- return True
- return False
-
-def process_file(filename, skipprnorep):
- """ stuff all of the functions into an array """
- allfuncs = []
- try:
- with open(filename, "r") as shellcode:
- # if the file contains a comment containing
- # only "SHELLDOC-IGNORE" then skip that file
- if marked_as_ignored(filename):
- return None
- funcdef = ShellFunction(filename)
- linenum = 0
- for line in shellcode:
- linenum = linenum + 1
- if line.startswith('## @description'):
- funcdef.adddesc(line)
- elif line.startswith('## @audience'):
- funcdef.setaudience(line)
- elif line.startswith('## @stability'):
- funcdef.setstability(line)
- elif line.startswith('## @replaceable'):
- funcdef.setreplace(line)
- elif line.startswith('## @param'):
- funcdef.addparam(line)
- elif line.startswith('## @return'):
- funcdef.addreturn(line)
- elif line.startswith('function') or FUNCTIONRE.match(line):
- funcdef.setname(line)
- funcdef.setlinenum(linenum)
- if skipprnorep and \
- funcdef.getaudience() == "Private" and \
- funcdef.getreplace() == "No":
- pass
- else:
- allfuncs.append(funcdef)
- funcdef = ShellFunction(filename)
- except IOError, err:
- print >> sys.stderr, "ERROR: Failed to read from file: %s. Skipping."
% err.filename
- return None
- return allfuncs
-
-def process_input(inputlist, skipprnorep):
- """ take the input and loop around it """
- allfuncs = []
- for filename in inputlist: #pylint: disable=too-many-nested-blocks
- if os.path.isdir(filename):
- for root, dirs, files in os.walk(filename): #pylint:
disable=unused-variable
- for fname in files:
- if fname.endswith('sh'):
- newfuncs = process_file(filename=os.path.join(root,
fname),
- skipprnorep=skipprnorep)
- if newfuncs:
- allfuncs = allfuncs + newfuncs
- else:
- newfuncs = process_file(filename=filename, skipprnorep=skipprnorep)
- if newfuncs:
- allfuncs = allfuncs + newfuncs
-
- if allfuncs is None:
- print >> sys.stderr, "ERROR: no functions found."
- sys.exit(1)
-
- allfuncs = sorted(allfuncs)
- return allfuncs
-
-def write_output(filename, functions):
- """ write the markdown file """
- try:
- directory = os.path.dirname(filename)
- if not os.path.exists(directory):
- os.makedirs(directory)
- except OSError as exc:
- if exc.errno == errno.EEXIST and os.path.isdir(directory):
- pass
- else:
- print "Unable to create output directory %s: %u, %s" % \
- (directory, exc.errno, exc.message)
- sys.exit(1)
-
- with open(filename, "w") as outfile:
- outfile.write(ASFLICENSE)
- for line in toc(functions):
- outfile.write(line)
- outfile.write("\n------\n\n")
-
- header = []
- for funcs in functions:
- if header != funcs.getinter():
- header = funcs.getinter()
- line = "## %s\n" % (funcs.headerbuild())
- outfile.write(line)
- outfile.write(funcs.getdocpage())
-
-def main():
- '''main entry point'''
- parser = OptionParser(
- usage="usage: %prog [--skipprnorep] " + "[--output OUTFILE|--lint] " +
- "--input INFILE " + "[--input INFILE ...]",
- epilog=
- "You can mark a file to be ignored by shelldocs by adding"
- " 'SHELLDOC-IGNORE' as comment in its own line."
- )
- parser.add_option("-o",
- "--output",
- dest="outfile",
- action="store",
- type="string",
- help="file to create",
- metavar="OUTFILE")
- parser.add_option("-i",
- "--input",
- dest="infile",
- action="append",
- type="string",
- help="file to read",
- metavar="INFILE")
- parser.add_option("--skipprnorep",
- dest="skipprnorep",
- action="store_true",
- help="Skip Private & Not Replaceable")
- parser.add_option("--lint",
- dest="lint",
- action="store_true",
- help="Enable lint mode")
- parser.add_option(
- "-V",
- "--version",
- dest="release_version",
- action="store_true",
- default=False,
- help="display version information for shelldocs and exit.")
-
- (options, dummy_args) = parser.parse_args()
-
- if options.release_version:
- with open(
- os.path.join(
- os.path.dirname(__file__), "../VERSION"), 'r') as ver_file:
- print ver_file.read()
- sys.exit(0)
-
- if options.infile is None:
- parser.error("At least one input file needs to be supplied")
- elif options.outfile is None and options.lint is None:
- parser.error(
- "At least one of output file and lint mode needs to be specified")
-
- allfuncs = process_input(options.infile, options.skipprnorep)
-
- if options.lint:
- for funcs in allfuncs:
- message = funcs.lint()
- if message:
- print message
-
- if options.outfile is not None:
- write_output(options.outfile, allfuncs)
-
-if __name__ == "__main__":
- main()
