[54/59] [abbrv] hadoop git commit: HADOOP-13374. Add the L verification script. Contributed by Allen Wittenauer

2018-02-26 Thread xyao 
HADOOP-13374. Add the L verification script. Contributed by Allen Wittenauer


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/329a4fdd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/329a4fdd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/329a4fdd

Branch: refs/heads/HDFS-7240
Commit: 329a4fdd07ab007615f34c8e0e651360f988064d
Parents: 033f9c6
Author: Chris Douglas 
Authored: Fri Feb 23 17:07:22 2018 -0800
Committer: Chris Douglas 
Committed: Fri Feb 23 17:07:22 2018 -0800

--
 dev-support/bin/verify-license-files | 145 ++
 1 file changed, 145 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/329a4fdd/dev-support/bin/verify-license-files
--
diff --git a/dev-support/bin/verify-license-files 
b/dev-support/bin/verify-license-files
new file mode 100755
index 000..1fd70a6
--- /dev/null
+++ b/dev-support/bin/verify-license-files
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+## @description  check a file
+## @audience private
+## @stabilityevolving
+## @replaceable  no
+## @paramfilename
+## @paramjarfile
+## @return   0 = destroy verify dir
+## @return   1 = keep verify dir
+function process_file
+{
+  declare check=$1
+  declare fqfn=$2
+  declare fn
+  declare keepdir
+  declare tf
+  declare count
+  declare valid
+
+  fn=$(basename "${fqfn}")
+  keepdir=false
+  valid=0
+  count=0
+
+  unzip -o -d "${WORK_DIR}/${fn}" "${fqfn}" '*'"${check}"'*' >/dev/null 2>&1
+
+  while read -r tf; do
+((count = count + 1))
+if diff -q "${DIST_DIR}/${check}.txt" "${tf}" >/dev/null 2>&1; then
+  ((valid = valid + 1))
+fi
+  done < <(find "${WORK_DIR}/${fn}" -name "${check}"'*')
+
+  if [[ "${count}" -eq 0 ]]; then
+hadoop_error "ERROR: ${fn}: Missing a ${check} file"
+  elif [[ "${count}" -gt 1 ]]; then
+hadoop_error "WARNING: ${fn}: Found ${count} ${check} files (${valid} were 
valid)"
+keepdir=true
+  fi
+
+  if [[ "${valid}" -eq 0 ]] && [[ "${count}" -gt 0 ]]; then
+  hadoop_error "ERROR: ${fn}: No valid ${check} found"
+  keepdir=true
+  fi
+
+  if [[ "${keepdir}" = "false" ]]; then
+return 0
+  else
+return 1
+  fi
+}
+
+
+## @description  check a jar
+## @audience private
+## @stabilityevolving
+## @replaceable  no
+## @paramjarfile
+## @return   0 - success
+## @return   1 - errors
+function process_jar
+{
+  declare fqfn=$1
+  declare fn
+  declare keepwork
+
+  fn=$(basename "${fqfn}")
+  keepwork=false
+
+  if [[ ! ${fn} =~ hadoop-.*-${PROJ_VERSION} ]]; then
+return
+  fi
+
+  mkdir -p "${WORK_DIR}/${fn}"
+
+  if ! process_file LICENSE "${fqfn}"; then
+keepwork=true
+  fi
+
+  if ! process_file NOTICE "${fqfn}"; then
+keepwork=true
+  fi
+
+  if [[ "${keepwork}" = "false" ]]; then
+rm -rf "${WORK_DIR:?}/${fn}"
+return 0
+  else
+hadoop_error ""
+return 1
+  fi
+}
+
+
+MYNAME="${BASH_SOURCE-$0}"
+#shellcheck disable=SC2034
+HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
+BINDIR=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
+#shellcheck disable=SC2034
+HADOOP_LIBEXEC_DIR="${BINDIR}/../../hadoop-common-project/hadoop-common/src/main/bin"
+
+#shellcheck disable=SC1090
+. "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh"
+
+HADOOP_LIBEXEC_DIR=$(hadoop_abs "${HADOOP_LIBEXEC_DIR}")
+BINDIR=$(hadoop_abs "${BINDIR}")
+BASEDIR=$(hadoop_abs "${BINDIR}/../..")
+
+pushd "${BASEDIR}" >/dev/null
+#shellcheck disable=SC2016
+PROJ_VERSION=$(mvn -q -Dexec.executable="echo" 
-Dexec.args='${project.version}' --non-recursive exec:exec)
+popd >/dev/null
+
+DIST_DIR="${BASEDIR}/hadoop-dist/target/hadoop-${PROJ_VERSION}"
+WORK_DIR="${BASEDIR}/patchprocess/verify"
+
+rm -rf "${WORK_DIR:?}"
+mkdir -p "${WORK_DIR}"
+
+while read -r filename; do
+  process_jar "${filename}"
+  ((ret = ret + $? ))
+done < <(find "${DIST_DIR}" \
+  -name

hadoop git commit: HADOOP-13374. Add the L verification script. Contributed by Allen Wittenauer

2018-02-23 Thread cdouglas 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 033f9c68e -> 329a4fdd0


HADOOP-13374. Add the L verification script. Contributed by Allen Wittenauer


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/329a4fdd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/329a4fdd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/329a4fdd

Branch: refs/heads/trunk
Commit: 329a4fdd07ab007615f34c8e0e651360f988064d
Parents: 033f9c6
Author: Chris Douglas 
Authored: Fri Feb 23 17:07:22 2018 -0800
Committer: Chris Douglas 
Committed: Fri Feb 23 17:07:22 2018 -0800

--
 dev-support/bin/verify-license-files | 145 ++
 1 file changed, 145 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/329a4fdd/dev-support/bin/verify-license-files
--
diff --git a/dev-support/bin/verify-license-files 
b/dev-support/bin/verify-license-files
new file mode 100755
index 000..1fd70a6
--- /dev/null
+++ b/dev-support/bin/verify-license-files
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+## @description  check a file
+## @audience private
+## @stabilityevolving
+## @replaceable  no
+## @paramfilename
+## @paramjarfile
+## @return   0 = destroy verify dir
+## @return   1 = keep verify dir
+function process_file
+{
+  declare check=$1
+  declare fqfn=$2
+  declare fn
+  declare keepdir
+  declare tf
+  declare count
+  declare valid
+
+  fn=$(basename "${fqfn}")
+  keepdir=false
+  valid=0
+  count=0
+
+  unzip -o -d "${WORK_DIR}/${fn}" "${fqfn}" '*'"${check}"'*' >/dev/null 2>&1
+
+  while read -r tf; do
+((count = count + 1))
+if diff -q "${DIST_DIR}/${check}.txt" "${tf}" >/dev/null 2>&1; then
+  ((valid = valid + 1))
+fi
+  done < <(find "${WORK_DIR}/${fn}" -name "${check}"'*')
+
+  if [[ "${count}" -eq 0 ]]; then
+hadoop_error "ERROR: ${fn}: Missing a ${check} file"
+  elif [[ "${count}" -gt 1 ]]; then
+hadoop_error "WARNING: ${fn}: Found ${count} ${check} files (${valid} were 
valid)"
+keepdir=true
+  fi
+
+  if [[ "${valid}" -eq 0 ]] && [[ "${count}" -gt 0 ]]; then
+  hadoop_error "ERROR: ${fn}: No valid ${check} found"
+  keepdir=true
+  fi
+
+  if [[ "${keepdir}" = "false" ]]; then
+return 0
+  else
+return 1
+  fi
+}
+
+
+## @description  check a jar
+## @audience private
+## @stabilityevolving
+## @replaceable  no
+## @paramjarfile
+## @return   0 - success
+## @return   1 - errors
+function process_jar
+{
+  declare fqfn=$1
+  declare fn
+  declare keepwork
+
+  fn=$(basename "${fqfn}")
+  keepwork=false
+
+  if [[ ! ${fn} =~ hadoop-.*-${PROJ_VERSION} ]]; then
+return
+  fi
+
+  mkdir -p "${WORK_DIR}/${fn}"
+
+  if ! process_file LICENSE "${fqfn}"; then
+keepwork=true
+  fi
+
+  if ! process_file NOTICE "${fqfn}"; then
+keepwork=true
+  fi
+
+  if [[ "${keepwork}" = "false" ]]; then
+rm -rf "${WORK_DIR:?}/${fn}"
+return 0
+  else
+hadoop_error ""
+return 1
+  fi
+}
+
+
+MYNAME="${BASH_SOURCE-$0}"
+#shellcheck disable=SC2034
+HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
+BINDIR=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
+#shellcheck disable=SC2034
+HADOOP_LIBEXEC_DIR="${BINDIR}/../../hadoop-common-project/hadoop-common/src/main/bin"
+
+#shellcheck disable=SC1090
+. "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh"
+
+HADOOP_LIBEXEC_DIR=$(hadoop_abs "${HADOOP_LIBEXEC_DIR}")
+BINDIR=$(hadoop_abs "${BINDIR}")
+BASEDIR=$(hadoop_abs "${BINDIR}/../..")
+
+pushd "${BASEDIR}" >/dev/null
+#shellcheck disable=SC2016
+PROJ_VERSION=$(mvn -q -Dexec.executable="echo" 
-Dexec.args='${project.version}' --non-recursive exec:exec)
+popd >/dev/null
+
+DIST_DIR="${BASEDIR}/hadoop-dist/target/hadoop-${PROJ_VERSION}"
+WORK_DIR="${BASEDIR}/patchprocess/verify"
+
+rm -rf "${WORK_DIR:?}"
+mkdir -p "${WORK_DIR}"
+
+while read -r filename; do
+  process_jar "${filename}"
+