This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hadoop-release-support.git
commit e46695365b8d00b853b0117935c7f8f399626ece
Author: Steve Loughran <ste...@cloudera.com>
AuthorDate: Tue May 10 19:37:16 2022 +0100
HADOOP-18198. Add validator project to verify that the client APIs work.
This includes a "purge" profile which invokes ant to delete the appropriate
hadoop artifacts from the local maven repositoory, so guaranteeing that the
files are not being picked up locally.
---
.asf.yaml | 22 --
.github/workflows/lint.yml | 89 -------
.gitignore | 7 +-
pom.xml | 281 +++++++++++++++++++++
.../github/steveloughran/validator/CompileFS.java | 9 +-
.../steveloughran/validator/TestRuntimeValid.java} | 26 +-
src/test/resources/log4j.properties | 22 ++
7 files changed, 316 insertions(+), 140 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
deleted file mode 100644
index 1f413ca..0000000
--- a/.asf.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
-# See:
https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features
-github:
- features:
- # Enable issue management
- issues: true
-
-notifications:
- commits: w...@apache.org
- issues: w...@apache.org
- pullrequests: w...@apache.org
-
-pelican:
- notify: w...@apache.org
- autobuild: preview/*
- target: asf-site
- theme: theme/apache
- whoami: main
-
-staging:
- profile: ~
- whoami: asf-site
- autostage: preview/*
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
deleted file mode 100644
index 96ffb3a..0000000
--- a/.github/workflows/lint.yml
+++ /dev/null
@@ -1,89 +0,0 @@
-name: Lint
-
-on:
- workflow_dispatch:
- inputs:
- logLevel:
- description: 'Log level'
- required: true
- default: 'warning'
- tags:
- description: 'Perform Lint Check'
-
-jobs:
- flake8:
- name: Flake8
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python-version: [3.8]
- steps:
- - uses: actions/checkout@v2
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install flake8
- if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- - name: Lint with flake8
- run: |
- # stop the build if there are Python syntax errors or undefined names
- flake8 . --count --ignore=E201,E501,W605 --show-source --statistics
- markdownlint:
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os:
- - ubuntu-latest
- node_version:
- - 14
- architecture:
- - x64
- name: Markdown Node ${{ matrix.node_version }} - ${{ matrix.architecture
}} on ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v2
- - name: Setup node
- uses: actions/setup-node@v2
- with:
- node-version: ${{ matrix.node_version }}
- architecture: ${{ matrix.architecture }}
- - run: npm install -g markdownlint-cli@0.27.1
- - run: markdownlint '**/*.md'
- misspell:
- name: Check Spelling
- runs-on: ubuntu-latest
- steps:
- - name: Check Out
- uses: actions/checkout@v2
- - name: Install
- run: |
- wget -O - -q https://git.io/misspell | sh -s -- -b .
- - name: Misspell
- run: |
- git ls-files --empty-directory | xargs ./misspell -error
- trailing-whitespace:
- name: Trailing whitespace
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - name: Check for trailing whitespace
- run: "! git grep -EIn $'[ \t]+$'"
- yamllint:
- name: YAML
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: '3.x' # Version range or exact version of a Python
version to use, using SemVer's version range syntax
- architecture: 'x64' # optional x64 or x86. Defaults to x64 if not
specified
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install yamllint
- - name: YAML Lint
- run: |
- yamllint --strict .
diff --git a/.gitignore b/.gitignore
index 6d69e7d..b83d222 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1 @@
-*.pyc
-*.swp
-__pycache__
-pelican.auto.py
-site-generated/
-.authtokens
+/target/
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..2eb0875
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,281 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.github.steveloughran</groupId>
+ <artifactId>client-validator</artifactId>
+ <version>1.0-SNAPSHOT</version>
+
+ <name>client-validator</name>
+
+ <url>https://github.com/steveloughran/validate-hadoop-client-artifacts</url>
+
+ <description>
+ Project to verify hadoop client jars are good.
+
+ To build with the client API
+
+ mvn clean package -Pclient
+
+ to force an update
+
+ mvn clean package -Pclient -Pstaging -U
+
+ to purge all artifacts of that version from your local maven repo
+ mvn clean -Ppurge
+
+ *do not use with the dependency declaration of -Pclient or things get
confused*
+ (it will try to resolve the artifacts, even as they are deleted)
+ </description>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <file.encoding>UTF-8</file.encoding>
+ <javac.version>1.8</javac.version>
+ <enforced.java.version>${javac.version}</enforced.java.version>
+ <maven-antrun-plugin.version>1.7</maven-antrun-plugin.version>
+
+
+ <hadoop.version>3.3.3</hadoop.version>
+
+ <!-- SLF4J/LOG4J version -->
+ <slf4j.version>1.7.36</slf4j.version>
+ <reload4j.version>1.2.18.3</reload4j.version>
+
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.13.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ <version>${reload4j.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jmx</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <!--
+ This is a profile to enable the use of the ASF snapshot and staging
repositories
+ during a build. It is useful when testing against nightly or RC releases of
dependencies.
+ -->
+ <profiles>
+ <profile>
+ <id>staging</id>
+ <properties>
+ <!-- override point for ASF staging/snapshot repos -->
+
<asf.staging>https://repository.apache.org/content/groups/staging/</asf.staging>
+
<asf.snapshots>https://repository.apache.org/content/repositories/snapshots/</asf.snapshots>
+ </properties>
+
+ <pluginRepositories>
+ <pluginRepository>
+ <id>ASF Staging</id>
+ <url>${asf.staging}</url>
+ </pluginRepository>
+ <pluginRepository>
+ <id>ASF Snapshots</id>
+ <url>${asf.snapshots}</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ </pluginRepository>
+
+ </pluginRepositories>
+ <repositories>
+ <repository>
+ <id>ASF Staging</id>
+ <url>${asf.staging}</url>
+ </repository>
+ <repository>
+ <id>ASF Snapshots</id>
+ <url>${asf.snapshots}</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ </repository>
+ </repositories>
+ </profile>
+ <profile>
+ <id>hadoop-3.3.3</id>
+ <properties>
+ <hadoop.version>3.3.3</hadoop.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>trunk</id>
+ <properties>
+ <hadoop.version>3.4.0-SNAPSHOT</hadoop.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>branch-3.3</id>
+ <properties>
+ <hadoop.version>3.3.4-SNAPSHOT</hadoop.version>
+ </properties>
+ </profile>
+
+ <!-- build with the client api-->
+ <profile>
+ <id>client</id>
+
+ <dependencies>
+
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ </dependencies>
+ </profile>
+
+
+ <profile>
+ <id>purge</id>
+
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>delete RC artifacts</id>
+ <phase>clean</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+
+ <property name="mvn.repo"
+ location="${user.home}/.m2/repository"/>
+ <property name="hadoop.artifacts"
+ location="${mvn.repo}/org/apache/hadoop"/>
+
+ <delete>
+ <fileset dir="${hadoop.artifacts}"
+ includes="**/${hadoop.version}/*"/>
+ </delete>
+
+
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ </profiles>
+
+
+ <build>
+ <pluginManagement><!-- lock down plugins versions to avoid using Maven
defaults (may be moved to parent pom) -->
+ <plugins>
+ <!-- clean lifecycle, see
https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle
-->
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>3.1.0</version>
+ </plugin>
+ <!-- default lifecycle, jar packaging: see
https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging
-->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>3.0.2</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.0</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.22.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.0.2</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-install-plugin</artifactId>
+ <version>2.5.2</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.8.2</version>
+ </plugin>
+ <!-- site lifecycle, see
https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle
-->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.7.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>3.0.0</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>${maven-antrun-plugin.version}</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+</project>
diff --git a/src/main/java/com/github/steveloughran/validator/CompileFS.java
b/src/main/java/com/github/steveloughran/validator/CompileFS.java
index b02ebac..a5bf59a 100644
--- a/src/main/java/com/github/steveloughran/validator/CompileFS.java
+++ b/src/main/java/com/github/steveloughran/validator/CompileFS.java
@@ -19,8 +19,9 @@
package com.github.steveloughran.validator;
import java.io.IOException;
-import java.util.logging.Logger;
-import java.util.logging.Level;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -30,11 +31,11 @@ import org.apache.hadoop.fs.FileSystem;
*/
public class CompileFS {
+ public static final Logger LOG = LoggerFactory.getLogger(CompileFS.class);
public FileSystem run() throws IOException {
final FileSystem fs = FileSystem.getLocal(new Configuration());
- Logger log = Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
- log.log(Level.INFO,"fs is {)", fs);
+ LOG.info("fs is {)", fs);
return fs;
}
public static void main(String[] args) throws Exception {
diff --git a/src/main/java/com/github/steveloughran/validator/CompileFS.java
b/src/test/java/com/github/steveloughran/validator/TestRuntimeValid.java
similarity index 61%
copy from src/main/java/com/github/steveloughran/validator/CompileFS.java
copy to src/test/java/com/github/steveloughran/validator/TestRuntimeValid.java
index b02ebac..aeb3cd4 100644
--- a/src/main/java/com/github/steveloughran/validator/CompileFS.java
+++ b/src/test/java/com/github/steveloughran/validator/TestRuntimeValid.java
@@ -18,29 +18,17 @@
package com.github.steveloughran.validator;
-import java.io.IOException;
-import java.util.logging.Logger;
-import java.util.logging.Level;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
+import org.junit.Test;
/**
- *
+ * Let's test that runtime.
*/
-public class CompileFS {
+public class TestRuntimeValid {
+ @Test
+ public void testRuntime() throws Throwable {
+ final CompileFS compileFS = new CompileFS();
+ compileFS.run();
- public FileSystem run() throws IOException {
- final FileSystem fs = FileSystem.getLocal(new Configuration());
- Logger log = Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
- log.log(Level.INFO,"fs is {)", fs);
- return fs;
- }
- public static void main(String[] args) throws Exception {
- new CompileFS().run();
}
-
}
-
-
diff --git a/src/test/resources/log4j.properties
b/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a4d1388
--- /dev/null
+++ b/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2}
(%F:%M(%L)) - %m%n
+log4j.logger.org.apache.hadoop=DEBUG
+log4j.logger.org.apache.hadoop.util.Shell=INFO
+log4j.logger.org.apache.hadoop.metrics2=WARN
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
