Github user HyukjinKwon commented on a diff in the pull request: https://github.com/apache/spark/pull/16405#discussion_r94273331 --- Diff: dev/lint-python --- @@ -19,10 +19,8 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" SPARK_ROOT_DIR="$(dirname "$SCRIPT_DIR")" -PATHS_TO_CHECK="./python/pyspark/ ./examples/src/main/python/ ./dev/sparktestsupport" -# TODO: fix pep8 errors with the rest of the Python scripts under dev -PATHS_TO_CHECK="$PATHS_TO_CHECK ./dev/run-tests.py ./python/*.py ./dev/run-tests-jenkins.py" -PATHS_TO_CHECK="$PATHS_TO_CHECK ./dev/pip-sanity-check.py" +# Exclude auto-geneated configuration file. +PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" -not -path "*python/docs/conf.py" )" --- End diff -- I tested this as below for sure, ```bash ./lint-python ./dev/lint-python ./spark/dev/lint-python ``` So, now it is relative paths which currently are up to 11K as below: ``` ./dev/create-release/generate-contributors.py ./dev/create-release/releaseutils.py ./dev/create-release/translate-contributors.py ./dev/github_jira_sync.py ./dev/merge_spark_pr.py ./dev/pep8-1.7.0.py ./dev/pip-sanity-check.py ./dev/run-tests-jenkins.py ./dev/run-tests.py ./dev/sparktestsupport/__init__.py ./dev/sparktestsupport/modules.py ./dev/sparktestsupport/shellutils.py ./dev/sparktestsupport/toposort.py ./examples/src/main/python/als.py ./examples/src/main/python/avro_inputformat.py ./examples/src/main/python/kmeans.py ./examples/src/main/python/logistic_regression.py ./examples/src/main/python/ml/aft_survival_regression.py ./examples/src/main/python/ml/als_example.py ./examples/src/main/python/ml/binarizer_example.py ./examples/src/main/python/ml/bisecting_k_means_example.py ./examples/src/main/python/ml/bucketizer_example.py ./examples/src/main/python/ml/chisq_selector_example.py ./examples/src/main/python/ml/count_vectorizer_example.py ./examples/src/main/python/ml/cross _validator.py ./examples/src/main/python/ml/dataframe_example.py ./examples/src/main/python/ml/dct_example.py ./examples/src/main/python/ml/decision_tree_classification_example.py ./examples/src/main/python/ml/decision_tree_regression_example.py ./examples/src/main/python/ml/elementwise_product_example.py ./examples/src/main/python/ml/estimator_transformer_param_example.py ./examples/src/main/python/ml/gaussian_mixture_example.py ./examples/src/main/python/ml/generalized_linear_regression_example.py ./examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py ./examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py ./examples/src/main/python/ml/index_to_string_example.py ./examples/src/main/python/ml/isotonic_regression_example.py ./examples/src/main/python/ml/kmeans_example.py ./examples/src/main/python/ml/lda_example.py ./examples/src/main/python/ml/linear_regression_with_elastic_net.py ./examples/src/main/python/ml/logistic_regression_summary_example. py ./examples/src/main/python/ml/logistic_regression_with_elastic_net.py ./examples/src/main/python/ml/max_abs_scaler_example.py ./examples/src/main/python/ml/min_max_scaler_example.py ./examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py ./examples/src/main/python/ml/multilayer_perceptron_classification.py ./examples/src/main/python/ml/n_gram_example.py ./examples/src/main/python/ml/naive_bayes_example.py ./examples/src/main/python/ml/normalizer_example.py ./examples/src/main/python/ml/one_vs_rest_example.py ./examples/src/main/python/ml/onehot_encoder_example.py ./examples/src/main/python/ml/pca_example.py ./examples/src/main/python/ml/pipeline_example.py ./examples/src/main/python/ml/polynomial_expansion_example.py ./examples/src/main/python/ml/quantile_discretizer_example.py ./examples/src/main/python/ml/random_forest_classifier_example.py ./examples/src/main/python/ml/random_forest_regressor_example.py ./examples/src/main/python/ml/rformula_example.py ./examples/src/main/python/ml/sql_transformer.py ./examples/src/main/python/ml/standard_scaler_example.py ./examples/src/main/python/ml/stopwords_remover_example.py ./examples/src/main/python/ml/string_indexer_example.py ./examples/src/main/python/ml/tf_idf_example.py ./examples/src/main/python/ml/tokenizer_example.py ./examples/src/main/python/ml/train_validation_split.py ./examples/src/main/python/ml/vector_assembler_example.py ./examples/src/main/python/ml/vector_indexer_example.py ./examples/src/main/python/ml/vector_slicer_example.py ./examples/src/main/python/ml/word2vec_example.py ./examples/src/main/python/mllib/binary_classification_metrics_example.py ./examples/src/main/python/mllib/bisecting_k_means_example.py ./examples/src/main/python/mllib/correlations.py ./examples/src/main/python/mllib/correlations_example.py ./examples/src/main/python/mllib/decision_tree_classification_example.py ./examples/src/main/python/mllib/decision_tree_regression_example.py ./examples/src/ma in/python/mllib/elementwise_product_example.py ./examples/src/main/python/mllib/fpgrowth_example.py ./examples/src/main/python/mllib/gaussian_mixture_example.py ./examples/src/main/python/mllib/gaussian_mixture_model.py ./examples/src/main/python/mllib/gradient_boosting_classification_example.py ./examples/src/main/python/mllib/gradient_boosting_regression_example.py ./examples/src/main/python/mllib/hypothesis_testing_example.py ./examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py ./examples/src/main/python/mllib/isotonic_regression_example.py ./examples/src/main/python/mllib/k_means_example.py ./examples/src/main/python/mllib/kernel_density_estimation_example.py ./examples/src/main/python/mllib/kmeans.py ./examples/src/main/python/mllib/latent_dirichlet_allocation_example.py ./examples/src/main/python/mllib/linear_regression_with_sgd_example.py ./examples/src/main/python/mllib/logistic_regression.py ./examples/src/main/python/mllib/logistic_regress ion_with_lbfgs_example.py ./examples/src/main/python/mllib/multi_class_metrics_example.py ./examples/src/main/python/mllib/multi_label_metrics_example.py ./examples/src/main/python/mllib/naive_bayes_example.py ./examples/src/main/python/mllib/normalizer_example.py ./examples/src/main/python/mllib/power_iteration_clustering_example.py ./examples/src/main/python/mllib/random_forest_classification_example.py ./examples/src/main/python/mllib/random_forest_regression_example.py ./examples/src/main/python/mllib/random_rdd_generation.py ./examples/src/main/python/mllib/ranking_metrics_example.py ./examples/src/main/python/mllib/recommendation_example.py ./examples/src/main/python/mllib/regression_metrics_example.py ./examples/src/main/python/mllib/sampled_rdds.py ./examples/src/main/python/mllib/standard_scaler_example.py ./examples/src/main/python/mllib/stratified_sampling_example.py ./examples/src/main/python/mllib/streaming_k_means_example.py ./examples/src/main/python/mllib/streaming_l inear_regression_example.py ./examples/src/main/python/mllib/summary_statistics_example.py ./examples/src/main/python/mllib/svm_with_sgd_example.py ./examples/src/main/python/mllib/tf_idf_example.py ./examples/src/main/python/mllib/word2vec.py ./examples/src/main/python/mllib/word2vec_example.py ./examples/src/main/python/pagerank.py ./examples/src/main/python/parquet_inputformat.py ./examples/src/main/python/pi.py ./examples/src/main/python/sort.py ./examples/src/main/python/sql/basic.py ./examples/src/main/python/sql/datasource.py ./examples/src/main/python/sql/hive.py ./examples/src/main/python/sql/streaming/structured_kafka_wordcount.py ./examples/src/main/python/sql/streaming/structured_network_wordcount.py ./examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py ./examples/src/main/python/status_api_demo.py ./examples/src/main/python/streaming/direct_kafka_wordcount.py ./examples/src/main/python/streaming/flume_wordcount.py ./examples/src/main/python/s treaming/hdfs_wordcount.py ./examples/src/main/python/streaming/kafka_wordcount.py ./examples/src/main/python/streaming/network_wordcount.py ./examples/src/main/python/streaming/network_wordjoinsentiments.py ./examples/src/main/python/streaming/queue_stream.py ./examples/src/main/python/streaming/recoverable_network_wordcount.py ./examples/src/main/python/streaming/sql_network_wordcount.py ./examples/src/main/python/streaming/stateful_network_wordcount.py ./examples/src/main/python/transitive_closure.py ./examples/src/main/python/wordcount.py ./external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py ./python/docs/epytext.py ./python/pyspark/__init__.py ./python/pyspark/accumulators.py ./python/pyspark/broadcast.py ./python/pyspark/cloudpickle.py ./python/pyspark/conf.py ./python/pyspark/context.py ./python/pyspark/daemon.py ./python/pyspark/files.py ./python/pyspark/find_spark_home.py ./python/pyspark/heapq3.py ./python/pyspark/java_gateway.py ./python/pyspa rk/join.py ./python/pyspark/ml/__init__.py ./python/pyspark/ml/base.py ./python/pyspark/ml/classification.py ./python/pyspark/ml/clustering.py ./python/pyspark/ml/common.py ./python/pyspark/ml/evaluation.py ./python/pyspark/ml/feature.py ./python/pyspark/ml/linalg/__init__.py ./python/pyspark/ml/param/__init__.py ./python/pyspark/ml/param/_shared_params_code_gen.py ./python/pyspark/ml/param/shared.py ./python/pyspark/ml/pipeline.py ./python/pyspark/ml/recommendation.py ./python/pyspark/ml/regression.py ./python/pyspark/ml/tests.py ./python/pyspark/ml/tuning.py ./python/pyspark/ml/util.py ./python/pyspark/ml/wrapper.py ./python/pyspark/mllib/__init__.py ./python/pyspark/mllib/classification.py ./python/pyspark/mllib/clustering.py ./python/pyspark/mllib/common.py ./python/pyspark/mllib/evaluation.py ./python/pyspark/mllib/feature.py ./python/pyspark/mllib/fpm.py ./python/pyspark/mllib/linalg/__init__.py ./python/pyspark/mllib/linalg/distributed.py ./python/pyspark/mllib/random.py ./py thon/pyspark/mllib/recommendation.py ./python/pyspark/mllib/regression.py ./python/pyspark/mllib/stat/__init__.py ./python/pyspark/mllib/stat/_statistics.py ./python/pyspark/mllib/stat/distribution.py ./python/pyspark/mllib/stat/KernelDensity.py ./python/pyspark/mllib/stat/test.py ./python/pyspark/mllib/tests.py ./python/pyspark/mllib/tree.py ./python/pyspark/mllib/util.py ./python/pyspark/profiler.py ./python/pyspark/rdd.py ./python/pyspark/rddsampler.py ./python/pyspark/resultiterable.py ./python/pyspark/serializers.py ./python/pyspark/shell.py ./python/pyspark/shuffle.py ./python/pyspark/sql/__init__.py ./python/pyspark/sql/catalog.py ./python/pyspark/sql/column.py ./python/pyspark/sql/conf.py ./python/pyspark/sql/context.py ./python/pyspark/sql/dataframe.py ./python/pyspark/sql/functions.py ./python/pyspark/sql/group.py ./python/pyspark/sql/readwriter.py ./python/pyspark/sql/session.py ./python/pyspark/sql/streaming.py ./python/pyspark/sql/tests.py ./python/pyspark/sql/types.py ./python/pyspark/sql/utils.py ./python/pyspark/sql/window.py ./python/pyspark/statcounter.py ./python/pyspark/status.py ./python/pyspark/storagelevel.py ./python/pyspark/streaming/__init__.py ./python/pyspark/streaming/context.py ./python/pyspark/streaming/dstream.py ./python/pyspark/streaming/flume.py ./python/pyspark/streaming/kafka.py ./python/pyspark/streaming/kinesis.py ./python/pyspark/streaming/listener.py ./python/pyspark/streaming/tests.py ./python/pyspark/streaming/util.py ./python/pyspark/taskcontext.py ./python/pyspark/tests.py ./python/pyspark/traceback_utils.py ./python/pyspark/version.py ./python/pyspark/worker.py ./python/run-tests.py ./python/setup.py ./python/test_support/SimpleHTTPServer.py ./python/test_support/userlibrary.py ./sql/hive/src/test/resources/data/scripts/cat.py ./sql/hive/src/test/resources/data/scripts/cat_error.py ./sql/hive/src/test/resources/data/scripts/doubleescapedtab.py ./sql/hive/src/test/resources/data/scripts/dumpdata_script.py ./sql/hive /src/test/resources/data/scripts/escapedcarriagereturn.py ./sql/hive/src/test/resources/data/scripts/escapednewline.py ./sql/hive/src/test/resources/data/scripts/escapedtab.py ./sql/hive/src/test/resources/data/scripts/input20_script.py ./sql/hive/src/test/resources/data/scripts/newline.py ./sql/hive/src/test/resources/data/scripts/test_transform.py ```
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org