[2/2] phoenix git commit: PHOENIX-2460 Implement scrutiny command to validate whether or not an index is in sync with the data table (Vincent Poon)

[jamestaylor]

PHOENIX-2460 Implement scrutiny command to validate whether
 or not an index is in sync with the data table (Vincent Poon)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/549d22da
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/549d22da
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/549d22da

Branch: refs/heads/4.x-HBase-1.2
Commit: 549d22da904ef25113b714425271e1ab1f26fdf4
Parents: 4a81d92
Author: James Taylor 
Authored: Mon Aug 28 15:32:20 2017 -0700
Committer: James Taylor 
Committed: Mon Aug 28 15:43:30 2017 -0700

--
 .../phoenix/end2end/IndexScrutinyToolIT.java| 612 +++
 .../mapreduce/index/IndexScrutinyMapper.java| 377 
 .../index/IndexScrutinyTableOutput.java | 345 +++
 .../mapreduce/index/IndexScrutinyTool.java  | 522 
 .../mapreduce/index/PhoenixIndexDBWritable.java |  11 +-
 .../index/PhoenixScrutinyJobCounters.java   |  41 ++
 .../index/SourceTargetColumnNames.java  | 195 ++
 .../mapreduce/util/IndexColumnNames.java| 239 
 .../util/PhoenixConfigurationUtil.java  | 128 +++-
 .../java/org/apache/phoenix/util/QueryUtil.java |  63 +-
 .../org/apache/phoenix/util/SchemaUtil.java |  21 +
 .../phoenix/mapreduce/index/BaseIndexTest.java  |  97 +++
 .../index/IndexScrutinyTableOutputTest.java |  87 +++
 .../mapreduce/util/IndexColumnNamesTest.java|  74 +++
 .../org/apache/phoenix/util/QueryUtilTest.java  |  18 +-
 15 files changed, 2817 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/549d22da/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
new file mode 100644
index 000..5068610
--- /dev/null
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
@@ -0,0 +1,612 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache 
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the 
License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
required by applicable
+ * law or agreed to in writing, software distributed under the License is 
distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 
implied. See the License
+ * for the specific language governing permissions and limitations under the 
License.
+ */
+package org.apache.phoenix.end2end;
+
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.BAD_COVERED_COL_VAL_COUNT;
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.BATCHES_PROCESSED_COUNT;
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.INVALID_ROW_COUNT;
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.VALID_ROW_COUNT;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.phoenix.mapreduce.CsvBulkImportUtil;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTableOutput;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.OutputFormat;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.SourceTable;
+import org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters;
+import org.apache.phoenix.mapreduce.index.SourceTargetColumnNames;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;

[2/2] phoenix git commit: PHOENIX-2460 Implement scrutiny command to validate whether or not an index is in sync with the data table (Vincent Poon)

[jamestaylor]

PHOENIX-2460 Implement scrutiny command to validate whether
 or not an index is in sync with the data table (Vincent Poon)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/124404ae
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/124404ae
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/124404ae

Branch: refs/heads/4.x-HBase-1.1
Commit: 124404ae2db6f49a82d92d5f07010c6390b16746
Parents: cce7ab2
Author: James Taylor 
Authored: Mon Aug 28 15:32:20 2017 -0700
Committer: James Taylor 
Committed: Mon Aug 28 15:42:13 2017 -0700

--
 .../phoenix/end2end/IndexScrutinyToolIT.java| 612 +++
 .../mapreduce/index/IndexScrutinyMapper.java| 377 
 .../index/IndexScrutinyTableOutput.java | 345 +++
 .../mapreduce/index/IndexScrutinyTool.java  | 522 
 .../mapreduce/index/PhoenixIndexDBWritable.java |  11 +-
 .../index/PhoenixScrutinyJobCounters.java   |  41 ++
 .../index/SourceTargetColumnNames.java  | 195 ++
 .../mapreduce/util/IndexColumnNames.java| 239 
 .../util/PhoenixConfigurationUtil.java  | 128 +++-
 .../java/org/apache/phoenix/util/QueryUtil.java |  63 +-
 .../org/apache/phoenix/util/SchemaUtil.java |  21 +
 .../phoenix/mapreduce/index/BaseIndexTest.java  |  97 +++
 .../index/IndexScrutinyTableOutputTest.java |  87 +++
 .../mapreduce/util/IndexColumnNamesTest.java|  74 +++
 .../org/apache/phoenix/util/QueryUtilTest.java  |  18 +-
 15 files changed, 2817 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/124404ae/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
new file mode 100644
index 000..5068610
--- /dev/null
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
@@ -0,0 +1,612 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache 
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the 
License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
required by applicable
+ * law or agreed to in writing, software distributed under the License is 
distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 
implied. See the License
+ * for the specific language governing permissions and limitations under the 
License.
+ */
+package org.apache.phoenix.end2end;
+
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.BAD_COVERED_COL_VAL_COUNT;
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.BATCHES_PROCESSED_COUNT;
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.INVALID_ROW_COUNT;
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.VALID_ROW_COUNT;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.phoenix.mapreduce.CsvBulkImportUtil;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTableOutput;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.OutputFormat;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.SourceTable;
+import org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters;
+import org.apache.phoenix.mapreduce.index.SourceTargetColumnNames;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;

[2/2] phoenix git commit: PHOENIX-2460 Implement scrutiny command to validate whether or not an index is in sync with the data table (Vincent Poon)

[jamestaylor]

PHOENIX-2460 Implement scrutiny command to validate whether
 or not an index is in sync with the data table (Vincent Poon)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/08e2a29f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/08e2a29f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/08e2a29f

Branch: refs/heads/4.x-HBase-0.98
Commit: 08e2a29f60bcedd107f96fc50cfeb8996296d56f
Parents: ce6810f
Author: James Taylor 
Authored: Mon Aug 28 15:32:20 2017 -0700
Committer: James Taylor 
Committed: Mon Aug 28 15:40:56 2017 -0700

--
 .../phoenix/end2end/IndexScrutinyToolIT.java| 612 +++
 .../mapreduce/index/IndexScrutinyMapper.java| 377 
 .../index/IndexScrutinyTableOutput.java | 345 +++
 .../mapreduce/index/IndexScrutinyTool.java  | 522 
 .../mapreduce/index/PhoenixIndexDBWritable.java |  11 +-
 .../index/PhoenixScrutinyJobCounters.java   |  41 ++
 .../index/SourceTargetColumnNames.java  | 195 ++
 .../mapreduce/util/IndexColumnNames.java| 239 
 .../util/PhoenixConfigurationUtil.java  | 128 +++-
 .../java/org/apache/phoenix/util/QueryUtil.java |  63 +-
 .../org/apache/phoenix/util/SchemaUtil.java |  21 +
 .../phoenix/mapreduce/index/BaseIndexTest.java  |  97 +++
 .../index/IndexScrutinyTableOutputTest.java |  87 +++
 .../mapreduce/util/IndexColumnNamesTest.java|  74 +++
 .../org/apache/phoenix/util/QueryUtilTest.java  |  18 +-
 15 files changed, 2817 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/08e2a29f/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
new file mode 100644
index 000..5068610
--- /dev/null
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
@@ -0,0 +1,612 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache 
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the 
License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
required by applicable
+ * law or agreed to in writing, software distributed under the License is 
distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 
implied. See the License
+ * for the specific language governing permissions and limitations under the 
License.
+ */
+package org.apache.phoenix.end2end;
+
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.BAD_COVERED_COL_VAL_COUNT;
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.BATCHES_PROCESSED_COUNT;
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.INVALID_ROW_COUNT;
+import static 
org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.VALID_ROW_COUNT;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.phoenix.mapreduce.CsvBulkImportUtil;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTableOutput;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.OutputFormat;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.SourceTable;
+import org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters;
+import org.apache.phoenix.mapreduce.index.SourceTargetColumnNames;
+import