PHOENIX-1791. Adding ability for Pherf Write Workloads to write to a multi-tenant view.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/42476da8 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/42476da8 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/42476da8 Branch: refs/heads/4.5-HBase-0.98 Commit: 42476da8357f971995d2504acf421dfc7c22e3a9 Parents: 060a00c Author: Jan <jferna...@salesforce.com> Authored: Thu Aug 6 11:12:52 2015 -0700 Committer: James Taylor <jtay...@salesforce.com> Committed: Tue Aug 11 18:36:52 2015 -0700 ---------------------------------------------------------------------- .../org/apache/phoenix/pherf/DataIngestIT.java | 67 +++++++++++++++++--- .../phoenix/pherf/configuration/Scenario.java | 25 ++++++-- .../pherf/configuration/XMLConfigParser.java | 23 ++++--- .../apache/phoenix/pherf/util/PhoenixUtil.java | 30 +++++---- .../phoenix/pherf/workload/WriteWorkload.java | 49 +++++++++----- .../datamodel/test_schema_mt_table.sql | 31 +++++++++ .../resources/datamodel/test_schema_mt_view.sql | 1 + .../test/resources/scenario/test_scenario.xml | 22 +++++++ 8 files changed, 198 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/42476da8/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java index 1defbb1..297f882 100644 --- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java +++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java @@ -22,6 +22,19 @@ import com.jcabi.jdbc.JdbcSession; import com.jcabi.jdbc.Outcome; import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import org.apache.phoenix.pherf.configuration.Column; import org.apache.phoenix.pherf.configuration.DataModel; import org.apache.phoenix.pherf.configuration.DataTypeMapping; @@ -35,15 +48,8 @@ import org.apache.phoenix.pherf.workload.WriteWorkload; import org.junit.Before; import org.junit.Test; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.*; +import com.jcabi.jdbc.JdbcSession; +import com.jcabi.jdbc.Outcome; public class DataIngestIT extends ResultBaseTestIT { @@ -146,4 +152,47 @@ public class DataIngestIT extends ResultBaseTestIT { fail("Failed to load data. An exception was thrown: " + e.getMessage()); } } + + + @Test + /** + * Validates that Pherf can write data to a Multi-Tenant View in addition to + * standard Phoenix tables. + */ + public void testMultiTenantViewWriteWorkload() throws Exception { + // Arrange + Scenario scenario = parser.getScenarioByName("testMTWriteScenario"); + WorkloadExecutor executor = new WorkloadExecutor(); + executor.add(new WriteWorkload(util, parser, scenario, GeneratePhoenixStats.NO)); + + // Act + try { + // Wait for data to load up. + executor.get(); + executor.shutdown(); + } catch (Exception e) { + fail("Failed to load data. An exception was thrown: " + e.getMessage()); + } + + assertExpectedNumberOfRecordsWritten(scenario); + } + + private void assertExpectedNumberOfRecordsWritten(Scenario scenario) throws Exception, + SQLException { + Connection connection = util.getConnection(scenario.getTenantId()); + String sql = "select count(*) from " + scenario.getTableName(); + Integer count = new JdbcSession(connection).sql(sql).select(new Outcome<Integer>() { + @Override public Integer handle(ResultSet resultSet, Statement statement) + throws SQLException { + while (resultSet.next()) { + return resultSet.getInt(1); + } + return null; + } + }); + assertNotNull("Could not retrieve count. " + count); + assertEquals("Expected 100 rows to have been inserted", + scenario.getRowCount(), count.intValue()); + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/42476da8/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java index 7de96cc..6c949d8 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java @@ -18,15 +18,16 @@ package org.apache.phoenix.pherf.configuration; -import org.apache.commons.lang.builder.HashCodeBuilder; -import org.apache.phoenix.pherf.util.PhoenixUtil; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.phoenix.pherf.util.PhoenixUtil; @XmlRootElement(namespace = "org.apache.phoenix.pherf.configuration.DataModel") public class Scenario { @@ -37,6 +38,7 @@ public class Scenario { private List<QuerySet> querySet = new ArrayList<>(); private WriteParams writeParams; private String name; + private String tenantId; public Scenario() { writeParams = new WriteParams(); @@ -162,6 +164,19 @@ public class Scenario { public void setName(String name) { this.name = name; } + + /** + * Tenant Id used by connection of this query + * @return + */ + @XmlAttribute + public String getTenantId() { + return tenantId; + } + + public void setTenantId(String tenantId) { + this.tenantId = tenantId; + } public WriteParams getWriteParams() { return writeParams; http://git-wip-us.apache.org/repos/asf/phoenix/blob/42476da8/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java index 393fa7e..93dc94c 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java @@ -18,16 +18,6 @@ package org.apache.phoenix.pherf.configuration; -import org.apache.phoenix.pherf.PherfConstants; -import org.apache.phoenix.pherf.exception.FileLoaderException; -import org.apache.phoenix.pherf.util.ResourceList; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; import java.io.OutputStream; import java.nio.file.Path; import java.util.ArrayList; @@ -35,6 +25,17 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.phoenix.pherf.PherfConstants; +import org.apache.phoenix.pherf.exception.FileLoaderException; +import org.apache.phoenix.pherf.util.ResourceList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class XMLConfigParser { private static final Logger logger = LoggerFactory.getLogger(XMLConfigParser.class); @@ -134,6 +135,8 @@ public class XMLConfigParser { if (fullTableName.contains(".")) { ret = fullTableName.substring(fullTableName.indexOf(".") + 1, fullTableName.length()); } + // Remove any quotes that may be needed for multi-tenant tables + ret = ret.replaceAll("\"", ""); return ret; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/42476da8/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java index 5b223b1..19b6bd2 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java @@ -18,21 +18,29 @@ package org.apache.phoenix.pherf.util; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + import org.apache.phoenix.pherf.PherfConstants; import org.apache.phoenix.pherf.configuration.Column; import org.apache.phoenix.pherf.configuration.DataTypeMapping; - -import java.sql.*; -import java.util.*; - import org.apache.phoenix.pherf.configuration.Query; import org.apache.phoenix.pherf.configuration.QuerySet; +import org.apache.phoenix.pherf.configuration.Scenario; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; - // TODO This class needs to be cleanup up a bit. I just wanted to get an initial placeholder in. public class PhoenixUtil { private static final Logger logger = LoggerFactory.getLogger(PhoenixUtil.class); @@ -80,11 +88,11 @@ public class PhoenixUtil { return DriverManager.getConnection(url, props); } - public boolean executeStatement(String sql) throws Exception { + public boolean executeStatement(String sql, Scenario scenario) throws Exception { Connection connection = null; boolean result = false; try { - connection = getConnection(); + connection = getConnection(scenario.getTenantId()); result = executeStatement(sql, connection); } finally { if (connection != null) { @@ -262,8 +270,8 @@ public class PhoenixUtil { * @param tableName * @throws Exception */ - public void updatePhoenixStats(String tableName) throws Exception { + public void updatePhoenixStats(String tableName, Scenario scenario) throws Exception { logger.info("Updating stats for " + tableName); - executeStatement("UPDATE STATISTICS " + tableName); + executeStatement("UPDATE STATISTICS " + tableName, scenario); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/42476da8/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java index b6686c6..d0b99af 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java @@ -18,6 +18,21 @@ package org.apache.phoenix.pherf.workload; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + import org.apache.phoenix.pherf.PherfConstants; import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import org.apache.phoenix.pherf.configuration.Column; @@ -35,17 +50,6 @@ import org.apache.phoenix.pherf.util.RowCalculator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigDecimal; -import java.sql.*; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - public class WriteWorkload implements Workload { private static final Logger logger = LoggerFactory.getLogger(WriteWorkload.class); private final PhoenixUtil pUtil; @@ -163,15 +167,18 @@ public class WriteWorkload implements Workload { List<Future> writeBatches = getBatches(dataLoadThreadTime, scenario); waitForBatches(dataLoadTimeSummary, scenario, start, writeBatches); - + // Update Phoenix Statistics if (this.generateStatistics == GeneratePhoenixStats.YES) { logger.info("Updating Phoenix table statistics..."); - pUtil.updatePhoenixStats(scenario.getTableName()); + pUtil.updatePhoenixStats(scenario.getTableName(), scenario); logger.info("Stats update done!"); } else { logger.info("Phoenix table stats update not requested."); } + + // always update stats for Phoenix base tables + updatePhoenixStats(scenario.getTableName(), scenario); } private List<Future> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario scenario) @@ -185,7 +192,7 @@ public class WriteWorkload implements Workload { List<Column> phxMetaCols = pUtil.getColumnsFromPhoenix(scenario.getSchemaName(), - scenario.getTableNameWithoutSchemaName(), pUtil.getConnection()); + scenario.getTableNameWithoutSchemaName(), pUtil.getConnection(scenario.getTenantId())); int threadRowCount = rowCalculator.getNext(); logger.info( "Kick off thread (#" + i + ")for upsert with (" + threadRowCount + ") rows."); @@ -221,6 +228,18 @@ public class WriteWorkload implements Workload { .add(scenario.getTableName(), sumRows, (int) (System.currentTimeMillis() - start)); } + /** + * TODO Move this method to PhoenixUtil + * Update Phoenix table stats + * + * @param tableName + * @throws Exception + */ + public void updatePhoenixStats(String tableName, Scenario scenario) throws Exception { + logger.info("Updating stats for " + tableName); + pUtil.executeStatement("UPDATE STATISTICS " + tableName, scenario); + } + public Future<Info> upsertData(final Scenario scenario, final List<Column> columns, final String tableName, final int rowCount, final DataLoadThreadTime dataLoadThreadTime) { @@ -231,7 +250,7 @@ public class WriteWorkload implements Workload { SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); Connection connection = null; try { - connection = pUtil.getConnection(); + connection = pUtil.getConnection(scenario.getTenantId()); long logStartTime = System.currentTimeMillis(); long maxDuration = http://git-wip-us.apache.org/repos/asf/phoenix/blob/42476da8/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql new file mode 100644 index 0000000..b6791bf --- /dev/null +++ b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql @@ -0,0 +1,31 @@ +/* + -- Licensed to the Apache Software Foundation (ASF) under one + -- or more contributor license agreements. See the NOTICE file + -- distributed with this work for additional information + -- regarding copyright ownership. The ASF licenses this file + -- to you under the Apache License, Version 2.0 (the + -- "License"); you may not use this file except in compliance + -- with the License. You may obtain a copy of the License at + -- + -- http://www.apache.org/licenses/LICENSE-2.0 + -- + -- Unless required by applicable law or agreed to in writing, software + -- distributed under the License is distributed on an "AS IS" BASIS, + -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + -- See the License for the specific language governing permissions and + -- limitations under the License. +*/ +CREATE TABLE IF NOT EXISTS PHERF.TEST_MULTI_TENANT_TABLE ( + TENANT_ID CHAR(15) NOT NULL, + IDENTIFIER CHAR(3) NOT NULL, + ID CHAR(15) NOT NULL, + CREATED_DATE DATE, + FIELD VARCHAR, + SOME_INT INTEGER + CONSTRAINT PK PRIMARY KEY + ( + TENANT_ID, + IDENTIFIER, + ID + ) +) VERSIONS=1,MULTI_TENANT=true http://git-wip-us.apache.org/repos/asf/phoenix/blob/42476da8/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql new file mode 100644 index 0000000..5f5d7ec --- /dev/null +++ b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql @@ -0,0 +1 @@ +CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/42476da8/phoenix-pherf/src/test/resources/scenario/test_scenario.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml index fddf022..b5fe564 100644 --- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml +++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml @@ -127,6 +127,25 @@ <name>NEWVAL_STRING</name> <prefix>TSTPRFX</prefix> </column> + <column> + <type>CHAR</type> + <length>3</length> + <userDefined>true</userDefined> + <dataSequence>LIST</dataSequence> + <name>IDENTIFIER</name> + <valuelist> + <!-- Distributes according to specified values. These must total 100 --> + <datavalue distribution="60"> + <value>ABC</value> + </datavalue> + <datavalue distribution="20"> + <value>XYZ</value> + </datavalue> + <datavalue distribution="20"> + <value>LMN</value> + </datavalue> + </valuelist> + </column> </datamapping> <scenarios> <scenario tableName="PHERF.TEST_TABLE" rowCount="100" name="testScenarioRW"> @@ -201,5 +220,8 @@ <query id="q4" statement="select sum(SOME_INT) from PHERF.TEST_TABLE"/> </querySet> </scenario> + <!-- Test writing to a Multi-tenant View --> + <scenario tableName="PHERF.TEST_VIEW" tenantId="abcdefghijklmno" rowCount="100" name="testMTWriteScenario"> + </scenario> </scenarios> </datamodel> \ No newline at end of file
