vikramahuja1001 commented on a change in pull request #3873: URL: https://github.com/apache/carbondata/pull/3873#discussion_r476216833
########## File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/IndexRepairCommand.scala ########## @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command.index + +import java.util + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.execution.command.DataCommand +import org.apache.spark.sql.hive.CarbonRelation +import org.apache.spark.sql.index.CarbonIndexUtil + +import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.metadata.index.IndexType +import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager} +import org.apache.carbondata.core.util.path.CarbonTablePath +import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel} + +/** + * Show indexes on the table + */ +case class IndexRepairCommand(indexname: Option[String], tableNameOp: TableIdentifier, Review comment: done ########## File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/IndexRepairCommand.scala ########## @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command.index + +import java.util + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.execution.command.DataCommand +import org.apache.spark.sql.hive.CarbonRelation +import org.apache.spark.sql.index.CarbonIndexUtil + +import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.metadata.index.IndexType +import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager} +import org.apache.carbondata.core.util.path.CarbonTablePath +import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel} + +/** + * Show indexes on the table Review comment: chnaged ########## File path: index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestIndexRepair.scala ########## @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.spark.testsuite.secondaryindex + +import org.scalatest.BeforeAndAfterAll + +import org.apache.spark.sql.test.util.QueryTest + +/** + * test cases for testing create index table + */ +class TestIndexRepair extends QueryTest with BeforeAndAfterAll { + + override def beforeAll { + sql("drop table if exists maintable") + sql("drop table if exists indextable1") + sql("drop table if exists indextable2") + } + + test("reindex command after deleting segments from SI table") { + sql("drop table if exists maintable") + sql("CREATE TABLE maintable(a INT, b STRING, c STRING) stored as carbondata") + sql("CREATE INDEX indextable1 on table maintable(c) as 'carbondata'") + sql("INSERT INTO maintable SELECT 1,'string1', 'string2'") + sql("INSERT INTO maintable SELECT 1,'string1', 'string2'") + val preDeleteSegments = sql("SHOW SEGMENTS FOR TABLE INDEXTABLE1").count() + sql("DELETE FROM TABLE INDEXTABLE1 WHERE SEGMENT.ID IN(0,1)") + sql("CLEAN FILES FOR TABLE INDEXTABLE1") + val postDeleteSegments = sql("SHOW SEGMENTS FOR TABLE INDEXTABLE1").count() + assert(preDeleteSegments!=postDeleteSegments) + sql("REINDEX INDEX TABLE indextable1 ON MAINTABLE") + val postRepairSegments = sql("SHOW SEGMENTS FOR TABLE INDEXTABLE1").count() + assert(preDeleteSegments == postRepairSegments) + sql("drop table if exists maintable") + } + + + test("reindex command after deleting segments from SI table on other database without use") { + sql("drop table if exists test.maintable") + sql("drop database if exists test cascade") + sql("create database test") + sql("CREATE TABLE test.maintable(a INT, b STRING, c STRING) stored as carbondata") + sql("CREATE INDEX indextable1 on table test.maintable(c) as 'carbondata'") + sql("INSERT INTO test.maintable SELECT 1,'string1', 'string2'") + sql("INSERT INTO test.maintable SELECT 1,'string1', 'string2'") + sql("INSERT INTO test.maintable SELECT 1,'string1', 'string2'") + + val preDeleteSegments = sql("SHOW SEGMENTS FOR TABLE test.INDEXTABLE1").count() + sql("DELETE FROM TABLE test.INDEXTABLE1 WHERE SEGMENT.ID IN(0,1,2)") + sql("CLEAN FILES FOR TABLE test.INDEXTABLE1") + val postDeleteSegments = sql("SHOW SEGMENTS FOR TABLE test.INDEXTABLE1").count() + assert(preDeleteSegments!=postDeleteSegments) + sql("REINDEX INDEX TABLE indextable1 ON test.MAINTABLE") + val postRepairSegments = sql("SHOW SEGMENTS FOR TABLE test.INDEXTABLE1").count() + assert(preDeleteSegments == postRepairSegments) + sql("drop table if exists test.maintable") + sql("drop database if exists test cascade") + } + + test("reindex command using segment.id after deleting segments from SI table") { + sql("drop table if exists maintable") + sql("CREATE TABLE maintable(a INT, b STRING, c STRING) stored as carbondata") + sql("CREATE INDEX indextable1 on table maintable(c) as 'carbondata'") + sql("INSERT INTO maintable SELECT 1,'string1', 'string2'") + sql("INSERT INTO maintable SELECT 1,'string1', 'string2'") + sql("INSERT INTO maintable SELECT 1,'string1', 'string2'") + + val preDeleteSegments = sql("SHOW SEGMENTS FOR TABLE INDEXTABLE1").count() + sql("DELETE FROM TABLE INDEXTABLE1 WHERE SEGMENT.ID IN(0,1,2)") + sql("CLEAN FILES FOR TABLE INDEXTABLE1") + val postDeleteSegments = sql("SHOW SEGMENTS FOR TABLE INDEXTABLE1").count() + assert(preDeleteSegments!=postDeleteSegments) + sql("REINDEX INDEX TABLE indextable1 ON MAINTABLE WHERE SEGMENT.ID IN (0,1)") + val postFirstRepair = sql("SHOW SEGMENTS FOR TABLE INDEXTABLE1").count() + assert(postDeleteSegments + 2 == postFirstRepair) + sql("REINDEX INDEX TABLE indextable1 ON MAINTABLE WHERE SEGMENT.ID IN (2)") Review comment: changed ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
