akashrn5 commented on a change in pull request #4020:
URL: https://github.com/apache/carbondata/pull/4020#discussion_r533944878
##########
File path:
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
##########
@@ -186,6 +187,134 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest
with BeforeAndAfterAll
}
+ def generateData(numOrders: Int = 100000): DataFrame = {
+ import sqlContext.implicits._
+ sqlContext.sparkContext.parallelize(1 to numOrders, 4)
+ .map { x => ("country" + x, x, "07/23/2015", "name" + x, "phonetype" + x
% 10,
+ "serialname" + x, x + 10000)
+ }.toDF("country", "ID", "date", "name", "phonetype", "serialname",
"salary")
+ }
+
+ test("test skip segment whose data size exceed threshold in minor compaction
" +
+ "in system level control and table level") {
+
CarbonProperties.getInstance().addProperty("carbon.compaction.level.threshold",
"2,2")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
+ // set threshold to 1MB in system level
+ CarbonProperties.getInstance().addProperty("carbon.minor.compaction.size",
"1")
+
+ sql("drop table if exists minor_threshold")
+ sql("drop table if exists tmp")
+ sql(
+ "CREATE TABLE IF NOT EXISTS minor_threshold (country String, ID Int,
date" +
+ " Timestamp, name String, phonetype String, serialname String, salary
Int) " +
+ "STORED AS carbondata"
+ )
+ sql(
+ "CREATE TABLE IF NOT EXISTS tmp (country String, ID Int, date
Timestamp," +
+ " name String, phonetype String, serialname String, salary Int) STORED
AS carbondata"
+ )
+ val initframe = generateData(100000)
+ initframe.write
+ .format("carbondata")
+ .option("tablename", "tmp")
+ .mode(SaveMode.Overwrite)
+ .save()
+ // load 3 segments
+ for (i <- 0 to 2) {
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE
minor_threshold" +
+ " OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ }
+ // insert a new segment(id is 3) data size exceed 1 MB
+ sql("insert into minor_threshold select * from tmp")
+ // load another 3 segments
+ for (i <- 0 to 2) {
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE
minor_threshold" +
+ " OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ }
+ // do minor compaction
+ sql("alter table minor_threshold compact 'minor'")
+ // check segment 3 whose size exceed the limit should not be compacted but
success
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(
+ CarbonCommonConstants.DATABASE_DEFAULT_NAME, "minor_threshold")
+ val carbonTablePath = carbonTable.getMetadataPath
+ val segments = SegmentStatusManager.readLoadMetadata(carbonTablePath);
+ assertResult(SegmentStatus.SUCCESS)(segments(3).getSegmentStatus)
+ assertResult(100030)(sql("select count(*) from
minor_threshold").collect().head.get(0))
+
+ // change the threshold to 5MB by dynamic table properties setting, then
the segment whose id is
+ // 3 should be included in minor compaction
+ sql("alter table minor_threshold set
TBLPROPERTIES('minor_compaction_size'='5')")
+ // reload some segments
+ for (i <- 0 to 2) {
+ sql("insert into minor_threshold select * from tmp")
+ }
+ // do minor compaction
+ sql("alter table minor_threshold compact 'minor'")
+ // check segment 3 whose size not exceed the new threshold limit should be
compacted now
+ val segments2 = SegmentStatusManager.readLoadMetadata(carbonTablePath);
+ assertResult(SegmentStatus.COMPACTED)(segments2(3).getSegmentStatus)
+ assertResult(400030)(sql("select count(*) from
minor_threshold").collect().head.get(0))
+
+ // reset to -1
+ CarbonProperties.getInstance().addProperty("carbon.minor.compaction.size",
"-1")
Review comment:
not only resetting this, reset all property which you have set, like
timestamp and threshold. Please check below test case, same for that also.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
