SilverNarcissus commented on a change in pull request #1693: URL: https://github.com/apache/incubator-iotdb/pull/1693#discussion_r494795361
########## File path: server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java ########## @@ -267,6 +269,279 @@ public void testSeqAndUnSeqSyncClose() } } + @Test + public void testEnableDiscardOutOfOrderDataForInsertRowPlan() + throws WriteProcessException, QueryProcessException, IllegalPathException, IOException { + IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + boolean defaultValue = config.isEnableDiscardOutOfOrderData(); + config.setEnableDiscardOutOfOrderData(true); + + for (int j = 21; j <= 30; j++) { + TSRecord record = new TSRecord(j, deviceId); + record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); + insertToStorageGroupProcessor(record); + processor.asyncCloseAllWorkingTsFileProcessors(); + } + processor.syncCloseAllWorkingTsFileProcessors(); + + for (int j = 10; j >= 1; j--) { + TSRecord record = new TSRecord(j, deviceId); + record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); + insertToStorageGroupProcessor(record); + processor.asyncCloseAllWorkingTsFileProcessors(); + } + + processor.syncCloseAllWorkingTsFileProcessors(); + + for (TsFileProcessor tsfileProcessor : processor.getWorkUnsequenceTsFileProcessor()) { + tsfileProcessor.syncFlush(); + } + + QueryDataSource queryDataSource = processor.query(new PartialPath(deviceId), measurementId, context, + null, null); + Assert.assertEquals(10, queryDataSource.getSeqResources().size()); + Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + for (TsFileResource resource : queryDataSource.getSeqResources()) { + Assert.assertTrue(resource.isClosed()); + } + for (TsFileResource resource : queryDataSource.getUnseqResources()) { + Assert.assertTrue(resource.isClosed()); + } + + config.setEnableDiscardOutOfOrderData(defaultValue); + } + + @Test + public void testEnableDiscardOutOfOrderDataForInsertTablet1() + throws QueryProcessException, IllegalPathException, IOException { + IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + boolean defaultValue = config.isEnableDiscardOutOfOrderData(); + config.setEnableDiscardOutOfOrderData(true); + long defaultTimePartition = IoTDBDescriptor.getInstance().getConfig().getPartitionInterval(); + config.setPartitionInterval(100); + + String[] measurements = new String[2]; + measurements[0] = "s0"; + measurements[1] = "s1"; + List<Integer> dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT32.ordinal()); + dataTypes.add(TSDataType.INT64.ordinal()); + + MeasurementMNode[] measurementMNodes = new MeasurementMNode[2]; + measurementMNodes[0] = new MeasurementMNode(null, "s0", + new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.PLAIN), null); + measurementMNodes[1] = new MeasurementMNode(null, "s1", + new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN), null); + + InsertTabletPlan insertTabletPlan1 = new InsertTabletPlan(new PartialPath("root.vehicle.d0"), measurements, + dataTypes); + + long[] times = new long[100]; + Object[] columns = new Object[2]; + columns[0] = new int[100]; + columns[1] = new long[100]; + + for (int r = 0; r < 100; r++) { + times[r] = r; + ((int[]) columns[0])[r] = 1; + ((long[]) columns[1])[r] = 1; + } + insertTabletPlan1.setTimes(times); + insertTabletPlan1.setColumns(columns); + insertTabletPlan1.setRowCount(times.length); + insertTabletPlan1.setMeasurementMNodes(measurementMNodes); + + processor.insertTablet(insertTabletPlan1); + processor.asyncCloseAllWorkingTsFileProcessors(); + + InsertTabletPlan insertTabletPlan2 = new InsertTabletPlan(new PartialPath("root.vehicle.d0"), measurements, + dataTypes); + + for (int r = 50; r < 149; r++) { + times[r - 50] = r; + ((int[]) columns[0])[r - 50] = 1; + ((long[]) columns[1])[r - 50] = 1; + } + insertTabletPlan2.setTimes(times); + insertTabletPlan2.setColumns(columns); + insertTabletPlan2.setRowCount(times.length); + insertTabletPlan2.setMeasurementMNodes(measurementMNodes); + + processor.insertTablet(insertTabletPlan2); + processor.asyncCloseAllWorkingTsFileProcessors(); + processor.syncCloseAllWorkingTsFileProcessors(); + + for (TsFileProcessor tsfileProcessor : processor.getWorkUnsequenceTsFileProcessor()) { + tsfileProcessor.syncFlush(); + } + + QueryDataSource queryDataSource = processor.query(new PartialPath(deviceId), measurementId, context, + null, null); + + Assert.assertEquals(2, queryDataSource.getSeqResources().size()); + Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + for (TsFileResource resource : queryDataSource.getSeqResources()) { + Assert.assertTrue(resource.isClosed()); + } + + config.setEnableDiscardOutOfOrderData(defaultValue); + config.setPartitionInterval(defaultTimePartition); + } + + @Test + public void testEnableDiscardOutOfOrderDataForInsertTablet2() + throws QueryProcessException, IllegalPathException, IOException { + IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + boolean defaultValue = config.isEnableDiscardOutOfOrderData(); + config.setEnableDiscardOutOfOrderData(true); + long defaultTimePartition = IoTDBDescriptor.getInstance().getConfig().getPartitionInterval(); + config.setPartitionInterval(500); + + String[] measurements = new String[2]; + measurements[0] = "s0"; + measurements[1] = "s1"; + List<Integer> dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT32.ordinal()); + dataTypes.add(TSDataType.INT64.ordinal()); + + MeasurementMNode[] measurementMNodes = new MeasurementMNode[2]; + measurementMNodes[0] = new MeasurementMNode(null, "s0", + new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.PLAIN), null); + measurementMNodes[1] = new MeasurementMNode(null, "s1", + new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN), null); + + InsertTabletPlan insertTabletPlan1 = new InsertTabletPlan(new PartialPath("root.vehicle.d0"), measurements, + dataTypes); + + long[] times = new long[100]; + Object[] columns = new Object[2]; + columns[0] = new int[100]; + columns[1] = new long[100]; + + for (int r = 0; r < 100; r++) { + times[r] = r; + ((int[]) columns[0])[r] = 1; + ((long[]) columns[1])[r] = 1; + } + insertTabletPlan1.setTimes(times); + insertTabletPlan1.setColumns(columns); + insertTabletPlan1.setRowCount(times.length); + insertTabletPlan1.setMeasurementMNodes(measurementMNodes); + + processor.insertTablet(insertTabletPlan1); + processor.asyncCloseAllWorkingTsFileProcessors(); + + InsertTabletPlan insertTabletPlan2 = new InsertTabletPlan(new PartialPath("root.vehicle.d0"), measurements, + dataTypes); + + for (int r = 50; r < 149; r++) { + times[r - 50] = r; + ((int[]) columns[0])[r - 50] = 1; + ((long[]) columns[1])[r - 50] = 1; + } + insertTabletPlan2.setTimes(times); + insertTabletPlan2.setColumns(columns); + insertTabletPlan2.setRowCount(times.length); + insertTabletPlan2.setMeasurementMNodes(measurementMNodes); + + processor.insertTablet(insertTabletPlan2); + processor.asyncCloseAllWorkingTsFileProcessors(); + processor.syncCloseAllWorkingTsFileProcessors(); + + for (TsFileProcessor tsfileProcessor : processor.getWorkUnsequenceTsFileProcessor()) { + tsfileProcessor.syncFlush(); + } + + QueryDataSource queryDataSource = processor.query(new PartialPath(deviceId), measurementId, context, + null, null); + + Assert.assertEquals(2, queryDataSource.getSeqResources().size()); + Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + for (TsFileResource resource : queryDataSource.getSeqResources()) { + Assert.assertTrue(resource.isClosed()); + } + + config.setEnableDiscardOutOfOrderData(defaultValue); + config.setPartitionInterval(defaultTimePartition); + } + + @Test + public void testEnableDiscardOutOfOrderDataForInsertTablet3() + throws QueryProcessException, IllegalPathException, IOException { + IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + boolean defaultValue = config.isEnableDiscardOutOfOrderData(); + config.setEnableDiscardOutOfOrderData(true); + long defaultTimePartition = IoTDBDescriptor.getInstance().getConfig().getPartitionInterval(); + config.setPartitionInterval(1000); Review comment: Sure, you set the time partition. But your insert time range only from 0 -150 which never cross the partition interval. By the way, you should also enable the partition like this: ``` config.setEnablePartition(true); ``` and do not forget: ``` config.setEnablePartition(false); ``` Thank you very much~ ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org