Change in asterixdb[master]: Add Checkpoint Test

2016-10-25 Thread Murtadha Hubail (Code Review) 
Murtadha Hubail has submitted this change and it was merged.

Change subject: Add Checkpoint Test
..


Add Checkpoint Test

This change adds a unit test case which validates that
checkpoints do not delete log files that are still required
for recovery, and delete those that are no longer needed.

Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1270
Sonar-Qube: Jenkins 
Tested-by: Jenkins 
Integration-Tests: Jenkins 
Reviewed-by: Ian Maxon 
---
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
A 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
M 
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
M 
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
6 files changed, 278 insertions(+), 47 deletions(-)

Approvals:
  Ian Maxon: Looks good to me, approved
  Jenkins: Verified; No violations found; Verified



diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index a806532..88ca736 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -32,11 +32,8 @@
 import org.apache.asterix.common.context.TransactionSubsystemProvider;
 import 
org.apache.asterix.common.dataflow.AsterixLSMInsertDeleteOperatorNodePushable;
 import 
org.apache.asterix.common.dataflow.AsterixLSMTreeInsertDeleteOperatorDescriptor;
-import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.exceptions.AsterixException;
 import 
org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
 import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
-import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
 import org.apache.asterix.common.transactions.ITransactionManager;
 import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
 import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
@@ -46,6 +43,7 @@
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
+import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
 import org.apache.asterix.test.runtime.ExecutionTestUtil;
 import 
org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory;
 import 
org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerProvider;
@@ -67,7 +65,6 @@
 import org.apache.hyracks.api.dataflow.value.ITypeTraits;
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.HyracksException;
 import org.apache.hyracks.api.job.JobId;
 import org.apache.hyracks.api.job.JobSpecification;
 import org.apache.hyracks.api.util.HyracksConstants;
@@ -115,8 +112,12 @@
 private JobId jobId;
 private long jobCounter = 0L;
 private IHyracksJobletContext jobletCtx;
+private final String testConfigFileName;
+private final boolean runHDFS;
 
-public TestNodeController() throws AsterixException, HyracksException, 
ACIDException {
+public TestNodeController(String testConfigFileName, boolean runHDFS) {
+this.testConfigFileName = testConfigFileName;
+this.runHDFS = runHDFS;
 }
 
 public void init() throws Exception {
@@ -125,7 +126,9 @@
 outdir.mkdirs();
 // remove library directory
 TestLibrarian.removeLibraryDir();
-ExecutionTestUtil.setUp(cleanupOnStart);
+ExecutionTestUtil.setUp(cleanupOnStart,
+testConfigFileName == null ? TEST_CONFIG_FILE_NAME : 
testConfigFileName,
+ExecutionTestUtil.integrationUtil, runHDFS);
 } catch (Throwable th) {
 th.printStackTrace();
 throw th;
@@ -299,7 +302,7 @@
 PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, 
primaryKeyTypes, recordType, metaType,
 mergePolicyFactory, mergePolicyProperties, filterFields);
 TreeIndexCreateOperatorDescriptor indexOpDesc =

Change in asterixdb[master]: Add Checkpoint Test

2016-10-25 Thread Murtadha Hubail (Code Review) 
Murtadha Hubail has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 3:

(1 comment)

https://asterix-gerrit.ics.uci.edu/#/c/1270/3/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java:

Line 139: while (logManager.getLogFileIds().size() == 1) {
> Why ==1 and not !=2 here now? Was it getting to 3 somehow and going forever
It was indeed going forever but not because of going to 3. It was caused by 
issue ASTERIXDB-1701. I changed to == 1 while debugging the issue and then 
decided to keep it like that; !=2 should be fine as well.


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: Till Westmann 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: Yes

Change in asterixdb[master]: Add Checkpoint Test

2016-10-24 Thread Ian Maxon (Code Review) 
Ian Maxon has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 3: Code-Review+2

(1 comment)

One question but other than that +2, since this test already proved its worth...

https://asterix-gerrit.ics.uci.edu/#/c/1270/3/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java:

Line 139: while (logManager.getLogFileIds().size() == 1) {
Why ==1 and not !=2 here now? Was it getting to 3 somehow and going forever?


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: Till Westmann 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: Yes

Change in asterixdb[master]: Add Checkpoint Test

2016-10-23 Thread Jenkins (Code Review) 
Jenkins has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 3: Integration-Tests+1

Integration Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1008/ 
: SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: Till Westmann 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: No

Change in asterixdb[master]: Add Checkpoint Test

2016-10-23 Thread Jenkins (Code Review) 
Jenkins has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 3:

Integration Tests Started 
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1008/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: Till Westmann 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: No

Change in asterixdb[master]: Add Checkpoint Test

2016-10-23 Thread Jenkins (Code Review) 
Jenkins has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 3: Integration-Tests-1

Integration Tests Failed

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1007/ 
: UNSTABLE

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: Till Westmann 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: No

Change in asterixdb[master]: Add Checkpoint Test

2016-10-23 Thread Jenkins (Code Review) 
Jenkins has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 3:

Integration Tests Started 
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1007/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: Till Westmann 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: No

Change in asterixdb[master]: Add Checkpoint Test

2016-10-23 Thread Murtadha Hubail (Code Review) 
Murtadha Hubail has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 2:

(2 comments)

https://asterix-gerrit.ics.uci.edu/#/c/1270/2/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java:

Line 193: private void cleanUp() {
> Move this to TestHelper as well?
Done


https://asterix-gerrit.ics.uci.edu/#/c/1270/2/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java:

Line 149: if (initialLowWaterMarkFileId == 
currentLowWaterMarkLogFileId) {
> I see, so it's to assure that it does indeed happen, either by flush or (mo
I added more comments on patchset 3. Hopefully things will be clear. I will be 
happy to answer any questions you might have there.


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: Till Westmann 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: Yes

Change in asterixdb[master]: Add Checkpoint Test

2016-10-23 Thread Murtadha Hubail (Code Review) 
Hello Ian Maxon, Jenkins,

I'd like you to reexamine a change.  Please visit

https://asterix-gerrit.ics.uci.edu/1270

to look at the new patch set (#3).

Change subject: Add Checkpoint Test
..

Add Checkpoint Test

This change adds a unit test case which validates that
checkpoints do not delete log files that are still required
for recovery, and delete those that are no longer needed.

Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
---
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
A 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
M 
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
M 
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
6 files changed, 278 insertions(+), 47 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb 
refs/changes/70/1270/3
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: Till Westmann 
Gerrit-Reviewer: abdullah alamoudi

Change in asterixdb[master]: Add Checkpoint Test

2016-10-20 Thread Till Westmann (Code Review) 
Till Westmann has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 2:

(1 comment)

https://asterix-gerrit.ics.uci.edu/#/c/1270/2/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java:

Line 193: private void cleanUp() {
Move this to TestHelper as well?


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: Till Westmann 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: Yes

Change in asterixdb[master]: Add Checkpoint Test

2016-10-19 Thread Ian Maxon (Code Review) 
Ian Maxon has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 2:

(1 comment)

https://asterix-gerrit.ics.uci.edu/#/c/1270/2/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java:

Line 149: if (initialLowWaterMarkFileId == 
currentLowWaterMarkLogFileId) {
> Even though there is no checkpoint yet, there is a tiny chance (depending o
I see, so it's to assure that it does indeed happen, either by flush or (more 
likely) by what's in this condition. What about the second part after this 
then, the next while loop? It would go something like the checkpoint would be 
done even though it isn't necessary to cause the low water mark to move to the 
2nd file, and then the deletion is checked?


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: Yes

Change in asterixdb[master]: Add Checkpoint Test

2016-10-19 Thread Murtadha Hubail (Code Review) 
Murtadha Hubail has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 2:

(2 comments)

https://asterix-gerrit.ics.uci.edu/#/c/1270/2/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java:

Line 129: ExecutionTestUtil.setUp(cleanupOnStart,
> Is this just to speed things up (i.e. allowing HDFS test cluster to be skip
Yes.


https://asterix-gerrit.ics.uci.edu/#/c/1270/2/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java:

Line 149: if (initialLowWaterMarkFileId == 
currentLowWaterMarkLogFileId) {
> What should happen here if the condition is not true? Isn't it expected as 
Even though there is no checkpoint yet, there is a tiny chance (depending on 
the dataset memory budget configuration) that the dataset is flushed right at 
the beginning of the second log file, so the lowWaterMarkLSN would be in the 
second log file. In that case, this part of the test case will be skipped but 
the log file deletion part will still be tested.


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: Murtadha Hubail 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: Yes

Change in asterixdb[master]: Add Checkpoint Test

2016-10-19 Thread Ian Maxon (Code Review) 
Ian Maxon has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 2: Code-Review+1

(2 comments)

Just a couple questions, otherwise looks great to me.

https://asterix-gerrit.ics.uci.edu/#/c/1270/2/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java:

Line 129: ExecutionTestUtil.setUp(cleanupOnStart,
Is this just to speed things up (i.e. allowing HDFS test cluster to be skipped)?


https://asterix-gerrit.ics.uci.edu/#/c/1270/2/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
File 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java:

Line 149: if (initialLowWaterMarkFileId == 
currentLowWaterMarkLogFileId) {
What should happen here if the condition is not true? Isn't it expected as a 
consequence of not having a checkpoint yet?


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: Yes

Change in asterixdb[master]: Add Checkpoint Test

2016-10-19 Thread Murtadha Hubail (Code Review) 
Hello Jenkins,

I'd like you to reexamine a change.  Please visit

https://asterix-gerrit.ics.uci.edu/1270

to look at the new patch set (#2).

Change subject: Add Checkpoint Test
..

Add Checkpoint Test

This change adds a unit test case which validates that
checkpoints do not delete log files that are still required
for recovery, and delete those that are no longer needed.

Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
---
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
A 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
M 
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
M 
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
6 files changed, 282 insertions(+), 44 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb 
refs/changes/70/1270/2
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: abdullah alamoudi

Change in asterixdb[master]: Add Checkpoint Test

2016-10-19 Thread Jenkins (Code Review) 
Jenkins has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 2:

Build Started 
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/3090/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Ian Maxon 
Gerrit-Reviewer: Jenkins 
Gerrit-Reviewer: Michael Blow 
Gerrit-Reviewer: abdullah alamoudi 
Gerrit-HasComments: No

Change in asterixdb[master]: Add Checkpoint Test

2016-10-12 Thread Jenkins (Code Review) 
Jenkins has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 1: Integration-Tests+1

Integration Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/894/ : 
SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Add Checkpoint Test

2016-10-12 Thread Jenkins (Code Review) 
Jenkins has posted comments on this change.

Change subject: Add Checkpoint Test
..


Patch Set 1:

Integration Tests Started 
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/894/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail 
Gerrit-Reviewer: Jenkins 
Gerrit-HasComments: No

Change in asterixdb[master]: Add Checkpoint Test

2016-10-12 Thread Murtadha Hubail (Code Review) 
Murtadha Hubail has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1270

Change subject: Add Checkpoint Test
..

Add Checkpoint Test

This change adds a unit test case which validates that
checkpoints do not delete log files that are still required
for recovery, and delete those that are no longer needed.

Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
---
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
A 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
M 
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
M 
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
6 files changed, 282 insertions(+), 44 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb 
refs/changes/70/1270/1

diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index a548b3a..bce556d 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -32,11 +32,8 @@
 import org.apache.asterix.common.context.TransactionSubsystemProvider;
 import 
org.apache.asterix.common.dataflow.AsterixLSMInsertDeleteOperatorNodePushable;
 import 
org.apache.asterix.common.dataflow.AsterixLSMTreeInsertDeleteOperatorDescriptor;
-import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.exceptions.AsterixException;
 import 
org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
 import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
-import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
 import org.apache.asterix.common.transactions.ITransactionManager;
 import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
 import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
@@ -46,6 +43,7 @@
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
+import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
 import org.apache.asterix.test.runtime.ExecutionTestUtil;
 import 
org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory;
 import 
org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerProvider;
@@ -67,7 +65,6 @@
 import org.apache.hyracks.api.dataflow.value.ITypeTraits;
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.HyracksException;
 import org.apache.hyracks.api.job.JobId;
 import org.apache.hyracks.api.job.JobSpecification;
 import org.apache.hyracks.api.util.HyracksConstants;
@@ -115,8 +112,12 @@
 private JobId jobId;
 private long jobCounter = 0L;
 private IHyracksJobletContext jobletCtx;
+private final String testConfigFileName;
+private final boolean runHDFS;
 
-public TestNodeController() throws AsterixException, HyracksException, 
ACIDException {
+public TestNodeController(String testConfigFileName, boolean runHDFS) {
+this.testConfigFileName = testConfigFileName;
+this.runHDFS = runHDFS;
 }
 
 public void init() throws Exception {
@@ -125,7 +126,9 @@
 outdir.mkdirs();
 // remove library directory
 TestLibrarian.removeLibraryDir();
-ExecutionTestUtil.setUp(cleanupOnStart);
+ExecutionTestUtil.setUp(cleanupOnStart,
+testConfigFileName == null ? TEST_CONFIG_FILE_NAME : 
testConfigFileName,
+ExecutionTestUtil.integrationUtil, runHDFS);
 } catch (Throwable th) {
 th.printStackTrace();
 throw th;
@@ -299,7 +302,7 @@
 PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, 
primaryKeyTypes, recordType, metaType,
 mergePolicyFactory, mergePolicyProperties, filterFields);
 TreeIndexCreateOperatorDescriptor indexOpDesc = 
getIndexCreateOpDesc(primaryIndexInfo);
-return getPrimaryIndexDataflowHelper(createTestContext(), 
primaryIndexInfo, indexOpDesc);
+return getPrimaryIndexDataflowHelper(createTestContext(true), 
primaryIndexInfo, indexOpDesc);
 }
 
 public void