Michael Blow has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/3327
Change subject: DO NOT MERGE: ignore me
......................................................................
DO NOT MERGE: ignore me
Change-Id: I044674f7171b63c24295dd68e1d5de1b82798b50
---
M asterixdb/asterix-app/pom.xml
M asterixdb/asterix-common/pom.xml
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
3 files changed, 26 insertions(+), 6 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/27/3327/1
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index c919856..a96e0a1 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -676,5 +676,9 @@
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-lsm-invertedindex</artifactId>
</dependency>
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml
index 760e027..b4681c1 100644
--- a/asterixdb/asterix-common/pom.xml
+++ b/asterixdb/asterix-common/pom.xml
@@ -290,5 +290,9 @@
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
</dependency>
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
index cb34600..45b6e96 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
@@ -19,27 +19,33 @@
package org.apache.asterix.common.storage;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.function.LongPredicate;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import it.unimi.dsi.fastutil.longs.Long2LongMap;
+import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
public class IndexCheckpoint {
private static final Logger LOGGER = LogManager.getLogger();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final long INITIAL_CHECKPOINT_ID = 0;
+ // TODO(mblow): remove this marker & related logic once we no longer are
able to read indexes prior to the fix
+ private static final long HAS_NULL_MISSING_VALUES_FIX = -1;
private long id;
private long validComponentSequence;
private long lowWatermark;
private long lastComponentId;
- private Map<Long, Long> masterNodeFlushMap;
+ private Long2LongMap masterNodeFlushMap;
public static IndexCheckpoint first(long lastComponentSequence, long
lowWatermark, long validComponentId) {
IndexCheckpoint firstCheckpoint = new IndexCheckpoint();
@@ -47,7 +53,8 @@
firstCheckpoint.lowWatermark = lowWatermark;
firstCheckpoint.validComponentSequence = lastComponentSequence;
firstCheckpoint.lastComponentId = validComponentId;
- firstCheckpoint.masterNodeFlushMap = new HashMap<>();
+ firstCheckpoint.masterNodeFlushMap = new Long2LongOpenHashMap();
+ firstCheckpoint.masterNodeFlushMap.put(HAS_NULL_MISSING_VALUES_FIX,
HAS_NULL_MISSING_VALUES_FIX);
return firstCheckpoint;
}
@@ -66,7 +73,8 @@
next.validComponentSequence = validComponentSequence;
next.masterNodeFlushMap = latest.getMasterNodeFlushMap();
// remove any lsn from the map that wont be used anymore
- next.masterNodeFlushMap.values().removeIf(lsn -> lsn <= lowWatermark);
+ next.masterNodeFlushMap.values()
+ .removeIf((LongPredicate) (lsn -> lsn <= lowWatermark && lsn
!= HAS_NULL_MISSING_VALUES_FIX));
return next;
}
@@ -86,7 +94,7 @@
return lastComponentId;
}
- public Map<Long, Long> getMasterNodeFlushMap() {
+ public Long2LongMap getMasterNodeFlushMap() {
return masterNodeFlushMap;
}
@@ -94,6 +102,10 @@
return id;
}
+ public boolean hasNullMissingValuesFix() {
+ return masterNodeFlushMap.containsKey(HAS_NULL_MISSING_VALUES_FIX);
+ }
+
public String asJson() throws HyracksDataException {
try {
return OBJECT_MAPPER.writeValueAsString(this);
--
To view, visit https://asterix-gerrit.ics.uci.edu/3327
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I044674f7171b63c24295dd68e1d5de1b82798b50
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <[email protected]>
