This is an automated email from the ASF dual-hosted git repository.
jin pushed a commit to branch master
in repository
https://gitbox.apache.org/repos/asf/incubator-hugegraph-toolchain.git
The following commit(s) were added to refs/heads/master by this push:
new e688036e feat(hbase): bypass server for hbase writing (BETA) (#347)
e688036e is described below
commit e688036ec693698f52d37dead2756c5347cd84f7
Author: YangJiaqi <[email protected]>
AuthorDate: Thu Nov 3 22:40:53 2022 +0800
feat(hbase): bypass server for hbase writing (BETA) (#347)
Co-authored-by: imbajin <[email protected]>
---
hugegraph-client/pom.xml | 6 +-
.../com/baidu/hugegraph/example/BatchExample.java | 2 +-
.../serializer/direct/HBaseSerializer.java | 124 ++++
.../direct/RocksDBSerializer.java} | 36 +-
.../serializer/direct/reuse/BytesDemo.java | 194 ++++++
.../serializer/direct/struct/DataType.java | 205 ++++++
.../serializer/direct/struct/HugeType.java | 190 ++++++
.../serializer/direct/util/BytesBuffer.java | 759 +++++++++++++++++++++
.../serializer/direct/util/GraphSchema.java | 105 +++
.../serializer/direct/util/HugeException.java | 56 ++
.../baidu/hugegraph/serializer/direct/util/Id.java | 88 +++
.../serializer/direct/util/IdGenerator.java | 440 ++++++++++++
.../hugegraph/serializer/direct/util/LZ4Util.java | 93 +++
.../direct/util/SplicingIdGenerator.java | 132 ++++
.../serializer/direct/util/StringEncoding.java | 153 +++++
.../baidu/hugegraph/structure/GraphElement.java | 4 +
.../hugegraph/structure/constant/Direction.java | 4 +-
.../hugegraph/structure/constant/HugeType.java | 4 +-
.../com/baidu/hugegraph/structure/graph/Edge.java | 1 +
.../baidu/hugegraph/structure/graph/Vertex.java | 1 +
.../hugegraph/loader/builder/EdgeBuilder.java | 4 +-
.../loader/flink/HugeGraphOutputFormat.java | 5 +-
.../baidu/hugegraph/loader/util/DataTypeUtil.java | 2 +-
23 files changed, 2571 insertions(+), 37 deletions(-)
diff --git a/hugegraph-client/pom.xml b/hugegraph-client/pom.xml
index 7321b4c3..95edf27f 100644
--- a/hugegraph-client/pom.xml
+++ b/hugegraph-client/pom.xml
@@ -46,7 +46,11 @@
<artifactId>hugegraph-common</artifactId>
<version>${hugegraph.common.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ <version>1.4.0</version>
+ </dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/example/BatchExample.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/example/BatchExample.java
index 94f59e5e..8f2da31f 100644
---
a/hugegraph-client/src/main/java/com/baidu/hugegraph/example/BatchExample.java
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/example/BatchExample.java
@@ -31,7 +31,7 @@ import com.baidu.hugegraph.structure.graph.Vertex;
public class BatchExample {
public static void main(String[] args) {
- // If connect failed will throw a exception.
+ // If connect failed will throw an exception.
HugeClient hugeClient = HugeClient.builder("http://localhost:8080";,
"hugegraph").build();
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/HBaseSerializer.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/HBaseSerializer.java
new file mode 100644
index 00000000..024618fa
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/HBaseSerializer.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct;
+
+import com.baidu.hugegraph.driver.HugeClient;
+import com.baidu.hugegraph.serializer.direct.struct.HugeType;
+import com.baidu.hugegraph.serializer.direct.util.BytesBuffer;
+import com.baidu.hugegraph.serializer.direct.util.GraphSchema;
+import com.baidu.hugegraph.serializer.direct.util.Id;
+import com.baidu.hugegraph.serializer.direct.util.IdGenerator;
+import com.baidu.hugegraph.structure.GraphElement;
+import com.baidu.hugegraph.structure.graph.Edge;
+import com.baidu.hugegraph.structure.schema.PropertyKey;
+
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * TODO: review later
+ */
+public class HBaseSerializer {
+
+ private int edgeLogicPartitions = 30;
+ private int vertexLogicPartitions = 10;
+ private HugeClient client;
+ private GraphSchema graphSchema;
+
+
+ public HBaseSerializer(HugeClient client, int vertexPartitions, int
edgePartitions){
+ this.client = client;
+ this.graphSchema = new GraphSchema(client);
+ this.edgeLogicPartitions = edgePartitions;
+ this.vertexLogicPartitions = vertexPartitions;
+ }
+
+ public byte[] getKeyBytes(GraphElement e) {
+ byte[] array = null;
+ if(e.type() == "vertex" && e.id() != null){
+ BytesBuffer buffer = BytesBuffer.allocate(2 + 1 +
e.id().toString().length());
+ buffer.writeShort(getPartition(HugeType.VERTEX,
IdGenerator.of(e.id())));
+ buffer.writeId(IdGenerator.of(e.id()));
+ array = buffer.bytes();
+ }else if ( e.type() == "edge" ){
+ BytesBuffer buffer = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID);
+ Edge edge = (Edge)e;
+ buffer.writeShort(getPartition(HugeType.EDGE,
IdGenerator.of(edge.sourceId())));
+ buffer.writeId(IdGenerator.of(edge.sourceId()));
+ buffer.write(HugeType.EDGE_OUT.code());
+
buffer.writeId(IdGenerator.of(graphSchema.getEdgeLabel(e.label()).id())); //出现错误
+ buffer.writeStringWithEnding("");
+ buffer.writeId(IdGenerator.of(edge.targetId()));
+ array = buffer.bytes();
+ }
+ return array;
+ }
+
+ public byte[] getValueBytes(GraphElement e) {
+ byte[] array = null;
+ if(e.type() == "vertex"){
+ int propsCount = e.properties().size() ;
//vertex.sizeOfProperties();
+ BytesBuffer buffer = BytesBuffer.allocate(8 + 16 * propsCount);
+
buffer.writeId(IdGenerator.of(graphSchema.getVertexLabel(e.label()).id()));
+ buffer.writeVInt(propsCount);
+ for(Map.Entry<String, Object> entry : e.properties().entrySet()){
+ PropertyKey propertyKey =
graphSchema.getPropertyKey(entry.getKey());
+ buffer.writeVInt(propertyKey.id().intValue());
+ buffer.writeProperty(propertyKey.dataType(),entry.getValue());
+ }
+ array = buffer.bytes();
+ } else if ( e.type() == "edge" ){
+ int propsCount = e.properties().size();
+ BytesBuffer buffer = BytesBuffer.allocate(4 + 16 * propsCount);
+ buffer.writeVInt(propsCount);
+ for(Map.Entry<String, Object> entry : e.properties().entrySet()){
+ PropertyKey propertyKey =
graphSchema.getPropertyKey(entry.getKey());
+ buffer.writeVInt(propertyKey.id().intValue());
+ buffer.writeProperty(propertyKey.dataType(),entry.getValue());
+ }
+ array = buffer.bytes();
+ }
+
+ return array;
+ }
+
+ public short getPartition(HugeType type, Id id) {
+ int hashcode = Arrays.hashCode(id.asBytes());
+ short partition = 1;
+ if (type.isEdge()) {
+ partition = (short) (hashcode % edgeLogicPartitions);
+ } else if (type.isVertex()) {
+ partition = (short) (hashcode % vertexLogicPartitions);
+ }
+ return partition > 0 ? partition : (short) -partition;
+ }
+
+ public int getEdgeLogicPartitions(){
+ return this.edgeLogicPartitions;
+ }
+
+ public int getVertexLogicPartitions(){
+ return this.vertexLogicPartitions;
+ }
+
+ public void close(){
+ this.client.close();
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/RocksDBSerializer.java
similarity index 65%
copy from
hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java
copy to
hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/RocksDBSerializer.java
index 7d93c211..de964f3c 100644
---
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/RocksDBSerializer.java
@@ -17,30 +17,16 @@
* under the License.
*/
-package com.baidu.hugegraph.structure.constant;
+package com.baidu.hugegraph.serializer.direct;
-public enum Direction {
-
- OUT(1, "out"),
-
- IN(2, "in"),
-
- BOTH(3, "both");
-
- private byte code = 0;
- private String name = null;
-
- Direction(int code, String name) {
- assert code < 256;
- this.code = (byte) code;
- this.name = name;
- }
-
- public byte code() {
- return this.code;
- }
-
- public String string() {
- return this.name;
- }
+/**
+ * TODO: review later
+ *
+ * In this serializer, we only support normal type now:
+ * - number
+ * - string
+ * And they will be transferred to bytes directly
+ **/
+public class RocksDBSerializer {
+ // TODO: Support write RocksDB directly
}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/reuse/BytesDemo.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/reuse/BytesDemo.java
new file mode 100644
index 00000000..dc3d9967
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/reuse/BytesDemo.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.reuse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.baidu.hugegraph.driver.GraphManager;
+import com.baidu.hugegraph.driver.HugeClient;
+import com.baidu.hugegraph.driver.SchemaManager;
+import com.baidu.hugegraph.serializer.direct.HBaseSerializer;
+import com.baidu.hugegraph.serializer.direct.RocksDBSerializer;
+import com.baidu.hugegraph.structure.graph.Edge;
+import com.baidu.hugegraph.structure.graph.Vertex;
+
+/**
+ * @author jin
+ * This class is a demo for rocksdb/HBase put(rowkey, values) which use
Client-Side's graph struct
+ * And we don't need to construct the graph element, just use it and transfer
them to bytes array
+ * instead of json format
+ */
+public class BytesDemo {
+
+ static HugeClient client;
+ boolean bypassServer = true;
+ RocksDBSerializer ser;
+ HBaseSerializer HBaseSer;
+
+ public static void main(String[] args) {
+ BytesDemo ins = new BytesDemo();
+ ins.initGraph();
+ }
+
+ void initGraph() {
+ int edgeLogicPartitions = 16;
+ int vertexLogicPartitions = 8;
+ // If connect failed will throw an exception.
+ client = HugeClient.builder("http://localhost:8081";,
"hugegraph").build();
+
+ SchemaManager schema = client.schema();
+
+
+ schema.propertyKey("name").asText().ifNotExist().create();
+ schema.propertyKey("age").asInt().ifNotExist().create();
+ schema.propertyKey("lang").asText().ifNotExist().create();
+ schema.propertyKey("date").asText().ifNotExist().create();
+ schema.propertyKey("price").asText().ifNotExist().create();
+
+ schema.vertexLabel("person")
+ .properties("name", "age")
+ .useCustomizeStringId()
+ .enableLabelIndex(false)
+ .ifNotExist()
+ .create();
+
+ schema.vertexLabel("personB")
+ .properties("price")
+ .nullableKeys("price")
+ .useCustomizeNumberId()
+ .enableLabelIndex(false)
+ .ifNotExist()
+ .create();
+
+ schema.vertexLabel("software")
+ .properties("name", "lang", "price")
+ .useCustomizeStringId()
+ .enableLabelIndex(false)
+ .ifNotExist()
+ .create();
+
+ schema.edgeLabel("knows")
+ .link("person", "person")
+ .properties("date")
+ .enableLabelIndex(false)
+ .ifNotExist()
+ .create();
+
+ schema.edgeLabel("created")
+ .link("person", "software")
+ .properties("date")
+ .enableLabelIndex(false)
+ .ifNotExist()
+ .create();
+
+ HBaseSer = new HBaseSerializer(client, vertexLogicPartitions,
edgeLogicPartitions);
+ writeGraphElements();
+
+ client.close();
+ }
+
+ private void writeGraphElements() {
+ GraphManager graph = client.graph();
+ // construct some vertexes & edges
+ Vertex peter = new Vertex("person");
+ peter.property("name", "peter");
+ peter.property("age", 35);
+ peter.id("peter");
+
+ Vertex lop = new Vertex("software");
+ lop.property("name", "lop");
+ lop.property("lang", "java");
+ lop.property("price", "328");
+ lop.id("lop");
+
+ Vertex vadasB = new Vertex("personB");
+ vadasB.property("price", "120");
+ vadasB.id(12345);
+
+ Edge peterCreateLop = new
Edge("created").source(peter).target(lop).property("date", "2017-03-24");
+
+ List<Vertex> vertices = new ArrayList<Vertex>(){{
+ add(peter);add(lop);add(vadasB);
+ }};
+
+
+ List<Edge> edges = new ArrayList<Edge>(){{
+ add(peterCreateLop);
+ }};
+
+ // Old way: encode to json then send to server
+ if (bypassServer) {
+ writeDirectly(vertices, edges);
+ } else {
+ writeByServer(graph, vertices, edges);
+ }
+ }
+
+ /* we transfer the vertex & edge into bytes array
+ * TODO: use a batch and send them together
+ * */
+ void writeDirectly(List<Vertex> vertices, List<Edge> edges) {
+ for (Vertex vertex : vertices) {
+ byte[] rowkey = HBaseSer.getKeyBytes(vertex);
+ byte[] values = HBaseSer.getValueBytes(vertex);
+ sendRpcToHBase("vertex", rowkey, values);
+ }
+
+ for (Edge edge: edges) {
+ byte[] rowkey = HBaseSer.getKeyBytes(edge);
+ byte[] values = HBaseSer.getValueBytes(edge);
+ sendRpcToHBase("edge", rowkey, values);
+ }
+ }
+
+ boolean sendRpcToRocksDB(byte[] rowkey, byte[] values) {
+ // here we call the rpc
+ boolean flag = false;
+ //flag = put(rowkey, values);
+ return flag;
+ }
+
+ void writeByServer(GraphManager graph, List<Vertex> vertices, List<Edge>
edges) {
+ vertices = graph.addVertices(vertices);
+ vertices.forEach(System.out::println);
+
+ edges = graph.addEdges(edges, false);
+ edges.forEach(System.out::println);
+ }
+
+ boolean sendRpcToHBase(String type, byte[] rowkey, byte[] values) {
+ boolean flag = false;
+ try {
+ flag = put(type, rowkey, values);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return flag;
+ }
+
+
+ boolean put (String type, byte[] rowkey, byte[] values) throws IOException
{
+ // TODO: put to HBase
+ return true;
+ }
+
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/DataType.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/DataType.java
new file mode 100644
index 00000000..a12aa460
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/DataType.java
@@ -0,0 +1,205 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.struct;
+
+import java.util.Date;
+import java.util.UUID;
+
+import com.baidu.hugegraph.serializer.direct.util.HugeException;
+import com.baidu.hugegraph.serializer.direct.util.StringEncoding;
+import com.baidu.hugegraph.util.CollectionUtil;
+import com.baidu.hugegraph.util.DateUtil;
+import com.baidu.hugegraph.util.E;
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.Table;
+
+public enum DataType {
+
+ UNKNOWN(0, "unknown", Object.class),
+ OBJECT(1, "object", Object.class),
+ BOOLEAN(2, "boolean", Boolean.class),
+ BYTE(3, "byte", Byte.class),
+ INT(4, "int", Integer.class),
+ LONG(5, "long", Long.class),
+ FLOAT(6, "float", Float.class),
+ DOUBLE(7, "double", Double.class),
+ TEXT(8, "text", String.class),
+ //BLOB(9, "blob", Blob.class),
+ DATE(10, "date", Date.class),
+ UUID(11, "uuid", UUID.class);
+
+ private final byte code;
+ private final String name;
+ private final Class<?> clazz;
+
+ static {
+ register(DataType.class);
+ }
+
+ static Table<Class<?>, Byte, DataType> TABLE = HashBasedTable.create();
+
+ static void register(Class<? extends DataType> clazz) {
+ Object enums;
+ try {
+ enums = clazz.getMethod("values").invoke(null);
+ } catch (Exception e) {
+ throw new HugeException("DataType invalid", e);
+ }
+ for (DataType e : CollectionUtil.<DataType>toList(enums)) {
+ TABLE.put(clazz, e.code(), e);
+ }
+ }
+
+ static <T extends DataType> T fromCode(Class<T> clazz, byte code) {
+ @SuppressWarnings("unchecked")
+ T value = (T) TABLE.get(clazz, code);
+ if (value == null) {
+ E.checkArgument(false, "Can't construct %s from code %s",
+ clazz.getSimpleName(), code);
+ }
+ return value;
+ }
+
+ DataType(int code, String name, Class<?> clazz) {
+ assert code < 256;
+ this.code = (byte) code;
+ this.name = name;
+ this.clazz = clazz;
+ }
+
+ public byte code() {
+ return this.code;
+ }
+
+ public String string() {
+ return this.name;
+ }
+
+ public Class<?> clazz() {
+ return this.clazz;
+ }
+
+ public boolean isText() {
+ return this == DataType.TEXT;
+ }
+
+ public boolean isNumber() {
+ return this == BYTE || this == INT || this == LONG ||
+ this == FLOAT || this == DOUBLE;
+ }
+
+ public boolean isNumber4() {
+ // Store index value of Byte using 4 bytes
+ return this == BYTE || this == INT || this == FLOAT;
+ }
+
+ public boolean isNumber8() {
+ return this == LONG || this == DOUBLE;
+ }
+
+ //public boolean isBlob() {
+ // return this == DataType.BLOB;
+ //}
+
+ public boolean isDate() {
+ return this == DataType.DATE;
+ }
+
+ public boolean isUUID() {
+ return this == DataType.UUID;
+ }
+
+ public <V> Number valueToNumber(V value) {
+ if (!(this.isNumber() && value instanceof Number)) {
+ return null;
+ }
+ if (this.clazz.isInstance(value)) {
+ return (Number) value;
+ }
+
+ Number number;
+ try {
+ switch (this) {
+ case BYTE:
+ number = Byte.valueOf(value.toString());
+ break;
+ case INT:
+ number = Integer.valueOf(value.toString());
+ break;
+ case LONG:
+ number = Long.valueOf(value.toString());
+ break;
+ case FLOAT:
+ number = Float.valueOf(value.toString());
+ break;
+ case DOUBLE:
+ number = Double.valueOf(value.toString());
+ break;
+ default:
+ throw new AssertionError(String.format(
+ "Number type only contains Byte, Integer, " +
+ "Long, Float, Double, but got %s",
this.clazz()));
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(String.format(
+ "Can't read '%s' as %s: %s",
+ value, this.name, e.getMessage()));
+ }
+ return number;
+ }
+
+ public <V> Date valueToDate(V value) {
+ if (!this.isDate()) {
+ return null;
+ }
+ if (value instanceof Date) {
+ return (Date) value;
+ } else if (value instanceof Integer) {
+ return new Date(((Number) value).intValue());
+ } else if (value instanceof Long) {
+ return new Date(((Number) value).longValue());
+ } else if (value instanceof String) {
+ return DateUtil.parse((String) value);
+ }
+ return null;
+ }
+
+ public <V> UUID valueToUUID(V value) {
+ if (!this.isUUID()) {
+ return null;
+ }
+ if (value instanceof UUID) {
+ return (UUID) value;
+ } else if (value instanceof String) {
+ return StringEncoding.uuid((String) value);
+ }
+ return null;
+ }
+
+
+ public static DataType fromClass(Class<?> clazz) {
+ for (DataType type : DataType.values()) {
+ if (type.clazz() == clazz) {
+ return type;
+ }
+ }
+ throw new HugeException("Unknown clazz '%s' for DataType", clazz);
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/HugeType.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/HugeType.java
new file mode 100644
index 00000000..7c88567a
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/HugeType.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.struct;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public enum HugeType {
+
+ UNKNOWN(0, "UNKNOWN"),
+
+ /* Schema types */
+ VERTEX_LABEL(1, "VL"),
+ EDGE_LABEL(2, "EL"),
+ PROPERTY_KEY(3, "PK"),
+ INDEX_LABEL(4, "IL"),
+
+ META(40, "M"),
+ COUNTER(50, "C"),
+
+ /* Data types */
+ VERTEX(101, "V"),
+ // System meta
+ SYS_PROPERTY(102, "S"),
+ // Property
+ PROPERTY(103, "U"),
+ // Vertex aggregate property
+ AGGR_PROPERTY_V(104, "VP"),
+ // Edge aggregate property
+ AGGR_PROPERTY_E(105, "EP"),
+ // Olap property
+ OLAP(106, "AP"),
+ // Edge
+ EDGE(120, "E"),
+ // Edge's direction is OUT for the specified vertex
+ EDGE_OUT(130, "O"),
+ // Edge's direction is IN for the specified vertex
+ EDGE_IN(140, "I"),
+
+ SECONDARY_INDEX(150, "SI"),
+ VERTEX_LABEL_INDEX(151, "VI"),
+ EDGE_LABEL_INDEX(152, "EI"),
+ RANGE_INT_INDEX(160, "II"),
+ RANGE_FLOAT_INDEX(161, "FI"),
+ RANGE_LONG_INDEX(162, "LI"),
+ RANGE_DOUBLE_INDEX(163, "DI"),
+ SEARCH_INDEX(170, "AI"),
+ SHARD_INDEX(175, "HI"),
+ UNIQUE_INDEX(178, "UI"),
+
+ TASK(180, "T"),
+
+ // System schema
+ SYS_SCHEMA(250, "SS"),
+
+ MAX_TYPE(255, "~");
+
+ private byte type = 0;
+ private String name;
+
+ private static final Map<String, HugeType> ALL_NAME;
+
+ static {
+ ALL_NAME = new HashMap<>();
+ for (HugeType type : values()) {
+ ALL_NAME.put(type.name, type);
+ }
+ }
+
+ HugeType(int type, String name) {
+ assert type < 256;
+ this.type = (byte) type;
+ this.name = name;
+ }
+
+ public byte code() {
+ return this.type;
+ }
+
+ public String string() {
+ return this.name;
+ }
+
+ public String readableName() {
+ return this.name().replace('_', ' ').toLowerCase();
+ }
+
+ public boolean isSchema() {
+ return this == HugeType.VERTEX_LABEL ||
+ this == HugeType.EDGE_LABEL ||
+ this == HugeType.PROPERTY_KEY ||
+ this == HugeType.INDEX_LABEL;
+ }
+
+ public boolean isGraph() {
+ return this.isVertex() || this.isEdge();
+ }
+
+ public boolean isVertex() {
+ return this == HugeType.VERTEX;
+ }
+
+ public boolean isEdge() {
+ return this == EDGE || this == EDGE_OUT || this == EDGE_IN;
+ }
+
+ public boolean isIndex() {
+ return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX ||
+ this == SECONDARY_INDEX || this == SEARCH_INDEX ||
+ this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX ||
+ this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX ||
+ this == SHARD_INDEX || this == UNIQUE_INDEX;
+ }
+
+ public boolean isStringIndex() {
+ return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX ||
+ this == SECONDARY_INDEX || this == SEARCH_INDEX ||
+ this == SHARD_INDEX || this == UNIQUE_INDEX;
+ }
+
+ public boolean isNumericIndex() {
+ return this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX ||
+ this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX ||
+ this == SHARD_INDEX;
+ }
+
+ public boolean isSecondaryIndex() {
+ return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX ||
+ this == SECONDARY_INDEX;
+ }
+
+ public boolean isSearchIndex() {
+ return this == SEARCH_INDEX;
+ }
+
+ public boolean isRangeIndex() {
+ return this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX ||
+ this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX;
+ }
+
+ public boolean isRange4Index() {
+ return this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX;
+ }
+
+ public boolean isRange8Index() {
+ return this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX;
+ }
+
+ public boolean isShardIndex() {
+ return this == SHARD_INDEX;
+ }
+
+ public boolean isUniqueIndex() {
+ return this == UNIQUE_INDEX;
+ }
+
+ public boolean isVertexAggregateProperty() {
+ return this == AGGR_PROPERTY_V;
+ }
+
+ public boolean isEdgeAggregateProperty() {
+ return this == AGGR_PROPERTY_E;
+ }
+
+ public boolean isAggregateProperty() {
+ return this.isVertexAggregateProperty() ||
+ this.isEdgeAggregateProperty();
+ }
+
+ public static HugeType fromString(String type) {
+ return ALL_NAME.get(type);
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/BytesBuffer.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/BytesBuffer.java
new file mode 100644
index 00000000..81320e86
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/BytesBuffer.java
@@ -0,0 +1,759 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.util;
+
+import com.baidu.hugegraph.serializer.direct.struct.HugeType;
+import com.baidu.hugegraph.util.*;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.UUID;
+
+/**
+ * class BytesBuffer is a util for read/write binary
+ */
+public final class BytesBuffer extends OutputStream {
+
+ public static final int BYTE_LEN = Byte.BYTES;
+ public static final int SHORT_LEN = Short.BYTES;
+ public static final int INT_LEN = Integer.BYTES;
+ public static final int LONG_LEN = Long.BYTES;
+ public static final int CHAR_LEN = Character.BYTES;
+ public static final int FLOAT_LEN = Float.BYTES;
+ public static final int DOUBLE_LEN = Double.BYTES;
+ public static final int BLOB_LEN = 4;
+
+ public static final int UINT8_MAX = ((byte) -1) & 0xff;
+ public static final int UINT16_MAX = ((short) -1) & 0xffff;
+ public static final long UINT32_MAX = (-1) & 0xffffffffL;
+
+ // NOTE: +1 to let code 0 represent length 1
+ public static final int ID_LEN_MASK = 0x7f;
+ public static final int ID_LEN_MAX = 0x7f + 1; // 128
+ public static final int BIG_ID_LEN_MAX = 0x7fff + 1; // 32768
+
+ public static final byte STRING_ENDING_BYTE = (byte) 0x00;
+ public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff;
+ public static final int STRING_LEN_MAX = UINT16_MAX;
+ public static final long BLOB_LEN_MAX = 1 * Bytes.GB;
+
+ // The value must be in range [8, ID_LEN_MAX]
+ public static final int INDEX_HASH_ID_THRESHOLD = 32;
+
+ public static final int DEFAULT_CAPACITY = 64;
+ public static final int MAX_BUFFER_CAPACITY = 128 * 1024 * 1024; // 128M
+
+ public static final int BUF_EDGE_ID = 128;
+ public static final int BUF_PROPERTY = 64;
+
+ private ByteBuffer buffer;
+ private final boolean resize;
+
+ public BytesBuffer() {
+ this(DEFAULT_CAPACITY);
+ }
+
+ public BytesBuffer(int capacity) {
+ E.checkArgument(capacity <= MAX_BUFFER_CAPACITY,
+ "Capacity exceeds max buffer capacity: %s",
+ MAX_BUFFER_CAPACITY);
+ this.buffer = ByteBuffer.allocate(capacity);
+ this.resize = true;
+ }
+
+ public BytesBuffer(ByteBuffer buffer) {
+ E.checkNotNull(buffer, "buffer");
+ this.buffer = buffer;
+ this.resize = false;
+ }
+
+ public static BytesBuffer allocate(int capacity) {
+ return new BytesBuffer(capacity);
+ }
+
+ public static BytesBuffer wrap(ByteBuffer buffer) {
+ return new BytesBuffer(buffer);
+ }
+
+ public static BytesBuffer wrap(byte[] array) {
+ return new BytesBuffer(ByteBuffer.wrap(array));
+ }
+
+ public static BytesBuffer wrap(byte[] array, int offset, int length) {
+ return new BytesBuffer(ByteBuffer.wrap(array, offset, length));
+ }
+
+ public ByteBuffer asByteBuffer() {
+ return this.buffer;
+ }
+
+ public BytesBuffer forReadWritten() {
+ this.buffer.flip();
+ return this;
+ }
+
+ public BytesBuffer forReadAll() {
+ this.buffer.position(this.buffer.limit());
+ return this;
+ }
+
+ public byte[] array() {
+ return this.buffer.array();
+ }
+
+ public byte[] bytes() {
+ byte[] bytes = this.buffer.array();
+ int position = this.buffer.position();
+ if (position == bytes.length) {
+ return bytes;
+ } else {
+ return Arrays.copyOf(bytes, position);
+ }
+ }
+
+ public int position() {
+ return this.buffer.position();
+ }
+
+ public BytesBuffer copyFrom(BytesBuffer other) {
+ this.write(other.bytes());
+ return this;
+ }
+
+ public int remaining() {
+ return this.buffer.remaining();
+ }
+
+ private void require(int size) {
+ // Does need to resize?
+ if (this.buffer.limit() - this.buffer.position() >= size) {
+ return;
+ }
+ // Can't resize for wrapped buffer since will change the origin ref
+ E.checkState(this.resize, "Can't resize for wrapped buffer");
+
+ // Extra capacity as buffer
+ int newcapacity = size + this.buffer.limit() + DEFAULT_CAPACITY;
+ E.checkArgument(newcapacity <= MAX_BUFFER_CAPACITY,
+ "Capacity exceeds max buffer capacity: %s",
+ MAX_BUFFER_CAPACITY);
+ ByteBuffer newBuffer = ByteBuffer.allocate(newcapacity);
+ this.buffer.flip();
+ newBuffer.put(this.buffer);
+ this.buffer = newBuffer;
+ }
+
+ public BytesBuffer write(byte val) {
+ require(BYTE_LEN);
+ this.buffer.put(val);
+ return this;
+ }
+
+ @Override
+ public void write(int val) {
+ assert val <= UINT8_MAX;
+ require(BYTE_LEN);
+ this.buffer.put((byte) val);
+ }
+
+ @Override
+ public void write(byte[] val) {
+ require(BYTE_LEN * val.length);
+ this.buffer.put(val);
+ }
+
+ @Override
+ public void write(byte[] val, int offset, int length) {
+ require(BYTE_LEN * length);
+ this.buffer.put(val, offset, length);
+ }
+
+ public BytesBuffer writeBoolean(boolean val) {
+ this.write(val ? 1 : 0);
+ return this;
+ }
+
+ public BytesBuffer writeChar(char val) {
+ require(CHAR_LEN);
+ this.buffer.putChar(val);
+ return this;
+ }
+
+ public BytesBuffer writeShort(short val) {
+ require(SHORT_LEN);
+ this.buffer.putShort(val);
+ return this;
+ }
+
+ public BytesBuffer writeInt(int val) {
+ require(INT_LEN);
+ this.buffer.putInt(val);
+ return this;
+ }
+
+ public BytesBuffer writeLong(long val) {
+ require(LONG_LEN);
+ this.buffer.putLong(val);
+ return this;
+ }
+
+ public BytesBuffer writeFloat(float val) {
+ require(FLOAT_LEN);
+ this.buffer.putFloat(val);
+ return this;
+ }
+
+ public BytesBuffer writeDouble(double val) {
+ require(DOUBLE_LEN);
+ this.buffer.putDouble(val);
+ return this;
+ }
+
+ public byte peek() {
+ return this.buffer.get(this.buffer.position());
+ }
+
+ public byte peekLast() {
+ return this.buffer.get(this.buffer.capacity() - 1);
+ }
+
+ public byte read() {
+ return this.buffer.get();
+ }
+
+ public byte[] read(int length) {
+ byte[] bytes = new byte[length];
+ this.buffer.get(bytes);
+ return bytes;
+ }
+
+ public boolean readBoolean() {
+ return this.buffer.get() != 0;
+ }
+
+ public char readChar() {
+ return this.buffer.getChar();
+ }
+
+ public short readShort() {
+ return this.buffer.getShort();
+ }
+
+ public int readInt() {
+ return this.buffer.getInt();
+ }
+
+ public long readLong() {
+ return this.buffer.getLong();
+ }
+
+ public float readFloat() {
+ return this.buffer.getFloat();
+ }
+
+ public double readDouble() {
+ return this.buffer.getDouble();
+ }
+
+ public BytesBuffer writeBytes(byte[] bytes) {
+ E.checkArgument(bytes.length <= UINT16_MAX,
+ "The max length of bytes is %s, but got %s",
+ UINT16_MAX, bytes.length);
+ require(SHORT_LEN + bytes.length);
+ this.writeVInt(bytes.length);
+ this.write(bytes);
+ return this;
+ }
+
+ public byte[] readBytes() {
+ int length = this.readVInt();
+ assert length >= 0;
+ byte[] bytes = this.read(length);
+ return bytes;
+ }
+
+ public BytesBuffer writeBigBytes(byte[] bytes) {
+ E.checkArgument(bytes.length <= BLOB_LEN_MAX,
+ "The max length of bytes is %s, but got %s",
+ BLOB_LEN_MAX, bytes.length);
+ require(BLOB_LEN + bytes.length);
+ this.writeVInt(bytes.length);
+ this.write(bytes);
+ return this;
+ }
+
+ public byte[] readBigBytes() {
+ int length = this.readVInt();
+ assert length >= 0;
+ byte[] bytes = this.read(length);
+ return bytes;
+ }
+
+ public BytesBuffer writeStringRaw(String val) {
+ this.write(StringEncoding.encode(val));
+ return this;
+ }
+
+ public BytesBuffer writeString(String val) {
+ byte[] bytes = StringEncoding.encode(val);
+ this.writeBytes(bytes);
+ return this;
+ }
+
+ public String readString() {
+ return StringEncoding.decode(this.readBytes());
+ }
+
+ public BytesBuffer writeStringWithEnding(String value) {
+ if (!value.isEmpty()) {
+ byte[] bytes = StringEncoding.encode(value);
+ /*
+ * assert '0x00'/'0xFF' not exist in string index id
+ * NOTE:
+ * 0x00 is NULL in UTF8(or ASCII) bytes
+ * 0xFF is not a valid byte in UTF8 bytes
+ */
+ assert !Bytes.contains(bytes, STRING_ENDING_BYTE_FF) :
+ "Invalid UTF8 bytes: " + value;
+ if (Bytes.contains(bytes, STRING_ENDING_BYTE)) {
+ E.checkArgument(false,
+ "Can't contains byte '0x00' in string: '%s'",
+ value);
+ }
+ this.write(bytes);
+ }
+ /*
+ * Choose 0x00 as ending symbol (see #1057)
+ * The following is out of date:
+ * A reasonable ending symbol should be 0x00(to ensure order), but
+ * considering that some backends like PG do not support 0x00 string,
+ * so choose 0xFF currently.
+ */
+ this.write(STRING_ENDING_BYTE);
+ return this;
+ }
+
+ public String readStringWithEnding() {
+ return StringEncoding.decode(this.readBytesWithEnding());
+ }
+
+ public BytesBuffer writeStringToRemaining(String value) {
+ byte[] bytes = StringEncoding.encode(value);
+ this.write(bytes);
+ return this;
+ }
+
+ public String readStringFromRemaining() {
+ byte[] bytes = new byte[this.buffer.remaining()];
+ this.buffer.get(bytes);
+ return StringEncoding.decode(bytes);
+ }
+
+ public BytesBuffer writeUInt8(int val) {
+ assert val <= UINT8_MAX;
+ this.write(val);
+ return this;
+ }
+
+ public int readUInt8() {
+ return this.read() & 0x000000ff;
+ }
+
+ public BytesBuffer writeUInt16(int val) {
+ assert val <= UINT16_MAX;
+ this.writeShort((short) val);
+ return this;
+ }
+
+ public int readUInt16() {
+ return this.readShort() & 0x0000ffff;
+ }
+
+ public BytesBuffer writeUInt32(long val) {
+ assert val <= UINT32_MAX;
+ this.writeInt((int) val);
+ return this;
+ }
+
+ public long readUInt32() {
+ return this.readInt() & 0xffffffffL;
+ }
+
+ public BytesBuffer writeVInt(int value) {
+ // NOTE: negative numbers are not compressed
+ if (value > 0x0fffffff || value < 0) {
+ this.write(0x80 | ((value >>> 28) & 0x7f));
+ }
+ if (value > 0x1fffff || value < 0) {
+ this.write(0x80 | ((value >>> 21) & 0x7f));
+ }
+ if (value > 0x3fff || value < 0) {
+ this.write(0x80 | ((value >>> 14) & 0x7f));
+ }
+ if (value > 0x7f || value < 0) {
+ this.write(0x80 | ((value >>> 7) & 0x7f));
+ }
+ this.write(value & 0x7f);
+
+ return this;
+ }
+
+ public int readVInt() {
+ byte leading = this.read();
+ E.checkArgument(leading != 0x80,
+ "Unexpected varint with leading byte '0x%s'",
+ Bytes.toHex(leading));
+ int value = leading & 0x7f;
+ if (leading >= 0) {
+ assert (leading & 0x80) == 0;
+ return value;
+ }
+
+ int i = 1;
+ for (; i < 5; i++) {
+ byte b = this.read();
+ if (b >= 0) {
+ value = b | (value << 7);
+ break;
+ } else {
+ value = (b & 0x7f) | (value << 7);
+ }
+ }
+
+ E.checkArgument(i < 5,
+ "Unexpected varint %s with too many bytes(%s)",
+ value, i + 1);
+ E.checkArgument(i < 4 || (leading & 0x70) == 0,
+ "Unexpected varint %s with leading byte '0x%s'",
+ value, Bytes.toHex(leading));
+ return value;
+ }
+
+ public BytesBuffer writeVLong(long value) {
+ if (value < 0) {
+ this.write((byte) 0x81);
+ }
+ if (value > 0xffffffffffffffL || value < 0L) {
+ this.write(0x80 | ((int) (value >>> 56) & 0x7f));
+ }
+ if (value > 0x1ffffffffffffL || value < 0L) {
+ this.write(0x80 | ((int) (value >>> 49) & 0x7f));
+ }
+ if (value > 0x3ffffffffffL || value < 0L) {
+ this.write(0x80 | ((int) (value >>> 42) & 0x7f));
+ }
+ if (value > 0x7ffffffffL || value < 0L) {
+ this.write(0x80 | ((int) (value >>> 35) & 0x7f));
+ }
+ if (value > 0xfffffffL || value < 0L) {
+ this.write(0x80 | ((int) (value >>> 28) & 0x7f));
+ }
+ if (value > 0x1fffffL || value < 0L) {
+ this.write(0x80 | ((int) (value >>> 21) & 0x7f));
+ }
+ if (value > 0x3fffL || value < 0L) {
+ this.write(0x80 | ((int) (value >>> 14) & 0x7f));
+ }
+ if (value > 0x7fL || value < 0L) {
+ this.write(0x80 | ((int) (value >>> 7) & 0x7f));
+ }
+ this.write((int) value & 0x7f);
+
+ return this;
+ }
+
+ public long readVLong() {
+ byte leading = this.read();
+ E.checkArgument(leading != 0x80,
+ "Unexpected varlong with leading byte '0x%s'",
+ Bytes.toHex(leading));
+ long value = leading & 0x7fL;
+ if (leading >= 0) {
+ assert (leading & 0x80) == 0;
+ return value;
+ }
+
+ int i = 1;
+ for (; i < 10; i++) {
+ byte b = this.read();
+ if (b >= 0) {
+ value = b | (value << 7);
+ break;
+ } else {
+ value = (b & 0x7f) | (value << 7);
+ }
+ }
+
+ E.checkArgument(i < 10,
+ "Unexpected varlong %s with too many bytes(%s)",
+ value, i + 1);
+ E.checkArgument(i < 9 || (leading & 0x7e) == 0,
+ "Unexpected varlong %s with leading byte '0x%s'",
+ value, Bytes.toHex(leading));
+ return value;
+ }
+
+ public BytesBuffer writeId(Id id) {
+ return this.writeId(id, false);
+ }
+
+ public BytesBuffer writeId(Id id, boolean big) {
+ switch (id.type()) {
+ case LONG:
+ // Number Id
+ long value = id.asLong();
+ this.writeNumber(value);
+ break;
+ case UUID:
+ // UUID Id
+ byte[] bytes = id.asBytes();
+ assert bytes.length == Id.UUID_LENGTH;
+ this.writeUInt8(0x7f); // 0b01111111 means UUID
+ this.write(bytes);
+ break;
+ default:
+ // String Id
+ bytes = id.asBytes();
+ int len = bytes.length;
+ E.checkArgument(len > 0, "Can't write empty id");
+ if (!big) {
+ E.checkArgument(len <= ID_LEN_MAX,
+ "Id max length is %s, but got %s {%s}",
+ ID_LEN_MAX, len, id);
+ len -= 1; // mapping [1, 128] to [0, 127]
+ this.writeUInt8(len | 0x80);
+ } else {
+ E.checkArgument(len <= BIG_ID_LEN_MAX,
+ "Big id max length is %s, but got %s {%s}",
+ BIG_ID_LEN_MAX, len, id);
+ len -= 1;
+ int high = len >> 8;
+ int low = len & 0xff;
+ this.writeUInt8(high | 0x80);
+ this.writeUInt8(low);
+ }
+ this.write(bytes);
+ break;
+ }
+ return this;
+ }
+
+ public BytesBuffer writeIndexId(Id id, HugeType type) {
+ return this.writeIndexId(id, type, true);
+ }
+
+ public BytesBuffer writeIndexId(Id id, HugeType type, boolean withEnding) {
+ byte[] bytes = id.asBytes();
+ int len = bytes.length;
+ E.checkArgument(len > 0, "Can't write empty id");
+
+ this.write(bytes);
+ if (type.isStringIndex()) {
+ if (Bytes.contains(bytes, STRING_ENDING_BYTE)) {
+ // Not allow STRING_ENDING_BYTE exist in string index id
+ E.checkArgument(false,
+ "The %s type index id can't contains " +
+ "byte '0x%s', but got: 0x%s", type,
+ Bytes.toHex(STRING_ENDING_BYTE),
+ Bytes.toHex(bytes));
+ }
+ if (withEnding) {
+ this.writeStringWithEnding("");
+ }
+ }
+ return this;
+ }
+
+ private void writeNumber(long val) {
+ /*
+ * 8 kinds of number, 2 ~ 9 bytes number:
+ * 0b 0kkksxxx X...
+ * 0(1 bit) + kind(3 bits) + signed(1 bit) + number(n bits)
+ *
+ * 2 byte : 0b 0000 1xxx X(8 bits) [0, 2047]
+ * 0b 0000 0xxx X(8 bits) [-2048, -1]
+ * 3 bytes: 0b 0001 1xxx X X [0, 524287]
+ * 0b 0001 0xxx X X [-524288, -1]
+ * 4 bytes: 0b 0010 1xxx X X X [0, 134217727]
+ * 0b 0010 0xxx X X X [-134217728, -1]
+ * 5 bytes: 0b 0011 1xxx X X X X [0, 2^35 - 1]
+ * 0b 0011 0xxx X X X X [-2^35, -1]
+ * 6 bytes: 0b 0100 1xxx X X X X X [0, 2^43 - 1]
+ * 0b 0100 0xxx X X X X X [-2^43, -1]
+ * 7 bytes: 0b 0101 1xxx X X X X X X [0, 2^51 - 1]
+ * 0b 0101 0xxx X X X X X X [-2^51, -1]
+ * 8 bytes: 0b 0110 1xxx X X X X X X X [0, 2^59 - 1]
+ * 0b 0110 0xxx X X X X X X X [-2^59, -1]
+ * 9 bytes: 0b 0111 1000 X X X X X X X X [0, 2^64 - 1]
+ * 0b 0111 0000 X X X X X X X X [-2^64, -1]
+ *
+ * NOTE: 0b 0111 1111 is used by 128 bits UUID
+ * 0b 0111 1110 is used by EdgeId
+ */
+ int positive = val >= 0 ? 0x08 : 0x00;
+ if (~0x7ffL <= val && val <= 0x7ffL) {
+ int high3bits = (int) (val >> 8) & 0x07;
+ this.writeUInt8(0x00 | positive | high3bits);
+ this.writeUInt8((byte) val);
+ } else if (~0x7ffffL <= val && val <= 0x7ffffL) {
+ int high3bits = (int) (val >> 16) & 0x07;
+ this.writeUInt8(0x10 | positive | high3bits);
+ this.writeShort((short) val);
+ } else if (~0x7ffffffL <= val && val <= 0x7ffffffL) {
+ int high3bits = (int) (val >> 24 & 0x07);
+ this.writeUInt8(0x20 | positive | high3bits);
+ this.write((byte) (val >> 16));
+ this.writeShort((short) val);
+ } else if (~0x7ffffffffL <= val && val <= 0x7ffffffffL) {
+ int high3bits = (int) (val >> 32) & 0x07;
+ this.writeUInt8(0x30 | positive | high3bits);
+ this.writeInt((int) val);
+ } else if (~0x7ffffffffffL <= val && val <= 0x7ffffffffffL) {
+ int high3bits = (int) (val >> 40) & 0x07;
+ this.writeUInt8(0x40 | positive | high3bits);
+ this.write((byte) (val >> 32));
+ this.writeInt((int) val);
+ } else if (~0x7ffffffffffffL <= val && val <= 0x7ffffffffffffL) {
+ int high3bits = (int) (val >> 48) & 0x07;
+ this.writeUInt8(0x50 | positive | high3bits);
+ this.writeShort((short) (val >> 32));
+ this.writeInt((int) val);
+ } else if (~0x7ffffffffffffffL <= val && val <= 0x7ffffffffffffffL) {
+ int high3bits = (int) (val >> 56) & 0x07;
+ this.writeUInt8(0x60 | positive | high3bits);
+ this.write((byte) (val >> 48));
+ this.writeShort((short) (val >> 32));
+ this.writeInt((int) val);
+ } else {
+ // high3bits is always 0b000 for 9 bytes number
+ this.writeUInt8(0x70 | positive);
+ this.writeLong(val);
+ }
+ }
+
+ private long readNumber(byte b) {
+ E.checkArgument((b & 0x80) == 0,
+ "Not a number type with prefix byte '0x%s'",
+ Bytes.toHex(b));
+ // Parse the kind from byte 0kkksxxx
+ int kind = b >>> 4;
+ boolean positive = (b & 0x08) > 0;
+ long high3bits = b & 0x07;
+ long value = high3bits << ((kind + 1) * 8);
+ switch (kind) {
+ case 0:
+ value |= this.readUInt8();
+ break;
+ case 1:
+ value |= this.readUInt16();
+ break;
+ case 2:
+ value |= this.readUInt8() << 16 | this.readUInt16();
+ break;
+ case 3:
+ value |= this.readUInt32();
+ break;
+ case 4:
+ value |= (long) this.readUInt8() << 32 | this.readUInt32();
+ break;
+ case 5:
+ value |= (long) this.readUInt16() << 32 | this.readUInt32();
+ break;
+ case 6:
+ value |= (long) this.readUInt8() << 48 |
+ (long) this.readUInt16() << 32 |
+ this.readUInt32();
+ break;
+ case 7:
+ assert high3bits == 0L;
+ value |= this.readLong();
+ break;
+ default:
+ throw new AssertionError("Invalid length of number: " + kind);
+ }
+ if (!positive && kind < 7) {
+ // Restore the bits of the original negative number
+ long mask = Long.MIN_VALUE >> (52 - kind * 8);
+ value |= mask;
+ }
+ return value;
+ }
+
+ private byte[] readBytesWithEnding() {
+ int start = this.buffer.position();
+ boolean foundEnding = false;
+ while (this.remaining() > 0) {
+ byte current = this.read();
+ if (current == STRING_ENDING_BYTE) {
+ foundEnding = true;
+ break;
+ }
+ }
+ E.checkArgument(foundEnding, "Not found ending '0x%s'",
+ Bytes.toHex(STRING_ENDING_BYTE));
+ int end = this.buffer.position() - 1;
+ int len = end - start;
+ byte[] bytes = new byte[len];
+ System.arraycopy(this.array(), start, bytes, 0, len);
+ return bytes;
+ }
+
+ public void writeProperty(com.baidu.hugegraph.structure.constant.DataType
dataType, Object value) {
+ switch (dataType) {
+ case BOOLEAN:
+ this.writeVInt(((Boolean) value) ? 1 : 0);
+ break;
+ case BYTE:
+ this.writeVInt((Byte) value);
+ break;
+ case INT:
+ this.writeVInt((Integer) value);
+ break;
+ case FLOAT:
+ this.writeFloat((Float) value);
+ break;
+ case LONG:
+ this.writeVLong((Long) value);
+ break;
+ case DATE:
+ this.writeVLong(((Date) value).getTime());
+ break;
+ case DOUBLE:
+ this.writeDouble((Double) value);
+ break;
+ case TEXT:
+ this.writeString((String) value);
+ break;
+
+ case UUID:
+ UUID uuid = (UUID) value;
+ // Generally writeVLong(uuid) can't save space
+ this.writeLong(uuid.getMostSignificantBits());
+ this.writeLong(uuid.getLeastSignificantBits());
+ break;
+ default:
+ //this.writeBytes(KryoUtil.toKryoWithType(value));
+ break;
+ }
+
+ }
+
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/GraphSchema.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/GraphSchema.java
new file mode 100644
index 00000000..af432f27
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/GraphSchema.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.baidu.hugegraph.driver.HugeClient;
+import com.baidu.hugegraph.exception.ServerException;
+import com.baidu.hugegraph.structure.schema.EdgeLabel;
+import com.baidu.hugegraph.structure.schema.PropertyKey;
+import com.baidu.hugegraph.structure.schema.VertexLabel;
+
+/**
+ * We could get all graph schema from server and cache/update it in
client(subset of SchemaManager)
+ */
+public class GraphSchema {
+
+ private final HugeClient client;
+ private final Map<String, PropertyKey> propertyKeys;
+ private final Map<String, VertexLabel> vertexLabels;
+ private final Map<String, EdgeLabel> edgeLabels;
+
+ public GraphSchema(HugeClient client) {
+ this.client = client;
+ this.propertyKeys = new HashMap<>();
+ this.vertexLabels = new HashMap<>();
+ this.edgeLabels = new HashMap<>();
+ // init all schema
+ this.updateAll();
+ }
+
+ public void updateAll() {
+ this.propertyKeys.clear();
+ this.vertexLabels.clear();
+ this.edgeLabels.clear();
+
+ client.schema().getPropertyKeys().forEach(pk ->
this.propertyKeys.put(pk.name(), pk));
+ client.schema().getVertexLabels().forEach(vl ->
this.vertexLabels.put(vl.name(), vl));
+ client.schema().getEdgeLabels().forEach(el ->
this.edgeLabels.put(el.name(), el));
+ }
+
+ public PropertyKey getPropertyKey(String name) {
+ PropertyKey propertyKey = this.propertyKeys.get(name);
+ if (propertyKey == null) {
+ try {
+ propertyKey = this.client.schema().getPropertyKey(name);
+ } catch (ServerException e) {
+ throw new HugeException("The property key '%s' doesn't exist",
name);
+ }
+ }
+ return propertyKey;
+ }
+
+ public VertexLabel getVertexLabel(String name) {
+ VertexLabel vertexLabel = this.vertexLabels.get(name);
+ if (vertexLabel == null) {
+ try {
+ vertexLabel = this.client.schema().getVertexLabel(name);
+ } catch (ServerException e) {
+ throw new HugeException("The vertex label '%s' doesn't exist",
name);
+ }
+ }
+ return vertexLabel;
+ }
+
+ public EdgeLabel getEdgeLabel(String name) {
+ EdgeLabel edgeLabel = this.edgeLabels.get(name);
+ if (edgeLabel == null) {
+ try {
+ edgeLabel = this.client.schema().getEdgeLabel(name);
+ } catch (ServerException e) {
+ throw new HugeException("The edge label '%s' doesn't exist",
name);
+ }
+ }
+ return edgeLabel;
+ }
+
+ public EdgeLabel getEdgeLabel(int id) {
+ for (EdgeLabel label : edgeLabels.values()) {
+ if (label.id() == id) {
+ return label;
+ }
+ }
+
+ throw new HugeException("The edge label id '%s' doesn't exist", id);
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/HugeException.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/HugeException.java
new file mode 100644
index 00000000..143e5ae2
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/HugeException.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.util;
+
+/**
+ * Used for fix Exception problems, simplify from server
+ */
+public class HugeException extends RuntimeException {
+
+ private static final long serialVersionUID = -8711375282196157051L;
+
+ public HugeException(String message) {
+ super(message);
+ }
+
+ public HugeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public HugeException(String message, Object... args) {
+ super(String.format(message, args));
+ }
+
+ public HugeException(String message, Throwable cause, Object... args) {
+ super(String.format(message, args), cause);
+ }
+
+ public Throwable rootCause() {
+ return rootCause(this);
+ }
+
+ public static Throwable rootCause(Throwable e) {
+ Throwable cause = e;
+ while (cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+ return cause;
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/Id.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/Id.java
new file mode 100644
index 00000000..0aee0d4f
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/Id.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.util;
+
+import com.baidu.hugegraph.util.E;
+
+public interface Id extends Comparable<Id> {
+
+ public static final int UUID_LENGTH = 16;
+
+ public Object asObject();
+
+ public String asString();
+
+ public long asLong();
+
+ public byte[] asBytes();
+
+ public int length();
+
+ public IdType type();
+
+ public default boolean number() {
+ return this.type() == IdType.LONG;
+ }
+
+ public default boolean uuid() {
+ return this.type() == IdType.UUID;
+ }
+
+ public default boolean string() {
+ return this.type() == IdType.STRING;
+ }
+
+ public default boolean edge() {
+ return this.type() == IdType.EDGE;
+ }
+
+ public enum IdType {
+
+ UNKNOWN,
+ LONG,
+ UUID,
+ STRING,
+ EDGE;
+
+ public char prefix() {
+ if (this == UNKNOWN) {
+ return 'N';
+ }
+ return this.name().charAt(0);
+ }
+
+ public static IdType valueOfPrefix(String id) {
+ E.checkArgument(id != null && id.length() > 0,
+ "Invalid id '%s'", id);
+ switch (id.charAt(0)) {
+ case 'L':
+ return IdType.LONG;
+ case 'U':
+ return IdType.UUID;
+ case 'S':
+ return IdType.STRING;
+ case 'E':
+ return IdType.EDGE;
+ default:
+ return IdType.UNKNOWN;
+ }
+ }
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/IdGenerator.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/IdGenerator.java
new file mode 100644
index 00000000..3e6c1e96
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/IdGenerator.java
@@ -0,0 +1,440 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.util;
+
+
+import com.baidu.hugegraph.util.E;
+import com.baidu.hugegraph.util.LongEncoding;
+import com.baidu.hugegraph.util.NumericUtil;
+import com.baidu.hugegraph.serializer.direct.util.Id.IdType;
+
+import java.util.Objects;
+import java.util.UUID;
+
+public abstract class IdGenerator {
+
+ public static final Id ZERO = IdGenerator.of(0L);
+
+ public final static Id of(String id) {
+ return new StringId(id);
+ }
+
+ public final static Id of(UUID id) {
+ return new UuidId(id);
+ }
+
+ public final static Id of(String id, boolean uuid) {
+ return uuid ? new UuidId(id) : new StringId(id);
+ }
+
+ public final static Id of(long id) {
+ return new LongId(id);
+ }
+
+ public static Id of(Object id) {
+ if (id instanceof Id) {
+ return (Id) id;
+ } else if (id instanceof String) {
+ return of((String) id);
+ } else if (id instanceof Number) {
+ return of(((Number) id).longValue());
+ } else if (id instanceof UUID) {
+ return of((UUID) id);
+ }
+ return new ObjectId(id);
+ }
+
+ public final static Id of(byte[] bytes, IdType type) {
+ switch (type) {
+ case LONG:
+ return new LongId(bytes);
+ case UUID:
+ return new UuidId(bytes);
+ case STRING:
+ return new StringId(bytes);
+ default:
+ throw new AssertionError("Invalid id type " + type);
+ }
+ }
+
+ public final static Id ofStoredString(String id, IdType type) {
+ switch (type) {
+ case LONG:
+ return of(LongEncoding.decodeSignedB64(id));
+ case UUID:
+ byte[] bytes = StringEncoding.decodeBase64(id);
+ return of(bytes, IdType.UUID);
+ case STRING:
+ return of(id);
+ default:
+ throw new AssertionError("Invalid id type " + type);
+ }
+ }
+
+ public final static String asStoredString(Id id) {
+ switch (id.type()) {
+ case LONG:
+ return LongEncoding.encodeSignedB64(id.asLong());
+ case UUID:
+ return StringEncoding.encodeBase64(id.asBytes());
+ case STRING:
+ return id.asString();
+ default:
+ throw new AssertionError("Invalid id type " + id.type());
+ }
+ }
+
+ public final static IdType idType(Id id) {
+ if (id instanceof LongId) {
+ return IdType.LONG;
+ }
+ if (id instanceof UuidId) {
+ return IdType.UUID;
+ }
+ if (id instanceof StringId) {
+ return IdType.STRING;
+ }
+
+ return IdType.UNKNOWN;
+ }
+
+ private final static int compareType(Id id1, Id id2) {
+ return idType(id1).ordinal() - idType(id2).ordinal();
+ }
+
+ /****************************** id defines ******************************/
+
+ public static final class StringId implements Id {
+
+ private final String id;
+
+ public StringId(String id) {
+ E.checkArgument(!id.isEmpty(), "The id can't be empty");
+ this.id = id;
+ }
+
+ public StringId(byte[] bytes) {
+ this.id = StringEncoding.decode(bytes);
+ }
+
+ @Override
+ public IdType type() {
+ return IdType.STRING;
+ }
+
+ @Override
+ public Object asObject() {
+ return this.id;
+ }
+
+ @Override
+ public String asString() {
+ return this.id;
+ }
+
+ @Override
+ public long asLong() {
+ return Long.parseLong(this.id);
+ }
+
+ @Override
+ public byte[] asBytes() {
+ return StringEncoding.encode(this.id);
+ }
+
+ @Override
+ public int length() {
+ return this.id.length();
+ }
+
+ @Override
+ public int compareTo(Id other) {
+ int cmp = compareType(this, other);
+ if (cmp != 0) {
+ return cmp;
+ }
+ return this.id.compareTo(other.asString());
+ }
+
+ @Override
+ public int hashCode() {
+ return this.id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof StringId)) {
+ return false;
+ }
+ return this.id.equals(((StringId) other).id);
+ }
+
+ @Override
+ public String toString() {
+ return this.id;
+ }
+ }
+
+ public static final class LongId extends Number implements Id {
+
+ private static final long serialVersionUID = -7732461469037400190L;
+
+ private final long id;
+
+ public LongId(long id) {
+ this.id = id;
+ }
+
+ public LongId(byte[] bytes) {
+ this.id = NumericUtil.bytesToLong(bytes);
+ }
+
+ @Override
+ public IdType type() {
+ return IdType.LONG;
+ }
+
+ @Override
+ public Object asObject() {
+ return this.id;
+ }
+
+ @Override
+ public String asString() {
+ // TODO: encode with base64
+ return Long.toString(this.id);
+ }
+
+ @Override
+ public long asLong() {
+ return this.id;
+ }
+
+ @Override
+ public byte[] asBytes() {
+ return NumericUtil.longToBytes(this.id);
+ }
+
+ @Override
+ public int length() {
+ return Long.BYTES;
+ }
+
+ @Override
+ public int compareTo(Id other) {
+ int cmp = compareType(this, other);
+ if (cmp != 0) {
+ return cmp;
+ }
+ return Long.compare(this.id, other.asLong());
+ }
+
+ @Override
+ public int hashCode() {
+ return Long.hashCode(this.id);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Number)) {
+ return false;
+ }
+ return this.id == ((Number) other).longValue();
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(this.id);
+ }
+
+ @Override
+ public int intValue() {
+ return (int) this.id;
+ }
+
+ @Override
+ public long longValue() {
+ return this.id;
+ }
+
+ @Override
+ public float floatValue() {
+ return this.id;
+ }
+
+ @Override
+ public double doubleValue() {
+ return this.id;
+ }
+ }
+
+ public static final class UuidId implements Id {
+
+ private final UUID uuid;
+
+ public UuidId(String string) {
+ this(StringEncoding.uuid(string));
+ }
+
+ public UuidId(byte[] bytes) {
+ this(fromBytes(bytes));
+ }
+
+ public UuidId(UUID uuid) {
+ E.checkArgument(uuid != null, "The uuid can't be null");
+ this.uuid = uuid;
+ }
+
+ @Override
+ public IdType type() {
+ return IdType.UUID;
+ }
+
+ @Override
+ public Object asObject() {
+ return this.uuid;
+ }
+
+ @Override
+ public String asString() {
+ return this.uuid.toString();
+ }
+
+ @Override
+ public long asLong() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public byte[] asBytes() {
+ BytesBuffer buffer = BytesBuffer.allocate(16);
+ buffer.writeLong(this.uuid.getMostSignificantBits());
+ buffer.writeLong(this.uuid.getLeastSignificantBits());
+ return buffer.bytes();
+ }
+
+ private static UUID fromBytes(byte[] bytes) {
+ E.checkArgument(bytes != null, "The UUID can't be null");
+ BytesBuffer buffer = BytesBuffer.wrap(bytes);
+ long high = buffer.readLong();
+ long low = buffer.readLong();
+ return new UUID(high, low);
+ }
+
+ @Override
+ public int length() {
+ return UUID_LENGTH;
+ }
+
+ @Override
+ public int compareTo(Id other) {
+ E.checkNotNull(other, "compare id");
+ int cmp = compareType(this, other);
+ if (cmp != 0) {
+ return cmp;
+ }
+ return this.uuid.compareTo(((UuidId) other).uuid);
+ }
+
+ @Override
+ public int hashCode() {
+ return this.uuid.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof UuidId)) {
+ return false;
+ }
+ return this.uuid.equals(((UuidId) other).uuid);
+ }
+
+ @Override
+ public String toString() {
+ return this.uuid.toString();
+ }
+ }
+
+ /**
+ * This class is just used by backend store for wrapper object as Id
+ */
+ private static final class ObjectId implements Id {
+
+ private final Object object;
+
+ public ObjectId(Object object) {
+ E.checkNotNull(object, "object");
+ this.object = object;
+ }
+
+ @Override
+ public IdType type() {
+ return IdType.UNKNOWN;
+ }
+
+ @Override
+ public Object asObject() {
+ return this.object;
+ }
+
+ @Override
+ public String asString() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long asLong() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public byte[] asBytes() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int length() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int compareTo(Id o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int hashCode() {
+ return this.object.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof ObjectId)) {
+ return false;
+ }
+ return Objects.equals(this.object, ((ObjectId) other).object);
+ }
+
+ @Override
+ public String toString() {
+ return this.object.toString();
+ }
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/LZ4Util.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/LZ4Util.java
new file mode 100644
index 00000000..32684369
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/LZ4Util.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.util;
+
+
+import net.jpountz.lz4.LZ4BlockOutputStream;
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+import net.jpountz.lz4.LZ4BlockInputStream;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+public class LZ4Util {
+
+ protected static final float DEFAULT_BUFFER_RATIO = 1.5f;
+
+ public static BytesBuffer compress(byte[] bytes, int blockSize) {
+ return compress(bytes, blockSize, DEFAULT_BUFFER_RATIO);
+ }
+
+ public static BytesBuffer compress(byte[] bytes, int blockSize,
+ float bufferRatio) {
+ float ratio = bufferRatio <= 0.0F ? DEFAULT_BUFFER_RATIO : bufferRatio;
+ LZ4Factory factory = LZ4Factory.fastestInstance();
+ LZ4Compressor compressor = factory.fastCompressor();
+ int initBufferSize = Math.round(bytes.length / ratio);
+ BytesBuffer buf = new BytesBuffer(initBufferSize);
+ LZ4BlockOutputStream lz4Output = new LZ4BlockOutputStream(
+ buf, blockSize, compressor);
+ try {
+ lz4Output.write(bytes);
+ lz4Output.close();
+ } catch (IOException e) {
+ throw new HugeException("Failed to compress", e);
+ }
+ /*
+ * If need to perform reading outside the method,
+ * remember to call forReadWritten()
+ */
+ return buf;
+ }
+
+ public static BytesBuffer decompress(byte[] bytes, int blockSize) {
+ return decompress(bytes, blockSize, DEFAULT_BUFFER_RATIO);
+ }
+
+ public static BytesBuffer decompress(byte[] bytes, int blockSize,
+ float bufferRatio) {
+ float ratio = bufferRatio <= 0.0F ? DEFAULT_BUFFER_RATIO : bufferRatio;
+ LZ4Factory factory = LZ4Factory.fastestInstance();
+ LZ4FastDecompressor decompressor = factory.fastDecompressor();
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ int initBufferSize = Math.min(Math.round(bytes.length * ratio),
+ BytesBuffer.MAX_BUFFER_CAPACITY);
+ BytesBuffer buf = new BytesBuffer(initBufferSize);
+ LZ4BlockInputStream lzInput = new LZ4BlockInputStream(bais,
+ decompressor);
+ int count;
+ byte[] buffer = new byte[blockSize];
+ try {
+ while ((count = lzInput.read(buffer)) != -1) {
+ buf.write(buffer, 0, count);
+ }
+ lzInput.close();
+ } catch (IOException e) {
+ throw new HugeException("Failed to decompress", e);
+ }
+ /*
+ * If need to perform reading outside the method,
+ * remember to call forReadWritten()
+ */
+ return buf;
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/SplicingIdGenerator.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/SplicingIdGenerator.java
new file mode 100644
index 00000000..ab504fb1
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/SplicingIdGenerator.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.util;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.baidu.hugegraph.util.IdUtil;
+
+/**
+ * This class is used for merge / parse id in primaryKey mode, simplify from
server
+ **/
+public class SplicingIdGenerator {
+
+ private static volatile SplicingIdGenerator instance;
+
+ public static SplicingIdGenerator instance() {
+ if (instance == null) {
+ synchronized (SplicingIdGenerator.class) {
+ if (instance == null) {
+ instance = new SplicingIdGenerator();
+ }
+ }
+ }
+ return instance;
+ }
+
+ /*
+ * The following defines can't be java regex special characters:
"\^$.|?*+()[{"
+ */
+ private static final char ESCAPE = '`';
+ private static final char IDS_SPLITOR = '>';
+ private static final char ID_SPLITOR = ':';
+ private static final char NAME_SPLITOR = '!';
+
+ public static final String ESCAPE_STR = String.valueOf(ESCAPE);
+ public static final String IDS_SPLITOR_STR = String.valueOf(IDS_SPLITOR);
+ public static final String ID_SPLITOR_STR = String.valueOf(ID_SPLITOR);
+
+ /**
+ * Generate a string id of HugeVertex from Vertex name
+ */
+// public Id generate(HugeVertex vertex) {
+// /*
+// * Hash for row-key which will be evenly distributed.
+// * We can also use LongEncoding.encode() to encode the int/long hash
+// * if needed.
+// * id = String.format("%s%s%s", HashUtil.hash(id), ID_SPLITOR, id);
+// */
+// // TODO: use binary Id with binary fields instead of string id
+// return splicing(vertex.schemaLabel().id().asString(), vertex.name());
+// }
+
+ /**
+ * Concat multiple ids into one composite id with IDS_SPLITOR
+ * @param ids the string id values to be contacted
+ * @return contacted string value
+ */
+ public static String concat(String... ids) {
+ // NOTE: must support string id when using this method
+ return IdUtil.escape(IDS_SPLITOR, ESCAPE, ids);
+ }
+
+ /**
+ * Split a composite id into multiple ids with IDS_SPLITOR
+ * @param ids the string id value to be split
+ * @return split string values
+ */
+ public static String[] split(String ids) {
+ return IdUtil.unescape(ids, IDS_SPLITOR_STR, ESCAPE_STR);
+ }
+
+ /**
+ * Concat property values with NAME_SPLITOR
+ * @param values the property values to be concatted
+ * @return concatted string value
+ */
+ public static String concatValues(List<?> values) {
+ // Convert the object list to string array
+ int valuesSize = values.size();
+ String[] parts = new String[valuesSize];
+ for (int i = 0; i < valuesSize; i++) {
+ parts[i] = values.get(i).toString();
+ }
+ return IdUtil.escape(NAME_SPLITOR, ESCAPE, parts);
+ }
+
+ /**
+ * Concat property values with NAME_SPLITOR
+ * @param values the property values to be contacted
+ * @return contacted string value
+ */
+ public static String concatValues(Object... values) {
+ return concatValues(Arrays.asList(values));
+ }
+
+ /**
+ * Concat multiple parts into a single id with ID_SPLITOR
+ * @param parts the string id values to be spliced
+ * @return spliced id object
+ */
+ public static Id splicing(String... parts) {
+ String escaped = IdUtil.escape(ID_SPLITOR, ESCAPE, parts);
+ return IdGenerator.of(escaped);
+ }
+
+ /**
+ * Parse a single id into multiple parts with ID_SPLITOR
+ * @param id the id object to be parsed
+ * @return parsed string id parts
+ */
+ public static String[] parse(Id id) {
+ return IdUtil.unescape(id.asString(), ID_SPLITOR_STR, ESCAPE_STR);
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/StringEncoding.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/StringEncoding.java
new file mode 100644
index 00000000..5668f14a
--- /dev/null
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/StringEncoding.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.serializer.direct.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+import java.util.UUID;
+
+import com.baidu.hugegraph.util.Bytes;
+import com.baidu.hugegraph.util.E;
+import com.google.common.base.CharMatcher;
+
+/**
+ * Used for encode / decode string values, simplify from server
+ */
+public final class StringEncoding {
+
+ private static final MessageDigest DIGEST;
+ private static final byte[] BYTES_EMPTY = new byte[0];
+ private static final String STRING_EMPTY = "";
+ private static final int BLOCK_SIZE = 4096;
+
+ static {
+ final String ALG = "SHA-256";
+ try {
+ DIGEST = MessageDigest.getInstance(ALG);
+ } catch (NoSuchAlgorithmException e) {
+ throw new HugeException("Failed to load algorithm %s", e, ALG);
+ }
+ }
+
+ private static final Base64.Encoder BASE64_ENCODER = Base64.getEncoder();
+ private static final Base64.Decoder BASE64_DECODER = Base64.getDecoder();
+
+ // Similar to {@link StringSerializer}
+ public static int writeAsciiString(byte[] array, int offset, String value)
{
+ E.checkArgument(CharMatcher.ascii().matchesAllOf(value),
+ "'%s' must be ASCII string", value);
+ int len = value.length();
+ if (len == 0) {
+ array[offset++] = (byte) 0x80;
+ return offset;
+ }
+
+ int i = 0;
+ do {
+ int c = value.charAt(i);
+ assert c <= 127;
+ byte b = (byte) c;
+ if (++i == len) {
+ b |= 0x80; // End marker
+ }
+ array[offset++] = b;
+ } while (i < len);
+
+ return offset;
+ }
+
+ public static String readAsciiString(byte[] array, int offset) {
+ StringBuilder sb = new StringBuilder();
+ int c = 0;
+ do {
+ c = 0xFF & array[offset++];
+ if (c != 0x80) {
+ sb.append((char) (c & 0x7F));
+ }
+ } while ((c & 0x80) <= 0);
+ return sb.toString();
+ }
+
+ public static int getAsciiByteLength(String value) {
+ E.checkArgument(CharMatcher.ascii().matchesAllOf(value),
+ "'%s' must be ASCII string", value);
+ return value.isEmpty() ? 1 : value.length();
+ }
+
+ public static byte[] encode(String value) {
+ return value.getBytes(UTF_8);
+ }
+
+ public static String decode(byte[] bytes) {
+ if (bytes.length == 0) {
+ return STRING_EMPTY;
+ }
+ return new String(bytes, UTF_8);
+ }
+
+ public static String decode(byte[] bytes, int offset, int length) {
+ if (length == 0) {
+ return STRING_EMPTY;
+ }
+ return new String(bytes, offset, length, UTF_8);
+ }
+
+ public static String encodeBase64(byte[] bytes) {
+ return BASE64_ENCODER.encodeToString(bytes);
+ }
+
+ public static byte[] decodeBase64(String value) {
+ if (value.isEmpty()) {
+ return BYTES_EMPTY;
+ }
+ return BASE64_DECODER.decode(value);
+ }
+
+ public static String sha256(String string) {
+ byte[] stringBytes = encode(string);
+ DIGEST.reset();
+ return StringEncoding.encodeBase64(DIGEST.digest(stringBytes));
+ }
+
+ public static String format(byte[] bytes) {
+ return String.format("%s[0x%s]", decode(bytes), Bytes.toHex(bytes));
+ }
+
+ public static UUID uuid(String value) {
+ E.checkArgument(value != null, "The UUID can't be null");
+ try {
+ if (value.contains("-") && value.length() == 36) {
+ return UUID.fromString(value);
+ }
+ // UUID represented by hex string
+ E.checkArgument(value.length() == 32,
+ "Invalid UUID string: %s", value);
+ String high = value.substring(0, 16);
+ String low = value.substring(16);
+ return new UUID(Long.parseUnsignedLong(high, 16),
+ Long.parseUnsignedLong(low, 16));
+ } catch (NumberFormatException ignored) {
+ throw new IllegalArgumentException("Invalid UUID string: " +
value);
+ }
+ }
+}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/GraphElement.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/GraphElement.java
index 042882c2..c4363b88 100644
---
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/GraphElement.java
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/GraphElement.java
@@ -93,4 +93,8 @@ public abstract class GraphElement extends Element implements
GraphAttachable {
protected abstract GraphElement setProperty(String key, Object value);
public abstract GraphElement removeProperty(String key);
+
+ public int sizeOfProperties() {
+ return properties.size();
+ }
}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java
index 7d93c211..676bd878 100644
---
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java
@@ -27,8 +27,8 @@ public enum Direction {
BOTH(3, "both");
- private byte code = 0;
- private String name = null;
+ private byte code;
+ private String name;
Direction(int code, String name) {
assert code < 256;
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/HugeType.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/HugeType.java
index cdfb1818..fe9de078 100644
---
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/HugeType.java
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/HugeType.java
@@ -65,8 +65,8 @@ public enum HugeType {
// Metrics
METRICS(240, "metrics");
- private int code;
- private String name = null;
+ private final int code;
+ private final String name;
HugeType(int code, String name) {
assert code < 256;
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Edge.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Edge.java
index 6d6c0dfa..baddda37 100644
---
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Edge.java
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Edge.java
@@ -56,6 +56,7 @@ public class Edge extends GraphElement {
this.name = null;
}
+ @Override
public String id() {
return this.id;
}
diff --git
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Vertex.java
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Vertex.java
index 0322410f..f0ce717f 100644
---
a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Vertex.java
+++
b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Vertex.java
@@ -38,6 +38,7 @@ public class Vertex extends GraphElement {
this.type = "vertex";
}
+ @Override
public Object id() {
return this.id;
}
diff --git
a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java
b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java
index 899a6ea0..864e4f74 100644
---
a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java
+++
b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java
@@ -45,7 +45,7 @@ public class EdgeBuilder extends ElementBuilder<Edge> {
private final VertexLabel sourceLabel;
private final VertexLabel targetLabel;
private final Collection<String> nonNullKeys;
- // Used to optimize access performace
+ // Used to optimize access performance
private VertexIdsIndex vertexIdsIndex;
private String[] lastNames;
@@ -57,7 +57,7 @@ public class EdgeBuilder extends ElementBuilder<Edge> {
this.sourceLabel = this.getVertexLabel(this.edgeLabel.sourceLabel());
this.targetLabel = this.getVertexLabel(this.edgeLabel.targetLabel());
this.nonNullKeys = this.nonNullableKeys(this.edgeLabel);
- // Ensure that the source/target id fileds are matched with id strategy
+ // Ensure that the source/target id fields are matched with id strategy
this.checkIdFields(this.sourceLabel, this.mapping.sourceFields());
this.checkIdFields(this.targetLabel, this.mapping.targetFields());
diff --git
a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java
b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java
index edc503fd..36b5dc23 100644
---
a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java
+++
b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java
@@ -19,7 +19,6 @@
package com.baidu.hugegraph.loader.flink;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -129,7 +128,7 @@ public class HugeGraphOutputFormat<T> extends
RichOutputFormat<T> {
}
@Override
- public synchronized void writeRecord(T row) throws IOException {
+ public synchronized void writeRecord(T row) {
for (Map.Entry<ElementBuilder, List<String>> builder :
this.builders.entrySet()) {
ElementMapping elementMapping = builder.getKey().mapping();
@@ -164,7 +163,7 @@ public class HugeGraphOutputFormat<T> extends
RichOutputFormat<T> {
return Tuple2.of(op, elementBuilder.build(fields, values));
}
- private void flush(ElementBuilder elementBuilder, List<String> rows) {
+ private void flush(ElementBuilder<GraphElement> elementBuilder,
List<String> rows) {
GraphManager g = this.loadContext.client().graph();
ElementMapping elementMapping = elementBuilder.mapping();
for (String row : rows) {
diff --git
a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/DataTypeUtil.java
b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/DataTypeUtil.java
index 87ff516b..6f4e4dba 100644
---
a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/DataTypeUtil.java
+++
b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/DataTypeUtil.java
@@ -94,7 +94,7 @@ public final class DataTypeUtil {
if (rawValue instanceof Number) {
return ((Number) rawValue).longValue();
} else if (rawValue instanceof String) {
- // trim() is a little time consuming
+ // trim() is a little time-consuming
return parseLong(((String) rawValue).trim());
}
throw new IllegalArgumentException(String.format(
