This is an automated email from the ASF dual-hosted git repository. xikai pushed a commit to branch memtable-poc in repository https://gitbox.apache.org/repos/asf/incubator-horaedb.git
commit c4d31f124d00498f23e235e31faee4a8cedf512d Author: xikai.wxk <[email protected]> AuthorDate: Wed Dec 27 14:47:42 2023 +0800 improve: use parquet with dictionary decoding enhancement --- Cargo.lock | 83 +++++++++++++++++++++++++------------------------------------- Cargo.toml | 18 +++++++------- 2 files changed, 43 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 325d0237..42fa8abc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -241,8 +241,7 @@ dependencies = [ [[package]] name = "arrow" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "2feeebd77b34b0bc88f224e06d01c27da4733997cc4789a4e056196656cdc59a" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "ahash 0.8.3", "arrow-arith 43.0.0", @@ -278,8 +277,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "7173f5dc49c0ecb5135f52565af33afd3fdc9a12d13bd6f9973e8b96305e4b2e" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "arrow-array 43.0.0", "arrow-buffer 43.0.0", @@ -309,8 +307,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "63d7ea725f7d1f8bb2cffc53ef538557e95fc802e217d5be25122d402e22f3d0" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "ahash 0.8.3", "arrow-buffer 43.0.0", @@ -336,8 +333,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "bdbe439e077f484e5000b9e1d47b5e4c0d15f2b311a8f5bcc682553d5d67a722" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "half 2.2.1", "num", @@ -362,8 +358,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "93913cc14875770aa1eef5e310765e855effa352c094cb1c7c00607d0f37b4e1" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "arrow-array 43.0.0", "arrow-buffer 43.0.0", @@ -399,8 +394,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "ef55b67c55ed877e6fe7b923121c19dae5e31ca70249ea2779a17b58fb0fbd9a" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "arrow-array 43.0.0", "arrow-buffer 43.0.0", @@ -430,8 +424,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "d4f4f4a3c54614126a71ab91f6631c9743eb4643d6e9318b74191da9dc6e028b" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "arrow-buffer 43.0.0", "arrow-schema 43.0.0", @@ -456,8 +449,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "d41a3659f984a524ef1c2981d43747b24d8eec78e2425267fcd0ef34ce71cd18" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "arrow-array 43.0.0", "arrow-buffer 43.0.0", @@ -490,8 +482,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "10b95faa95a378f56ef32d84cc0104ea998c39ef7cd1faaa6b4cebf8ea92846d" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "arrow-array 43.0.0", "arrow-buffer 43.0.0", @@ -525,8 +516,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "c68549a4284d9f8b39586afb8d5ff8158b8f0286353a4844deb1d11cf1ba1f26" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "arrow-array 43.0.0", "arrow-buffer 43.0.0", @@ -555,8 +545,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "0a75a4a757afc301ce010adadff54d79d66140c4282ed3de565f6ccb716a5cf3" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "ahash 0.8.3", "arrow-array 43.0.0", @@ -576,8 +565,7 @@ checksum = "bc85923d8d6662cc66ac6602c7d1876872e671002d60993dfdf492a6badeae92" [[package]] name = "arrow-schema" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "2bebcb57eef570b15afbcf2d07d813eb476fde9f6dd69c81004d6476c197e87e" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; [[package]] name = "arrow-select" @@ -595,8 +583,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "f6e2943fa433a48921e914417173816af64eef61c0a3d448280e6c40a62df221" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "arrow-array 43.0.0", "arrow-buffer 43.0.0", @@ -623,8 +610,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "bbc92ed638851774f6d7af1ad900b92bc1486746497511868b4298fcbcfa35af" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "arrow-array 43.0.0", "arrow-buffer 43.0.0", @@ -649,7 +635,7 @@ dependencies = [ [[package]] name = "arrow_util" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "ahash 0.8.3", "arrow 43.0.0", @@ -2089,7 +2075,7 @@ dependencies = [ [[package]] name = "datafusion" version = "27.0.0" -source = "git+https://github.com/CeresDB/arrow-datafusion.git?rev=9c3a537e25e5ab3299922864034f67fb2f79805d#9c3a537e25e5ab3299922864034f67fb2f79805d"; +source = "git+https://github.com/ShiKaiWi/arrow-datafusion.git?rev=f4440be8f9c4a50f5e2e8224e18c498c12da37ee#f4440be8f9c4a50f5e2e8224e18c498c12da37ee"; dependencies = [ "ahash 0.8.3", "arrow 43.0.0", @@ -2137,7 +2123,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "27.0.0" -source = "git+https://github.com/CeresDB/arrow-datafusion.git?rev=9c3a537e25e5ab3299922864034f67fb2f79805d#9c3a537e25e5ab3299922864034f67fb2f79805d"; +source = "git+https://github.com/ShiKaiWi/arrow-datafusion.git?rev=f4440be8f9c4a50f5e2e8224e18c498c12da37ee#f4440be8f9c4a50f5e2e8224e18c498c12da37ee"; dependencies = [ "arrow 43.0.0", "arrow-array 43.0.0", @@ -2151,7 +2137,7 @@ dependencies = [ [[package]] name = "datafusion-execution" version = "27.0.0" -source = "git+https://github.com/CeresDB/arrow-datafusion.git?rev=9c3a537e25e5ab3299922864034f67fb2f79805d#9c3a537e25e5ab3299922864034f67fb2f79805d"; +source = "git+https://github.com/ShiKaiWi/arrow-datafusion.git?rev=f4440be8f9c4a50f5e2e8224e18c498c12da37ee#f4440be8f9c4a50f5e2e8224e18c498c12da37ee"; dependencies = [ "dashmap 5.4.0", "datafusion-common", @@ -2168,7 +2154,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "27.0.0" -source = "git+https://github.com/CeresDB/arrow-datafusion.git?rev=9c3a537e25e5ab3299922864034f67fb2f79805d#9c3a537e25e5ab3299922864034f67fb2f79805d"; +source = "git+https://github.com/ShiKaiWi/arrow-datafusion.git?rev=f4440be8f9c4a50f5e2e8224e18c498c12da37ee#f4440be8f9c4a50f5e2e8224e18c498c12da37ee"; dependencies = [ "ahash 0.8.3", "arrow 43.0.0", @@ -2182,7 +2168,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "27.0.0" -source = "git+https://github.com/CeresDB/arrow-datafusion.git?rev=9c3a537e25e5ab3299922864034f67fb2f79805d#9c3a537e25e5ab3299922864034f67fb2f79805d"; +source = "git+https://github.com/ShiKaiWi/arrow-datafusion.git?rev=f4440be8f9c4a50f5e2e8224e18c498c12da37ee#f4440be8f9c4a50f5e2e8224e18c498c12da37ee"; dependencies = [ "arrow 43.0.0", "async-trait", @@ -2199,7 +2185,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "27.0.0" -source = "git+https://github.com/CeresDB/arrow-datafusion.git?rev=9c3a537e25e5ab3299922864034f67fb2f79805d#9c3a537e25e5ab3299922864034f67fb2f79805d"; +source = "git+https://github.com/ShiKaiWi/arrow-datafusion.git?rev=f4440be8f9c4a50f5e2e8224e18c498c12da37ee#f4440be8f9c4a50f5e2e8224e18c498c12da37ee"; dependencies = [ "ahash 0.8.3", "arrow 43.0.0", @@ -2234,7 +2220,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "27.0.0" -source = "git+https://github.com/CeresDB/arrow-datafusion.git?rev=9c3a537e25e5ab3299922864034f67fb2f79805d#9c3a537e25e5ab3299922864034f67fb2f79805d"; +source = "git+https://github.com/ShiKaiWi/arrow-datafusion.git?rev=f4440be8f9c4a50f5e2e8224e18c498c12da37ee#f4440be8f9c4a50f5e2e8224e18c498c12da37ee"; dependencies = [ "arrow 43.0.0", "chrono", @@ -2248,7 +2234,7 @@ dependencies = [ [[package]] name = "datafusion-row" version = "27.0.0" -source = "git+https://github.com/CeresDB/arrow-datafusion.git?rev=9c3a537e25e5ab3299922864034f67fb2f79805d#9c3a537e25e5ab3299922864034f67fb2f79805d"; +source = "git+https://github.com/ShiKaiWi/arrow-datafusion.git?rev=f4440be8f9c4a50f5e2e8224e18c498c12da37ee#f4440be8f9c4a50f5e2e8224e18c498c12da37ee"; dependencies = [ "arrow 43.0.0", "datafusion-common", @@ -2259,7 +2245,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "27.0.0" -source = "git+https://github.com/CeresDB/arrow-datafusion.git?rev=9c3a537e25e5ab3299922864034f67fb2f79805d#9c3a537e25e5ab3299922864034f67fb2f79805d"; +source = "git+https://github.com/ShiKaiWi/arrow-datafusion.git?rev=f4440be8f9c4a50f5e2e8224e18c498c12da37ee#f4440be8f9c4a50f5e2e8224e18c498c12da37ee"; dependencies = [ "arrow 43.0.0", "arrow-schema 43.0.0", @@ -2272,7 +2258,7 @@ dependencies = [ [[package]] name = "datafusion_util" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "async-trait", "datafusion", @@ -2862,7 +2848,7 @@ checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" [[package]] name = "generated_types" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "pbjson", "pbjson-build", @@ -3300,7 +3286,7 @@ dependencies = [ [[package]] name = "influxdb_influxql_parser" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "chrono", "chrono-tz", @@ -3393,7 +3379,7 @@ dependencies = [ [[package]] name = "iox_query" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "arrow 43.0.0", "arrow_util", @@ -3417,7 +3403,7 @@ dependencies = [ [[package]] name = "iox_query_influxql" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "arrow 43.0.0", "chrono", @@ -4517,7 +4503,7 @@ dependencies = [ [[package]] name = "observability_deps" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "tracing", ] @@ -4642,8 +4628,7 @@ dependencies = [ [[package]] name = "parquet" version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "ec7267a9607c3f955d4d0ac41b88a67cecc0d8d009173ad3da390699a6cb3750" +source = "git+https://github.com/ShiKaiWi/arrow-rs.git?rev=d4e905a6cc337f10c61f47d75f264df82fc97242#d4e905a6cc337f10c61f47d75f264df82fc97242"; dependencies = [ "ahash 0.8.3", "arrow-array 43.0.0", @@ -5485,7 +5470,7 @@ dependencies = [ [[package]] name = "query_functions" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "arrow 43.0.0", "chrono", @@ -6140,7 +6125,7 @@ dependencies = [ [[package]] name = "schema" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "arrow 43.0.0", "hashbrown 0.13.2", @@ -6911,7 +6896,7 @@ dependencies = [ [[package]] name = "test_helpers" version = "0.1.0" -source = "git+https://github.com/CeresDB/influxql.git?rev=a905863#a9058633c03f018607dc1e4f6ca090b82d46a30c"; +source = "git+https://github.com/ShiKaiWi/influxql.git?rev=309590aacf39939b007abbfa29bfd2237ab5f40a#309590aacf39939b007abbfa29bfd2237ab5f40a"; dependencies = [ "dotenvy", "observability_deps", diff --git a/Cargo.toml b/Cargo.toml index 3433300a..b6d9f710 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,8 +81,8 @@ members = [ [workspace.dependencies] alloc_tracker = { path = "components/alloc_tracker" } -arrow = { version = "43.0.0", features = ["prettyprint"] } -arrow_ipc = { version = "43.0.0" } +arrow = { git = "https://github.com/ShiKaiWi/arrow-rs.git";, rev = "d4e905a6cc337f10c61f47d75f264df82fc97242", features = ["prettyprint"] } +arrow_ipc = { git = "https://github.com/ShiKaiWi/arrow-rs.git";, rev = "d4e905a6cc337f10c61f47d75f264df82fc97242" } arrow_ext = { path = "components/arrow_ext" } analytic_engine = { path = "analytic_engine" } arena = { path = "components/arena" } @@ -102,8 +102,8 @@ clru = "0.6.1" cluster = { path = "cluster" } criterion = "0.5" common_types = { path = "common_types" } -datafusion = { git = "https://github.com/CeresDB/arrow-datafusion.git";, rev = "9c3a537e25e5ab3299922864034f67fb2f79805d" } -datafusion-proto = { git = "https://github.com/CeresDB/arrow-datafusion.git";, rev = "9c3a537e25e5ab3299922864034f67fb2f79805d" } +datafusion = { git = "https://github.com/ShiKaiWi/arrow-datafusion.git";, rev = "f4440be8f9c4a50f5e2e8224e18c498c12da37ee" } +datafusion-proto = { git = "https://github.com/ShiKaiWi/arrow-datafusion.git";, rev = "f4440be8f9c4a50f5e2e8224e18c498c12da37ee" } df_operator = { path = "df_operator" } df_engine_extensions = { path = "df_engine_extensions" } future_ext = { path = "components/future_ext" } @@ -115,10 +115,10 @@ hash_ext = { path = "components/hash_ext" } hex = "0.4.3" hyperloglog = { git = "https://github.com/jedisct1/rust-hyperloglog.git";, rev = "425487ce910f26636fbde8c4d640b538431aad50" } id_allocator = { path = "components/id_allocator" } -influxql-logical-planner = { git = "https://github.com/CeresDB/influxql.git";, rev = "a905863", package = "iox_query_influxql" } -influxql-parser = { git = "https://github.com/CeresDB/influxql.git";, rev = "a905863", package = "influxdb_influxql_parser" } -influxql-query = { git = "https://github.com/CeresDB/influxql.git";, rev = "a905863", package = "iox_query" } -influxql-schema = { git = "https://github.com/CeresDB/influxql.git";, rev = "a905863", package = "schema" } +influxql-logical-planner = { git = "https://github.com/ShiKaiWi/influxql.git";, rev = "309590aacf39939b007abbfa29bfd2237ab5f40a", package = "iox_query_influxql" } +influxql-parser = { git = "https://github.com/ShiKaiWi/influxql.git";, rev = "309590aacf39939b007abbfa29bfd2237ab5f40a", package = "influxdb_influxql_parser" } +influxql-query = { git = "https://github.com/ShiKaiWi/influxql.git";, rev = "309590aacf39939b007abbfa29bfd2237ab5f40a", package = "iox_query" } +influxql-schema = { git = "https://github.com/ShiKaiWi/influxql.git";, rev = "309590aacf39939b007abbfa29bfd2237ab5f40a", package = "schema" } interpreters = { path = "interpreters" } itertools = "0.10.5" lz4_flex = { version = "0.11", default-features = false, features = ["frame"] } @@ -135,7 +135,7 @@ panic_ext = { path = "components/panic_ext" } partitioned_lock = { path = "components/partitioned_lock" } partition_table_engine = { path = "partition_table_engine" } parquet_ext = { path = "components/parquet_ext" } -parquet = { version = "43.0.0" } +parquet = { git = "https://github.com/ShiKaiWi/arrow-rs.git";, rev = "d4e905a6cc337f10c61f47d75f264df82fc97242" } paste = "1.0" pin-project-lite = "0.2.8" pprof = "0.12.1" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
