Repository: arrow Updated Branches: refs/heads/master b4886da0f -> 00994b820
ARROW-920: [GLib] Add Lua examples Author: Kouhei Sutou <[email protected]> Closes #620 from kou/glib-lua-examples and squashes the following commits: 491c0e4 [Kouhei Sutou] [GLib] Add Lua examples Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/00994b82 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/00994b82 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/00994b82 Branch: refs/heads/master Commit: 00994b82015365fec8474605bf09bd11637859af Parents: b4886da Author: Kouhei Sutou <[email protected]> Authored: Sun Apr 30 13:35:29 2017 -0400 Committer: Wes McKinney <[email protected]> Committed: Sun Apr 30 13:35:29 2017 -0400 ---------------------------------------------------------------------- c_glib/README.md | 2 + c_glib/configure.ac | 1 + c_glib/example/Makefile.am | 6 ++- c_glib/example/README.md | 5 ++- c_glib/example/lua/Makefile.am | 24 +++++++++++ c_glib/example/lua/README.md | 45 +++++++++++++++++++ c_glib/example/lua/read-batch.lua | 44 +++++++++++++++++++ c_glib/example/lua/read-stream.lua | 51 ++++++++++++++++++++++ c_glib/example/lua/write-batch.lua | 74 ++++++++++++++++++++++++++++++++ c_glib/example/lua/write-stream.lua | 74 ++++++++++++++++++++++++++++++++ 10 files changed, 323 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/README.md ---------------------------------------------------------------------- diff --git a/c_glib/README.md b/c_glib/README.md index b253d32..6eadb79 100644 --- a/c_glib/README.md +++ b/c_glib/README.md @@ -194,10 +194,12 @@ You can use Arrow GLib with non C languages with GObject Introspection based bindings. Here are languages that support GObject Introspection: * Ruby: [red-arrow gem](https://rubygems.org/gems/red-arrow) should be used. + * Examples: https://github.com/red-data-tools/red-arrow/tree/master/example * Python: [PyGObject](https://wiki.gnome.org/Projects/PyGObject) should be used. (Note that you should use PyArrow than Arrow GLib.) * Lua: [LGI](https://github.com/pavouk/lgi) should be used. + * Examples: `example/lua/` directory. * Go: [Go-gir-generator](https://github.com/linuxdeepin/go-gir-generator) should be used. http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/configure.ac ---------------------------------------------------------------------- diff --git a/c_glib/configure.ac b/c_glib/configure.ac index f367192..e010d96 100644 --- a/c_glib/configure.ac +++ b/c_glib/configure.ac @@ -97,6 +97,7 @@ AC_CONFIG_FILES([ doc/Makefile doc/reference/Makefile example/Makefile + example/lua/Makefile ]) AC_OUTPUT http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/Makefile.am ---------------------------------------------------------------------- diff --git a/c_glib/example/Makefile.am b/c_glib/example/Makefile.am index 8bf3c15..66d2cdd 100644 --- a/c_glib/example/Makefile.am +++ b/c_glib/example/Makefile.am @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +SUBDIRS = \ + lua + AM_CPPFLAGS = \ -I$(top_builddir) \ -I$(top_srcdir) @@ -41,7 +44,8 @@ read_batch_SOURCES = \ read_stream_SOURCES = \ read-stream.c -example_DATA = \ +dist_example_DATA = \ + README.md \ $(build_SOURCES) \ $(read_batch_SOURCES) \ $(read_stream_SOURCES) http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/README.md ---------------------------------------------------------------------- diff --git a/c_glib/example/README.md b/c_glib/example/README.md index b1ba259..99730d5 100644 --- a/c_glib/example/README.md +++ b/c_glib/example/README.md @@ -16,7 +16,9 @@ There are example codes in this directory. -C example codes exist in this directory. +C example codes exist in this directory. Language bindings example +codes exists in sub directories. For example, Lua example codes exists +in `lua/` sub directory. ## C example codes @@ -39,4 +41,3 @@ Here are example codes in this directory: * `read-stream.c`: It shows how to read Arrow array from file in stream mode. - http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/Makefile.am ---------------------------------------------------------------------- diff --git a/c_glib/example/lua/Makefile.am b/c_glib/example/lua/Makefile.am new file mode 100644 index 0000000..9019d24 --- /dev/null +++ b/c_glib/example/lua/Makefile.am @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +lua_exampledir = $(exampledir)/lua +dist_lua_example_DATA = \ + README.md \ + read-batch.lua \ + read-stream.lua \ + write-batch.lua \ + write-stream.lua http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/README.md ---------------------------------------------------------------------- diff --git a/c_glib/example/lua/README.md b/c_glib/example/lua/README.md new file mode 100644 index 0000000..d127573 --- /dev/null +++ b/c_glib/example/lua/README.md @@ -0,0 +1,45 @@ +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# Arrow Lua example + +There are Lua example codes in this directory. + +## How to run + +All example codes use [LGI](https://github.com/pavouk/lgi) to use +Arrow GLib based bindings. + +Here are command lines to install LGI on Debian GNU/Linux and Ubuntu: + +```text +% sudo apt install -y luarocks +% sudo luarocks install lgi +``` + +## Lua example codes + +Here are example codes in this directory: + + * `write-batch.lua`: It shows how to write Arrow array to file in + batch mode. + + * `read-batch.lua`: It shows how to read Arrow array from file in + batch mode. + + * `write-stream.lua`: It shows how to write Arrow array to file in + stream mode. + + * `read-stream.lua`: It shows how to read Arrow array from file in + stream mode. http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/read-batch.lua ---------------------------------------------------------------------- diff --git a/c_glib/example/lua/read-batch.lua b/c_glib/example/lua/read-batch.lua new file mode 100644 index 0000000..b28d346 --- /dev/null +++ b/c_glib/example/lua/read-batch.lua @@ -0,0 +1,44 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +local lgi = require 'lgi' +local Arrow = lgi.Arrow + +local input_path = arg[1] or "/tmp/batch.arrow"; + +local input = Arrow.MemoryMappedFile.open(input_path, Arrow.FileMode.READ) +local reader = Arrow.FileReader.open(input) + +for i = 0, reader:get_n_record_batches() - 1 do + local record_batch = reader:get_record_batch(i) + print(string.rep("=", 40)) + print("record-batch["..i.."]:") + for j = 0, record_batch:get_n_columns() - 1 do + local column = record_batch:get_column(j) + local column_name = record_batch:get_column_name(j) + io.write(" "..column_name..": [") + for k = 0, record_batch:get_n_rows() - 1 do + if k > 0 then + io.write(", ") + end + io.write(column:get_value(k)) + end + print("]") + end +end + +input:close() http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/read-stream.lua ---------------------------------------------------------------------- diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua new file mode 100644 index 0000000..3b08206 --- /dev/null +++ b/c_glib/example/lua/read-stream.lua @@ -0,0 +1,51 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +local lgi = require 'lgi' +local Arrow = lgi.Arrow + +local input_path = arg[1] or "/tmp/stream.arrow"; + +local input = Arrow.MemoryMappedFile.open(input_path, Arrow.FileMode.READ) +local reader = Arrow.StreamReader.open(input) + +local i = 0 +while true do + local record_batch = reader:get_next_record_batch(i) + if not record_batch then + break + end + + print(string.rep("=", 40)) + print("record-batch["..i.."]:") + for j = 0, record_batch:get_n_columns() - 1 do + local column = record_batch:get_column(j) + local column_name = record_batch:get_column_name(j) + io.write(" "..column_name..": [") + for k = 0, record_batch:get_n_rows() - 1 do + if k > 0 then + io.write(", ") + end + io.write(column:get_value(k)) + end + print("]") + end + + i = i + 1 +end + +input:close() http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/write-batch.lua ---------------------------------------------------------------------- diff --git a/c_glib/example/lua/write-batch.lua b/c_glib/example/lua/write-batch.lua new file mode 100644 index 0000000..3a22cd5 --- /dev/null +++ b/c_glib/example/lua/write-batch.lua @@ -0,0 +1,74 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +local lgi = require 'lgi' +local Arrow = lgi.Arrow + +local output_path = arg[1] or "/tmp/batch.arrow"; + +local fields = { + Arrow.Field.new("uint8", Arrow.UInt8DataType.new()), + Arrow.Field.new("uint16", Arrow.UInt16DataType.new()), + Arrow.Field.new("uint32", Arrow.UInt32DataType.new()), + Arrow.Field.new("uint64", Arrow.UInt64DataType.new()), + Arrow.Field.new("int8", Arrow.Int8DataType.new()), + Arrow.Field.new("int16", Arrow.Int16DataType.new()), + Arrow.Field.new("int32", Arrow.Int32DataType.new()), + Arrow.Field.new("int64", Arrow.Int64DataType.new()), + Arrow.Field.new("float", Arrow.FloatDataType.new()), + Arrow.Field.new("double", Arrow.DoubleDataType.new()), +} +local schema = Arrow.Schema.new(fields) + +local output = Arrow.FileOutputStream.open(output_path, false) +local writer = Arrow.FileWriter.open(output, schema) + +function build_array(builder, values) + for _, value in pairs(values) do + builder:append(value) + end + return builder:finish() +end + +local uints = {1, 2, 4, 8} +local ints = {1, -2, 4, -8} +local floats = {1.1, -2.2, 4.4, -8.8} +local columns = { + build_array(Arrow.UInt8ArrayBuilder.new(), uints), + build_array(Arrow.UInt16ArrayBuilder.new(), uints), + build_array(Arrow.UInt32ArrayBuilder.new(), uints), + build_array(Arrow.UInt64ArrayBuilder.new(), uints), + build_array(Arrow.Int8ArrayBuilder.new(), ints), + build_array(Arrow.Int16ArrayBuilder.new(), ints), + build_array(Arrow.Int32ArrayBuilder.new(), ints), + build_array(Arrow.Int64ArrayBuilder.new(), ints), + build_array(Arrow.FloatArrayBuilder.new(), floats), + build_array(Arrow.DoubleArrayBuilder.new(), floats), +} + +local record_batch = Arrow.RecordBatch.new(schema, 4, columns) +writer:write_record_batch(record_batch) + +local sliced_columns = {} +for i, column in pairs(columns) do + sliced_columns[i] = column:slice(1, 3) +end +record_batch = Arrow.RecordBatch.new(schema, 3, sliced_columns) +writer:write_record_batch(record_batch) + +writer:close() +output:close() http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/write-stream.lua ---------------------------------------------------------------------- diff --git a/c_glib/example/lua/write-stream.lua b/c_glib/example/lua/write-stream.lua new file mode 100644 index 0000000..37c6bb5 --- /dev/null +++ b/c_glib/example/lua/write-stream.lua @@ -0,0 +1,74 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +local lgi = require 'lgi' +local Arrow = lgi.Arrow + +local output_path = arg[1] or "/tmp/stream.arrow"; + +local fields = { + Arrow.Field.new("uint8", Arrow.UInt8DataType.new()), + Arrow.Field.new("uint16", Arrow.UInt16DataType.new()), + Arrow.Field.new("uint32", Arrow.UInt32DataType.new()), + Arrow.Field.new("uint64", Arrow.UInt64DataType.new()), + Arrow.Field.new("int8", Arrow.Int8DataType.new()), + Arrow.Field.new("int16", Arrow.Int16DataType.new()), + Arrow.Field.new("int32", Arrow.Int32DataType.new()), + Arrow.Field.new("int64", Arrow.Int64DataType.new()), + Arrow.Field.new("float", Arrow.FloatDataType.new()), + Arrow.Field.new("double", Arrow.DoubleDataType.new()), +} +local schema = Arrow.Schema.new(fields) + +local output = Arrow.FileOutputStream.open(output_path, false) +local writer = Arrow.StreamWriter.open(output, schema) + +function build_array(builder, values) + for _, value in pairs(values) do + builder:append(value) + end + return builder:finish() +end + +local uints = {1, 2, 4, 8} +local ints = {1, -2, 4, -8} +local floats = {1.1, -2.2, 4.4, -8.8} +local columns = { + build_array(Arrow.UInt8ArrayBuilder.new(), uints), + build_array(Arrow.UInt16ArrayBuilder.new(), uints), + build_array(Arrow.UInt32ArrayBuilder.new(), uints), + build_array(Arrow.UInt64ArrayBuilder.new(), uints), + build_array(Arrow.Int8ArrayBuilder.new(), ints), + build_array(Arrow.Int16ArrayBuilder.new(), ints), + build_array(Arrow.Int32ArrayBuilder.new(), ints), + build_array(Arrow.Int64ArrayBuilder.new(), ints), + build_array(Arrow.FloatArrayBuilder.new(), floats), + build_array(Arrow.DoubleArrayBuilder.new(), floats), +} + +local record_batch = Arrow.RecordBatch.new(schema, 4, columns) +writer:write_record_batch(record_batch) + +local sliced_columns = {} +for i, column in pairs(columns) do + sliced_columns[i] = column:slice(1, 3) +end +record_batch = Arrow.RecordBatch.new(schema, 3, sliced_columns) +writer:write_record_batch(record_batch) + +writer:close() +output:close()
