arrow git commit: ARROW-852: Also search for ARROW libs when pkg-config provided the path

2017-04-18 Thread wesm 
Repository: arrow
Updated Branches:
  refs/heads/master a94c03a02 -> 59cd801a7


ARROW-852: Also search for ARROW libs when pkg-config provided the path

Change-Id: Ic7fb227342782dfed5885f8fc5e73418fd31d504

Author: Uwe L. Korn 

Closes #563 from xhochy/ARROW-852 and squashes the following commits:

9630352 [Uwe L. Korn] Remove ARROW_HOME
5fc43ce [Uwe L. Korn] Always search for libs


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/59cd801a
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/59cd801a
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/59cd801a

Branch: refs/heads/master
Commit: 59cd801a7645783c0c33ed2435be08db4ffcd378
Parents: a94c03a
Author: Uwe L. Korn 
Authored: Tue Apr 18 22:32:59 2017 -0400
Committer: Wes McKinney 
Committed: Tue Apr 18 22:32:59 2017 -0400

--
 python/cmake_modules/FindArrow.cmake | 61 ---
 python/manylinux1/build_arrow.sh |  1 -
 2 files changed, 31 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/59cd801a/python/cmake_modules/FindArrow.cmake
--
diff --git a/python/cmake_modules/FindArrow.cmake 
b/python/cmake_modules/FindArrow.cmake
index fbe4545..9fb1355 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -34,6 +34,7 @@ if ("$ENV{ARROW_HOME}" STREQUAL "")
 message(STATUS "Arrow SO version: ${ARROW_SO_VERSION}")
 set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
 set(ARROW_LIBS ${ARROW_LIBRARY_DIRS})
+set(ARROW_SEARCH_LIB_PATH ${ARROW_LIBRARY_DIRS})
   endif()
 else()
   set(ARROW_HOME "$ENV{ARROW_HOME}")
@@ -51,42 +52,42 @@ else()
 # make sure we don't accidentally pick up a different version
 NO_DEFAULT_PATH
 )
+endif()
 
-  find_library(ARROW_LIB_PATH NAMES arrow
-PATHS
-${ARROW_SEARCH_LIB_PATH}
-NO_DEFAULT_PATH)
-  get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
+find_library(ARROW_LIB_PATH NAMES arrow
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
 
-  find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
-PATHS
-${ARROW_SEARCH_LIB_PATH}
-NO_DEFAULT_PATH)
+find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
 
-  find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
-PATHS
-${ARROW_SEARCH_LIB_PATH}
-NO_DEFAULT_PATH)
+find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
 
-  if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
-set(ARROW_FOUND TRUE)
+if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
+  set(ARROW_FOUND TRUE)
 
-if (MSVC)
-  set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
-  set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
-  set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH})
-  set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
-  set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
-  set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB})
-else()
-  set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
-  set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
-  set(ARROW_JEMALLOC_STATIC_LIB 
${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
+  if (MSVC)
+set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
+set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
+set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH})
+set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
+set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
+set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB})
+  else()
+set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
+set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
+set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
 
-  set(ARROW_SHARED_LIB 
${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
-  set(ARROW_JEMALLOC_SHARED_LIB 
${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
-  set(ARROW_PYTHON_SHARED_LIB 
${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
-endif()
+set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+set(ARROW_JEMALLOC_SHARED_LIB 
${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
+set(ARROW_PYTHON_SHARED_LIB 
${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
 endif()
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/59cd801a/python/manylinux1/build_arrow.sh
--
diff --git

arrow git commit: ARROW-844: [Format] Update README documents in format/

2017-04-18 Thread wesm 
Repository: arrow
Updated Branches:
  refs/heads/master 7f20f6e73 -> 38efabea9


ARROW-844: [Format] Update README documents in format/

Added a section reflecting specification maturity and stability.

Author: Wes McKinney 

Closes #556 from wesm/ARROW-844 and squashes the following commits:

03dbb71 [Wes McKinney] Update README documents in format/


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/38efabea
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/38efabea
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/38efabea

Branch: refs/heads/master
Commit: 38efabea9bbc8d6386f96a635a95c53ba70e6149
Parents: 7f20f6e
Author: Wes McKinney 
Authored: Tue Apr 18 11:43:13 2017 -0400
Committer: Wes McKinney 
Committed: Tue Apr 18 11:43:13 2017 -0400

--
 format/README.md | 20 +++-
 1 file changed, 15 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/38efabea/format/README.md
--
diff --git a/format/README.md b/format/README.md
index 048badb..3aa8fdd 100644
--- a/format/README.md
+++ b/format/README.md
@@ -14,16 +14,14 @@
 
 ## Arrow specification documents
 
-> **Work-in-progress specification documents**. These are discussion documents
-> created by the Arrow developers during late 2015 and in no way represents a
-> finalized specification.
-
 Currently, the Arrow specification consists of these pieces:
 
 - Metadata specification (see Metadata.md)
 - Physical memory layout specification (see Layout.md)
-- Metadata serialized representation (see Message.fbs)
+- Logical Types, Schemas, and Record Batch Metadata (see Schema.fbs)
+- Encapsulated Messages (see Message.fbs)
 - Mechanics of messaging between Arrow systems (IPC, RPC, etc.) (see IPC.md)
+- Tensor (Multi-dimensional array) Metadata (see Tensor.fbs)
 
 The metadata currently uses Google's [flatbuffers library][1] for serializing a
 couple related pieces of information:
@@ -35,4 +33,16 @@ couple related pieces of information:
schema, and enable a system to send and receive Arrow row batches in a form
that can be precisely disassembled or reconstructed.
 
+## Arrow Format Maturity and Stability
+
+We have made significant progress hardening the Arrow in-memory format and
+Flatbuffer metadata since the project started in February 2016. We have
+integration tests which verify binary compatibility between the Java and C++
+implementations, for example.
+
+Major versions may still include breaking changes to the memory format or
+metadata, so it is recommended to use the same released version of all
+libraries in your applications for maximum compatibility. Data stored in the
+Arrow IPC formats should not be used for long term storage.
+
 [1]: http://github.com/google/flatbuffers

arrow git commit: ARROW-818: [Python] Expand Sphinx API docs, pyarrow.* namespace. Add factory functions for time32, time64

2017-04-18 Thread uwe 
Repository: arrow
Updated Branches:
  refs/heads/master bb287e203 -> 7f20f6e73


ARROW-818: [Python] Expand Sphinx API docs, pyarrow.* namespace. Add factory 
functions for time32, time64

Author: Wes McKinney 

Closes #557 from wesm/ARROW-818 and squashes the following commits:

96ce436 [Wes McKinney] Expand Sphinx API docs, pyarrow.* namespace. Add factory 
functions for time32, time64


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/7f20f6e7
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/7f20f6e7
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/7f20f6e7

Branch: refs/heads/master
Commit: 7f20f6e738a2e163b0b753416ee4c4ed00998f4b
Parents: bb287e2
Author: Wes McKinney 
Authored: Tue Apr 18 16:37:03 2017 +0200
Committer: Uwe L. Korn 
Committed: Tue Apr 18 16:37:03 2017 +0200

--
 python/doc/source/api.rst| 69 +++-
 python/pyarrow/__init__.py   | 33 ++
 python/pyarrow/_array.pxd| 10 +
 python/pyarrow/_array.pyx| 74 ++-
 python/pyarrow/_io.pyx   |  6 +--
 python/pyarrow/includes/libarrow.pxd |  3 ++
 python/pyarrow/tests/test_io.py  |  4 +-
 python/pyarrow/tests/test_schema.py  | 21 +
 8 files changed, 195 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/7f20f6e7/python/doc/source/api.rst
--
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 92e248b..08a0694 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -24,8 +24,8 @@ API Reference
 
 .. _api.functions:
 
-Type Metadata and Schemas
--
+Type and Schema Factory Functions
+-
 
 .. autosummary::
:toctree: generated/
@@ -43,6 +43,8 @@ Type Metadata and Schemas
float16
float32
float64
+   time32
+   time64
timestamp
date32
date64
@@ -53,10 +55,8 @@ Type Metadata and Schemas
struct
dictionary
field
-   DataType
-   Field
-   Schema
schema
+   from_numpy_dtype
 
 Scalar Value Types
 --
@@ -68,6 +68,7 @@ Scalar Value Types
NAType
Scalar
ArrayValue
+   BooleanValue
Int8Value
Int16Value
Int32Value
@@ -82,6 +83,11 @@ Scalar Value Types
BinaryValue
StringValue
FixedSizeBinaryValue
+   Date32Value
+   Date64Value
+   TimestampValue
+   DecimalValue
+
 
 Array Types and Constructors
 
@@ -91,21 +97,30 @@ Array Types and Constructors
 
array
Array
-   NullArray
-   NumericArray
-   IntegerArray
-   FloatingPointArray
BooleanArray
+   DictionaryArray
+   FloatingPointArray
+   IntegerArray
Int8Array
Int16Array
Int32Array
Int64Array
+   NullArray
+   NumericArray
UInt8Array
UInt16Array
UInt32Array
UInt64Array
-   DictionaryArray
+   BinaryArray
+   FixedSizeBinaryArray
StringArray
+   Time32Array
+   Time64Array
+   Date32Array
+   Date64Array
+   TimestampArray
+   DecimalArray
+   ListArray
 
 Tables and Record Batches
 -
@@ -113,9 +128,11 @@ Tables and Record Batches
 .. autosummary::
:toctree: generated/
 
+   ChunkedArray
Column
RecordBatch
Table
+   get_record_batch_size
 
 Tensor type and Functions
 -
@@ -141,7 +158,7 @@ Input / Output and Shared Memory
MemoryMappedFile
memory_map
create_memory_map
-   PythonFileInterface
+   PythonFile
 
 Interprocess Communication and Messaging
 
@@ -165,3 +182,33 @@ Memory Pools
jemalloc_memory_pool
total_allocated_bytes
set_memory_pool
+
+Type Classes
+
+
+.. autosummary::
+   :toctree: generated/
+
+   DataType
+   DecimalType
+   DictionaryType
+   FixedSizeBinaryType
+   Time32Type
+   Time64Type
+   TimestampType
+   Field
+   Schema
+
+.. currentmodule:: pyarrow.parquet
+
+Apache Parquet
+--
+
+.. autosummary::
+   :toctree: generated/
+
+   ParquetDataset
+   ParquetFile
+   read_table
+   write_metadata
+   write_table

http://git-wip-us.apache.org/repos/asf/arrow/blob/7f20f6e7/python/pyarrow/__init__.py
--
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 87f2352..4d8da9f 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -31,12 +31,20 @@ from pyarrow._config import cpu_count, set_cpu_count
 from pyarrow._array import (null, bool_,
 int8, int16, int32, int64,
 uint8, uint16, uint32, uint64,
-