Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-jsonlines for
openSUSE:Factory checked in at 2022-09-30 17:57:57
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-jsonlines (Old)
and /work/SRC/openSUSE:Factory/.python-jsonlines.new.2275 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-jsonlines"
Fri Sep 30 17:57:57 2022 rev:3 rq:1007080 version:3.1.0
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-jsonlines/python-jsonlines.changes
2021-11-08 17:25:21.996739896 +0100
+++
/work/SRC/openSUSE:Factory/.python-jsonlines.new.2275/python-jsonlines.changes
2022-09-30 17:58:16.101309395 +0200
@@ -1,0 +2,10 @@
+Thu Sep 29 14:16:46 UTC 2022 - Yogalakshmi Arunachalam <yarunacha...@suse.com>
+
+- Update to 3.0.0
+ * add type annotations; adopt mypy in strict mode (#58, #62)
+ * ignore UTF-8 BOM sequences in various scenarios (#69)
+ * support dumps() callables returning bytes again (#64)
+ * add basic support for rfc7464 text sequences (#61)
+ * drop support for numbers.Number in type= arguments (#63)
+
+-------------------------------------------------------------------
Old:
----
2.0.0.tar.gz
New:
----
3.1.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-jsonlines.spec ++++++
--- /var/tmp/diff_new_pack.WBnVUk/_old 2022-09-30 17:58:16.573310404 +0200
+++ /var/tmp/diff_new_pack.WBnVUk/_new 2022-09-30 17:58:16.577310412 +0200
@@ -1,7 +1,7 @@
#
# spec file for package python-jsonlines
#
-# Copyright (c) 2021 SUSE LLC
+# Copyright (c) 2022 SUSE LLC
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
@@ -19,7 +19,7 @@
%{?!python_module:%define python_module() python-%{**} python3-%{**}}
%global skip_python2 1
Name: python-jsonlines
-Version: 2.0.0
+Version: 3.1.0
Release: 0
Summary: Library with helpers for the jsonlines file format
License: BSD-3-Clause
++++++ 2.0.0.tar.gz -> 3.1.0.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/.gitignore
new/jsonlines-3.1.0/.gitignore
--- old/jsonlines-2.0.0/.gitignore 2021-01-04 17:03:18.000000000 +0100
+++ new/jsonlines-3.1.0/.gitignore 2022-07-01 18:35:18.000000000 +0200
@@ -4,8 +4,9 @@
# Testing
/.coverage
-/.tox/
+/coverage.xml
/htmlcov/
+/.tox/
# Packaging
/.cache/
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/LICENSE.rst
new/jsonlines-3.1.0/LICENSE.rst
--- old/jsonlines-2.0.0/LICENSE.rst 2021-01-04 17:03:18.000000000 +0100
+++ new/jsonlines-3.1.0/LICENSE.rst 2022-07-01 18:35:18.000000000 +0200
@@ -1,6 +1,6 @@
*(This is the OSI approved 3-clause "New BSD License".)*
-Copyright ?? 2016, Wouter Bolsterlee
+Copyright ?? 2016, wouter bolsterlee
All rights reserved.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/README.rst
new/jsonlines-3.1.0/README.rst
--- old/jsonlines-2.0.0/README.rst 2021-01-04 17:03:18.000000000 +0100
+++ new/jsonlines-3.1.0/README.rst 2022-07-01 18:35:18.000000000 +0200
@@ -7,6 +7,9 @@
.. image:: https://pepy.tech/badge/jsonlines/month
:target: https://pepy.tech/project/jsonlines
+.. image:: https://anaconda.org/anaconda/anaconda/badges/installer/conda.svg
+ :target: https://anaconda.org/anaconda/jsonlines
+
=========
jsonlines
=========
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/doc/index.rst
new/jsonlines-3.1.0/doc/index.rst
--- old/jsonlines-2.0.0/doc/index.rst 2021-01-04 17:03:18.000000000 +0100
+++ new/jsonlines-3.1.0/doc/index.rst 2022-07-01 18:35:18.000000000 +0200
@@ -179,6 +179,34 @@
Version history
===============
+* 3.1.0, released at 2022-07-01
+
+ * Return number of chars/bytes written by :py:meth:`Writer.write()`
+ and :py:meth:`~Writer.write_all()`
+ (`#73 <https://github.com/wbolster/jsonlines/pull/73>`_)
+
+ * allow ``mode='x'`` in :py:func:`~jsonlines.open()`
+ to open a file for exclusive creation
+ (`#74 <https://github.com/wbolster/jsonlines/issues/74>`_)
+
+* 3.0.0, released at 2021-12-04
+
+ * add type annotations; adopt mypy in strict mode
+ (`#58 <https://github.com/wbolster/jsonlines/pull/58>`_,
+ `#62 <https://github.com/wbolster/jsonlines/pull/62>`_)
+
+ * ignore UTF-8 BOM sequences in various scenarios
+ (`#69 <https://github.com/wbolster/jsonlines/pull/69>`_)
+
+ * support ``dumps()`` callables returning bytes again
+ (`#64 <https://github.com/wbolster/jsonlines/issues/64>`_)
+
+ * add basic support for rfc7464 text sequences
+ (`#61 <https://github.com/wbolster/jsonlines/pull/61>`_)
+
+ * drop support for ``numbers.Number`` in ``type=`` arguments
+ (`#63 <https://github.com/wbolster/jsonlines/issues/63>`_)
+
* 2.0.0, released at 2021-01-04
* drop support for end-of-life Python versions; this package is now
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/jsonlines/jsonlines.py
new/jsonlines-3.1.0/jsonlines/jsonlines.py
--- old/jsonlines-2.0.0/jsonlines/jsonlines.py 2021-01-04 17:03:18.000000000
+0100
+++ new/jsonlines-3.1.0/jsonlines/jsonlines.py 2022-07-01 18:35:18.000000000
+0200
@@ -3,9 +3,36 @@
"""
import builtins
+import codecs
+import enum
+import io
import json
-import numbers
+import os
+import sys
+import types
+import typing
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Iterable,
+ Iterator,
+ List,
+ Optional,
+ Tuple,
+ Type,
+ TypeVar,
+ Union,
+ cast,
+ overload,
+)
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal # pragma: no cover
+import attr
VALID_TYPES = {
bool,
@@ -13,19 +40,59 @@
float,
int,
list,
- numbers.Number,
str,
}
+# Characters to skip at the beginning of a line. Note: at most one such
+# character is skipped per line.
+SKIPPABLE_SINGLE_INITIAL_CHARS = (
+ "\x1e", # RFC7464 text sequence
+ codecs.BOM_UTF8.decode(),
+)
+
+class DumpsResultConversion(enum.Enum):
+ LeaveAsIs = enum.auto()
+ EncodeToBytes = enum.auto()
+ DecodeToString = enum.auto()
+
+
+# https://docs.python.org/3/library/functions.html#open
+Openable = Union[str, bytes, int, os.PathLike]
+
+LoadsCallable = Callable[[Union[str, bytes]], Any]
+DumpsCallable = Callable[[Any], Union[str, bytes]]
+
+# Currently, JSON structures cannot be typed properly:
+# - https://github.com/python/typing/issues/182
+# - https://github.com/python/mypy/issues/731
+JSONCollection = Union[Dict[str, Any], List[Any]]
+JSONScalar = Union[bool, float, int, str]
+JSONValue = Union[JSONCollection, JSONScalar]
+TJSONValue = TypeVar("TJSONValue", bound=JSONValue)
+
+TRW = TypeVar("TRW", bound="ReaderWriterBase")
+
+default_loads = json.loads
+
+
+def default_dumps(obj: Any) -> str:
+ """
+ Fake dumps() function to use as a default marker.
+ """
+ raise NotImplementedError # pragma: no cover
+
+
+@attr.s(auto_exc=True, auto_attribs=True)
class Error(Exception):
"""
Base error class.
"""
- pass
+ message: str
+@attr.s(auto_exc=True, auto_attribs=True, init=False)
class InvalidLineError(Error, ValueError):
"""
Error raised when an invalid line is encountered.
@@ -42,24 +109,30 @@
"""
#: The invalid line
- line = None
+ line: Union[str, bytes]
#: The line number
- lineno = None
+ lineno: int
- def __init__(self, msg, line, lineno):
- msg = f"{msg} (line {lineno})"
+ def __init__(self, message: str, line: Union[str, bytes], lineno: int) ->
None:
self.line = line.rstrip()
self.lineno = lineno
- super().__init__(msg)
+ super().__init__(f"{message} (line {lineno})")
+@attr.s(auto_attribs=True, repr=False)
class ReaderWriterBase:
"""
Base class with shared behaviour for both the reader and writer.
"""
- def close(self):
+ _fp: Union[typing.IO[str], typing.IO[bytes], None] = attr.ib(
+ default=None, init=False
+ )
+ _closed: bool = attr.ib(default=False, init=False)
+ _should_close_fp: bool = attr.ib(default=False, init=False)
+
+ def close(self) -> None:
"""
Close this reader/writer.
@@ -70,27 +143,30 @@
if self._closed:
return
self._closed = True
- if self._should_close_fp:
+ if self._fp is not None and self._should_close_fp:
self._fp.close()
- def __repr__(self):
- name = getattr(self._fp, "name", None)
- if name:
- wrapping = repr(name)
- else:
- wrapping = "<{} at 0x{:x}>".format(type(self._fp).__name__,
id(self._fp))
- return "<jsonlines.{} at 0x{:x} wrapping {}>".format(
- type(self).__name__, id(self), wrapping
- )
+ def __repr__(self) -> str:
+ cls_name = type(self).__name__
+ wrapped = self._repr_for_wrapped()
+ return f"<jsonlines.{cls_name} at 0x{id(self):x} wrapping {wrapped}>"
- def __enter__(self):
+ def _repr_for_wrapped(self) -> str:
+ raise NotImplementedError # pragma: no cover
+
+ def __enter__(self: TRW) -> TRW:
return self
- def __exit__(self, *exc_info):
+ def __exit__(
+ self,
+ exc_type: Optional[Type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[types.TracebackType],
+ ) -> None:
self.close()
- return False
+@attr.s(auto_attribs=True, repr=False)
class Reader(ReaderWriterBase):
"""
Reader for the jsonlines format.
@@ -100,33 +176,100 @@
an open file or an ``io.TextIO`` instance, but it can also be
something else as long as it yields strings when iterated over.
+ Instances are iterable and can be used as a context manager.
+
The `loads` argument can be used to replace the standard json
decoder. If specified, it must be a callable that accepts a
(unicode) string and returns the decoded object.
- Instances are iterable and can be used as a context manager.
-
- :param file-like iterable: iterable yielding lines as strings
- :param callable loads: custom json decoder callable
- """
+ :param file_or_iterable: file-like object or iterable yielding lines as
+ strings
+ :param loads: custom json decoder callable
+ """
+
+ _file_or_iterable: Union[
+ typing.IO[str], typing.IO[bytes], Iterable[Union[str, bytes]]
+ ]
+ _line_iter: Iterator[Tuple[int, Union[bytes, str]]] = attr.ib(init=False)
+ _loads: LoadsCallable = attr.ib(default=default_loads, kw_only=True)
+
+ def __attrs_post_init__(self) -> None:
+ if isinstance(self._file_or_iterable, io.IOBase):
+ self._fp = cast(
+ Union[typing.IO[str], typing.IO[bytes]],
+ self._file_or_iterable,
+ )
- def __init__(self, iterable, loads=None):
- self._fp = iterable
- self._should_close_fp = False
- self._closed = False
- if loads is None:
- loads = json.loads
- self._loads = loads
- self._line_iter = enumerate(iterable, 1)
+ self._line_iter = enumerate(self._file_or_iterable, 1)
- def read(self, type=None, allow_none=False, skip_empty=False):
+ # No type specified, None not allowed
+ @overload
+ def read(
+ self,
+ *,
+ type: Literal[None] = ...,
+ allow_none: Literal[False] = ...,
+ skip_empty: bool = ...,
+ ) -> JSONValue:
+ ... # pragma: no cover
+
+ # No type specified, None allowed
+ @overload
+ def read(
+ self,
+ *,
+ type: Literal[None] = ...,
+ allow_none: Literal[True],
+ skip_empty: bool = ...,
+ ) -> Optional[JSONValue]:
+ ... # pragma: no cover
+
+ # Type specified, None not allowed
+ @overload
+ def read(
+ self,
+ *,
+ type: Type[TJSONValue],
+ allow_none: Literal[False] = ...,
+ skip_empty: bool = ...,
+ ) -> TJSONValue:
+ ... # pragma: no cover
+
+ # Type specified, None allowed
+ @overload
+ def read(
+ self,
+ *,
+ type: Type[TJSONValue],
+ allow_none: Literal[True],
+ skip_empty: bool = ...,
+ ) -> Optional[TJSONValue]:
+ ... # pragma: no cover
+
+ # Generic definition
+ @overload
+ def read(
+ self,
+ *,
+ type: Optional[Type[Any]] = ...,
+ allow_none: bool = ...,
+ skip_empty: bool = ...,
+ ) -> Optional[JSONValue]:
+ ... # pragma: no cover
+
+ def read(
+ self,
+ *,
+ type: Optional[Type[Any]] = None,
+ allow_none: bool = False,
+ skip_empty: bool = False,
+ ) -> Optional[JSONValue]:
"""
Read and decode a line.
The optional `type` argument specifies the expected data type.
Supported types are ``dict``, ``list``, ``str``, ``int``,
- ``float``, ``numbers.Number`` (accepts both integers and
- floats), and ``bool``. When specified, non-conforming lines
+ ``float``, and ``bool``. When specified, non-conforming lines
result in :py:exc:`InvalidLineError`.
By default, input lines containing ``null`` (in JSON) are
@@ -158,8 +301,11 @@
)
raise exc from orig_exc
+ if line.startswith(SKIPPABLE_SINGLE_INITIAL_CHARS):
+ line = line[1:]
+
try:
- value = self._loads(line)
+ value: JSONValue = self._loads(line)
except ValueError as orig_exc:
exc = InvalidLineError(
f"line contains invalid json: {orig_exc}", line, lineno
@@ -173,8 +319,9 @@
if type is not None:
valid = isinstance(value, type)
- if type in (int, numbers.Number):
- valid = valid and not isinstance(value, bool)
+ if type is int and isinstance(value, bool):
+ # isinstance() is not sufficient, since bool is an int subclass
+ valid = False
if not valid:
raise InvalidLineError(
"line does not match requested type", line, lineno
@@ -182,7 +329,73 @@
return value
- def iter(self, type=None, allow_none=False, skip_empty=False,
skip_invalid=False):
+ # No type specified, None not allowed
+ @overload
+ def iter(
+ self,
+ *,
+ type: Literal[None] = ...,
+ allow_none: Literal[False] = ...,
+ skip_empty: bool = ...,
+ skip_invalid: bool = ...,
+ ) -> Iterator[JSONValue]:
+ ... # pragma: no cover
+
+ # No type specified, None allowed
+ @overload
+ def iter(
+ self,
+ *,
+ type: Literal[None] = ...,
+ allow_none: Literal[True],
+ skip_empty: bool = ...,
+ skip_invalid: bool = ...,
+ ) -> Iterator[JSONValue]:
+ ... # pragma: no cover
+
+ # Type specified, None not allowed
+ @overload
+ def iter(
+ self,
+ *,
+ type: Type[TJSONValue],
+ allow_none: Literal[False] = ...,
+ skip_empty: bool = ...,
+ skip_invalid: bool = ...,
+ ) -> Iterator[TJSONValue]:
+ ... # pragma: no cover
+
+ # Type specified, None allowed
+ @overload
+ def iter(
+ self,
+ *,
+ type: Type[TJSONValue],
+ allow_none: Literal[True],
+ skip_empty: bool = ...,
+ skip_invalid: bool = ...,
+ ) -> Iterator[Optional[TJSONValue]]:
+ ... # pragma: no cover
+
+ # Generic definition
+ @overload
+ def iter(
+ self,
+ *,
+ type: Optional[Type[TJSONValue]] = ...,
+ allow_none: bool = ...,
+ skip_empty: bool = ...,
+ skip_invalid: bool = ...,
+ ) -> Iterator[Optional[TJSONValue]]:
+ ... # pragma: no cover
+
+ def iter(
+ self,
+ type: Optional[Type[Any]] = None,
+ allow_none: bool = False,
+ skip_empty: bool = False,
+ skip_invalid: bool = False,
+ ) -> Iterator[Optional[JSONValue]]:
"""
Iterate over all lines.
@@ -209,17 +422,26 @@
except EOFError:
pass
- def __iter__(self):
+ def __iter__(self) -> Iterator[Any]:
"""
See :py:meth:`~Reader.iter()`.
"""
return self.iter()
+ def _repr_for_wrapped(self) -> str:
+ if self._fp is not None:
+ return repr_for_fp(self._fp)
+ class_name = type(self._file_or_iterable).__name__
+ return f"<{class_name} at 0x{id(self._file_or_iterable):x}>"
+
+@attr.s(auto_attribs=True, repr=False)
class Writer(ReaderWriterBase):
"""
Writer for the jsonlines format.
+ Instances can be used as a context manager.
+
The `fp` argument must be a file-like object with a ``.write()``
method accepting either text (unicode) or bytes.
@@ -235,63 +457,144 @@
When the `flush` argument is set to ``True``, the writer will call
``fp.flush()`` after each written line.
- Instances can be used as a context manager.
-
- :param file-like fp: writable file-like object
- :param bool compact: whether to use a compact output format
- :param bool sort_keys: whether to sort object keys
- :param callable dumps: custom encoder callable
- :param bool flush: whether to flush the file-like object after
- writing each line
- """
+ :param fp: writable file-like object
+ :param compact: whether to use a compact output format
+ :param sort_keys: whether to sort object keys
+ :param dumps: custom encoder callable
+ :param flush: whether to flush the file-like object after writing each line
+ """
+
+ _fp: Union[typing.IO[str], typing.IO[bytes]] = attr.ib(default=None)
+ _fp_is_binary: bool = attr.ib(default=False, init=False)
+ _compact: bool = attr.ib(default=False, kw_only=True)
+ _sort_keys: bool = attr.ib(default=False, kw_only=True)
+ _flush: bool = attr.ib(default=False, kw_only=True)
+ _dumps: DumpsCallable = attr.ib(default=default_dumps, kw_only=True)
+ _dumps_result_conversion: DumpsResultConversion = attr.ib(
+ default=DumpsResultConversion.LeaveAsIs, init=False
+ )
- def __init__(self, fp, compact=False, sort_keys=False, dumps=None,
flush=False):
- self._closed = False
- try:
- fp.write("")
+ def __attrs_post_init__(self) -> None:
+ if isinstance(self._fp, io.TextIOBase):
self._fp_is_binary = False
- except TypeError:
+ elif isinstance(self._fp, io.IOBase):
self._fp_is_binary = True
- if dumps is None:
- encoder_kwargs = dict(ensure_ascii=False, sort_keys=sort_keys)
- if compact:
+ else:
+ try:
+ self._fp.write("") # type: ignore[arg-type]
+ except TypeError:
+ self._fp_is_binary = True
+ else:
+ self._fp_is_binary = False
+
+ if self._dumps is default_dumps:
+ encoder_kwargs: Dict[str, Any] = dict(
+ ensure_ascii=False,
+ sort_keys=self._sort_keys,
+ )
+ if self._compact:
encoder_kwargs.update(separators=(",", ":"))
- dumps = json.JSONEncoder(**encoder_kwargs).encode
- self._fp = fp
- self._should_close_fp = False
- self._dumps = dumps
- self._flush = flush
+ self._dumps = json.JSONEncoder(**encoder_kwargs).encode
- def write(self, obj):
+ # Detect if str-to-bytes conversion (or vice versa) is needed for the
+ # combination of this file-like object and the used dumps() callable.
+ # This avoids checking this for each .write(). Note that this
+ # deliberately does not support ???dynamic??? return types that depend
on
+ # input and dump options, like simplejson on Python 2 in some cases.
+ sample_dumps_result = self._dumps({})
+ if isinstance(sample_dumps_result, str) and self._fp_is_binary:
+ self._dumps_result_conversion = DumpsResultConversion.EncodeToBytes
+ elif isinstance(sample_dumps_result, bytes) and not self._fp_is_binary:
+ self._dumps_result_conversion =
DumpsResultConversion.DecodeToString
+
+ def write(self, obj: Any) -> int:
"""
Encode and write a single object.
:param obj: the object to encode and write
+ :return: number of characters or bytes written
"""
if self._closed:
raise RuntimeError("writer is closed")
+
line = self._dumps(obj)
- if self._fp_is_binary:
- line = line.encode("utf-8")
- self._fp.write(line)
- self._fp.write(b"\n")
- else:
- self._fp.write(line)
- self._fp.write("\n")
+
+ # This handles either str or bytes, but the type checker does not know
+ # that this code always passes the right type of arguments.
+ if self._dumps_result_conversion ==
DumpsResultConversion.EncodeToBytes:
+ line = line.encode() # type: ignore[union-attr]
+ elif self._dumps_result_conversion ==
DumpsResultConversion.DecodeToString:
+ line = line.decode() # type: ignore[union-attr]
+
+ fp = self._fp
+ fp.write(line) # type: ignore[arg-type]
+ fp.write(b"\n" if self._fp_is_binary else "\n") # type:
ignore[arg-type]
+
if self._flush:
- self._fp.flush()
+ fp.flush()
+
+ return len(line) + 1 # including newline
- def write_all(self, iterable):
+ def write_all(self, iterable: Iterable[Any]) -> int:
"""
Encode and write multiple objects.
:param iterable: an iterable of objects
+ :return: number of characters or bytes written
"""
- for obj in iterable:
- self.write(obj)
+ return sum(self.write(obj) for obj in iterable)
+ def _repr_for_wrapped(self) -> str:
+ return repr_for_fp(self._fp)
-def open(name, mode="r", **kwargs):
+
+@overload
+def open(
+ file: Openable,
+ mode: Literal["r"] = ...,
+ *,
+ loads: Optional[LoadsCallable] = ...,
+) -> Reader:
+ ... # pragma: no cover
+
+
+@overload
+def open(
+ file: Openable,
+ mode: Literal["w", "a", "x"],
+ *,
+ dumps: Optional[DumpsCallable] = ...,
+ compact: Optional[bool] = ...,
+ sort_keys: Optional[bool] = ...,
+ flush: Optional[bool] = ...,
+) -> Writer:
+ ... # pragma: no cover
+
+
+@overload
+def open(
+ file: Openable,
+ mode: str = ...,
+ *,
+ loads: Optional[LoadsCallable] = ...,
+ dumps: Optional[DumpsCallable] = ...,
+ compact: Optional[bool] = ...,
+ sort_keys: Optional[bool] = ...,
+ flush: Optional[bool] = ...,
+) -> Union[Reader, Writer]:
+ ... # pragma: no cover
+
+
+def open(
+ file: Openable,
+ mode: str = "r",
+ *,
+ loads: Optional[LoadsCallable] = None,
+ dumps: Optional[DumpsCallable] = None,
+ compact: Optional[bool] = None,
+ sort_keys: Optional[bool] = None,
+ flush: Optional[bool] = None,
+) -> Union[Reader, Writer]:
"""
Open a jsonlines file for reading or writing.
@@ -312,17 +615,35 @@
with jsonlines.open('out.jsonl', mode='w') as writer:
writer.write(...)
- :param file-like fp: name of the file to open
- :param str mode: whether to open the file for reading (``r``),
+ :param file: name or ???path-like object??? of the file to open
+ :param mode: whether to open the file for reading (``r``),
writing (``w``) or appending (``a``).
- :param **kwargs: additional arguments, forwarded to the reader or writer
"""
- if mode not in {"r", "w", "a"}:
- raise ValueError("'mode' must be either 'r', 'w', or 'a'")
- fp = builtins.open(name, mode=mode + "t", encoding="utf-8")
- if mode == "r":
- instance = Reader(fp, **kwargs)
- else:
- instance = Writer(fp, **kwargs)
+ if mode not in {"r", "w", "a", "x"}:
+ raise ValueError("'mode' must be either 'r', 'w', 'a', or 'x'")
+
+ cls = Reader if mode == "r" else Writer
+ encoding = "utf-8-sig" if mode == "r" else "utf-8"
+ fp = builtins.open(file, mode=mode + "t", encoding=encoding)
+ kwargs = dict(
+ loads=loads,
+ dumps=dumps,
+ compact=compact,
+ sort_keys=sort_keys,
+ flush=flush,
+ )
+ kwargs = {key: value for key, value in kwargs.items() if value is not None}
+ instance: Union[Reader, Writer] = cls(fp, **kwargs)
instance._should_close_fp = True
return instance
+
+
+def repr_for_fp(fp: typing.IO[Any]) -> str:
+ """
+ Helper to make a useful repr() for a file-like object.
+ """
+ name = getattr(fp, "name", None)
+ if name is not None:
+ return repr(name)
+ else:
+ return repr(fp)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/mypy.ini new/jsonlines-3.1.0/mypy.ini
--- old/jsonlines-2.0.0/mypy.ini 1970-01-01 01:00:00.000000000 +0100
+++ new/jsonlines-3.1.0/mypy.ini 2022-07-01 18:35:18.000000000 +0200
@@ -0,0 +1,16 @@
+[mypy]
+check_untyped_defs = True
+disallow_any_generics = True
+disallow_incomplete_defs = True
+disallow_subclassing_any = True
+disallow_untyped_calls = True
+disallow_untyped_decorators = True
+disallow_untyped_defs = True
+no_implicit_optional = True
+no_implicit_reexport = True
+show_error_codes = True
+strict_equality = True
+warn_redundant_casts = True
+warn_return_any = True
+warn_unused_configs = True
+warn_unused_ignores = True
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/requirements-dev.txt
new/jsonlines-3.1.0/requirements-dev.txt
--- old/jsonlines-2.0.0/requirements-dev.txt 2021-01-04 17:03:18.000000000
+0100
+++ new/jsonlines-3.1.0/requirements-dev.txt 2022-07-01 18:35:18.000000000
+0200
@@ -1,5 +1,6 @@
black
flake8
+mypy
pytest>=3
pytest-cov
sphinx
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/setup.cfg
new/jsonlines-3.1.0/setup.cfg
--- old/jsonlines-2.0.0/setup.cfg 2021-01-04 17:03:18.000000000 +0100
+++ new/jsonlines-3.1.0/setup.cfg 2022-07-01 18:35:18.000000000 +0200
@@ -1,7 +1,7 @@
[metadata]
name = jsonlines
-version = 2.0.0
-author = Wouter Bolsterlee
+version = 3.1.0
+author = wouter bolsterlee
author_email = wou...@bolsterl.ee
license = BSD
license_file = LICENSE.rst
@@ -24,6 +24,12 @@
[options]
packages = jsonlines
python_requires = >=3.6
+install_requires =
+ attrs>=19.2.0
+ typing_extensions; python_version < "3.8"
+
+[options.package_data]
+jsonlines = py.typed
[build_sphinx]
source-dir = doc/
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/jsonlines-2.0.0/tests/sample-crlf-line-separators.jsonl
new/jsonlines-3.1.0/tests/sample-crlf-line-separators.jsonl
--- old/jsonlines-2.0.0/tests/sample-crlf-line-separators.jsonl 2021-01-04
17:03:18.000000000 +0100
+++ new/jsonlines-3.1.0/tests/sample-crlf-line-separators.jsonl 1970-01-01
01:00:00.000000000 +0100
@@ -1,2 +0,0 @@
-{"a": 1}
-{"b": 2}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/tests/sample-no-eol-at-eof.jsonl
new/jsonlines-3.1.0/tests/sample-no-eol-at-eof.jsonl
--- old/jsonlines-2.0.0/tests/sample-no-eol-at-eof.jsonl 2021-01-04
17:03:18.000000000 +0100
+++ new/jsonlines-3.1.0/tests/sample-no-eol-at-eof.jsonl 1970-01-01
01:00:00.000000000 +0100
@@ -1,2 +0,0 @@
-{"a": 1}
-{"b": 2}
\ No newline at end of file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/tests/sample.jsonl
new/jsonlines-3.1.0/tests/sample.jsonl
--- old/jsonlines-2.0.0/tests/sample.jsonl 2021-01-04 17:03:18.000000000
+0100
+++ new/jsonlines-3.1.0/tests/sample.jsonl 1970-01-01 01:00:00.000000000
+0100
@@ -1,2 +0,0 @@
-{"a": 1}
-{"b": 2}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/tests/test_jsonlines.py
new/jsonlines-3.1.0/tests/test_jsonlines.py
--- old/jsonlines-2.0.0/tests/test_jsonlines.py 2021-01-04 17:03:18.000000000
+0100
+++ new/jsonlines-3.1.0/tests/test_jsonlines.py 2022-07-01 18:35:18.000000000
+0200
@@ -2,11 +2,13 @@
Tests for the jsonlines library.
"""
+import codecs
import collections
import io
-import jsonlines
+import json
import tempfile
+import jsonlines
import pytest
@@ -14,7 +16,7 @@
SAMPLE_TEXT = SAMPLE_BYTES.decode("utf-8")
-def test_reader():
+def test_reader() -> None:
fp = io.BytesIO(SAMPLE_BYTES)
with jsonlines.Reader(fp) as reader:
it = iter(reader)
@@ -26,13 +28,61 @@
reader.read()
-def test_reading_from_iterable():
+def test_reading_from_iterable() -> None:
with jsonlines.Reader(["1", b"{}"]) as reader:
assert list(reader) == [1, {}]
assert "wrapping <list at " in repr(reader)
-def test_writer_text():
+def test_reader_rfc7464_text_sequences() -> None:
+ fp = io.BytesIO(b'\x1e"a"\x0a\x1e"b"\x0a')
+ with jsonlines.Reader(fp) as reader:
+ assert list(reader) == ["a", "b"]
+
+
+def test_reader_utf8_bom_bytes() -> None:
+ """
+ UTF-8 BOM is ignored, even if it occurs in the middle of a stream.
+ """
+ chunks = [
+ codecs.BOM_UTF8,
+ b"1\n",
+ codecs.BOM_UTF8,
+ b"2\n",
+ ]
+ fp = io.BytesIO(b"".join(chunks))
+ with jsonlines.Reader(fp) as reader:
+ assert list(reader) == [1, 2]
+
+
+def test_reader_utf8_bom_text() -> None:
+ """
+ Text version of ``test_reader_utf8_bom_bytes()``.
+ """
+ chunks = [
+ "1\n",
+ codecs.BOM_UTF8.decode(),
+ "2\n",
+ ]
+ fp = io.StringIO("".join(chunks))
+ with jsonlines.Reader(fp) as reader:
+ assert list(reader) == [1, 2]
+
+
+def test_reader_utf8_bom_bom_bom() -> None:
+ """
+ Too many UTF-8 BOM BOM BOM chars cause BOOM ???? BOOM.
+ """
+ reader = jsonlines.Reader([codecs.BOM_UTF8.decode() * 3 + "1\n"])
+ with pytest.raises(jsonlines.InvalidLineError) as excinfo:
+ reader.read()
+
+ exc = excinfo.value
+ assert "invalid json" in str(exc)
+ assert isinstance(exc.__cause__, json.JSONDecodeError)
+
+
+def test_writer_text() -> None:
fp = io.StringIO()
with jsonlines.Writer(fp) as writer:
writer.write({"a": 1})
@@ -40,7 +90,7 @@
assert fp.getvalue() == SAMPLE_TEXT
-def test_writer_binary():
+def test_writer_binary() -> None:
fp = io.BytesIO()
with jsonlines.Writer(fp) as writer:
writer.write_all(
@@ -52,7 +102,7 @@
assert fp.getvalue() == SAMPLE_BYTES
-def test_closing():
+def test_closing() -> None:
reader = jsonlines.Reader([])
reader.close()
with pytest.raises(RuntimeError):
@@ -64,7 +114,7 @@
writer.write(123)
-def test_invalid_lines():
+def test_invalid_lines() -> None:
data = "[1, 2"
with jsonlines.Reader(io.StringIO(data)) as reader:
with pytest.raises(jsonlines.InvalidLineError) as excinfo:
@@ -72,9 +122,10 @@
exc = excinfo.value
assert "invalid json" in str(exc)
assert exc.line == data
+ assert isinstance(exc.__cause__, json.JSONDecodeError)
-def test_skip_invalid():
+def test_skip_invalid() -> None:
fp = io.StringIO("12\ninvalid\n34")
reader = jsonlines.Reader(fp)
it = reader.iter(skip_invalid=True)
@@ -82,7 +133,7 @@
assert next(it) == 34
-def test_empty_strings_in_iterable():
+def test_empty_strings_in_iterable() -> None:
input = ["123", "", "456"]
it = iter(jsonlines.Reader(input))
assert next(it) == 123
@@ -94,14 +145,14 @@
assert list(it) == [123, 456]
-def test_invalid_utf8():
+def test_invalid_utf8() -> None:
with jsonlines.Reader([b"\xff\xff"]) as reader:
with pytest.raises(jsonlines.InvalidLineError) as excinfo:
reader.read()
assert "line is not valid utf-8" in str(excinfo.value)
-def test_empty_lines():
+def test_empty_lines() -> None:
data_with_empty_line = b"1\n\n2\n"
with jsonlines.Reader(io.BytesIO(data_with_empty_line)) as reader:
assert reader.read()
@@ -114,9 +165,16 @@
assert list(reader.iter(skip_empty=True)) == [1, 2]
-def test_typed_reads():
- with jsonlines.Reader(io.StringIO('12\n"foo"\n')) as reader:
+def test_typed_reads() -> None:
+ with jsonlines.Reader(io.StringIO('12\ntrue\n"foo"\n')) as reader:
assert reader.read(type=int) == 12
+
+ with pytest.raises(jsonlines.InvalidLineError) as excinfo:
+ reader.read(type=int)
+ exc = excinfo.value
+ assert "does not match requested type" in str(exc)
+ assert exc.line == 'true'
+
with pytest.raises(jsonlines.InvalidLineError) as excinfo:
reader.read(type=float)
exc = excinfo.value
@@ -124,7 +182,15 @@
assert exc.line == '"foo"'
-def test_typed_iteration():
+def test_typed_read_invalid_type() -> None:
+ reader = jsonlines.Reader([])
+ with pytest.raises(ValueError) as excinfo:
+ reader.read(type="nope") # type: ignore[call-overload]
+ exc = excinfo.value
+ assert str(exc) == "invalid type specified"
+
+
+def test_typed_iteration() -> None:
fp = io.StringIO("1\n2\n")
with jsonlines.Reader(fp) as reader:
actual = list(reader.iter(type=int))
@@ -139,7 +205,7 @@
assert "does not match requested type" in str(exc)
-def test_writer_flags():
+def test_writer_flags() -> None:
fp = io.BytesIO()
with jsonlines.Writer(fp, compact=True, sort_keys=True) as writer:
writer.write(
@@ -153,21 +219,36 @@
assert fp.getvalue() == b'{"a":1,"b":2}\n'
-def test_custom_dumps():
+def test_custom_dumps() -> None:
fp = io.BytesIO()
writer = jsonlines.Writer(fp, dumps=lambda obj: "oh hai")
with writer:
- writer.write({})
+ nbytes = writer.write({})
+ assert nbytes == len(b"oh hai\n")
+
assert fp.getvalue() == b"oh hai\n"
-def test_custom_loads():
+def test_custom_dumps_bytes() -> None:
+ """
+ A custom dump function that returns bytes (e.g. ???orjson???) should work.
+ """
+
+ fp = io.BytesIO()
+ writer = jsonlines.Writer(fp, dumps=lambda obj: b"some bytes")
+ with writer:
+ writer.write(123)
+
+ assert fp.getvalue() == b"some bytes\n"
+
+
+def test_custom_loads() -> None:
fp = io.BytesIO(b"{}\n")
with jsonlines.Reader(fp, loads=lambda s: "uh what") as reader:
assert reader.read() == "uh what"
-def test_open_reading():
+def test_open_reading() -> None:
with tempfile.NamedTemporaryFile("wb") as fp:
fp.write(b"123\n")
fp.flush()
@@ -175,7 +256,19 @@
assert list(reader) == [123]
-def test_open_writing():
+def test_open_reading_with_utf8_bom() -> None:
+ """
+ The ``.open()`` helper ignores a UTF-8 BOM.
+ """
+ with tempfile.NamedTemporaryFile("wb") as fp:
+ fp.write(codecs.BOM_UTF8)
+ fp.write(b"123\n")
+ fp.flush()
+ with jsonlines.open(fp.name) as reader:
+ assert list(reader) == [123]
+
+
+def test_open_writing() -> None:
with tempfile.NamedTemporaryFile("w+b") as fp:
with jsonlines.open(fp.name, mode="w") as writer:
writer.write(123)
@@ -183,16 +276,25 @@
assert fp.name in repr(writer)
-def test_open_and_append_writing():
+def test_open_and_append_writing() -> None:
with tempfile.NamedTemporaryFile("w+b") as fp:
with jsonlines.open(fp.name, mode="w") as writer:
- writer.write(123)
+ nbytes = writer.write(123)
+ assert nbytes == len(str(123)) + 1
with jsonlines.open(fp.name, mode="a") as writer:
- writer.write(456)
+ nbytes = writer.write(456)
+ assert nbytes == len(str(456)) + 1
assert fp.read() == b"123\n456\n"
-def test_open_invalid_mode():
+def test_open_invalid_mode() -> None:
with pytest.raises(ValueError) as excinfo:
jsonlines.open("foo", mode="foo")
assert "mode" in str(excinfo.value)
+
+
+def test_single_char_stripping() -> None:
+ """ "
+ Sanity check that a helper constant actually contains single-char strings.
+ """
+ assert all(len(s) == 1 for s in
jsonlines.jsonlines.SKIPPABLE_SINGLE_INITIAL_CHARS)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/tests/test_typing.py
new/jsonlines-3.1.0/tests/test_typing.py
--- old/jsonlines-2.0.0/tests/test_typing.py 1970-01-01 01:00:00.000000000
+0100
+++ new/jsonlines-3.1.0/tests/test_typing.py 2022-07-01 18:35:18.000000000
+0200
@@ -0,0 +1,96 @@
+"""
+This file should give any type checking errors.
+"""
+
+import io
+import json
+import random
+
+import numbers
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional
+
+
+if not TYPE_CHECKING:
+
+ def reveal_type(obj: Any) -> None:
+ pass
+
+
+import jsonlines
+
+
+def something_with_reader() -> None:
+
+ reader: jsonlines.Reader
+ reader = jsonlines.Reader(io.StringIO())
+ reader = jsonlines.Reader(io.BytesIO())
+ reader = jsonlines.Reader(['"text"'])
+ reader = jsonlines.Reader([b'"bytes"'])
+
+ r1 = reader.read()
+ r2 = reader.read(allow_none=True)
+ r3: numbers.Number = reader.read(type=random.choice([int, float]))
+
+ # For debugging:
+ # reveal_type(r1)
+ # reveal_type(r2)
+ # reveal_type(r3)
+
+ some_int: int = reader.read(type=int)
+ maybe_int: Optional[int] = reader.read(type=int, allow_none=True)
+
+ some_float: float = reader.read(type=float)
+ maybe_float: Optional[float] = reader.read(type=float, allow_none=True)
+
+ some_bool: bool = reader.read(type=bool)
+ maybe_bool: Optional[bool] = reader.read(type=bool, allow_none=True)
+
+ some_dict: Dict[str, Any] = reader.read(type=dict)
+ optional_dict: Optional[Dict[str, Any]] = reader.read(type=dict,
allow_none=True)
+
+ some_list: List[Any] = reader.read(type=list)
+ maybe_list: Optional[List[Any]] = reader.read(type=list, allow_none=True)
+
+ iter_int: Iterable[int] = reader.iter(type=int)
+ iter_str: Iterable[str] = reader.iter(type=str)
+ iter_dict: Iterable[Dict[str, Any]] = reader.iter(type=dict)
+ iter_optional_str: Iterable[Optional[str]] = reader.iter(type=str,
allow_none=True)
+
+ locals() # Silence flake8 F841
+
+
+def something_with_writer() -> None:
+ writer: jsonlines.Writer
+ writer = jsonlines.Writer(io.StringIO())
+ writer = jsonlines.Writer(io.BytesIO())
+
+ locals() # Silence flake8 F841
+
+
+def something_with_open() -> None:
+ name = "/nonexistent"
+
+ reader: jsonlines.Reader
+ reader = jsonlines.open(name)
+ reader = jsonlines.open(name, "r")
+ reader = jsonlines.open(name, mode="r")
+ reader = jsonlines.open(
+ name,
+ mode="r",
+ loads=json.loads,
+ )
+
+ writer: jsonlines.Writer
+ writer = jsonlines.open(name, "w")
+ writer = jsonlines.open(name, mode="w")
+ writer = jsonlines.open(name, "a")
+ writer = jsonlines.open(
+ name,
+ mode="w",
+ dumps=json.dumps,
+ compact=True,
+ sort_keys=True,
+ flush=True,
+ )
+
+ locals() # Silence flake8 F841
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/jsonlines-2.0.0/tox.ini new/jsonlines-3.1.0/tox.ini
--- old/jsonlines-2.0.0/tox.ini 2021-01-04 17:03:18.000000000 +0100
+++ new/jsonlines-3.1.0/tox.ini 2022-07-01 18:35:18.000000000 +0200
@@ -1,5 +1,5 @@
[tox]
-envlist = py36,py37,py38,py39,flake8
+envlist = py310,py39,py38,py37,py36,linters
[testenv]
deps = -rrequirements-dev.txt
@@ -11,3 +11,4 @@
commands =
flake8 jsonlines/ tests/
black --check jsonlines/ tests/
+ mypy --strict jsonlines/ tests/
