Changeset: c069c8238a35 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c069c8238a35
Modified Files:
clients/iotclient/requirements.txt
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/tests/datatypesinsertstests.py
clients/iotclient/tests/main.py
Branch: iot
Log Message:
Fixed imports, tests and time types
diffs (truncated from 592 to 300 lines):
diff --git a/clients/iotclient/requirements.txt
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -1,7 +1,6 @@
fake-factory==0.5.7
Flask-RESTful==0.3.5
IPy==0.83
-iso8601==0.1.11
jsonschema==2.5.1
python-dateutil==2.5.3
python-monetdb==11.24.0
diff --git a/clients/iotclient/src/Streams/datatypes.py
b/clients/iotclient/src/Streams/datatypes.py
--- a/clients/iotclient/src/Streams/datatypes.py
+++ b/clients/iotclient/src/Streams/datatypes.py
@@ -5,12 +5,12 @@ from collections import OrderedDict
from copy import deepcopy
from datetime import datetime, timedelta
from dateutil import parser
-from iso8601 import parse_date
from itertools import chain
from json import dumps
from math import ceil, log10
from re import compile, split
-from jsonschemas import UUID_REGEX, MAC_ADDRESS_REGEX, TIME_REGEX, IPV4_REGEX
+from jsonschemas import UUID_REGEX, MAC_ADDRESS_REGEX, TIME_REGEX, IPV4_REGEX,
TIME_WITH_TIMEZONE_TYPE_INTERNAL,\
+ TIME_WITH_TIMEZONE_TYPE_EXTERNAL, TIMESTAMP_WITH_TIMEZONE_TYPE_INTERNAL,
TIMESTAMP_WITH_TIMEZONE_TYPE_EXTERNAL
# The null constants might change from system to system due to different CPU's
limits
LITTLE_ENDIAN_ALIGNMENT = '<' # for now it is little-endian
@@ -671,18 +671,24 @@ class DateType(BaseDateTimeType): # Sto
return struct.pack(LITTLE_ENDIAN_ALIGNMENT + str(counter) + 'I',
*extracted_values)
-class TimeWithoutTimeZoneType(BaseDateTimeType): # Stored as an uint with the
number of milliseconds since hour00:00:00
+class TimeType(BaseDateTimeType): # Stored as an uint with the number of
milliseconds since hour00:00:00
"""Covers: TIME"""
def __init__(self, **kwargs):
- super(TimeWithoutTimeZoneType, self).__init__(**kwargs)
+ super(TimeType, self).__init__(**kwargs)
def add_json_schema_entry(self, schema):
- super(TimeWithoutTimeZoneType, self).add_json_schema_entry(schema)
+ super(TimeType, self).add_json_schema_entry(schema)
schema[self._column_name]['pattern'] = TIME_REGEX
def parse_entry(self, entry):
- return parser.parse(entry)
+ parsed = parser.parse(entry)
+ string = parsed.strftime("%z")
+ delta = timedelta(hours=int(string[1:3]), minutes=int(string[3:5]))
+ if string[0] == '-':
+ delta = -delta
+ parsed = parsed.replace(tzinfo=None) - delta
+ return parsed
def pack_next_value(self, parsed, counter, parameters, errors):
if parsed is None:
@@ -694,12 +700,22 @@ class TimeWithoutTimeZoneType(BaseDateTi
def pack_parsed_values(self, extracted_values, counter, parameters):
return struct.pack(LITTLE_ENDIAN_ALIGNMENT + str(counter) + 'I',
*extracted_values)
+ def to_json_representation(self):
+ json_value = super(TimeType, self).to_json_representation()
+ if self._data_type == TIME_WITH_TIMEZONE_TYPE_INTERNAL:
+ json_value['type'] = TIME_WITH_TIMEZONE_TYPE_EXTERNAL
+ return json_value
-class TimeWithTimeZoneType(TimeWithoutTimeZoneType):
- """Covers: TIME WITH TIME ZONE"""
+
+class TimestampType(BaseDateTimeType): # It is represented with the two
integers from time and date
+ """Covers: TIMESTAMP"""
def __init__(self, **kwargs):
- super(TimeWithTimeZoneType, self).__init__(**kwargs)
+ super(TimestampType, self).__init__(**kwargs)
+
+ def add_json_schema_entry(self, schema):
+ super(TimestampType, self).add_json_schema_entry(schema)
+ schema[self._column_name]['format'] = 'date-time'
def parse_entry(self, entry):
parsed = parser.parse(entry)
@@ -710,25 +726,6 @@ class TimeWithTimeZoneType(TimeWithoutTi
parsed = parsed.replace(tzinfo=None) - delta
return parsed
- def to_json_representation(self):
- json_value = super(TimeWithTimeZoneType, self).to_json_representation()
- json_value['type'] = 'time with time zone'
- return json_value
-
-
-class TimestampWithoutTimeZoneType(BaseDateTimeType): # It is represented
with the two integers from time and date
- """Covers: TIMESTAMP"""
-
- def __init__(self, **kwargs):
- super(TimestampWithoutTimeZoneType, self).__init__(**kwargs)
-
- def add_json_schema_entry(self, schema):
- super(TimestampWithoutTimeZoneType, self).add_json_schema_entry(schema)
- schema[self._column_name]['format'] = 'date-time'
-
- def parse_entry(self, entry):
- return parse_date(entry)
-
def pack_next_value(self, parsed, counter, parameters, errors):
if parsed is None:
return [0, 0x80000000]
@@ -743,25 +740,10 @@ class TimestampWithoutTimeZoneType(BaseD
concat_array = list(chain(*extracted_values))
return struct.pack(LITTLE_ENDIAN_ALIGNMENT + str(counter << 1) + 'I',
*concat_array)
-
-class TimestampWithTimeZoneType(TimestampWithoutTimeZoneType):
- """Covers: TIMESTAMP WITH TIME ZONE"""
-
- def __init__(self, **kwargs):
- super(TimestampWithTimeZoneType, self).__init__(**kwargs)
-
- def parse_entry(self, entry):
- parsed = parse_date(entry)
- string = parsed.strftime("%z")
- delta = timedelta(hours=int(string[1:3]), minutes=int(string[3:5]))
- if string[0] == '-':
- delta = -delta
- parsed = parsed.replace(tzinfo=None) - delta
- return parsed
-
def to_json_representation(self):
- json_value = super(TimestampWithTimeZoneType,
self).to_json_representation()
- json_value['type'] = 'timestamp with time zone'
+ json_value = super(TimestampType, self).to_json_representation()
+ if self._data_type == TIMESTAMP_WITH_TIMEZONE_TYPE_INTERNAL:
+ json_value['type'] = TIMESTAMP_WITH_TIMEZONE_TYPE_EXTERNAL
return json_value
@@ -770,11 +752,11 @@ class IntervalType(NumberBaseType):
def __init__(self, **kwargs):
interval = kwargs['type'][9:].split(" to ")[-1]
- self._multiplier = {'second': 1000, 'minute': 60000, 'hour': 3600000,
'day': 86400000, 'month': 1, 'year': 12} \
+ self._multiplier = {'second': 1000, 'minute': 60000, 'hour': 3600000,
'day': 86400000, 'month': 1, 'year': 12}\
.get(interval)
self._nullable_constant = {'second': INT64_MIN, 'minute': INT64_MIN,
'hour': INT64_MIN, 'day': INT64_MIN,
'month': INT32_MIN, 'year':
INT32_MIN}.get(interval)
- self._pack_sym = {'second': 'q', 'minute': 'q', 'hour': 'q', 'day':
'q', 'month': 'i', 'year': 'i'} \
+ self._pack_sym = {'second': 'q', 'minute': 'q', 'hour': 'q', 'day':
'q', 'month': 'i', 'year': 'i'}\
.get(interval)
super(IntervalType, self).__init__(**kwargs)
diff --git a/clients/iotclient/src/Streams/streampolling.py
b/clients/iotclient/src/Streams/streampolling.py
--- a/clients/iotclient/src/Streams/streampolling.py
+++ b/clients/iotclient/src/Streams/streampolling.py
@@ -1,15 +1,13 @@
from collections import OrderedDict, defaultdict
from json import dumps
from jsonschema import Draft4Validator, FormatChecker
-from .datatypes import TextType, LimitedTextType, SmallIntegerType,
HugeIntegerType, FloatType, DecimalType, DateType, \
- TimeWithoutTimeZoneType, TimeWithTimeZoneType,
TimestampWithoutTimeZoneType, TimestampWithTimeZoneType, \
- IntervalType, BooleanType, INetType, INetSixType, MACType, URLType,
UUIDType, RegexType, EnumType,\
- ENUM_TYPE_SEPARATOR
+from .datatypes import TextType, LimitedTextType, SmallIntegerType,
HugeIntegerType, FloatType, DecimalType, DateType,\
+ TimeType, TimestampType, IntervalType, BooleanType, INetType, INetSixType,
MACType, URLType, UUIDType, RegexType,\
+ EnumType, ENUM_TYPE_SEPARATOR
from .jsonschemas import UNBOUNDED_TEXT_TYPE, BOUNDED_TEXT_TYPES,
SMALL_INTEGERS_TYPES, HUGE_INTEGER_TYPE,\
- FLOATING_POINT_PRECISION_TYPES, DECIMAL_TYPE, DATE_TYPE,
TIME_WITHOUT_TIMEZONE_TYPE,\
- TIME_WITH_TIMEZONE_TYPE_INTERNAL, TIMESTAMP_WITHOUT_TIMEZONE_TYPE,
TIMESTAMP_WITH_TIMEZONE_TYPE_INTERNAL, \
- INTERVAL_INPUTS, BOOLEAN_TYPE, INET_TYPE, URL_TYPE, UUID_TYPE, INET6_TYPE,
MAC_TYPE, REGEX_TYPE, ENUM_TYPE, \
- SECOND_INTERVAL_TYPE, MONTH_INTERVAL_TYPE
+ FLOATING_POINT_PRECISION_TYPES, DECIMAL_TYPE, DATE_TYPE,
TIME_WITHOUT_TIMEZONE_TYPE, MONTH_INTERVAL_TYPE,\
+ TIME_WITH_TIMEZONE_TYPE_INTERNAL, TIMESTAMP_WITHOUT_TIMEZONE_TYPE,
TIMESTAMP_WITH_TIMEZONE_TYPE_INTERNAL,\
+ BOOLEAN_TYPE, INET_TYPE, URL_TYPE, UUID_TYPE, INET6_TYPE, MAC_TYPE,
REGEX_TYPE, ENUM_TYPE, SECOND_INTERVAL_TYPE
from .streams import TupleBasedStream, TimeBasedStream, AutoFlushedStream,
IMPLICIT_TIMESTAMP_COLUMN_NAME,\
HOST_IDENTIFIER_COLUMN_NAME
from .streamscontext import get_streams_context
@@ -24,11 +22,9 @@ Switcher = [{'types': [UNBOUNDED_TEXT_TY
{'types': [DECIMAL_TYPE], 'class': DecimalType},
{'types': [BOOLEAN_TYPE], 'class': BooleanType},
{'types': [DATE_TYPE], 'class': DateType},
- {'types': [TIME_WITHOUT_TIMEZONE_TYPE], 'class':
TimeWithoutTimeZoneType},
- {'types': [TIME_WITH_TIMEZONE_TYPE_INTERNAL], 'class':
TimeWithTimeZoneType},
- {'types': [TIMESTAMP_WITHOUT_TIMEZONE_TYPE], 'class':
TimestampWithoutTimeZoneType},
- {'types': [TIMESTAMP_WITH_TIMEZONE_TYPE_INTERNAL], 'class':
TimestampWithTimeZoneType},
- {'types': INTERVAL_INPUTS, 'class': IntervalType},
+ {'types': [TIME_WITHOUT_TIMEZONE_TYPE,
TIME_WITH_TIMEZONE_TYPE_INTERNAL], 'class': TimeType},
+ {'types': [TIMESTAMP_WITHOUT_TIMEZONE_TYPE,
TIMESTAMP_WITH_TIMEZONE_TYPE_INTERNAL], 'class': TimestampType},
+ {'types': [SECOND_INTERVAL_TYPE, MONTH_INTERVAL_TYPE], 'class':
IntervalType},
{'types': [URL_TYPE], 'class': URLType},
{'types': [INET_TYPE], 'class': INetType},
{'types': [UUID_TYPE], 'class': UUIDType},
diff --git a/clients/iotclient/src/Streams/streamscreator.py
b/clients/iotclient/src/Streams/streamscreator.py
--- a/clients/iotclient/src/Streams/streamscreator.py
+++ b/clients/iotclient/src/Streams/streamscreator.py
@@ -1,13 +1,11 @@
from collections import OrderedDict
from json import dumps
from jsonschema import Draft4Validator, FormatChecker
-from .datatypes import TextType, LimitedTextType, SmallIntegerType, FloatType,
DecimalType, DateType, \
- TimeWithoutTimeZoneType, TimeWithTimeZoneType,
TimestampWithoutTimeZoneType, TimestampWithTimeZoneType, \
- IntervalType, BooleanType, INetType, INetSixType, MACType, URLType,
UUIDType, RegexType, EnumType
-from .jsonschemas import UNBOUNDED_TEXT_INPUTS, BOUNDED_TEXT_INPUTS,
SMALL_INTEGERS_INPUTS, HUGE_INTEGER_TYPE, \
- FLOATING_POINT_PRECISION_INPUTS, DECIMAL_INPUTS, DATE_TYPE,
TIME_WITHOUT_TIMEZONE_TYPE, \
- TIME_WITH_TIMEZONE_TYPE_EXTERNAL, TIMESTAMP_WITHOUT_TIMEZONE_TYPE,
TIMESTAMP_WITH_TIMEZONE_TYPE_EXTERNAL, \
- INTERVAL_INPUTS, BOOLEAN_INPUTS, INET_TYPE, INET6_TYPE, MAC_TYPE,
URL_TYPE, UUID_TYPE, REGEX_TYPE, ENUM_TYPE, \
+from .datatypes import TextType, LimitedTextType, SmallIntegerType, FloatType,
DecimalType, DateType, EnumType,\
+ TimeType, TimestampType, IntervalType, BooleanType, INetType, INetSixType,
MACType, URLType, UUIDType, RegexType
+from .jsonschemas import UNBOUNDED_TEXT_INPUTS, BOUNDED_TEXT_INPUTS,
SMALL_INTEGERS_INPUTS, HUGE_INTEGER_TYPE,\
+ FLOATING_POINT_PRECISION_INPUTS, DECIMAL_INPUTS, DATE_TYPE, TIME_INPUTS,
TIMESTAMP_INPUTS, INTERVAL_INPUTS,\
+ BOOLEAN_INPUTS, INET_TYPE, INET6_TYPE, MAC_TYPE, URL_TYPE, UUID_TYPE,
REGEX_TYPE, ENUM_TYPE,\
TIMED_FLUSH_IDENTIFIER, TUPLE_FLUSH_IDENTIFIER
from .streams import TupleBasedStream, TimeBasedStream, AutoFlushedStream,
IMPLICIT_TIMESTAMP_COLUMN_NAME,\
HOST_IDENTIFIER_COLUMN_NAME
@@ -19,10 +17,8 @@ Switcher = [{'types': UNBOUNDED_TEXT_INP
{'types': FLOATING_POINT_PRECISION_INPUTS, 'class': FloatType},
{'types': DECIMAL_INPUTS, 'class': DecimalType},
{'types': [DATE_TYPE], 'class': DateType},
- {'types': [TIME_WITHOUT_TIMEZONE_TYPE], 'class':
TimeWithoutTimeZoneType},
- {'types': [TIME_WITH_TIMEZONE_TYPE_EXTERNAL], 'class':
TimeWithTimeZoneType},
- {'types': [TIMESTAMP_WITHOUT_TIMEZONE_TYPE], 'class':
TimestampWithoutTimeZoneType},
- {'types': [TIMESTAMP_WITH_TIMEZONE_TYPE_EXTERNAL], 'class':
TimestampWithTimeZoneType},
+ {'types': TIME_INPUTS, 'class': TimeType},
+ {'types': TIMESTAMP_INPUTS, 'class': TimestampType},
{'types': INTERVAL_INPUTS, 'class': IntervalType},
{'types': BOOLEAN_INPUTS, 'class': BooleanType},
{'types': [INET_TYPE], 'class': INetType},
diff --git a/clients/iotclient/tests/datatypesinsertstests.py
b/clients/iotclient/tests/datatypesinsertstests.py
--- a/clients/iotclient/tests/datatypesinsertstests.py
+++ b/clients/iotclient/tests/datatypesinsertstests.py
@@ -1,11 +1,10 @@
-import os
-import random
-import uuid
-import unittest
-
from abc import ABCMeta, abstractmethod
from faker import Factory
from pytz import timezone
+from os import path, remove
+from random import choice, randint, uniform
+from unittest import TestCase, skip
+from uuid import uuid4
def load_src(name, fpath):
@@ -18,14 +17,13 @@ def load_src(name, fpath):
load_src("jsonschemas", "../src/Streams/jsonschemas.py")
load_src("datatypes", "../src/Streams/datatypes.py")
-from datatypes import TextType, URLType, INetType, UUIDType, BooleanType,
SmallIntegerType,\
- HugeIntegerType, FloatType, DecimalType, DateType,
TimeWithoutTimeZoneType, TimeWithTimeZoneType,\
- TimestampWithoutTimeZoneType, TimestampWithTimeZoneType, IntervalType
+from datatypes import TextType, URLType, INetType, UUIDType, BooleanType,
SmallIntegerType, HugeIntegerType, FloatType,\
+ DecimalType, DateType, TimeType, TimestampType, IntervalType
faker = Factory.create()
-class DataTypesTest(unittest.TestCase):
+class DataTypesTest(TestCase):
__metaclass__ = ABCMeta
def __init__(self, **kwargs):
@@ -34,7 +32,7 @@ class DataTypesTest(unittest.TestCase):
self._number_inserts = kwargs['number_inserts']
self._data_type = self.get_data_type()
self._serializer = self.get_serializer()
- self._temp_path = os.path.join(kwargs['temp_path'],
self._data_type.replace(' ', '_'))
+ self._temp_path = path.join(kwargs['temp_path'],
self._data_type.replace(' ', '_'))
@abstractmethod
def get_data_type(self):
@@ -84,7 +82,7 @@ class DataTypesTest(unittest.TestCase):
self._mapi_connection.execute("DROP TABLE binary_insert")
try:
- os.remove(self._temp_path)
+ remove(self._temp_path)
except:
pass
self.assertListEqual(sql_inserts, binary_inserts) # the lists must be
equal!!
@@ -129,10 +127,6 @@ class URLTest(BaseStringText):
def get_next_batch(self, number_inserts):
return [faker.uri() for _ in xrange(number_inserts)]
- # @unittest.skip("Data conversion problem")
- # def runTest(self):
- # super(URLTest, self).runTest()
-
class INetTest(BaseStringText):
@@ -161,7 +155,7 @@ class UUIDTest(BaseStringText):
return UUIDType(**{'name': 'val', 'type': 'uuid'})
def get_next_batch(self, number_inserts):
- return [str(uuid.uuid4()) for _ in xrange(number_inserts)]
+ return [str(uuid4()) for _ in xrange(number_inserts)]
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
