[ https://issues.apache.org/jira/browse/ASTERIXDB-1609?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15439529#comment-15439529 ]
Wail Alkowaileet commented on ASTERIXDB-1609: --------------------------------------------- I have another question: Why parsing the Tweets using JSONObject ? why not using the ADMParser instead? > TwitterParser does parse closed-nullable fields. > ------------------------------------------------ > > Key: ASTERIXDB-1609 > URL: https://issues.apache.org/jira/browse/ASTERIXDB-1609 > Project: Apache AsterixDB > Issue Type: Bug > Components: Data Model, Feeds > Reporter: Wail Alkowaileet > > TwitterParser doesn't support types NULL/MISSING > To reproduce, connect TwitterFeed using the defined type below. I > roundtrip-ed it using load statement and it was successfully loaded. > DDL: > {noformat} > drop dataverse feeds if exists; > create dataverse feeds; > use dataverse feeds; > create type userType1 as open { > 'utc_offset':int64?, > 'friends_count':int64?, > 'profile_image_url_https':string?, > 'listed_count':int64?, > 'profile_background_image_url':string?, > 'default_profile_image':boolean?, > 'favourites_count':int64?, > 'description':string?, > 'created_at':string?, > 'is_translator':boolean?, > 'profile_background_image_url_https':string?, > 'protected':boolean?, > 'screen_name':string?, > 'id_str':string?, > 'profile_link_color':string?, > 'id':int64?, > 'geo_enabled':boolean?, > 'profile_background_color':string?, > 'lang':string?, > 'profile_sidebar_border_color':string?, > 'profile_text_color':string?, > 'verified':boolean?, > 'profile_image_url':string?, > 'time_zone':string?, > 'contributors_enabled':boolean?, > 'profile_background_tile':boolean?, > 'profile_banner_url':string?, > 'statuses_count':int64?, > 'followers_count':int64?, > 'profile_use_background_image':boolean?, > 'default_profile':boolean?, > 'name':string?, > 'location':string?, > 'profile_sidebar_fill_color':string?, > 'url':string? > } > create type smallType1 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type largeType1 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type thumbType1 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type mediumType1 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type sizesType1 as open { > 'small':smallType1?, > 'large':largeType1?, > 'thumb':thumbType1?, > 'medium':mediumType1? > } > create type listType2 as open { > 'content_type':string?, > 'bitrate':int64?, > 'url':string? > } > create type video_infoType1 as open { > 'aspect_ratio':[int64]?, > 'duration_millis':int64?, > 'variants':[listType2]? > } > create type listType1 as open { > 'display_url':string?, > 'indices':[int64]?, > 'sizes':sizesType1?, > 'id_str':string?, > 'expanded_url':string?, > 'media_url_https':string?, > 'id':int64?, > 'type':string?, > 'media_url':string?, > 'url':string?, > 'video_info':video_infoType1?, > 'source_user_id':int64?, > 'source_status_id':int64?, > 'source_status_id_str':string?, > 'source_user_id_str':string? > } > create type extended_entitiesType1 as open { > 'media':[listType1]? > } > create type smallType2 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type largeType2 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type thumbType2 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type mediumType2 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type sizesType2 as open { > 'small':smallType2?, > 'large':largeType2?, > 'thumb':thumbType2?, > 'medium':mediumType2? > } > create type listType3 as open { > 'display_url':string?, > 'indices':[int64]?, > 'sizes':sizesType2?, > 'id_str':string?, > 'expanded_url':string?, > 'media_url_https':string?, > 'id':int64?, > 'type':string?, > 'media_url':string?, > 'url':string?, > 'source_user_id':int64?, > 'source_status_id':int64?, > 'source_status_id_str':string?, > 'source_user_id_str':string? > } > create type listType4 as open { > 'display_url':string?, > 'indices':[int64]?, > 'expanded_url':string?, > 'url':string? > } > create type listType5 as open { > 'indices':[int64]?, > 'text':string? > } > create type listType6 as open { > 'indices':[int64]?, > 'screen_name':string?, > 'id_str':string?, > 'name':string?, > 'id':int64? > } > create type entitiesType1 as open { > 'media':[listType3]?, > 'urls':[listType4]?, > 'hashtags':[listType5]?, > 'user_mentions':[listType6]? > } > create type bounding_boxType1 as open { > 'coordinates':[[[double]]]?, > 'type':string? > } > create type placeType1 as open { > 'country_code':string?, > 'country':string?, > 'full_name':string?, > 'bounding_box':bounding_boxType1?, > 'place_type':string?, > 'name':string?, > 'id':string?, > 'url':string? > } > create type userType2 as open { > 'utc_offset':int64?, > 'friends_count':int64?, > 'profile_image_url_https':string?, > 'listed_count':int64?, > 'profile_background_image_url':string?, > 'default_profile_image':boolean?, > 'favourites_count':int64?, > 'description':string?, > 'created_at':string?, > 'is_translator':boolean?, > 'profile_background_image_url_https':string?, > 'protected':boolean?, > 'screen_name':string?, > 'id_str':string?, > 'profile_link_color':string?, > 'id':int64?, > 'geo_enabled':boolean?, > 'profile_background_color':string?, > 'lang':string?, > 'profile_sidebar_border_color':string?, > 'profile_text_color':string?, > 'verified':boolean?, > 'profile_image_url':string?, > 'time_zone':string?, > 'contributors_enabled':boolean?, > 'profile_background_tile':boolean?, > 'profile_banner_url':string?, > 'statuses_count':int64?, > 'followers_count':int64?, > 'profile_use_background_image':boolean?, > 'default_profile':boolean?, > 'name':string?, > 'profile_sidebar_fill_color':string?, > 'location':string?, > 'url':string? > } > create type listType7 as open { > 'display_url':string?, > 'indices':[int64]?, > 'expanded_url':string?, > 'url':string? > } > create type listType8 as open { > 'indices':[int64]?, > 'text':string? > } > create type listType9 as open { > 'indices':[int64]?, > 'screen_name':string?, > 'id_str':string?, > 'name':string?, > 'id':int64? > } > create type smallType3 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type largeType3 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type thumbType3 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type mediumType3 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type sizesType3 as open { > 'small':smallType3?, > 'large':largeType3?, > 'thumb':thumbType3?, > 'medium':mediumType3? > } > create type listType10 as open { > 'display_url':string?, > 'indices':[int64]?, > 'sizes':sizesType3?, > 'id_str':string?, > 'expanded_url':string?, > 'media_url_https':string?, > 'id':int64?, > 'type':string?, > 'media_url':string?, > 'url':string?, > 'source_user_id':int64?, > 'source_status_id':int64?, > 'source_status_id_str':string?, > 'source_user_id_str':string? > } > create type entitiesType2 as open { > 'urls':[listType7]?, > 'hashtags':[listType8]?, > 'user_mentions':[listType9]?, > 'media':[listType10]? > } > create type smallType4 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type largeType4 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type thumbType4 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type mediumType4 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type sizesType4 as open { > 'small':smallType4?, > 'large':largeType4?, > 'thumb':thumbType4?, > 'medium':mediumType4? > } > create type listType12 as open { > 'content_type':string?, > 'url':string?, > 'bitrate':int64? > } > create type video_infoType2 as open { > 'aspect_ratio':[int64]?, > 'duration_millis':int64?, > 'variants':[listType12]? > } > create type listType11 as open { > 'display_url':string?, > 'indices':[int64]?, > 'sizes':sizesType4?, > 'id_str':string?, > 'expanded_url':string?, > 'media_url_https':string?, > 'id':int64?, > 'type':string?, > 'media_url':string?, > 'url':string?, > 'source_user_id':int64?, > 'source_status_id':int64?, > 'source_status_id_str':string?, > 'source_user_id_str':string?, > 'video_info':video_infoType2? > } > create type extended_entitiesType2 as open { > 'media':[listType11]? > } > create type bounding_boxType2 as open { > 'coordinates':[[[double]]]?, > 'type':string? > } > create type placeType2 as open { > 'country_code':string?, > 'country':string?, > 'full_name':string?, > 'bounding_box':bounding_boxType2?, > 'place_type':string?, > 'name':string?, > 'id':string?, > 'url':string? > } > create type quoted_statusType1 as open { > 'created_at':string?, > 'truncated':boolean?, > 'source':string?, > 'retweet_count':int64?, > 'retweeted':boolean?, > 'filter_level':string?, > 'is_quote_status':boolean?, > 'id_str':string?, > 'favorite_count':int64?, > 'id':int64?, > 'text':string?, > 'lang':string?, > 'user':userType2?, > 'favorited':boolean?, > 'quoted_status_id':int64?, > 'possibly_sensitive':boolean?, > 'entities':entitiesType2?, > 'quoted_status_id_str':string?, > 'extended_entities':extended_entitiesType2?, > 'in_reply_to_status_id_str':string?, > 'in_reply_to_status_id':int64?, > 'in_reply_to_user_id_str':string?, > 'in_reply_to_screen_name':string?, > 'in_reply_to_user_id':int64?, > 'place':placeType2? > } > create type coordinatesType1 as open { > 'coordinates':[double]?, > 'type':string? > } > create type geoType1 as open { > 'coordinates':[double]?, > 'type':string? > } > create type scopesType1 as open { > 'followers':boolean? > } > create type retweeted_statusType1 as open { > 'created_at':string?, > 'truncated':boolean?, > 'source':string?, > 'retweet_count':int64?, > 'retweeted':boolean?, > 'filter_level':string?, > 'is_quote_status':boolean?, > 'id_str':string?, > 'favorite_count':int64?, > 'id':int64?, > 'text':string?, > 'lang':string?, > 'user':userType1?, > 'favorited':boolean?, > 'extended_entities':extended_entitiesType1?, > 'possibly_sensitive':boolean?, > 'entities':entitiesType1?, > 'place':placeType1?, > 'quoted_status':quoted_statusType1?, > 'quoted_status_id':int64?, > 'quoted_status_id_str':string?, > 'in_reply_to_status_id_str':string?, > 'in_reply_to_status_id':int64?, > 'in_reply_to_user_id_str':string?, > 'in_reply_to_screen_name':string?, > 'in_reply_to_user_id':int64?, > 'coordinates':coordinatesType1?, > 'geo':geoType1?, > 'scopes':scopesType1? > } > create type listType13 as open { > 'indices':[int64]?, > 'screen_name':string?, > 'id_str':string?, > 'name':string?, > 'id':int64? > } > create type smallType5 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type largeType5 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type thumbType5 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type mediumType5 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type sizesType5 as open { > 'small':smallType5?, > 'large':largeType5?, > 'thumb':thumbType5?, > 'medium':mediumType5? > } > create type listType14 as open { > 'display_url':string?, > 'indices':[int64]?, > 'sizes':sizesType5?, > 'id_str':string?, > 'expanded_url':string?, > 'media_url_https':string?, > 'id':int64?, > 'type':string?, > 'media_url':string?, > 'url':string?, > 'source_user_id':int64?, > 'source_status_id':int64?, > 'source_status_id_str':string?, > 'source_user_id_str':string? > } > create type listType15 as open { > 'display_url':string?, > 'indices':[int64]?, > 'expanded_url':string?, > 'url':string? > } > create type listType16 as open { > 'indices':[int64]?, > 'text':string? > } > create type listType17 as open { > 'indices':[int64]?, > 'text':string? > } > create type entitiesType3 as open { > 'user_mentions':[listType13]?, > 'media':[listType14]?, > 'urls':[listType15]?, > 'hashtags':[listType16]?, > 'symbols':[listType17]? > } > create type userType3 as open { > 'friends_count':int64?, > 'profile_image_url_https':string?, > 'listed_count':int64?, > 'profile_background_image_url':string?, > 'default_profile_image':boolean?, > 'favourites_count':int64?, > 'description':string?, > 'created_at':string?, > 'is_translator':boolean?, > 'profile_background_image_url_https':string?, > 'protected':boolean?, > 'screen_name':string?, > 'id_str':string?, > 'profile_link_color':string?, > 'id':int64?, > 'geo_enabled':boolean?, > 'profile_background_color':string?, > 'lang':string?, > 'profile_sidebar_border_color':string?, > 'profile_text_color':string?, > 'verified':boolean?, > 'profile_image_url':string?, > 'contributors_enabled':boolean?, > 'profile_background_tile':boolean?, > 'profile_banner_url':string?, > 'statuses_count':int64?, > 'followers_count':int64?, > 'profile_use_background_image':boolean?, > 'default_profile':boolean?, > 'name':string?, > 'location':string?, > 'profile_sidebar_fill_color':string?, > 'utc_offset':int64?, > 'time_zone':string?, > 'url':string? > } > create type smallType6 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type largeType6 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type thumbType6 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type mediumType6 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type sizesType6 as open { > 'small':smallType6?, > 'large':largeType6?, > 'thumb':thumbType6?, > 'medium':mediumType6? > } > create type listType19 as open { > 'content_type':string?, > 'bitrate':int64?, > 'url':string? > } > create type video_infoType3 as open { > 'aspect_ratio':[int64]?, > 'duration_millis':int64?, > 'variants':[listType19]? > } > create type listType18 as open { > 'display_url':string?, > 'indices':[int64]?, > 'sizes':sizesType6?, > 'id_str':string?, > 'expanded_url':string?, > 'media_url_https':string?, > 'id':int64?, > 'type':string?, > 'media_url':string?, > 'url':string?, > 'source_user_id':int64?, > 'source_status_id':int64?, > 'source_status_id_str':string?, > 'source_user_id_str':string?, > 'video_info':video_infoType3? > } > create type extended_entitiesType3 as open { > 'media':[listType18]? > } > create type userType4 as open { > 'utc_offset':int64?, > 'friends_count':int64?, > 'profile_image_url_https':string?, > 'listed_count':int64?, > 'profile_background_image_url':string?, > 'default_profile_image':boolean?, > 'favourites_count':int64?, > 'description':string?, > 'created_at':string?, > 'is_translator':boolean?, > 'profile_background_image_url_https':string?, > 'protected':boolean?, > 'screen_name':string?, > 'id_str':string?, > 'profile_link_color':string?, > 'id':int64?, > 'geo_enabled':boolean?, > 'profile_background_color':string?, > 'lang':string?, > 'profile_sidebar_border_color':string?, > 'profile_text_color':string?, > 'verified':boolean?, > 'profile_image_url':string?, > 'time_zone':string?, > 'contributors_enabled':boolean?, > 'profile_background_tile':boolean?, > 'profile_banner_url':string?, > 'statuses_count':int64?, > 'followers_count':int64?, > 'profile_use_background_image':boolean?, > 'default_profile':boolean?, > 'name':string?, > 'profile_sidebar_fill_color':string?, > 'location':string?, > 'url':string? > } > create type smallType7 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type largeType7 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type thumbType7 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type mediumType7 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type sizesType7 as open { > 'small':smallType7?, > 'large':largeType7?, > 'thumb':thumbType7?, > 'medium':mediumType7? > } > create type listType21 as open { > 'content_type':string?, > 'bitrate':int64?, > 'url':string? > } > create type video_infoType4 as open { > 'aspect_ratio':[int64]?, > 'variants':[listType21]?, > 'duration_millis':int64? > } > create type listType20 as open { > 'display_url':string?, > 'source_user_id':int64?, > 'type':string?, > 'media_url':string?, > 'source_status_id':int64?, > 'url':string?, > 'indices':[int64]?, > 'sizes':sizesType7?, > 'id_str':string?, > 'expanded_url':string?, > 'source_status_id_str':string?, > 'media_url_https':string?, > 'id':int64?, > 'source_user_id_str':string?, > 'video_info':video_infoType4? > } > create type extended_entitiesType4 as open { > 'media':[listType20]? > } > create type smallType8 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type largeType8 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type thumbType8 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type mediumType8 as open { > 'w':int64?, > 'h':int64?, > 'resize':string? > } > create type sizesType8 as open { > 'small':smallType8?, > 'large':largeType8?, > 'thumb':thumbType8?, > 'medium':mediumType8? > } > create type listType22 as open { > 'display_url':string?, > 'source_user_id':int64?, > 'type':string?, > 'media_url':string?, > 'source_status_id':int64?, > 'url':string?, > 'indices':[int64]?, > 'sizes':sizesType8?, > 'id_str':string?, > 'expanded_url':string?, > 'source_status_id_str':string?, > 'media_url_https':string?, > 'id':int64?, > 'source_user_id_str':string? > } > create type listType23 as open { > 'display_url':string?, > 'indices':[int64]?, > 'expanded_url':string?, > 'url':string? > } > create type listType24 as open { > 'indices':[int64]?, > 'screen_name':string?, > 'id_str':string?, > 'name':string?, > 'id':int64? > } > create type listType25 as open { > 'indices':[int64]?, > 'text':string? > } > create type entitiesType4 as open { > 'media':[listType22]?, > 'urls':[listType23]?, > 'user_mentions':[listType24]?, > 'hashtags':[listType25]? > } > create type bounding_boxType3 as open { > 'coordinates':[[[double]]]?, > 'type':string? > } > create type placeType3 as open { > 'country_code':string?, > 'country':string?, > 'full_name':string?, > 'bounding_box':bounding_boxType3?, > 'place_type':string?, > 'name':string?, > 'id':string?, > 'url':string? > } > create type quoted_statusType2 as open { > 'created_at':string?, > 'truncated':boolean?, > 'source':string?, > 'retweet_count':int64?, > 'retweeted':boolean?, > 'filter_level':string?, > 'is_quote_status':boolean?, > 'id_str':string?, > 'favorite_count':int64?, > 'id':int64?, > 'text':string?, > 'lang':string?, > 'user':userType4?, > 'favorited':boolean?, > 'extended_entities':extended_entitiesType4?, > 'possibly_sensitive':boolean?, > 'entities':entitiesType4?, > 'quoted_status_id':int64?, > 'quoted_status_id_str':string?, > 'in_reply_to_status_id_str':string?, > 'in_reply_to_status_id':int64?, > 'in_reply_to_user_id_str':string?, > 'in_reply_to_screen_name':string?, > 'in_reply_to_user_id':int64?, > 'place':placeType3? > } > create type bounding_boxType4 as open { > 'coordinates':[[[double]]]?, > 'type':string? > } > create type placeType4 as open { > 'country_code':string?, > 'country':string?, > 'full_name':string?, > 'bounding_box':bounding_boxType4?, > 'place_type':string?, > 'name':string?, > 'id':string?, > 'url':string? > } > create type geoType2 as open { > 'coordinates':[double]?, > 'type':string? > } > create type coordinatesType2 as open { > 'coordinates':[double]?, > 'type':string? > } > create type Tweet as open { > 'id':string, > 'created_at':string?, > 'source':string?, > 'retweeted_status':retweeted_statusType1?, > 'retweet_count':int64?, > 'retweeted':boolean?, > 'filter_level':string?, > 'is_quote_status':boolean?, > 'id_str':string?, > 'favorite_count':int64?, > 'text':string?, > 'lang':string?, > 'favorited':boolean?, > 'truncated':boolean?, > 'timestamp_ms':string?, > 'entities':entitiesType3?, > 'user':userType3?, > 'extended_entities':extended_entitiesType3?, > 'in_reply_to_status_id_str':string?, > 'in_reply_to_status_id':int64?, > 'in_reply_to_user_id_str':string?, > 'in_reply_to_screen_name':string?, > 'in_reply_to_user_id':int64?, > 'possibly_sensitive':boolean?, > 'quoted_status':quoted_statusType2?, > 'quoted_status_id':int64?, > 'quoted_status_id_str':string?, > 'place':placeType4?, > 'geo':geoType2?, > 'coordinates':coordinatesType2? > } > create dataset Tweets (Tweet) > primary key id; > {noformat} -- This message was sent by Atlassian JIRA (v6.3.4#6332)