HIVE-18048: Vectorization: Support Struct type with vectorization (Colin Ma, reviewed by Ferdinand Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ac721836 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ac721836 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ac721836 Branch: refs/heads/master Commit: ac721836f182b6f4cbeb07c2dd1f56692276cbc0 Parents: 1bcc88f Author: Ferdinand Xu <cheng.a...@intel.com> Authored: Tue Feb 6 10:46:08 2018 +0800 Committer: Ferdinand Xu <cheng.a...@intel.com> Committed: Tue Feb 6 10:46:08 2018 +0800 ---------------------------------------------------------------------- data/files/struct_type.txt | 1025 ++++++++++++++++++ .../test/resources/testconfiguration.properties | 2 + .../ql/exec/vector/VectorizationContext.java | 45 +- .../expressions/VectorUDFStructField.java | 98 ++ .../ql/udf/generic/GenericUDFStructField.java | 53 + .../orc_struct_type_vectorization.q | 65 ++ .../parquet_struct_type_vectorization.q | 65 ++ .../llap/orc_struct_type_vectorization.q.out | 535 +++++++++ .../parquet_struct_type_vectorization.q.out | 535 +++++++++ .../llap/vector_complex_all.q.out | 63 +- .../orc_struct_type_vectorization.q.out | 503 +++++++++ .../parquet_struct_type_vectorization.q.out | 503 +++++++++ 12 files changed, 3484 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ac721836/data/files/struct_type.txt ---------------------------------------------------------------------- diff --git a/data/files/struct_type.txt b/data/files/struct_type.txt new file mode 100644 index 0000000..bbe9e94 --- /dev/null +++ b/data/files/struct_type.txt @@ -0,0 +1,1025 @@ +1|1,str1|2001,str2001 +2|2,str2|2002,str2002 +3|3,str3|2003,str2003 +4|4,str4|2004,str2004 +5|5,str5|2005,str2005 +6|6,str6|2006,str2006 +7|7,str7|2007,str2007 +8|8,str8|2008,str2008 +9|9,str9|2009,str2009 +10|10,str10|2010,str2010 +11|11,str11|2011,str2011 +12|12,str12|2012,str2012 +13|13,str13|2013,str2013 +14|14,str14|2014,str2014 +15|15,str15|2015,str2015 +16|16,str16|2016,str2016 +17|17,str17|2017,str2017 +18|18,str18|2018,str2018 +19|19,str19|2019,str2019 +20|20,str20|2020,str2020 +21|21,str21|2021,str2021 +22|22,str22|2022,str2022 +23|23,str23|2023,str2023 +24|24,str24|2024,str2024 +25|25,str25|2025,str2025 +26|26,str26|2026,str2026 +27|27,str27|2027,str2027 +28|28,str28|2028,str2028 +29|29,str29|2029,str2029 +30|30,str30|2030,str2030 +31|31,str31|2031,str2031 +32|32,str32|2032,str2032 +33|33,str33|2033,str2033 +34|34,str34|2034,str2034 +35|35,str35|2035,str2035 +36|36,str36|2036,str2036 +37|37,str37|2037,str2037 +38|38,str38|2038,str2038 +39|39,str39|2039,str2039 +40|40,str40|2040,str2040 +41|41,str41|2041,str2041 +42|42,str42|2042,str2042 +43|43,str43|2043,str2043 +44|44,str44|2044,str2044 +45|45,str45|2045,str2045 +46|46,str46|2046,str2046 +47|47,str47|2047,str2047 +48|48,str48|2048,str2048 +49|49,str49|2049,str2049 +50|50,str50|2050,str2050 +51|51,str51|2051,str2051 +52|52,str52|2052,str2052 +53|53,str53|2053,str2053 +54|54,str54|2054,str2054 +55|55,str55|2055,str2055 +56|56,str56|2056,str2056 +57|57,str57|2057,str2057 +58|58,str58|2058,str2058 +59|59,str59|2059,str2059 +60|60,str60|2060,str2060 +61|61,str61|2061,str2061 +62|62,str62|2062,str2062 +63|63,str63|2063,str2063 +64|64,str64|2064,str2064 +65|65,str65|2065,str2065 +66|66,str66|2066,str2066 +67|67,str67|2067,str2067 +68|68,str68|2068,str2068 +69|69,str69|2069,str2069 +70|70,str70|2070,str2070 +71|71,str71|2071,str2071 +72|72,str72|2072,str2072 +73|73,str73|2073,str2073 +74|74,str74|2074,str2074 +75|75,str75|2075,str2075 +76|76,str76|2076,str2076 +77|77,str77|2077,str2077 +78|78,str78|2078,str2078 +79|79,str79|2079,str2079 +80|80,str80|2080,str2080 +81|81,str81|2081,str2081 +82|82,str82|2082,str2082 +83|83,str83|2083,str2083 +84|84,str84|2084,str2084 +85|85,str85|2085,str2085 +86|86,str86|2086,str2086 +87|87,str87|2087,str2087 +88|88,str88|2088,str2088 +89|89,str89|2089,str2089 +90|90,str90|2090,str2090 +91|91,str91|2091,str2091 +92|92,str92|2092,str2092 +93|93,str93|2093,str2093 +94|94,str94|2094,str2094 +95|95,str95|2095,str2095 +96|96,str96|2096,str2096 +97|97,str97|2097,str2097 +98|98,str98|2098,str2098 +99|99,str99|2099,str2099 +100|100,str100|2100,str2100 +101|101,str101|2101,str2101 +102|102,str102|2102,str2102 +103|103,str103|2103,str2103 +104|104,str104|2104,str2104 +105|105,str105|2105,str2105 +106|106,str106|2106,str2106 +107|107,str107|2107,str2107 +108|108,str108|2108,str2108 +109|109,str109|2109,str2109 +110|110,str110|2110,str2110 +111|111,str111|2111,str2111 +112|112,str112|2112,str2112 +113|113,str113|2113,str2113 +114|114,str114|2114,str2114 +115|115,str115|2115,str2115 +116|116,str116|2116,str2116 +117|117,str117|2117,str2117 +118|118,str118|2118,str2118 +119|119,str119|2119,str2119 +120|120,str120|2120,str2120 +121|121,str121|2121,str2121 +122|122,str122|2122,str2122 +123|123,str123|2123,str2123 +124|124,str124|2124,str2124 +125|125,str125|2125,str2125 +126|126,str126|2126,str2126 +127|127,str127|2127,str2127 +128|128,str128|2128,str2128 +129|129,str129|2129,str2129 +130|130,str130|2130,str2130 +131|131,str131|2131,str2131 +132|132,str132|2132,str2132 +133|133,str133|2133,str2133 +134|134,str134|2134,str2134 +135|135,str135|2135,str2135 +136|136,str136|2136,str2136 +137|137,str137|2137,str2137 +138|138,str138|2138,str2138 +139|139,str139|2139,str2139 +140|140,str140|2140,str2140 +141|141,str141|2141,str2141 +142|142,str142|2142,str2142 +143|143,str143|2143,str2143 +144|144,str144|2144,str2144 +145|145,str145|2145,str2145 +146|146,str146|2146,str2146 +147|147,str147|2147,str2147 +148|148,str148|2148,str2148 +149|149,str149|2149,str2149 +150|150,str150|2150,str2150 +151|151,str151|2151,str2151 +152|152,str152|2152,str2152 +153|153,str153|2153,str2153 +154|154,str154|2154,str2154 +155|155,str155|2155,str2155 +156|156,str156|2156,str2156 +157|157,str157|2157,str2157 +158|158,str158|2158,str2158 +159|159,str159|2159,str2159 +160|160,str160|2160,str2160 +161|161,str161|2161,str2161 +162|162,str162|2162,str2162 +163|163,str163|2163,str2163 +164|164,str164|2164,str2164 +165|165,str165|2165,str2165 +166|166,str166|2166,str2166 +167|167,str167|2167,str2167 +168|168,str168|2168,str2168 +169|169,str169|2169,str2169 +170|170,str170|2170,str2170 +171|171,str171|2171,str2171 +172|172,str172|2172,str2172 +173|173,str173|2173,str2173 +174|174,str174|2174,str2174 +175|175,str175|2175,str2175 +176|176,str176|2176,str2176 +177|177,str177|2177,str2177 +178|178,str178|2178,str2178 +179|179,str179|2179,str2179 +180|180,str180|2180,str2180 +181|181,str181|2181,str2181 +182|182,str182|2182,str2182 +183|183,str183|2183,str2183 +184|184,str184|2184,str2184 +185|185,str185|2185,str2185 +186|186,str186|2186,str2186 +187|187,str187|2187,str2187 +188|188,str188|2188,str2188 +189|189,str189|2189,str2189 +190|190,str190|2190,str2190 +191|191,str191|2191,str2191 +192|192,str192|2192,str2192 +193|193,str193|2193,str2193 +194|194,str194|2194,str2194 +195|195,str195|2195,str2195 +196|196,str196|2196,str2196 +197|197,str197|2197,str2197 +198|198,str198|2198,str2198 +199|199,str199|2199,str2199 +200|200,str200|2200,str2200 +201|201,str201|2201,str2201 +202|202,str202|2202,str2202 +203|203,str203|2203,str2203 +204|204,str204|2204,str2204 +205|205,str205|2205,str2205 +206|206,str206|2206,str2206 +207|207,str207|2207,str2207 +208|208,str208|2208,str2208 +209|209,str209|2209,str2209 +210|210,str210|2210,str2210 +211|211,str211|2211,str2211 +212|212,str212|2212,str2212 +213|213,str213|2213,str2213 +214|214,str214|2214,str2214 +215|215,str215|2215,str2215 +216|216,str216|2216,str2216 +217|217,str217|2217,str2217 +218|218,str218|2218,str2218 +219|219,str219|2219,str2219 +220|220,str220|2220,str2220 +221|221,str221|2221,str2221 +222|222,str222|2222,str2222 +223|223,str223|2223,str2223 +224|224,str224|2224,str2224 +225|225,str225|2225,str2225 +226|226,str226|2226,str2226 +227|227,str227|2227,str2227 +228|228,str228|2228,str2228 +229|229,str229|2229,str2229 +230|230,str230|2230,str2230 +231|231,str231|2231,str2231 +232|232,str232|2232,str2232 +233|233,str233|2233,str2233 +234|234,str234|2234,str2234 +235|235,str235|2235,str2235 +236|236,str236|2236,str2236 +237|237,str237|2237,str2237 +238|238,str238|2238,str2238 +239|239,str239|2239,str2239 +240|240,str240|2240,str2240 +241|241,str241|2241,str2241 +242|242,str242|2242,str2242 +243|243,str243|2243,str2243 +244|244,str244|2244,str2244 +245|245,str245|2245,str2245 +246|246,str246|2246,str2246 +247|247,str247|2247,str2247 +248|248,str248|2248,str2248 +249|249,str249|2249,str2249 +250|250,str250|2250,str2250 +251|251,str251|2251,str2251 +252|252,str252|2252,str2252 +253|253,str253|2253,str2253 +254|254,str254|2254,str2254 +255|255,str255|2255,str2255 +256|256,str256|2256,str2256 +257|257,str257|2257,str2257 +258|258,str258|2258,str2258 +259|259,str259|2259,str2259 +260|260,str260|2260,str2260 +261|261,str261|2261,str2261 +262|262,str262|2262,str2262 +263|263,str263|2263,str2263 +264|264,str264|2264,str2264 +265|265,str265|2265,str2265 +266|266,str266|2266,str2266 +267|267,str267|2267,str2267 +268|268,str268|2268,str2268 +269|269,str269|2269,str2269 +270|270,str270|2270,str2270 +271|271,str271|2271,str2271 +272|272,str272|2272,str2272 +273|273,str273|2273,str2273 +274|274,str274|2274,str2274 +275|275,str275|2275,str2275 +276|276,str276|2276,str2276 +277|277,str277|2277,str2277 +278|278,str278|2278,str2278 +279|279,str279|2279,str2279 +280|280,str280|2280,str2280 +281|281,str281|2281,str2281 +282|282,str282|2282,str2282 +283|283,str283|2283,str2283 +284|284,str284|2284,str2284 +285|285,str285|2285,str2285 +286|286,str286|2286,str2286 +287|287,str287|2287,str2287 +288|288,str288|2288,str2288 +289|289,str289|2289,str2289 +290|290,str290|2290,str2290 +291|291,str291|2291,str2291 +292|292,str292|2292,str2292 +293|293,str293|2293,str2293 +294|294,str294|2294,str2294 +295|295,str295|2295,str2295 +296|296,str296|2296,str2296 +297|297,str297|2297,str2297 +298|298,str298|2298,str2298 +299|299,str299|2299,str2299 +300|300,str300|2300,str2300 +301|301,str301|2301,str2301 +302|302,str302|2302,str2302 +303|303,str303|2303,str2303 +304|304,str304|2304,str2304 +305|305,str305|2305,str2305 +306|306,str306|2306,str2306 +307|307,str307|2307,str2307 +308|308,str308|2308,str2308 +309|309,str309|2309,str2309 +310|310,str310|2310,str2310 +311|311,str311|2311,str2311 +312|312,str312|2312,str2312 +313|313,str313|2313,str2313 +314|314,str314|2314,str2314 +315|315,str315|2315,str2315 +316|316,str316|2316,str2316 +317|317,str317|2317,str2317 +318|318,str318|2318,str2318 +319|319,str319|2319,str2319 +320|320,str320|2320,str2320 +321|321,str321|2321,str2321 +322|322,str322|2322,str2322 +323|323,str323|2323,str2323 +324|324,str324|2324,str2324 +325|325,str325|2325,str2325 +326|326,str326|2326,str2326 +327|327,str327|2327,str2327 +328|328,str328|2328,str2328 +329|329,str329|2329,str2329 +330|330,str330|2330,str2330 +331|331,str331|2331,str2331 +332|332,str332|2332,str2332 +333|333,str333|2333,str2333 +334|334,str334|2334,str2334 +335|335,str335|2335,str2335 +336|336,str336|2336,str2336 +337|337,str337|2337,str2337 +338|338,str338|2338,str2338 +339|339,str339|2339,str2339 +340|340,str340|2340,str2340 +341|341,str341|2341,str2341 +342|342,str342|2342,str2342 +343|343,str343|2343,str2343 +344|344,str344|2344,str2344 +345|345,str345|2345,str2345 +346|346,str346|2346,str2346 +347|347,str347|2347,str2347 +348|348,str348|2348,str2348 +349|349,str349|2349,str2349 +350|350,str350|2350,str2350 +351|351,str351|2351,str2351 +352|352,str352|2352,str2352 +353|353,str353|2353,str2353 +354|354,str354|2354,str2354 +355|355,str355|2355,str2355 +356|356,str356|2356,str2356 +357|357,str357|2357,str2357 +358|358,str358|2358,str2358 +359|359,str359|2359,str2359 +360|360,str360|2360,str2360 +361|361,str361|2361,str2361 +362|362,str362|2362,str2362 +363|363,str363|2363,str2363 +364|364,str364|2364,str2364 +365|365,str365|2365,str2365 +366|366,str366|2366,str2366 +367|367,str367|2367,str2367 +368|368,str368|2368,str2368 +369|369,str369|2369,str2369 +370|370,str370|2370,str2370 +371|371,str371|2371,str2371 +372|372,str372|2372,str2372 +373|373,str373|2373,str2373 +374|374,str374|2374,str2374 +375|375,str375|2375,str2375 +376|376,str376|2376,str2376 +377|377,str377|2377,str2377 +378|378,str378|2378,str2378 +379|379,str379|2379,str2379 +380|380,str380|2380,str2380 +381|381,str381|2381,str2381 +382|382,str382|2382,str2382 +383|383,str383|2383,str2383 +384|384,str384|2384,str2384 +385|385,str385|2385,str2385 +386|386,str386|2386,str2386 +387|387,str387|2387,str2387 +388|388,str388|2388,str2388 +389|389,str389|2389,str2389 +390|390,str390|2390,str2390 +391|391,str391|2391,str2391 +392|392,str392|2392,str2392 +393|393,str393|2393,str2393 +394|394,str394|2394,str2394 +395|395,str395|2395,str2395 +396|396,str396|2396,str2396 +397|397,str397|2397,str2397 +398|398,str398|2398,str2398 +399|399,str399|2399,str2399 +400|400,str400|2400,str2400 +401|401,str401|2401,str2401 +402|402,str402|2402,str2402 +403|403,str403|2403,str2403 +404|404,str404|2404,str2404 +405|405,str405|2405,str2405 +406|406,str406|2406,str2406 +407|407,str407|2407,str2407 +408|408,str408|2408,str2408 +409|409,str409|2409,str2409 +410|410,str410|2410,str2410 +411|411,str411|2411,str2411 +412|412,str412|2412,str2412 +413|413,str413|2413,str2413 +414|414,str414|2414,str2414 +415|415,str415|2415,str2415 +416|416,str416|2416,str2416 +417|417,str417|2417,str2417 +418|418,str418|2418,str2418 +419|419,str419|2419,str2419 +420|420,str420|2420,str2420 +421|421,str421|2421,str2421 +422|422,str422|2422,str2422 +423|423,str423|2423,str2423 +424|424,str424|2424,str2424 +425|425,str425|2425,str2425 +426|426,str426|2426,str2426 +427|427,str427|2427,str2427 +428|428,str428|2428,str2428 +429|429,str429|2429,str2429 +430|430,str430|2430,str2430 +431|431,str431|2431,str2431 +432|432,str432|2432,str2432 +433|433,str433|2433,str2433 +434|434,str434|2434,str2434 +435|435,str435|2435,str2435 +436|436,str436|2436,str2436 +437|437,str437|2437,str2437 +438|438,str438|2438,str2438 +439|439,str439|2439,str2439 +440|440,str440|2440,str2440 +441|441,str441|2441,str2441 +442|442,str442|2442,str2442 +443|443,str443|2443,str2443 +444|444,str444|2444,str2444 +445|445,str445|2445,str2445 +446|446,str446|2446,str2446 +447|447,str447|2447,str2447 +448|448,str448|2448,str2448 +449|449,str449|2449,str2449 +450|450,str450|2450,str2450 +451|451,str451|2451,str2451 +452|452,str452|2452,str2452 +453|453,str453|2453,str2453 +454|454,str454|2454,str2454 +455|455,str455|2455,str2455 +456|456,str456|2456,str2456 +457|457,str457|2457,str2457 +458|458,str458|2458,str2458 +459|459,str459|2459,str2459 +460|460,str460|2460,str2460 +461|461,str461|2461,str2461 +462|462,str462|2462,str2462 +463|463,str463|2463,str2463 +464|464,str464|2464,str2464 +465|465,str465|2465,str2465 +466|466,str466|2466,str2466 +467|467,str467|2467,str2467 +468|468,str468|2468,str2468 +469|469,str469|2469,str2469 +470|470,str470|2470,str2470 +471|471,str471|2471,str2471 +472|472,str472|2472,str2472 +473|473,str473|2473,str2473 +474|474,str474|2474,str2474 +475|475,str475|2475,str2475 +476|476,str476|2476,str2476 +477|477,str477|2477,str2477 +478|478,str478|2478,str2478 +479|479,str479|2479,str2479 +480|480,str480|2480,str2480 +481|481,str481|2481,str2481 +482|482,str482|2482,str2482 +483|483,str483|2483,str2483 +484|484,str484|2484,str2484 +485|485,str485|2485,str2485 +486|486,str486|2486,str2486 +487|487,str487|2487,str2487 +488|488,str488|2488,str2488 +489|489,str489|2489,str2489 +490|490,str490|2490,str2490 +491|491,str491|2491,str2491 +492|492,str492|2492,str2492 +493|493,str493|2493,str2493 +494|494,str494|2494,str2494 +495|495,str495|2495,str2495 +496|496,str496|2496,str2496 +497|497,str497|2497,str2497 +498|498,str498|2498,str2498 +499|499,str499|2499,str2499 +500|500,str500|2500,str2500 +501|501,str501|2501,str2501 +502|502,str502|2502,str2502 +503|503,str503|2503,str2503 +504|504,str504|2504,str2504 +505|505,str505|2505,str2505 +506|506,str506|2506,str2506 +507|507,str507|2507,str2507 +508|508,str508|2508,str2508 +509|509,str509|2509,str2509 +510|510,str510|2510,str2510 +511|511,str511|2511,str2511 +512|512,str512|2512,str2512 +513|513,str513|2513,str2513 +514|514,str514|2514,str2514 +515|515,str515|2515,str2515 +516|516,str516|2516,str2516 +517|517,str517|2517,str2517 +518|518,str518|2518,str2518 +519|519,str519|2519,str2519 +520|520,str520|2520,str2520 +521|521,str521|2521,str2521 +522|522,str522|2522,str2522 +523|523,str523|2523,str2523 +524|524,str524|2524,str2524 +525|525,str525|2525,str2525 +526|526,str526|2526,str2526 +527|527,str527|2527,str2527 +528|528,str528|2528,str2528 +529|529,str529|2529,str2529 +530|530,str530|2530,str2530 +531|531,str531|2531,str2531 +532|532,str532|2532,str2532 +533|533,str533|2533,str2533 +534|534,str534|2534,str2534 +535|535,str535|2535,str2535 +536|536,str536|2536,str2536 +537|537,str537|2537,str2537 +538|538,str538|2538,str2538 +539|539,str539|2539,str2539 +540|540,str540|2540,str2540 +541|541,str541|2541,str2541 +542|542,str542|2542,str2542 +543|543,str543|2543,str2543 +544|544,str544|2544,str2544 +545|545,str545|2545,str2545 +546|546,str546|2546,str2546 +547|547,str547|2547,str2547 +548|548,str548|2548,str2548 +549|549,str549|2549,str2549 +550|550,str550|2550,str2550 +551|551,str551|2551,str2551 +552|552,str552|2552,str2552 +553|553,str553|2553,str2553 +554|554,str554|2554,str2554 +555|555,str555|2555,str2555 +556|556,str556|2556,str2556 +557|557,str557|2557,str2557 +558|558,str558|2558,str2558 +559|559,str559|2559,str2559 +560|560,str560|2560,str2560 +561|561,str561|2561,str2561 +562|562,str562|2562,str2562 +563|563,str563|2563,str2563 +564|564,str564|2564,str2564 +565|565,str565|2565,str2565 +566|566,str566|2566,str2566 +567|567,str567|2567,str2567 +568|568,str568|2568,str2568 +569|569,str569|2569,str2569 +570|570,str570|2570,str2570 +571|571,str571|2571,str2571 +572|572,str572|2572,str2572 +573|573,str573|2573,str2573 +574|574,str574|2574,str2574 +575|575,str575|2575,str2575 +576|576,str576|2576,str2576 +577|577,str577|2577,str2577 +578|578,str578|2578,str2578 +579|579,str579|2579,str2579 +580|580,str580|2580,str2580 +581|581,str581|2581,str2581 +582|582,str582|2582,str2582 +583|583,str583|2583,str2583 +584|584,str584|2584,str2584 +585|585,str585|2585,str2585 +586|586,str586|2586,str2586 +587|587,str587|2587,str2587 +588|588,str588|2588,str2588 +589|589,str589|2589,str2589 +590|590,str590|2590,str2590 +591|591,str591|2591,str2591 +592|592,str592|2592,str2592 +593|593,str593|2593,str2593 +594|594,str594|2594,str2594 +595|595,str595|2595,str2595 +596|596,str596|2596,str2596 +597|597,str597|2597,str2597 +598|598,str598|2598,str2598 +599|599,str599|2599,str2599 +600|600,str600|2600,str2600 +601|601,str601|2601,str2601 +602|602,str602|2602,str2602 +603|603,str603|2603,str2603 +604|604,str604|2604,str2604 +605|605,str605|2605,str2605 +606|606,str606|2606,str2606 +607|607,str607|2607,str2607 +608|608,str608|2608,str2608 +609|609,str609|2609,str2609 +610|610,str610|2610,str2610 +611|611,str611|2611,str2611 +612|612,str612|2612,str2612 +613|613,str613|2613,str2613 +614|614,str614|2614,str2614 +615|615,str615|2615,str2615 +616|616,str616|2616,str2616 +617|617,str617|2617,str2617 +618|618,str618|2618,str2618 +619|619,str619|2619,str2619 +620|620,str620|2620,str2620 +621|621,str621|2621,str2621 +622|622,str622|2622,str2622 +623|623,str623|2623,str2623 +624|624,str624|2624,str2624 +625|625,str625|2625,str2625 +626|626,str626|2626,str2626 +627|627,str627|2627,str2627 +628|628,str628|2628,str2628 +629|629,str629|2629,str2629 +630|630,str630|2630,str2630 +631|631,str631|2631,str2631 +632|632,str632|2632,str2632 +633|633,str633|2633,str2633 +634|634,str634|2634,str2634 +635|635,str635|2635,str2635 +636|636,str636|2636,str2636 +637|637,str637|2637,str2637 +638|638,str638|2638,str2638 +639|639,str639|2639,str2639 +640|640,str640|2640,str2640 +641|641,str641|2641,str2641 +642|642,str642|2642,str2642 +643|643,str643|2643,str2643 +644|644,str644|2644,str2644 +645|645,str645|2645,str2645 +646|646,str646|2646,str2646 +647|647,str647|2647,str2647 +648|648,str648|2648,str2648 +649|649,str649|2649,str2649 +650|650,str650|2650,str2650 +651|651,str651|2651,str2651 +652|652,str652|2652,str2652 +653|653,str653|2653,str2653 +654|654,str654|2654,str2654 +655|655,str655|2655,str2655 +656|656,str656|2656,str2656 +657|657,str657|2657,str2657 +658|658,str658|2658,str2658 +659|659,str659|2659,str2659 +660|660,str660|2660,str2660 +661|661,str661|2661,str2661 +662|662,str662|2662,str2662 +663|663,str663|2663,str2663 +664|664,str664|2664,str2664 +665|665,str665|2665,str2665 +666|666,str666|2666,str2666 +667|667,str667|2667,str2667 +668|668,str668|2668,str2668 +669|669,str669|2669,str2669 +670|670,str670|2670,str2670 +671|671,str671|2671,str2671 +672|672,str672|2672,str2672 +673|673,str673|2673,str2673 +674|674,str674|2674,str2674 +675|675,str675|2675,str2675 +676|676,str676|2676,str2676 +677|677,str677|2677,str2677 +678|678,str678|2678,str2678 +679|679,str679|2679,str2679 +680|680,str680|2680,str2680 +681|681,str681|2681,str2681 +682|682,str682|2682,str2682 +683|683,str683|2683,str2683 +684|684,str684|2684,str2684 +685|685,str685|2685,str2685 +686|686,str686|2686,str2686 +687|687,str687|2687,str2687 +688|688,str688|2688,str2688 +689|689,str689|2689,str2689 +690|690,str690|2690,str2690 +691|691,str691|2691,str2691 +692|692,str692|2692,str2692 +693|693,str693|2693,str2693 +694|694,str694|2694,str2694 +695|695,str695|2695,str2695 +696|696,str696|2696,str2696 +697|697,str697|2697,str2697 +698|698,str698|2698,str2698 +699|699,str699|2699,str2699 +700|700,str700|2700,str2700 +701|701,str701|2701,str2701 +702|702,str702|2702,str2702 +703|703,str703|2703,str2703 +704|704,str704|2704,str2704 +705|705,str705|2705,str2705 +706|706,str706|2706,str2706 +707|707,str707|2707,str2707 +708|708,str708|2708,str2708 +709|709,str709|2709,str2709 +710|710,str710|2710,str2710 +711|711,str711|2711,str2711 +712|712,str712|2712,str2712 +713|713,str713|2713,str2713 +714|714,str714|2714,str2714 +715|715,str715|2715,str2715 +716|716,str716|2716,str2716 +717|717,str717|2717,str2717 +718|718,str718|2718,str2718 +719|719,str719|2719,str2719 +720|720,str720|2720,str2720 +721|721,str721|2721,str2721 +722|722,str722|2722,str2722 +723|723,str723|2723,str2723 +724|724,str724|2724,str2724 +725|725,str725|2725,str2725 +726|726,str726|2726,str2726 +727|727,str727|2727,str2727 +728|728,str728|2728,str2728 +729|729,str729|2729,str2729 +730|730,str730|2730,str2730 +731|731,str731|2731,str2731 +732|732,str732|2732,str2732 +733|733,str733|2733,str2733 +734|734,str734|2734,str2734 +735|735,str735|2735,str2735 +736|736,str736|2736,str2736 +737|737,str737|2737,str2737 +738|738,str738|2738,str2738 +739|739,str739|2739,str2739 +740|740,str740|2740,str2740 +741|741,str741|2741,str2741 +742|742,str742|2742,str2742 +743|743,str743|2743,str2743 +744|744,str744|2744,str2744 +745|745,str745|2745,str2745 +746|746,str746|2746,str2746 +747|747,str747|2747,str2747 +748|748,str748|2748,str2748 +749|749,str749|2749,str2749 +750|750,str750|2750,str2750 +751|751,str751|2751,str2751 +752|752,str752|2752,str2752 +753|753,str753|2753,str2753 +754|754,str754|2754,str2754 +755|755,str755|2755,str2755 +756|756,str756|2756,str2756 +757|757,str757|2757,str2757 +758|758,str758|2758,str2758 +759|759,str759|2759,str2759 +760|760,str760|2760,str2760 +761|761,str761|2761,str2761 +762|762,str762|2762,str2762 +763|763,str763|2763,str2763 +764|764,str764|2764,str2764 +765|765,str765|2765,str2765 +766|766,str766|2766,str2766 +767|767,str767|2767,str2767 +768|768,str768|2768,str2768 +769|769,str769|2769,str2769 +770|770,str770|2770,str2770 +771|771,str771|2771,str2771 +772|772,str772|2772,str2772 +773|773,str773|2773,str2773 +774|774,str774|2774,str2774 +775|775,str775|2775,str2775 +776|776,str776|2776,str2776 +777|777,str777|2777,str2777 +778|778,str778|2778,str2778 +779|779,str779|2779,str2779 +780|780,str780|2780,str2780 +781|781,str781|2781,str2781 +782|782,str782|2782,str2782 +783|783,str783|2783,str2783 +784|784,str784|2784,str2784 +785|785,str785|2785,str2785 +786|786,str786|2786,str2786 +787|787,str787|2787,str2787 +788|788,str788|2788,str2788 +789|789,str789|2789,str2789 +790|790,str790|2790,str2790 +791|791,str791|2791,str2791 +792|792,str792|2792,str2792 +793|793,str793|2793,str2793 +794|794,str794|2794,str2794 +795|795,str795|2795,str2795 +796|796,str796|2796,str2796 +797|797,str797|2797,str2797 +798|798,str798|2798,str2798 +799|799,str799|2799,str2799 +800|800,str800|2800,str2800 +801|801,str801|2801,str2801 +802|802,str802|2802,str2802 +803|803,str803|2803,str2803 +804|804,str804|2804,str2804 +805|805,str805|2805,str2805 +806|806,str806|2806,str2806 +807|807,str807|2807,str2807 +808|808,str808|2808,str2808 +809|809,str809|2809,str2809 +810|810,str810|2810,str2810 +811|811,str811|2811,str2811 +812|812,str812|2812,str2812 +813|813,str813|2813,str2813 +814|814,str814|2814,str2814 +815|815,str815|2815,str2815 +816|816,str816|2816,str2816 +817|817,str817|2817,str2817 +818|818,str818|2818,str2818 +819|819,str819|2819,str2819 +820|820,str820|2820,str2820 +821|821,str821|2821,str2821 +822|822,str822|2822,str2822 +823|823,str823|2823,str2823 +824|824,str824|2824,str2824 +825|825,str825|2825,str2825 +826|826,str826|2826,str2826 +827|827,str827|2827,str2827 +828|828,str828|2828,str2828 +829|829,str829|2829,str2829 +830|830,str830|2830,str2830 +831|831,str831|2831,str2831 +832|832,str832|2832,str2832 +833|833,str833|2833,str2833 +834|834,str834|2834,str2834 +835|835,str835|2835,str2835 +836|836,str836|2836,str2836 +837|837,str837|2837,str2837 +838|838,str838|2838,str2838 +839|839,str839|2839,str2839 +840|840,str840|2840,str2840 +841|841,str841|2841,str2841 +842|842,str842|2842,str2842 +843|843,str843|2843,str2843 +844|844,str844|2844,str2844 +845|845,str845|2845,str2845 +846|846,str846|2846,str2846 +847|847,str847|2847,str2847 +848|848,str848|2848,str2848 +849|849,str849|2849,str2849 +850|850,str850|2850,str2850 +851|851,str851|2851,str2851 +852|852,str852|2852,str2852 +853|853,str853|2853,str2853 +854|854,str854|2854,str2854 +855|855,str855|2855,str2855 +856|856,str856|2856,str2856 +857|857,str857|2857,str2857 +858|858,str858|2858,str2858 +859|859,str859|2859,str2859 +860|860,str860|2860,str2860 +861|861,str861|2861,str2861 +862|862,str862|2862,str2862 +863|863,str863|2863,str2863 +864|864,str864|2864,str2864 +865|865,str865|2865,str2865 +866|866,str866|2866,str2866 +867|867,str867|2867,str2867 +868|868,str868|2868,str2868 +869|869,str869|2869,str2869 +870|870,str870|2870,str2870 +871|871,str871|2871,str2871 +872|872,str872|2872,str2872 +873|873,str873|2873,str2873 +874|874,str874|2874,str2874 +875|875,str875|2875,str2875 +876|876,str876|2876,str2876 +877|877,str877|2877,str2877 +878|878,str878|2878,str2878 +879|879,str879|2879,str2879 +880|880,str880|2880,str2880 +881|881,str881|2881,str2881 +882|882,str882|2882,str2882 +883|883,str883|2883,str2883 +884|884,str884|2884,str2884 +885|885,str885|2885,str2885 +886|886,str886|2886,str2886 +887|887,str887|2887,str2887 +888|888,str888|2888,str2888 +889|889,str889|2889,str2889 +890|890,str890|2890,str2890 +891|891,str891|2891,str2891 +892|892,str892|2892,str2892 +893|893,str893|2893,str2893 +894|894,str894|2894,str2894 +895|895,str895|2895,str2895 +896|896,str896|2896,str2896 +897|897,str897|2897,str2897 +898|898,str898|2898,str2898 +899|899,str899|2899,str2899 +900|900,str900|2900,str2900 +901|901,str901|2901,str2901 +902|902,str902|2902,str2902 +903|903,str903|2903,str2903 +904|904,str904|2904,str2904 +905|905,str905|2905,str2905 +906|906,str906|2906,str2906 +907|907,str907|2907,str2907 +908|908,str908|2908,str2908 +909|909,str909|2909,str2909 +910|910,str910|2910,str2910 +911|911,str911|2911,str2911 +912|912,str912|2912,str2912 +913|913,str913|2913,str2913 +914|914,str914|2914,str2914 +915|915,str915|2915,str2915 +916|916,str916|2916,str2916 +917|917,str917|2917,str2917 +918|918,str918|2918,str2918 +919|919,str919|2919,str2919 +920|920,str920|2920,str2920 +921|921,str921|2921,str2921 +922|922,str922|2922,str2922 +923|923,str923|2923,str2923 +924|924,str924|2924,str2924 +925|925,str925|2925,str2925 +926|926,str926|2926,str2926 +927|927,str927|2927,str2927 +928|928,str928|2928,str2928 +929|929,str929|2929,str2929 +930|930,str930|2930,str2930 +931|931,str931|2931,str2931 +932|932,str932|2932,str2932 +933|933,str933|2933,str2933 +934|934,str934|2934,str2934 +935|935,str935|2935,str2935 +936|936,str936|2936,str2936 +937|937,str937|2937,str2937 +938|938,str938|2938,str2938 +939|939,str939|2939,str2939 +940|940,str940|2940,str2940 +941|941,str941|2941,str2941 +942|942,str942|2942,str2942 +943|943,str943|2943,str2943 +944|944,str944|2944,str2944 +945|945,str945|2945,str2945 +946|946,str946|2946,str2946 +947|947,str947|2947,str2947 +948|948,str948|2948,str2948 +949|949,str949|2949,str2949 +950|950,str950|2950,str2950 +951|951,str951|2951,str2951 +952|952,str952|2952,str2952 +953|953,str953|2953,str2953 +954|954,str954|2954,str2954 +955|955,str955|2955,str2955 +956|956,str956|2956,str2956 +957|957,str957|2957,str2957 +958|958,str958|2958,str2958 +959|959,str959|2959,str2959 +960|960,str960|2960,str2960 +961|961,str961|2961,str2961 +962|962,str962|2962,str2962 +963|963,str963|2963,str2963 +964|964,str964|2964,str2964 +965|965,str965|2965,str2965 +966|966,str966|2966,str2966 +967|967,str967|2967,str2967 +968|968,str968|2968,str2968 +969|969,str969|2969,str2969 +970|970,str970|2970,str2970 +971|971,str971|2971,str2971 +972|972,str972|2972,str2972 +973|973,str973|2973,str2973 +974|974,str974|2974,str2974 +975|975,str975|2975,str2975 +976|976,str976|2976,str2976 +977|977,str977|2977,str2977 +978|978,str978|2978,str2978 +979|979,str979|2979,str2979 +980|980,str980|2980,str2980 +981|981,str981|2981,str2981 +982|982,str982|2982,str2982 +983|983,str983|2983,str2983 +984|984,str984|2984,str2984 +985|985,str985|2985,str2985 +986|986,str986|2986,str2986 +987|987,str987|2987,str2987 +988|988,str988|2988,str2988 +989|989,str989|2989,str2989 +990|990,str990|2990,str2990 +991|991,str991|2991,str2991 +992|992,str992|2992,str2992 +993|993,str993|2993,str2993 +994|994,str994|2994,str2994 +995|995,str995|2995,str2995 +996|996,str996|2996,str2996 +997|997,str997|2997,str2997 +998|998,str998|2998,str2998 +999|999,str999|2999,str2999 +1000|1000,str1000|3000,str3000 +1001|1001,str1001|3001,str3001 +1002|1002,str1002|3002,str3002 +1003|1003,str1003|3003,str3003 +1004|1004,str1004|3004,str3004 +1005|1005,str1005|3005,str3005 +1006|1006,str1006|3006,str3006 +1007|1007,str1007|3007,str3007 +1008|1008,str1008|3008,str3008 +1009|1009,str1009|3009,str3009 +1010|1010,str1010|3010,str3010 +1011|1011,str1011|3011,str3011 +1012|1012,str1012|3012,str3012 +1013|1013,str1013|3013,str3013 +1014|1014,str1014|3014,str3014 +1015|1015,str1015|3015,str3015 +1016|1016,str1016|3016,str3016 +1017|1017,str1017|3017,str3017 +1018|1018,str1018|3018,str3018 +1019|1019,str1019|3019,str3019 +1020|1020,str1020|3020,str3020 +1021|1021,str1021|3021,str3021 +1022|1022,str1022|3022,str3022 +1023|1023,str1023|3023,str3023 +1024|1024,str1024|3024,str3024 +1025|1025,str1025|3025,str3025 http://git-wip-us.apache.org/repos/asf/hive/blob/ac721836/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index fed9394..32b7551 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -82,6 +82,8 @@ minillap.shared.query.files=insert_into1.q,\ parquet_types_vectorization.q,\ parquet_complex_types_vectorization.q,\ parquet_map_type_vectorization.q,\ + parquet_struct_type_vectorization.q,\ + orc_struct_type_vectorization.q,\ union_type_chk.q,\ cte_2.q,\ cte_4.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/ac721836/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index d46eb8d..8264e8a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -63,6 +63,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.udf.*; import org.apache.hadoop.hive.ql.udf.generic.*; import org.apache.hadoop.hive.serde2.ByteStream.Output; @@ -812,6 +813,10 @@ public class VectorizationContext { mode); } else if (exprDesc instanceof ExprNodeDynamicValueDesc) { ve = getDynamicValueVectorExpression((ExprNodeDynamicValueDesc) exprDesc, mode); + } else if (exprDesc instanceof ExprNodeFieldDesc) { + // Get the GenericUDFStructField to process the field of Struct type + ve = getGenericUDFStructField((ExprNodeFieldDesc)exprDesc, + mode, exprDesc.getTypeInfo()); } if (ve == null) { throw new HiveException( @@ -824,6 +829,41 @@ public class VectorizationContext { return ve; } + private VectorExpression getGenericUDFStructField(ExprNodeFieldDesc exprNodeFieldDesc, + VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException { + // set the arguments for GenericUDFStructField + List<ExprNodeDesc> children = new ArrayList<>(2); + children.add(exprNodeFieldDesc.getDesc()); + children.add(new ExprNodeConstantDesc(getStructFieldIndex(exprNodeFieldDesc))); + + return getVectorExpressionForUdf(null, GenericUDFStructField.class, children, mode, returnType); + } + + /** + * The field of Struct is stored in StructColumnVector.fields[index]. + * Check the StructTypeInfo.getAllStructFieldNames() and compare to the field name, get the index. + */ + private int getStructFieldIndex(ExprNodeFieldDesc exprNodeFieldDesc) throws HiveException { + ExprNodeDesc structNodeDesc = exprNodeFieldDesc.getDesc(); + String fieldName = exprNodeFieldDesc.getFieldName(); + StructTypeInfo structTypeInfo = (StructTypeInfo) structNodeDesc.getTypeInfo(); + int index = 0; + boolean isFieldExist = false; + for (String fn : structTypeInfo.getAllStructFieldNames()) { + if (fieldName.equals(fn)) { + isFieldExist = true; + break; + } + index++; + } + if (isFieldExist) { + return index; + } else { + throw new HiveException("Could not vectorize expression:" + exprNodeFieldDesc.toString() + + ", the field " + fieldName + " doesn't exist."); + } + } + /** * Given a udf and its children, return the common type to which the children's type should be * cast. @@ -1654,7 +1694,8 @@ public class VectorizationContext { throw new HiveException("No match for type string " + childTypeString + " from undecorated type name method"); } builder.setArgumentType(i, undecoratedTypeName); - if ((child instanceof ExprNodeGenericFuncDesc) || (child instanceof ExprNodeColumnDesc)) { + if ((child instanceof ExprNodeGenericFuncDesc) || (child instanceof ExprNodeColumnDesc) + || (child instanceof ExprNodeFieldDesc)) { builder.setInputExpressionType(i, InputExpressionType.COLUMN); } else if (child instanceof ExprNodeConstantDesc) { builder.setInputExpressionType(i, InputExpressionType.SCALAR); @@ -1732,7 +1773,7 @@ public class VectorizationContext { inputTypeInfos[i] = childTypeInfo; inputDataTypePhysicalVariations[i] = DataTypePhysicalVariation.NONE; // Assume. - if (child instanceof ExprNodeGenericFuncDesc) { + if ((child instanceof ExprNodeGenericFuncDesc) || (child instanceof ExprNodeFieldDesc)) { VectorExpression vChild = getVectorExpression(child, childrenMode); children.add(vChild); arguments[i] = vChild.getOutputColumnNum(); http://git-wip-us.apache.org/repos/asf/hive/blob/ac721836/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFStructField.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFStructField.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFStructField.java new file mode 100644 index 0000000..0507fa5 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFStructField.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +/** + * Vectorized instruction to get the field of Struct type with field name and put + * the result in an output column. + */ +public class VectorUDFStructField extends VectorExpression { + + private int structColumnNum; + private int fieldIndex; + + public VectorUDFStructField() { + super(); + } + + public VectorUDFStructField(int structColumnNum, int fieldIndex, int outputColumnNum) { + super(outputColumnNum); + this.structColumnNum = structColumnNum; + this.fieldIndex = fieldIndex; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + ColumnVector outV = batch.cols[outputColumnNum]; + StructColumnVector structColumnVector = (StructColumnVector) batch.cols[structColumnNum]; + ColumnVector fieldColumnVector = structColumnVector.fields[fieldIndex]; + + outV.noNulls = true; + if (structColumnVector.isRepeating) { + if (structColumnVector.isNull[0]) { + outV.isNull[0] = true; + outV.noNulls = false; + } else { + outV.setElement(0, 0, fieldColumnVector); + outV.isNull[0] = false; + } + outV.isRepeating = true; + } else { + for (int i = 0; i < batch.size; i++) { + int j = (batch.selectedInUse) ? batch.selected[i] : i; + if (structColumnVector.isNull[j]) { + outV.isNull[j] = true; + outV.noNulls = false; + } else { + outV.setElement(j, j, fieldColumnVector); + outV.isNull[j] = false; + } + } + outV.isRepeating = false; + } + } + + @Override + public String vectorExpressionParameters() { + return getColumnParamString(0, structColumnNum) + ", " + getColumnParamString(1, fieldIndex); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.STRUCT, + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ac721836/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStructField.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStructField.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStructField.java new file mode 100644 index 0000000..a668489 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStructField.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFStructField; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +/** + * The class shouldn't be used, and only to align the implementation of vectorization UDF for + * struct field. + */ +@Description( + name = "structField", + value = "The class shouldn't be used, and only to align the implementation" + + " of vectorization UDF for struct field") +@VectorizedExpressions({VectorUDFStructField.class}) +public class GenericUDFStructField extends GenericUDF { + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + throw new UDFArgumentException("GenericUDFStructField shouldn't be used directly."); + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + throw new HiveException("GenericUDFStructField shouldn't be used directly."); + } + + @Override + public String getDisplayString(String[] children) { + throw new RuntimeException("GenericUDFStructField shouldn't be used directly."); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ac721836/ql/src/test/queries/clientpositive/orc_struct_type_vectorization.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/orc_struct_type_vectorization.q b/ql/src/test/queries/clientpositive/orc_struct_type_vectorization.q new file mode 100644 index 0000000..362ae77 --- /dev/null +++ b/ql/src/test/queries/clientpositive/orc_struct_type_vectorization.q @@ -0,0 +1,65 @@ +set hive.mapred.mode=nonstrict; +set hive.vectorized.execution.enabled=true; +set hive.fetch.task.conversion=none; + +DROP TABLE orc_struct_type_staging; +DROP TABLE orc_struct_type; + +CREATE TABLE orc_struct_type_staging ( +id int, +st1 struct<f1:int, f2:string>, +st2 struct<f1:int, f3:string> +) ROW FORMAT DELIMITED + FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ','; + +CREATE TABLE orc_struct_type ( +id int, +st1 struct<f1:int, f2:string>, +st2 struct<f1:int, f3:string> +) STORED AS ORC; + +LOAD DATA LOCAL INPATH '../../data/files/struct_type.txt' OVERWRITE INTO TABLE orc_struct_type_staging; +-- test data size < 1024 +INSERT OVERWRITE TABLE orc_struct_type +SELECT id, st1, st2 FROM orc_struct_type_staging where id < 1024; + +-- verify the row number +select count(*) from orc_struct_type; +-- test select all columns and fields +explain vectorization expression select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10; +select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10; +-- test select fields only +select st1.f1, st2.f1, st2.f3 from orc_struct_type limit 10; +select st1.f1, st2.f1 from orc_struct_type limit 10; +-- test complex select with list +explain vectorization expression select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 limit 10; +select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10; + +-- test data size = 1024 +INSERT OVERWRITE TABLE orc_struct_type +SELECT id, st1, st2 FROM orc_struct_type_staging where id < 1025; + +-- verify the row number +select count(*) from orc_struct_type; +-- test select all columns and fields +select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10; +-- test select fields only +select st1.f1, st2.f1, st2.f3 from orc_struct_type limit 10; +select st1.f1, st2.f1 from orc_struct_type limit 10; +-- test complex select with list +select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10; + +-- test data size = 1025 +INSERT OVERWRITE TABLE orc_struct_type +SELECT id, st1, st2 FROM orc_struct_type_staging where id < 1026; + +-- verify the row number +select count(*) from orc_struct_type; +-- test select all columns and fields +select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10; +-- test select fields only +select st1.f1, st2.f1, st2.f3 from orc_struct_type limit 10; +select st1.f1, st2.f1 from orc_struct_type limit 10; +-- test complex select with list +select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/ac721836/ql/src/test/queries/clientpositive/parquet_struct_type_vectorization.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/parquet_struct_type_vectorization.q b/ql/src/test/queries/clientpositive/parquet_struct_type_vectorization.q new file mode 100644 index 0000000..9922b39 --- /dev/null +++ b/ql/src/test/queries/clientpositive/parquet_struct_type_vectorization.q @@ -0,0 +1,65 @@ +set hive.mapred.mode=nonstrict; +set hive.vectorized.execution.enabled=true; +set hive.fetch.task.conversion=none; + +DROP TABLE parquet_struct_type_staging; +DROP TABLE parquet_struct_type; + +CREATE TABLE parquet_struct_type_staging ( +id int, +st1 struct<f1:int, f2:string>, +st2 struct<f1:int, f3:string> +) ROW FORMAT DELIMITED + FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ','; + +CREATE TABLE parquet_struct_type ( +id int, +st1 struct<f1:int, f2:string>, +st2 struct<f1:int, f3:string> +) STORED AS PARQUET; + +LOAD DATA LOCAL INPATH '../../data/files/struct_type.txt' OVERWRITE INTO TABLE parquet_struct_type_staging; +-- test data size < 1024 +INSERT OVERWRITE TABLE parquet_struct_type +SELECT id, st1, st2 FROM parquet_struct_type_staging where id < 1024; + +-- verify the row number +select count(*) from parquet_struct_type; +-- test select all columns and fields +explain vectorization expression select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from parquet_struct_type limit 10; +select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from parquet_struct_type limit 10; +-- test select fields only +select st1.f1, st2.f1, st2.f3 from parquet_struct_type limit 10; +select st1.f1, st2.f1 from parquet_struct_type limit 10; +-- test complex select with list +explain vectorization expression select sum(st1.f1), st1.f1 from parquet_struct_type where st1.f1 > 500 group by st1.f1 limit 10; +select sum(st1.f1), st1.f1 from parquet_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10; + +-- test data size = 1024 +INSERT OVERWRITE TABLE parquet_struct_type +SELECT id, st1, st2 FROM parquet_struct_type_staging where id < 1025; + +-- verify the row number +select count(*) from parquet_struct_type; +-- test select all columns and fields +select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from parquet_struct_type limit 10; +-- test select fields only +select st1.f1, st2.f1, st2.f3 from parquet_struct_type limit 10; +select st1.f1, st2.f1 from parquet_struct_type limit 10; +-- test complex select with list +select sum(st1.f1), st1.f1 from parquet_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10; + +-- test data size = 1025 +INSERT OVERWRITE TABLE parquet_struct_type +SELECT id, st1, st2 FROM parquet_struct_type_staging where id < 1026; + +-- verify the row number +select count(*) from parquet_struct_type; +-- test select all columns and fields +select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from parquet_struct_type limit 10; +-- test select fields only +select st1.f1, st2.f1, st2.f3 from parquet_struct_type limit 10; +select st1.f1, st2.f1 from parquet_struct_type limit 10; +-- test complex select with list +select sum(st1.f1), st1.f1 from parquet_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/ac721836/ql/src/test/results/clientpositive/llap/orc_struct_type_vectorization.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_struct_type_vectorization.q.out b/ql/src/test/results/clientpositive/llap/orc_struct_type_vectorization.q.out new file mode 100644 index 0000000..4cd56f8 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/orc_struct_type_vectorization.q.out @@ -0,0 +1,535 @@ +PREHOOK: query: DROP TABLE orc_struct_type_staging +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orc_struct_type_staging +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE orc_struct_type +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orc_struct_type +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orc_struct_type_staging ( +id int, +st1 struct<f1:int, f2:string>, +st2 struct<f1:int, f3:string> +) ROW FORMAT DELIMITED + FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ',' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_struct_type_staging +POSTHOOK: query: CREATE TABLE orc_struct_type_staging ( +id int, +st1 struct<f1:int, f2:string>, +st2 struct<f1:int, f3:string> +) ROW FORMAT DELIMITED + FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ',' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_struct_type_staging +PREHOOK: query: CREATE TABLE orc_struct_type ( +id int, +st1 struct<f1:int, f2:string>, +st2 struct<f1:int, f3:string> +) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_struct_type +POSTHOOK: query: CREATE TABLE orc_struct_type ( +id int, +st1 struct<f1:int, f2:string>, +st2 struct<f1:int, f3:string> +) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_struct_type +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/struct_type.txt' OVERWRITE INTO TABLE orc_struct_type_staging +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@orc_struct_type_staging +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/struct_type.txt' OVERWRITE INTO TABLE orc_struct_type_staging +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@orc_struct_type_staging +PREHOOK: query: INSERT OVERWRITE TABLE orc_struct_type +SELECT id, st1, st2 FROM orc_struct_type_staging where id < 1024 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type_staging +PREHOOK: Output: default@orc_struct_type +POSTHOOK: query: INSERT OVERWRITE TABLE orc_struct_type +SELECT id, st1, st2 FROM orc_struct_type_staging where id < 1024 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type_staging +POSTHOOK: Output: default@orc_struct_type +POSTHOOK: Lineage: orc_struct_type.id SIMPLE [(orc_struct_type_staging)orc_struct_type_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: orc_struct_type.st1 SIMPLE [(orc_struct_type_staging)orc_struct_type_staging.FieldSchema(name:st1, type:struct<f1:int,f2:string>, comment:null), ] +POSTHOOK: Lineage: orc_struct_type.st2 SIMPLE [(orc_struct_type_staging)orc_struct_type_staging.FieldSchema(name:st2, type:struct<f1:int,f3:string>, comment:null), ] +PREHOOK: query: select count(*) from orc_struct_type +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from orc_struct_type +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +1023 +PREHOOK: query: explain vectorization expression select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization expression select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orc_struct_type + Statistics: Num rows: 1023 Data size: 459256 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Select Operator + expressions: st1 (type: struct<f1:int,f2:string>), st1.f1 (type: int), st1.f2 (type: string), st2 (type: struct<f1:int,f3:string>), st2.f1 (type: int), st2.f3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1, 4, 5, 2, 6, 7] + selectExpressions: VectorUDFStructField(col 1:struct<f1:int,f2:string>, col 0:int) -> 4:int, VectorUDFStructField(col 1:struct<f1:int,f2:string>, col 1:int) -> 5:string, VectorUDFStructField(col 2:struct<f1:int,f3:string>, col 0:int) -> 6:int, VectorUDFStructField(col 2:struct<f1:int,f3:string>, col 1:int) -> 7:string + Statistics: Num rows: 1023 Data size: 459256 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Limit Vectorization: + className: VectorLimitOperator + native: true + Statistics: Num rows: 10 Data size: 4480 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 10 Data size: 4480 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +{"f1":1,"f2":"str1"} 1 str1 {"f1":2001,"f3":"str2001"} 2001 str2001 +{"f1":2,"f2":"str2"} 2 str2 {"f1":2002,"f3":"str2002"} 2002 str2002 +{"f1":3,"f2":"str3"} 3 str3 {"f1":2003,"f3":"str2003"} 2003 str2003 +{"f1":4,"f2":"str4"} 4 str4 {"f1":2004,"f3":"str2004"} 2004 str2004 +{"f1":5,"f2":"str5"} 5 str5 {"f1":2005,"f3":"str2005"} 2005 str2005 +{"f1":6,"f2":"str6"} 6 str6 {"f1":2006,"f3":"str2006"} 2006 str2006 +{"f1":7,"f2":"str7"} 7 str7 {"f1":2007,"f3":"str2007"} 2007 str2007 +{"f1":8,"f2":"str8"} 8 str8 {"f1":2008,"f3":"str2008"} 2008 str2008 +{"f1":9,"f2":"str9"} 9 str9 {"f1":2009,"f3":"str2009"} 2009 str2009 +{"f1":10,"f2":"str10"} 10 str10 {"f1":2010,"f3":"str2010"} 2010 str2010 +PREHOOK: query: select st1.f1, st2.f1, st2.f3 from orc_struct_type limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select st1.f1, st2.f1, st2.f3 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 2001 str2001 +2 2002 str2002 +3 2003 str2003 +4 2004 str2004 +5 2005 str2005 +6 2006 str2006 +7 2007 str2007 +8 2008 str2008 +9 2009 str2009 +10 2010 str2010 +PREHOOK: query: select st1.f1, st2.f1 from orc_struct_type limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select st1.f1, st2.f1 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 2001 +2 2002 +3 2003 +4 2004 +5 2005 +6 2006 +7 2007 +8 2008 +9 2009 +10 2010 +PREHOOK: query: explain vectorization expression select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 limit 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization expression select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 limit 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orc_struct_type + Pruned Column Paths: st1.f1 + Statistics: Num rows: 1023 Data size: 229628 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColGreaterLongScalar(col 4:int, val 500)(children: VectorUDFStructField(col 1:struct<f1:int,f2:string>, col 0:int) -> 4:int) + predicate: (st1.f1 > 500) (type: boolean) + Statistics: Num rows: 341 Data size: 76542 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: st1.f1 (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [4] + selectExpressions: VectorUDFStructField(col 1:struct<f1:int,f2:string>, col 0:int) -> 4:int + Statistics: Num rows: 341 Data size: 76542 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumLong(col 4:int) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 4:int + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 341 Data size: 76542 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 341 Data size: 76542 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFSumLong(col 1:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + keyExpressions: col 0:int + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumnNums: [0] + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 170 Data size: 38158 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: bigint), _col0 (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1, 0] + Statistics: Num rows: 170 Data size: 38158 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Limit Vectorization: + className: VectorLimitOperator + native: true + Statistics: Num rows: 10 Data size: 2240 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 10 Data size: 2240 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +501 501 +502 502 +503 503 +504 504 +505 505 +506 506 +507 507 +508 508 +509 509 +510 510 +PREHOOK: query: INSERT OVERWRITE TABLE orc_struct_type +SELECT id, st1, st2 FROM orc_struct_type_staging where id < 1025 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type_staging +PREHOOK: Output: default@orc_struct_type +POSTHOOK: query: INSERT OVERWRITE TABLE orc_struct_type +SELECT id, st1, st2 FROM orc_struct_type_staging where id < 1025 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type_staging +POSTHOOK: Output: default@orc_struct_type +POSTHOOK: Lineage: orc_struct_type.id SIMPLE [(orc_struct_type_staging)orc_struct_type_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: orc_struct_type.st1 SIMPLE [(orc_struct_type_staging)orc_struct_type_staging.FieldSchema(name:st1, type:struct<f1:int,f2:string>, comment:null), ] +POSTHOOK: Lineage: orc_struct_type.st2 SIMPLE [(orc_struct_type_staging)orc_struct_type_staging.FieldSchema(name:st2, type:struct<f1:int,f3:string>, comment:null), ] +PREHOOK: query: select count(*) from orc_struct_type +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from orc_struct_type +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +1024 +PREHOOK: query: select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +{"f1":1,"f2":"str1"} 1 str1 {"f1":2001,"f3":"str2001"} 2001 str2001 +{"f1":2,"f2":"str2"} 2 str2 {"f1":2002,"f3":"str2002"} 2002 str2002 +{"f1":3,"f2":"str3"} 3 str3 {"f1":2003,"f3":"str2003"} 2003 str2003 +{"f1":4,"f2":"str4"} 4 str4 {"f1":2004,"f3":"str2004"} 2004 str2004 +{"f1":5,"f2":"str5"} 5 str5 {"f1":2005,"f3":"str2005"} 2005 str2005 +{"f1":6,"f2":"str6"} 6 str6 {"f1":2006,"f3":"str2006"} 2006 str2006 +{"f1":7,"f2":"str7"} 7 str7 {"f1":2007,"f3":"str2007"} 2007 str2007 +{"f1":8,"f2":"str8"} 8 str8 {"f1":2008,"f3":"str2008"} 2008 str2008 +{"f1":9,"f2":"str9"} 9 str9 {"f1":2009,"f3":"str2009"} 2009 str2009 +{"f1":10,"f2":"str10"} 10 str10 {"f1":2010,"f3":"str2010"} 2010 str2010 +PREHOOK: query: select st1.f1, st2.f1, st2.f3 from orc_struct_type limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select st1.f1, st2.f1, st2.f3 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 2001 str2001 +2 2002 str2002 +3 2003 str2003 +4 2004 str2004 +5 2005 str2005 +6 2006 str2006 +7 2007 str2007 +8 2008 str2008 +9 2009 str2009 +10 2010 str2010 +PREHOOK: query: select st1.f1, st2.f1 from orc_struct_type limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select st1.f1, st2.f1 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 2001 +2 2002 +3 2003 +4 2004 +5 2005 +6 2006 +7 2007 +8 2008 +9 2009 +10 2010 +PREHOOK: query: select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +501 501 +502 502 +503 503 +504 504 +505 505 +506 506 +507 507 +508 508 +509 509 +510 510 +PREHOOK: query: INSERT OVERWRITE TABLE orc_struct_type +SELECT id, st1, st2 FROM orc_struct_type_staging where id < 1026 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type_staging +PREHOOK: Output: default@orc_struct_type +POSTHOOK: query: INSERT OVERWRITE TABLE orc_struct_type +SELECT id, st1, st2 FROM orc_struct_type_staging where id < 1026 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type_staging +POSTHOOK: Output: default@orc_struct_type +POSTHOOK: Lineage: orc_struct_type.id SIMPLE [(orc_struct_type_staging)orc_struct_type_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: orc_struct_type.st1 SIMPLE [(orc_struct_type_staging)orc_struct_type_staging.FieldSchema(name:st1, type:struct<f1:int,f2:string>, comment:null), ] +POSTHOOK: Lineage: orc_struct_type.st2 SIMPLE [(orc_struct_type_staging)orc_struct_type_staging.FieldSchema(name:st2, type:struct<f1:int,f3:string>, comment:null), ] +PREHOOK: query: select count(*) from orc_struct_type +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from orc_struct_type +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +1025 +PREHOOK: query: select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select st1, st1.f1, st1.f2, st2, st2.f1, st2.f3 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +{"f1":1,"f2":"str1"} 1 str1 {"f1":2001,"f3":"str2001"} 2001 str2001 +{"f1":2,"f2":"str2"} 2 str2 {"f1":2002,"f3":"str2002"} 2002 str2002 +{"f1":3,"f2":"str3"} 3 str3 {"f1":2003,"f3":"str2003"} 2003 str2003 +{"f1":4,"f2":"str4"} 4 str4 {"f1":2004,"f3":"str2004"} 2004 str2004 +{"f1":5,"f2":"str5"} 5 str5 {"f1":2005,"f3":"str2005"} 2005 str2005 +{"f1":6,"f2":"str6"} 6 str6 {"f1":2006,"f3":"str2006"} 2006 str2006 +{"f1":7,"f2":"str7"} 7 str7 {"f1":2007,"f3":"str2007"} 2007 str2007 +{"f1":8,"f2":"str8"} 8 str8 {"f1":2008,"f3":"str2008"} 2008 str2008 +{"f1":9,"f2":"str9"} 9 str9 {"f1":2009,"f3":"str2009"} 2009 str2009 +{"f1":10,"f2":"str10"} 10 str10 {"f1":2010,"f3":"str2010"} 2010 str2010 +PREHOOK: query: select st1.f1, st2.f1, st2.f3 from orc_struct_type limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select st1.f1, st2.f1, st2.f3 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 2001 str2001 +2 2002 str2002 +3 2003 str2003 +4 2004 str2004 +5 2005 str2005 +6 2006 str2006 +7 2007 str2007 +8 2008 str2008 +9 2009 str2009 +10 2010 str2010 +PREHOOK: query: select st1.f1, st2.f1 from orc_struct_type limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select st1.f1, st2.f1 from orc_struct_type limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 2001 +2 2002 +3 2003 +4 2004 +5 2005 +6 2006 +7 2007 +8 2008 +9 2009 +10 2010 +PREHOOK: query: select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_struct_type +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select sum(st1.f1), st1.f1 from orc_struct_type where st1.f1 > 500 group by st1.f1 order by st1.f1 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_struct_type +POSTHOOK: Output: hdfs://### HDFS PATH ### +501 501 +502 502 +503 503 +504 504 +505 505 +506 506 +507 507 +508 508 +509 509 +510 510