Hi Kunal,
Please find below cluster/platform details :-
Number of Nodes : 5
RAM/Node : 32GBCore/Node : 8DRILL_MAX_DIRECT_MEMORY="20G"DRILL_HEAP="8G"DRILL
VERSION = 1.12.0HADOOP VERSION = 2.7.3ZOOKEEPER VERSION = 3.4.8(Installed in
Distributed Mode on 3 nodes)planner.memory.max_query_memory_per_node = 4G
Please find below query which was giving error before setting "alter session set
`drill.exec.hashagg.fallback.enabled`=TRUE;" and its json profile for
success(variable is set) and failure(variable is not set). Please note this
query was working fine in Drill 1.10.0 in same environment.
Query :-
create table a_games_log_visit_utm asselectdistinctglv.sessionid,case when
(UFG('utms=', glv.url, '&') <> 'null') then UFG('utms=', glv.url, '&') else
UFG('utms=',glv.hcookie, '&') end as utms,
case when (UFG('utms=', glv.url, '&') <> 'null') then UFG('utmm=', glv.url, '&')
else UFG('utmm=',glv.hcookie, '&') end as utmm,
case when (UFG('utms=', glv.url, '&') <> 'null') then UFG('utmc=', glv.url, '&')
else UFG('utmc=',glv.hcookie, '&') end as utmc,
case when (UFG('utms=', glv.url, '&') <> 'null') then UFG('utmt=', glv.url, '&')
else UFG('utmt=',glv.hcookie, '&') end as utmt,
case when (UFG('utms=', glv.url, '&') <> 'null') then UFG('utmcmpn=', glv.url,
'&') else UFG('utmcmpn=',glv.hcookie, '&') end as utmcmpn,
case when (UFG('utms=', glv.url, '&') <> 'null') then UFG('utmplnt=', glv.url,
'&') else UFG('utmplnt=',glv.hcookie, '&') end as utmplnt,
case when (UFG('utms=', glv.url, '&') <> 'null') then UFG('uid=', glv.url, '&')
else UFG('uid=', glv.hcookie, '&') end as uid,
case when (UFG('utms=', glv.url, '&') <> 'null') then UFG('cid=', glv.url, '&')
else UFG('cid=', glv.hcookie, '&') end as cid,
case when (UFG('utms=', glv.url, '&') <> 'null') then UFG('gclid=', glv.url,
'&') else UFG('gclid=', glv.hcookie, '&') end as gclid
fromglv_utm_1 a,a_games_log_visit_base glvwherea.sessionid = glv.sessionidand
a.max_serverTime = glv.serverTimeand (STRPOS(glv.url,'utms=') > 0 OR
STRPOS(glv.hcookie,'utms=') > 0) and glv.url is not null ;
JSON Profile when Failed :-
{"id":{"part1":2690698323716856156,"part2":-8447970446519047466},"type":1,"start":1521006624202,"end":1521006722666,"query":"create
table a_games_log_visit_utm as\nselect\ndistinct\nglv.sessionid,\ncase when
(UFG('utms=', glv.url, '&') <> 'null') then UFG('utms=', glv.url, '&') else
UFG('utms=',glv.hcookie, '&') end as utms,\n\ncase when (UFG('utms=', glv.url,
'&') <> 'null') then UFG('utmm=', glv.url, '&') else UFG('utmm=',glv.hcookie,
'&') end as utmm,\n\ncase when (UFG('utms=', glv.url, '&') <> 'null') then
UFG('utmc=', glv.url, '&') else UFG('utmc=',glv.hcookie, '&') end as
utmc,\n\ncase when (UFG('utms=', glv.url, '&') <> 'null') then UFG('utmt=',
glv.url, '&') else UFG('utmt=',glv.hcookie, '&') end as utmt,\n\ncase when
(UFG('utms=', glv.url, '&') <> 'null') then UFG('utmcmpn=', glv.url, '&') else
UFG('utmcmpn=',glv.hcookie, '&') end as utmcmpn,\n\ncase when (UFG('utms=',
glv.url, '&') <> 'null') then UFG('utmplnt=', glv.url, '&') else
UFG('utmplnt=',glv.hcookie, '&') end as utmplnt,\n\ncase when (UFG('utms=',
glv.url, '&') <> 'null') then UFG('uid=', glv.url, '&') else UFG('uid=',
glv.hcookie, '&') end as uid,\n\ncase when (UFG('utms=', glv.url, '&') <>
'null') then UFG('cid=', glv.url, '&') else UFG('cid=', glv.hcookie, '&') end as
cid,\n\ncase when (UFG('utms=', glv.url, '&') <> 'null') then UFG('gclid=',
glv.url, '&') else UFG('gclid=', glv.hcookie, '&') end as
gclid\n\nfrom\nglv_utm_1 a,\na_games_log_visit_base glv\nwhere\na.sessionid =
glv.sessionid\nand a.max_serverTime = glv.serverTime\nand
(STRPOS(glv.url,'utms=') > 0 OR STRPOS(glv.hcookie,'utms=') > 0)\nand glv.url is
not null ","plan":"00-00 Screen : rowType = RecordType(VARCHAR(255) Fragment,
BIGINT Number of records written): rowcount = 402043.18275, cumulative cost =
{5.354375750937748E8 rows, 7.821369551257525E9 cpu, 0.0 io, 1.88842215292928E11
network, 4.283794390722E9 memory}, id = 7755274\n00-01 Project(Fragment=[$0],
Number of records written=[$1]) : rowType = RecordType(VARCHAR(255) Fragment,
BIGINT Number of records written): rowcount = 402043.18275, cumulative cost =
{5.353973707754998E8 rows, 7.82132934693925E9 cpu, 0.0 io, 1.88842215292928E11
network, 4.283794390722E9 memory}, id = 7755273\n00-02 UnionExchange : rowType =
RecordType(VARCHAR(255) Fragment, BIGINT Number of records written): rowcount =
402043.18275, cumulative cost = {5.353973707754998E8 rows, 7.82132934693925E9
cpu, 0.0 io, 1.88842215292928E11 network, 4.283794390722E9 memory}, id =
7755272\n01-01 Writer : rowType = RecordType(VARCHAR(255) Fragment, BIGINT
Number of records written): rowcount = 402043.18275, cumulative cost =
{5.3499532759274983E8 rows, 7.81811300147725E9 cpu, 0.0 io, 1.8554867753984E11
network, 4.283794390722E9 memory}, id = 7755271\n01-02
ProjectAllowDup(sessionid=[$0], utms=[$1], utmm=[$2], utmc=[$3], utmt=[$4],
utmcmpn=[$5], utmplnt=[$6], uid=[$7], cid=[$8], gclid=[$9]) : rowType =
RecordType(ANY sessionid, ANY utms, ANY utmm, ANY utmc, ANY utmt, ANY utmcmpn,
ANY utmplnt, ANY uid, ANY cid, ANY gclid): rowcount = 402043.18275, cumulative
cost = {5.3459328440999985E8 rows, 7.8177109582945E9 cpu, 0.0 io,
1.8554867753984E11 network, 4.283794390722E9 memory}, id = 7755270\n01-03
Project(sessionid=[$0], utms=[$1], utmm=[$2], utmc=[$3], utmt=[$4],
utmcmpn=[$5], utmplnt=[$6], uid=[$7], cid=[$8], gclid=[$9]) : rowType =
RecordType(ANY sessionid, ANY utms, ANY utmm, ANY utmc, ANY utmt, ANY utmcmpn,
ANY utmplnt, ANY uid, ANY cid, ANY gclid): rowcount = 402043.18275, cumulative
cost = {5.3459328440999985E8 rows, 7.8177109582945E9 cpu, 0.0 io,
1.8554867753984E11 network, 4.283794390722E9 memory}, id = 7755269\n01-04
HashAgg(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}]) : rowType = RecordType(ANY
sessionid, ANY utms, ANY utmm, ANY utmc, ANY utmt, ANY utmcmpn, ANY utmplnt, ANY
uid, ANY cid, ANY gclid): rowcount = 402043.18275, cumulative cost =
{5.3459328440999985E8 rows, 7.8177109582945E9 cpu, 0.0 io, 1.8554867753984E11
network, 4.283794390722E9 memory}, id = 7755268\n01-05 Project(sessionid=[$0],
utms=[$1], utmm=[$2], utmc=[$3], utmt=[$4], utmcmpn=[$5], utmplnt=[$6],
uid=[$7], cid=[$8], gclid=[$9]) : rowType = RecordType(ANY sessionid, ANY utms,
ANY utmm, ANY utmc, ANY utmt, ANY utmcmpn, ANY utmplnt, ANY uid, ANY cid, ANY
gclid): rowcount = 4020431.8274999997, cumulative cost = {5.3057285258249986E8
rows, 7.496076412094501E9 cpu, 0.0 io, 1.8554867753984E11 network,
3.89461658982E9 memory}, id = 7755267\n01-06 HashToRandomExchange(dist0=[[$0]],
dist1=[[$1]], dist2=[[$2]], dist3=[[$3]], dist4=[[$4]], dist5=[[$5]],
dist6=[[$6]], dist7=[[$7]], dist8=[[$8]], dist9=[[$9]]) : rowType =
RecordType(ANY sessionid, ANY utms, ANY utmm, ANY utmc, ANY utmt, ANY utmcmpn,
ANY utmplnt, ANY uid, ANY cid, ANY gclid, ANY E_X_P_R_H_A_S_H_F_I_E_L_D):
rowcount = 4020431.8274999997, cumulative cost = {5.3057285258249986E8 rows,
7.496076412094501E9 cpu, 0.0 io, 1.8554867753984E11 network, 3.89461658982E9
memory}, id = 7755266\n02-01 UnorderedMuxExchange : rowType = RecordType(ANY
sessionid, ANY utms, ANY utmm, ANY utmc, ANY utmt, ANY utmcmpn, ANY utmplnt, ANY
uid, ANY cid, ANY gclid, ANY E_X_P_R_H_A_S_H_F_I_E_L_D): rowcount =
4020431.8274999997, cumulative cost = {5.265524207549999E8 rows,
7.460696612012501E9 cpu, 0.0 io, 4.40410112E9 network, 3.89461658982E9 memory},
id = 7755265\n03-01 Project(sessionid=[$0], utms=[$1], utmm=[$2], utmc=[$3],
utmt=[$4], utmcmpn=[$5], utmplnt=[$6], uid=[$7], cid=[$8], gclid=[$9],
E_X_P_R_H_A_S_H_F_I_E_L_D=[hash32AsDouble($9, hash32AsDouble($8,
hash32AsDouble($7, hash32AsDouble($6, hash32AsDouble($5, hash32AsDouble($4,
hash32AsDouble($3, hash32AsDouble($2, hash32AsDouble($1, hash32AsDouble($0,
1301011))))))))))]) : rowType = RecordType(ANY sessionid, ANY utms, ANY utmm,
ANY utmc, ANY utmt, ANY utmcmpn, ANY utmplnt, ANY uid, ANY cid, ANY gclid, ANY
E_X_P_R_H_A_S_H_F_I_E_L_D): rowcount = 4020431.8274999997, cumulative cost =
{5.225319889274999E8 rows, 7.456676180185E9 cpu, 0.0 io, 4.40410112E9 network,
3.89461658982E9 memory}, id = 7755264\n03-02 HashAgg(group=[{0, 1, 2, 3, 4, 5,
6, 7, 8, 9}]) : rowType = RecordType(ANY sessionid, ANY utms, ANY utmm, ANY
utmc, ANY utmt, ANY utmcmpn, ANY utmplnt, ANY uid, ANY cid, ANY gclid): rowcount
= 4020431.8274999997, cumulative cost = {5.185115570999999E8 rows,
7.440594452875E9 cpu, 0.0 io, 4.40410112E9 network, 3.89461658982E9 memory}, id
= 7755263\n03-03 Project(sessionid=[$2], utms=[CASE(<>(UFG('utms=', $4, '&'),
'null'), UFG('utms=', $4, '&'), UFG('utms=', $5, '&'))],
utmm=[CASE(<>(UFG('utms=', $4, '&'), 'null'), UFG('utmm=', $4, '&'),
UFG('utmm=', $5, '&'))], utmc=[CASE(<>(UFG('utms=', $4, '&'), 'null'),
UFG('utmc=', $4, '&'), UFG('utmc=', $5, '&'))], utmt=[CASE(<>(UFG('utms=', $4,
'&'), 'null'), UFG('utmt=', $4, '&'), UFG('utmt=', $5, '&'))],
utmcmpn=[CASE(<>(UFG('utms=', $4, '&'), 'null'), UFG('utmcmpn=', $4, '&'),
UFG('utmcmpn=', $5, '&'))], utmplnt=[CASE(<>(UFG('utms=', $4, '&'), 'null'),
UFG('utmplnt=', $4, '&'), UFG('utmplnt=', $5, '&'))], uid=[CASE(<>(UFG('utms=',
$4, '&'), 'null'), UFG('uid=', $4, '&'), UFG('uid=', $5, '&'))],
cid=[CASE(<>(UFG('utms=', $4, '&'), 'null'), UFG('cid=', $4, '&'), UFG('cid=',
$5, '&'))], gclid=[CASE(<>(UFG('utms=', $4, '&'), 'null'), UFG('gclid=', $4,
'&'), UFG('gclid=', $5, '&'))]) : rowType = RecordType(ANY sessionid, ANY utms,
ANY utmm, ANY utmc, ANY utmt, ANY utmcmpn, ANY utmplnt, ANY uid, ANY cid, ANY
gclid): rowcount = 4.0204318275E7, cumulative cost = {4.783072388249999E8 rows,
4.224248990875E9 cpu, 0.0 io, 4.40410112E9 network, 2838580.8000000003 memory},
id = 7755262\n03-04 Project(sessionid=[$4], max_serverTime=[$5],
sessionid0=[$0], serverTime=[$1], url=[$2], hcookie=[$3]) : rowType =
RecordType(ANY sessionid, ANY max_serverTime, ANY sessionid0, ANY serverTime,
ANY url, ANY hcookie): rowcount = 4.0204318275E7, cumulative cost =
{4.3810292054999995E8 rows, 2.776893532975E9 cpu, 0.0 io, 4.40410112E9 network,
2838580.8000000003 memory}, id = 7755261\n03-05 HashJoin(condition=[AND(=($4,
$0), =($5, $1))], joinType=[inner]) : rowType = RecordType(ANY sessionid, ANY
serverTime, ANY url, ANY hcookie, ANY sessionid0, ANY max_serverTime): rowcount
= 4.0204318275E7, cumulative cost = {4.3810292054999995E8 rows, 2.776893532975E9
cpu, 0.0 io, 4.40410112E9 network, 2838580.8000000003 memory}, id =
7755260\n03-07 SelectionVectorRemover : rowType = RecordType(ANY sessionid, ANY
serverTime, ANY url, ANY hcookie): rowcount = 4.0204318275E7, cumulative cost =
{3.97576036275E8 rows, 1.809194322375E9 cpu, 0.0 io, 0.0 network, 0.0 memory},
id = 7755256\n03-09 Filter(condition=[AND(OR(>(STRPOS($2, 'utms='), 0),
>(STRPOS($3, 'utms='), 0)), IS NOT NULL($2))]) : rowType = RecordType(ANY
sessionid, ANY serverTime, ANY url, ANY hcookie): rowcount = 4.0204318275E7,
cumulative cost = {3.57371718E8 rows, 1.7689900041E9 cpu, 0.0 io, 0.0 network,
0.0 memory}, id = 7755255\n03-10 Scan(groupscan=[ParquetGroupScan
[entries=[ReadEntryWithPath
[path=hdfs://prod-hadoop-xx1.bom-prod.com:9000/usr/drill/warehouse/a_games_log_visit_base]],
selectionRoot=hdfs://prod-hadoop-xx1.bom-prod.com:9000/usr/drill/warehouse/a_games_log_visit_base,
numFiles=1, numRowGroups=232, usedMetadataFile=false, columns=[`sessionid`,
`serverTime`, `url`, `hcookie`]]]) : rowType = RecordType(ANY sessionid, ANY
serverTime, ANY url, ANY hcookie): rowcount = 1.78685859E8, cumulative cost =
{1.78685859E8 rows, 7.14743436E8 cpu, 0.0 io, 0.0 network, 0.0 memory}, id =
7755254\n03-06 Project(sessionid0=[$0], max_serverTime=[$1]) : rowType =
RecordType(ANY sessionid0, ANY max_serverTime): rowcount = 107522.0, cumulative
cost = {215044.0 rows, 1075220.0 cpu, 0.0 io, 4.40410112E9 network, 0.0 memory},
id = 7755259\n03-08 BroadcastExchange : rowType = RecordType(ANY sessionid, ANY
max_serverTime): rowcount = 107522.0, cumulative cost = {215044.0 rows,
1075220.0 cpu, 0.0 io, 4.40410112E9 network, 0.0 memory}, id = 7755258\n04-01
Scan(groupscan=[ParquetGroupScan [entries=[ReadEntryWithPath
[path=hdfs://prod-hadoop-xx1.bom-prod.com:9000/usr/drill/warehouse/glv_utm_1]],
selectionRoot=hdfs://prod-hadoop-xx1.bom-prod.com:9000/usr/drill/warehouse/glv_utm_1,
numFiles=1, numRowGroups=30, usedMetadataFile=false, columns=[`sessionid`,
`max_serverTime`]]]) : rowType = RecordType(ANY sessionid, ANY max_serverTime):
rowcount = 107522.0, cumulative cost = {107522.0 rows, 215044.0 cpu, 0.0 io, 0.0
network, 0.0 memory}, id =
7755257\n","foreman":{"address":"prod-hadoop-xx1.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":0},"state":4,"totalFragments":68,"finishedFragments":42,"fragmentProfile":[{"majorFragmentId":0,"minorFragmentProfile":[{"state":0,"minorFragmentId":0,"endpoint":{"address":"prod-hadoop-xx1.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":0},"lastUpdate":1521006628333,"lastProgress":1521006628333}]},{"majorFragmentId":3,"minorFragmentProfile":[{"state":3,"minorFragmentId":0,"operatorProfile":[{"inputProfile":[{"records":6186060,"batches":200,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":17802732925,"peakLocalMemoryAllocated":101089280,"metric":[{"metricId":0,"longValue":0},{"metricId":5,"longValue":0},{"metricId":7,"longValue":0},{"metricId":4,"longValue":8010},{"metricId":11,"longValue":11687847},{"metricId":6,"longValue":0},{"metricId":12,"longValue":0},{"metricId":16,"longValue":0},{"metricId":8,"longValue":1569776733},{"metricId":19,"longValue":81395966725},{"metricId":10,"longValue":63696348545},{"metricId":15,"longValue":63696348545},{"metricId":9,"longValue":0},{"metricId":2,"longValue":8010},{"metricId":13,"longValue":0},{"metricId":1,"longValue":8010},{"metricId":14,"longValue":4099331644},{"metricId":3,"longValue":0},{"metricId":17,"longValue":70718},{"metricId":18,"longValue":81395481538}],"waitNanos":63690635089},{"inputProfile":[{"records":6186060,"batches":192,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":2167676,"processNanos":9481199735,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":47601,"batches":192,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":376514,"processNanos":961402725,"peakLocalMemoryAllocated":1896448,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":796948,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":1972510452},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":94400,"processNanos":256436,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":47601,"batches":192,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":106350538,"peakLocalMemoryAllocated":26697728,"metric":[{"metricId":0,"longValue":262144},{"metricId":1,"longValue":107522},{"metricId":2,"longValue":2},{"metricId":3,"longValue":14}],"waitNanos":0},{"inputProfile":[{"records":3468,"batches":2,"schemas":1}],"operatorId":4,"operatorType":10,"setupNanos":69408,"processNanos":59257,"peakLocalMemoryAllocated":6021120,"waitNanos":0},{"inputProfile":[{"records":3468,"batches":2,"schemas":1}],"operatorId":3,"operatorType":10,"setupNanos":13988583,"processNanos":96946739,"peakLocalMemoryAllocated":1024000,"waitNanos":0},{"inputProfile":[{"records":3468,"batches":2,"schemas":1}],"operatorId":2,"operatorType":3,"setupNanos":163742,"processNanos":24203075,"peakLocalMemoryAllocated":48627712,"metric":[{"metricId":0,"longValue":65536},{"metricId":2,"longValue":0},{"metricId":3,"longValue":0},{"metricId":7,"longValue":0},{"metricId":4,"longValue":1},{"metricId":6,"longValue":0},{"metricId":1,"longValue":3462}],"waitNanos":0},{"inputProfile":[{"records":3463,"batches":3,"schemas":1}],"operatorId":1,"operatorType":10,"setupNanos":2865181,"processNanos":2660709,"peakLocalMemoryAllocated":47857664,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":47857664,"waitNanos":0},{"inputProfile":[{"records":3463,"batches":3,"schemas":1}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":216584,"peakLocalMemoryAllocated":47857664,"metric":[{"metricId":0,"longValue":730877}],"waitNanos":1256571}],"startTime":1521006628339,"endTime":1521006722561,"memoryUsed":0,"maxMemoryUsed":159717376,"endpoint":{"address":"prod-hadoop-xx5.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":1},"lastUpdate":1521006722561,"lastProgress":1521006722561},{"state":2,"minorFragmentId":2,"operatorProfile":[{"inputProfile":[{"records":5847418,"batches":189,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":17568225997,"peakLocalMemoryAllocated":101089280,"metric":[{"metricId":0,"longValue":0},{"metricId":17,"longValue":56174},{"metricId":18,"longValue":75411719920},{"metricId":13,"longValue":0},{"metricId":1,"longValue":6969},{"metricId":15,"longValue":59830547344},{"metricId":16,"longValue":0},{"metricId":11,"longValue":7388846},{"metricId":14,"longValue":3521451380},{"metricId":3,"longValue":0},{"metricId":5,"longValue":0},{"metricId":10,"longValue":59830547344},{"metricId":9,"longValue":0},{"metricId":8,"longValue":1412404599},{"metricId":6,"longValue":0},{"metricId":12,"longValue":0},{"metricId":7,"longValue":0},{"metricId":4,"longValue":6969},{"metricId":2,"longValue":6969},{"metricId":19,"longValue":75412289227}],"waitNanos":64058389010},{"inputProfile":[{"records":5847418,"batches":182,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":1753035,"processNanos":9140686073,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":49978,"batches":181,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":270460,"processNanos":729011104,"peakLocalMemoryAllocated":3346432,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":671832,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":1746848211},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":76821,"processNanos":237739,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":49978,"batches":181,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":98954835,"peakLocalMemoryAllocated":24862720,"metric":[{"metricId":0,"longValue":262144},{"metricId":3,"longValue":8},{"metricId":2,"longValue":2},{"metricId":1,"longValue":107522}],"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":4,"operatorType":10,"setupNanos":67901,"processNanos":19445,"peakLocalMemoryAllocated":319488,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":3,"operatorType":10,"setupNanos":14114279,"processNanos":114198,"peakLocalMemoryAllocated":532480,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":2,"operatorType":3,"setupNanos":169547,"processNanos":88422,"peakLocalMemoryAllocated":100,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":1,"operatorType":10,"setupNanos":2801259,"processNanos":31428,"peakLocalMemoryAllocated":100,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":0,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":37696,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0}],"waitNanos":22253}],"startTime":1521006628336,"endTime":1521006721768,"memoryUsed":107174464,"maxMemoryUsed":156047360,"endpoint":{"address":"prod-hadoop-xx2.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":1},"lastUpdate":1521006721772,"lastProgress":1521006721772},{"state":3,"minorFragmentId":7,"operatorProfile":[{"inputProfile":[{"records":5436947,"batches":178,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":16076368516,"peakLocalMemoryAllocated":101089280,"metric":[{"metricId":0,"longValue":0},{"metricId":16,"longValue":0},{"metricId":7,"longValue":0},{"metricId":13,"longValue":0},{"metricId":5,"longValue":0},{"metricId":9,"longValue":0},{"metricId":4,"longValue":7050},{"metricId":1,"longValue":7050},{"metricId":17,"longValue":59090},{"metricId":3,"longValue":0},{"metricId":14,"longValue":3532399855},{"metricId":19,"longValue":79685657618},{"metricId":8,"longValue":1390856453},{"metricId":2,"longValue":7050},{"metricId":12,"longValue":0},{"metricId":10,"longValue":63782980256},{"metricId":11,"longValue":8093990},{"metricId":15,"longValue":63782980256},{"metricId":6,"longValue":0},{"metricId":18,"longValue":79685280894}],"waitNanos":63778234790},{"inputProfile":[{"records":5436947,"batches":170,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":1909071,"processNanos":9762620840,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":49331,"batches":170,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":387561,"processNanos":881571738,"peakLocalMemoryAllocated":1777664,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":846055,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":2284223832},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":93530,"processNanos":507491,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":49331,"batches":170,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":116026703,"peakLocalMemoryAllocated":23486464,"metric":[{"metricId":0,"longValue":262144},{"metricId":2,"longValue":2},{"metricId":1,"longValue":107522},{"metricId":3,"longValue":7}],"waitNanos":0},{"inputProfile":[{"records":3471,"batches":2,"schemas":1}],"operatorId":4,"operatorType":10,"setupNanos":112653,"processNanos":69506,"peakLocalMemoryAllocated":6021120,"waitNanos":0},{"inputProfile":[{"records":3471,"batches":2,"schemas":1}],"operatorId":3,"operatorType":10,"setupNanos":13982159,"processNanos":58947356,"peakLocalMemoryAllocated":1024000,"waitNanos":0},{"inputProfile":[{"records":3471,"batches":2,"schemas":1}],"operatorId":2,"operatorType":3,"setupNanos":158821,"processNanos":61630100,"peakLocalMemoryAllocated":48627712,"metric":[{"metricId":0,"longValue":65536},{"metricId":4,"longValue":1},{"metricId":7,"longValue":0},{"metricId":6,"longValue":0},{"metricId":2,"longValue":0},{"metricId":1,"longValue":3464},{"metricId":3,"longValue":0}],"waitNanos":0},{"inputProfile":[{"records":3465,"batches":3,"schemas":1}],"operatorId":1,"operatorType":10,"setupNanos":2899498,"processNanos":1936582,"peakLocalMemoryAllocated":47857664,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":47857664,"waitNanos":0},{"inputProfile":[{"records":3465,"batches":3,"schemas":1}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":209657,"peakLocalMemoryAllocated":47857664,"metric":[{"metricId":0,"longValue":727943}],"waitNanos":3824128}],"startTime":1521006628337,"endTime":1521006721438,"memoryUsed":0,"maxMemoryUsed":156833792,"endpoint":{"address":"prod-hadoop-xx3.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":1},"lastUpdate":1521006721441,"lastProgress":1521006721441},{"state":2,"minorFragmentId":13,"operatorProfile":[{"inputProfile":[{"records":4913490,"batches":159,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":16161055409,"peakLocalMemoryAllocated":114720768,"metric":[{"metricId":0,"longValue":0},{"metricId":3,"longValue":0},{"metricId":7,"longValue":0},{"metricId":13,"longValue":0},{"metricId":14,"longValue":3059465534},{"metricId":6,"longValue":0},{"metricId":10,"longValue":60896126241},{"metricId":16,"longValue":0},{"metricId":17,"longValue":68704},{"metricId":19,"longValue":75593545859},{"metricId":15,"longValue":60896126241},{"metricId":9,"longValue":0},{"metricId":5,"longValue":0},{"metricId":8,"longValue":1204159762},{"metricId":18,"longValue":75593164897},{"metricId":1,"longValue":6151},{"metricId":4,"longValue":6151},{"metricId":2,"longValue":6151},{"metricId":11,"longValue":7817449},{"metricId":12,"longValue":0}],"waitNanos":65849770288},{"inputProfile":[{"records":4913490,"batches":153,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":1945569,"processNanos":7872627200,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":42830,"batches":153,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":362199,"processNanos":762036527,"peakLocalMemoryAllocated":1896448,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":813849,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":398149836},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":91516,"processNanos":275118,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":42830,"batches":153,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":116159411,"peakLocalMemoryAllocated":22700032,"metric":[{"metricId":0,"longValue":262144},{"metricId":2,"longValue":2},{"metricId":3,"longValue":14},{"metricId":1,"longValue":107522}],"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":4,"operatorType":10,"setupNanos":72039,"processNanos":21450,"peakLocalMemoryAllocated":319488,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":3,"operatorType":10,"setupNanos":14467684,"processNanos":138210,"peakLocalMemoryAllocated":532480,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":2,"operatorType":3,"setupNanos":197149,"processNanos":90764,"peakLocalMemoryAllocated":100,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":1,"operatorType":10,"setupNanos":2996530,"processNanos":37511,"peakLocalMemoryAllocated":100,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":0,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":42664,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0}],"waitNanos":26896}],"startTime":1521006628337,"endTime":1521006719596,"memoryUsed":75192896,"maxMemoryUsed":166860800,"endpoint":{"address":"prod-hadoop-xx4.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":1},"lastUpdate":1521006719599,"lastProgress":1521006719599},{"state":3,"minorFragmentId":20,"operatorProfile":[{"inputProfile":[{"records":5758732,"batches":186,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":14753064057,"peakLocalMemoryAllocated":102531072,"metric":[{"metricId":0,"longValue":0},{"metricId":3,"longValue":0},{"metricId":17,"longValue":76240},{"metricId":1,"longValue":7442},{"metricId":2,"longValue":7442},{"metricId":13,"longValue":0},{"metricId":16,"longValue":0},{"metricId":6,"longValue":0},{"metricId":5,"longValue":0},{"metricId":19,"longValue":76042440626},{"metricId":8,"longValue":1487463051},{"metricId":11,"longValue":10346359},{"metricId":18,"longValue":76041985564},{"metricId":9,"longValue":0},{"metricId":4,"longValue":7442},{"metricId":14,"longValue":3688806289},{"metricId":15,"longValue":61377397640},{"metricId":10,"longValue":61377397640},{"metricId":7,"longValue":0},{"metricId":12,"longValue":0}],"waitNanos":61372362497},{"inputProfile":[{"records":5758732,"batches":179,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":2345259,"processNanos":8343301523,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":47638,"batches":179,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":330337,"processNanos":598639052,"peakLocalMemoryAllocated":1851392,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":744028,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":2018401193},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":87917,"processNanos":273785,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":47638,"batches":179,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":98771363,"peakLocalMemoryAllocated":23748608,"metric":[{"metricId":0,"longValue":262144},{"metricId":1,"longValue":107522},{"metricId":2,"longValue":2},{"metricId":3,"longValue":10}],"waitNanos":0},{"inputProfile":[{"records":3380,"batches":2,"schemas":1}],"operatorId":4,"operatorType":10,"setupNanos":70049,"processNanos":48246,"peakLocalMemoryAllocated":6021120,"waitNanos":0},{"inputProfile":[{"records":3380,"batches":2,"schemas":1}],"operatorId":3,"operatorType":10,"setupNanos":78700463,"processNanos":52297956,"peakLocalMemoryAllocated":1024000,"waitNanos":0},{"inputProfile":[{"records":3380,"batches":2,"schemas":1}],"operatorId":2,"operatorType":3,"setupNanos":185515,"processNanos":14679198,"peakLocalMemoryAllocated":48627712,"metric":[{"metricId":0,"longValue":65536},{"metricId":6,"longValue":0},{"metricId":1,"longValue":3371},{"metricId":4,"longValue":1},{"metricId":3,"longValue":0},{"metricId":2,"longValue":0},{"metricId":7,"longValue":0}],"waitNanos":0},{"inputProfile":[{"records":3372,"batches":3,"schemas":1}],"operatorId":1,"operatorType":10,"setupNanos":2972145,"processNanos":1503086,"peakLocalMemoryAllocated":47857664,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":47857664,"waitNanos":0},{"inputProfile":[{"records":3372,"batches":3,"schemas":1}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":160236,"peakLocalMemoryAllocated":47857664,"metric":[{"metricId":0,"longValue":703268}],"waitNanos":1145163}],"startTime":1521006628339,"endTime":1521006715738,"memoryUsed":0,"maxMemoryUsed":158472192,"endpoint":{"address":"prod-hadoop-xx5.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":1},"lastUpdate":1521006715738,"lastProgress":1521006715738},{"state":3,"minorFragmentId":16,"operatorProfile":[{"inputProfile":[{"records":5745026,"batches":187,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":14973716148,"peakLocalMemoryAllocated":101089280,"metric":[{"metricId":0,"longValue":0},{"metricId":15,"longValue":58734209864},{"metricId":18,"longValue":73616025152},{"metricId":19,"longValue":73616468495},{"metricId":9,"longValue":0},{"metricId":11,"longValue":10021869},{"metricId":2,"longValue":7488},{"metricId":12,"longValue":0},{"metricId":16,"longValue":0},{"metricId":14,"longValue":3555357260},{"metricId":13,"longValue":0},{"metricId":6,"longValue":0},{"metricId":5,"longValue":0},{"metricId":10,"longValue":58734209864},{"metricId":17,"longValue":65222},{"metricId":4,"longValue":7488},{"metricId":7,"longValue":0},{"metricId":3,"longValue":0},{"metricId":1,"longValue":7488},{"metricId":8,"longValue":1420991420}],"waitNanos":58729371283},{"inputProfile":[{"records":5745026,"batches":179,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":2077524,"processNanos":8819158792,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":49255,"batches":179,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":322279,"processNanos":288516387,"peakLocalMemoryAllocated":1851392,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":663778,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":1540398648},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":96144,"processNanos":141315,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":49255,"batches":179,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":99997088,"peakLocalMemoryAllocated":23224320,"metric":[{"metricId":0,"longValue":262144},{"metricId":3,"longValue":10},{"metricId":2,"longValue":2},{"metricId":1,"longValue":107522}],"waitNanos":0},{"inputProfile":[{"records":3555,"batches":2,"schemas":1}],"operatorId":4,"operatorType":10,"setupNanos":67416,"processNanos":58380,"peakLocalMemoryAllocated":6021120,"waitNanos":0},{"inputProfile":[{"records":3555,"batches":2,"schemas":1}],"operatorId":3,"operatorType":10,"setupNanos":14020547,"processNanos":86441801,"peakLocalMemoryAllocated":1056768,"waitNanos":0},{"inputProfile":[{"records":3555,"batches":2,"schemas":1}],"operatorId":2,"operatorType":3,"setupNanos":168666,"processNanos":15267452,"peakLocalMemoryAllocated":48627712,"metric":[{"metricId":0,"longValue":65536},{"metricId":6,"longValue":0},{"metricId":2,"longValue":0},{"metricId":3,"longValue":0},{"metricId":1,"longValue":3546},{"metricId":7,"longValue":0},{"metricId":4,"longValue":1}],"waitNanos":0},{"inputProfile":[{"records":3547,"batches":3,"schemas":1}],"operatorId":1,"operatorType":10,"setupNanos":2823840,"processNanos":1584526,"peakLocalMemoryAllocated":47857664,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":47857664,"waitNanos":0},{"inputProfile":[{"records":3547,"batches":3,"schemas":1}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":162364,"peakLocalMemoryAllocated":47857664,"metric":[{"metricId":0,"longValue":743221}],"waitNanos":1594361}],"startTime":1521006628339,"endTime":1521006712979,"memoryUsed":0,"maxMemoryUsed":156571648,"endpoint":{"address":"prod-hadoop-xx5.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":1},"lastUpdate":1521006712979,"lastProgress":1521006712979},{"state":2,"minorFragmentId":28,"operatorProfile":[{"inputProfile":[{"records":3541038,"batches":113,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":26134913852,"peakLocalMemoryAllocated":102531072,"metric":[{"metricId":0,"longValue":0},{"metricId":17,"longValue":37365},{"metricId":5,"longValue":0},{"metricId":15,"longValue":47192448048},{"metricId":2,"longValue":4250},{"metricId":13,"longValue":0},{"metricId":12,"longValue":0},{"metricId":19,"longValue":71047212271},{"metricId":8,"longValue":852912272},{"metricId":1,"longValue":4250},{"metricId":11,"longValue":4648324},{"metricId":4,"longValue":4250},{"metricId":3,"longValue":0},{"metricId":16,"longValue":0},{"metricId":6,"longValue":0},{"metricId":18,"longValue":71046942098},{"metricId":9,"longValue":0},{"metricId":10,"longValue":47192448048},{"metricId":7,"longValue":0},{"metricId":14,"longValue":2246289869}],"waitNanos":54214989547},{"inputProfile":[{"records":3541038,"batches":109,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":1397588,"processNanos":6083637353,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":30993,"batches":109,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":281141,"processNanos":2792123565,"peakLocalMemoryAllocated":1765376,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":725054,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":872268018},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":73730,"processNanos":146704,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":30572,"batches":108,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":90412977,"peakLocalMemoryAllocated":23224320,"metric":[{"metricId":0,"longValue":262144},{"metricId":1,"longValue":107522},{"metricId":2,"longValue":2},{"metricId":3,"longValue":9}],"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":4,"operatorType":10,"setupNanos":66142,"processNanos":20082,"peakLocalMemoryAllocated":319488,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":3,"operatorType":10,"setupNanos":14426188,"processNanos":161709,"peakLocalMemoryAllocated":532480,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":2,"operatorType":3,"setupNanos":143054,"processNanos":87937,"peakLocalMemoryAllocated":100,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":1,"operatorType":10,"setupNanos":2951276,"processNanos":33516,"peakLocalMemoryAllocated":100,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":0,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":34621,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0}],"waitNanos":16681}],"startTime":1521006628338,"endTime":1521006718677,"memoryUsed":44556288,"maxMemoryUsed":155719680,"endpoint":{"address":"prod-hadoop-xx1.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":1},"lastUpdate":1521006718677,"lastProgress":1521006718677},{"state":2,"minorFragmentId":15,"operatorProfile":[{"inputProfile":[{"records":5592132,"batches":180,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":15256777986,"peakLocalMemoryAllocated":102400000,"metric":[{"metricId":0,"longValue":0},{"metricId":4,"longValue":6549},{"metricId":16,"longValue":0},{"metricId":10,"longValue":60264671479},{"metricId":1,"longValue":6549},{"metricId":19,"longValue":72560534277},{"metricId":3,"longValue":0},{"metricId":5,"longValue":0},{"metricId":13,"longValue":0},{"metricId":12,"longValue":0},{"metricId":9,"longValue":0},{"metricId":8,"longValue":1271594981},{"metricId":15,"longValue":60264671479},{"metricId":14,"longValue":3204221039},{"metricId":17,"longValue":54496},{"metricId":2,"longValue":6549},{"metricId":6,"longValue":0},{"metricId":7,"longValue":0},{"metricId":11,"longValue":7452940},{"metricId":18,"longValue":72560207199}],"waitNanos":64273784113},{"inputProfile":[{"records":5592132,"batches":174,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":2138431,"processNanos":9302016956,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":45618,"batches":174,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":388783,"processNanos":811697156,"peakLocalMemoryAllocated":1851392,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":700196,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":1704687712},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":87388,"processNanos":205982,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":45618,"batches":174,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":105436515,"peakLocalMemoryAllocated":22962176,"metric":[{"metricId":0,"longValue":262144},{"metricId":2,"longValue":2},{"metricId":3,"longValue":10},{"metricId":1,"longValue":107522}],"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":4,"operatorType":10,"setupNanos":68823,"processNanos":21104,"peakLocalMemoryAllocated":319488,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":3,"operatorType":10,"setupNanos":14495567,"processNanos":131782,"peakLocalMemoryAllocated":532480,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":2,"operatorType":3,"setupNanos":157464,"processNanos":85825,"peakLocalMemoryAllocated":100,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":1,"operatorType":10,"setupNanos":2878003,"processNanos":45117,"peakLocalMemoryAllocated":100,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":0,"waitNanos":0},{"inputProfile":[{"records":0,"batches":1,"schemas":1}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":46937,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0}],"waitNanos":25164}],"startTime":1521006628337,"endTime":1521006719896,"memoryUsed":129980992,"maxMemoryUsed":156243968,"endpoint":{"address":"prod-hadoop-xx3.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":1},"lastUpdate":1521006719899,"lastProgress":1521006719899},{"state":3,"minorFragmentId":12,"operatorProfile":[{"inputProfile":[{"records":6653071,"batches":215,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":18555717000,"peakLocalMemoryAllocated":102137856,"metric":[{"metricId":0,"longValue":0},{"metricId":17,"longValue":79222},{"metricId":18,"longValue":81677187403},{"metricId":10,"longValue":63224064165},{"metricId":3,"longValue":0},{"metricId":8,"longValue":1639751501},{"metricId":4,"longValue":8662},{"metricId":6,"longValue":0},{"metricId":12,"longValue":0},{"metricId":11,"longValue":12424972},{"metricId":5,"longValue":0},{"metricId":9,"longValue":0},{"metricId":14,"longValue":4197363153},{"metricId":2,"longValue":8662},{"metricId":16,"longValue":0},{"metricId":19,"longValue":81677704517},{"metricId":13,"longValue":0},{"metricId":1,"longValue":8662},{"metricId":15,"longValue":63224064165},{"metricId":7,"longValue":0}],"waitNanos":63218142215},{"inputProfile":[{"records":6653071,"batches":207,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":2000052,"processNanos":9999072510,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":56828,"batches":207,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":352362,"processNanos":802467745,"peakLocalMemoryAllocated":1886208,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":711471,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":1443194289},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":100939,"processNanos":1254125,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":56828,"batches":207,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":109746105,"peakLocalMemoryAllocated":27746304,"metric":[{"metricId":0,"longValue":262144},{"metricId":2,"longValue":2},{"metricId":3,"longValue":14},{"metricId":1,"longValue":107522}],"waitNanos":0},{"inputProfile":[{"records":4141,"batches":3,"schemas":1}],"operatorId":4,"operatorType":10,"setupNanos":68930,"processNanos":77132,"peakLocalMemoryAllocated":6021120,"waitNanos":0},{"inputProfile":[{"records":4141,"batches":3,"schemas":1}],"operatorId":3,"operatorType":10,"setupNanos":14040113,"processNanos":76142393,"peakLocalMemoryAllocated":1187840,"waitNanos":0},{"inputProfile":[{"records":4141,"batches":3,"schemas":1}],"operatorId":2,"operatorType":3,"setupNanos":174343,"processNanos":16425885,"peakLocalMemoryAllocated":48627712,"metric":[{"metricId":0,"longValue":65536},{"metricId":6,"longValue":0},{"metricId":4,"longValue":1},{"metricId":1,"longValue":4131},{"metricId":3,"longValue":0},{"metricId":7,"longValue":0},{"metricId":2,"longValue":0}],"waitNanos":0},{"inputProfile":[{"records":4132,"batches":3,"schemas":1}],"operatorId":1,"operatorType":10,"setupNanos":2808446,"processNanos":1851787,"peakLocalMemoryAllocated":47874048,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":47874048,"waitNanos":0},{"inputProfile":[{"records":4132,"batches":3,"schemas":1}],"operatorId":0,"operatorType":0,"setupNanos":0,"processNanos":161419,"peakLocalMemoryAllocated":47874048,"metric":[{"metricId":0,"longValue":875136}],"waitNanos":1358511}],"startTime":1521006628339,"endTime":1521006722649,"memoryUsed":0,"maxMemoryUsed":204002752,"endpoint":{"address":"prod-hadoop-xx5.bom-prod.com","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0","state":1},"lastUpdate":1521006722652,"lastProgress":1521006722652},{"state":2,"minorFragmentId":24,"operatorProfile":[{"inputProfile":[{"records":3713262,"batches":120,"schemas":1}],"operatorId":10,"operatorType":21,"setupNanos":0,"processNanos":27609872621,"peakLocalMemoryAllocated":101089280,"metric":[{"metricId":0,"longValue":0},{"metricId":5,"longValue":0},{"metricId":7,"longValue":0},{"metricId":14,"longValue":2253910786},{"metricId":13,"longValue":0},{"metricId":6,"longValue":0},{"metricId":4,"longValue":4116},{"metricId":10,"longValue":42008980115},{"metricId":15,"longValue":42008980115},{"metricId":17,"longValue":46575},{"metricId":2,"longValue":4116},{"metricId":11,"longValue":4608605},{"metricId":16,"longValue":0},{"metricId":12,"longValue":0},{"metricId":1,"longValue":4116},{"metricId":18,"longValue":64529990547},{"metricId":8,"longValue":849540998},{"metricId":3,"longValue":0},{"metricId":19,"longValue":64530271641},{"metricId":9,"longValue":0}],"waitNanos":52413870835},{"inputProfile":[{"records":3713262,"batches":116,"schemas":1}],"operatorId":9,"operatorType":2,"setupNanos":2153757,"processNanos":6539090922,"peakLocalMemoryAllocated":59965440,"waitNanos":0},{"inputProfile":[{"records":33998,"batches":116,"schemas":1}],"operatorId":7,"operatorType":14,"setupNanos":351935,"processNanos":2699222110,"peakLocalMemoryAllocated":1851392,"waitNanos":0},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":8,"operatorType":11,"setupNanos":0,"processNanos":689000,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":2}],"waitNanos":875366632},{"inputProfile":[{"records":107522,"batches":30,"schemas":1}],"operatorId":6,"operatorType":10,"setupNanos":95342,"processNanos":198747,"peakLocalMemoryAllocated":262144,"waitNanos":0},{"inputProfile":[{"records":33998,"batches":116,"schemas":1},{"records":107522,"batches":30,"schemas":1}],"operatorId":5,"operatorType":4,"setupNanos":0,"processNanos":96247611,"peakLocalMemoryAllocated":22437888,"metric":[{"metricId":0,"longValue":262144},{"metricId":3,"longValue":9},{"metricId":1,"longValue":107522},{"metricId":2,"lon
