[FLINK-2358] [dashboard] Add first stub of angular.js based dashboard. Add README.md that outlines build instructions.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d59cebd8 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d59cebd8 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d59cebd8 Branch: refs/heads/master Commit: d59cebd8c3f643dc6da88924300e78f65f26c640 Parents: e86f451 Author: Stephan Ewen <se...@apache.org> Authored: Mon Jul 20 22:39:01 2015 +0200 Committer: Stephan Ewen <se...@apache.org> Committed: Tue Jul 21 17:58:14 2015 +0200 ---------------------------------------------------------------------- .gitignore | 2 + flink-runtime-web/README.md | 124 + flink-runtime-web/web-dashboard/app/index.jade | 44 + .../app/partials/jobs/completed-jobs.jade | 30 + .../app/partials/jobs/job.exceptions.jade | 52 + .../web-dashboard/app/partials/jobs/job.jade | 39 + .../app/partials/jobs/job.plan.jade | 4 + .../app/partials/jobs/job.plan.node.jade | 134 + .../app/partials/jobs/job.statistics.jade | 18 + .../app/partials/jobs/job.timeline.jade | 4 + .../app/partials/jobs/job.timeline.vertex.jade | 14 + .../app/partials/jobs/running-jobs.jade | 30 + .../web-dashboard/app/partials/overview.jade | 118 + .../app/scripts/common/directives.coffee | 39 + .../app/scripts/common/filters.coffee | 11 + .../web-dashboard/app/scripts/index.coffee | 104 + .../app/scripts/modules/jobs/jobs.ctrl.coffee | 56 + .../app/scripts/modules/jobs/jobs.dir.coffee | 441 + .../app/scripts/modules/jobs/jobs.svc.coffee | 155 + .../modules/overview/overview.ctrl.coffee | 12 + .../modules/overview/overview.svc.coffee | 16 + .../app/styles/bootstrap_custom.less | 65 + .../web-dashboard/app/styles/graph.styl | 74 + .../web-dashboard/app/styles/index.styl | 253 + .../web-dashboard/app/styles/job.styl | 43 + .../web-dashboard/app/styles/timeline.styl | 14 + .../web-dashboard/assets/fonts/FontAwesome.otf | Bin 0 -> 93888 bytes .../assets/fonts/fontawesome-webfont.eot | Bin 0 -> 60767 bytes .../assets/fonts/fontawesome-webfont.svg | 565 + .../assets/fonts/fontawesome-webfont.ttf | Bin 0 -> 122092 bytes .../assets/fonts/fontawesome-webfont.woff | Bin 0 -> 71508 bytes .../assets/fonts/fontawesome-webfont.woff2 | Bin 0 -> 56780 bytes .../web-dashboard/assets/images/flink-logo.png | Bin 0 -> 6096 bytes flink-runtime-web/web-dashboard/bower.json | 41 + flink-runtime-web/web-dashboard/gulpfile.js | 156 + flink-runtime-web/web-dashboard/package.json | 37 + flink-runtime-web/web-dashboard/server.js | 45 + .../web-dashboard/vendor-local/d3-timeline.js | 567 + .../web-dashboard/web/css/index.css | 410 + .../web-dashboard/web/css/vendor.css | 8565 ++ .../web-dashboard/web/fonts/FontAwesome.otf | Bin 0 -> 93888 bytes .../web/fonts/fontawesome-webfont.eot | Bin 0 -> 60767 bytes .../web/fonts/fontawesome-webfont.svg | 565 + .../web/fonts/fontawesome-webfont.ttf | Bin 0 -> 122092 bytes .../web/fonts/fontawesome-webfont.woff | Bin 0 -> 71508 bytes .../web/fonts/fontawesome-webfont.woff2 | Bin 0 -> 56780 bytes .../web-dashboard/web/images/flink-logo.png | Bin 0 -> 6096 bytes flink-runtime-web/web-dashboard/web/index.html | 33 + flink-runtime-web/web-dashboard/web/js/index.js | 804 + .../web-dashboard/web/js/vendor.js | 74193 +++++++++++++++++ .../web/partials/jobs/completed-jobs.html | 35 + .../web/partials/jobs/job.exceptions.html | 52 + .../web-dashboard/web/partials/jobs/job.html | 30 + .../web/partials/jobs/job.plan.html | 5 + .../web/partials/jobs/job.plan.node.html | 157 + .../web/partials/jobs/job.statistics.html | 22 + .../web/partials/jobs/job.timeline.html | 5 + .../web/partials/jobs/job.timeline.vertex.html | 12 + .../web/partials/jobs/running-jobs.html | 35 + .../web-dashboard/web/partials/overview.html | 129 + 60 files changed, 88359 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 36d98e6..638f71f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,6 @@ _site docs/api build-target flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/ +flink-runtime-web/web-dashboard/node_modules/ +flink-runtime-web/web-dashboard/bower_components/ atlassian-ide-plugin.xml http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/README.md ---------------------------------------------------------------------- diff --git a/flink-runtime-web/README.md b/flink-runtime-web/README.md new file mode 100644 index 0000000..2e8c23e --- /dev/null +++ b/flink-runtime-web/README.md @@ -0,0 +1,124 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +# Apache Flink Web Dashboard + +The web dashboard is work in progress towards the new Flink runtime monitor. In particular, it will +provide the following missing features of the current web dashboard: + + - Live progress monitoring (via life accumulators) + - A graph view of the program, as it is executed. + - A REST style API to access the status of individual jobs. + - A more modular design + +The web dashboard can be activated by adding/uncommenting the config parameter +`jobmanager.new-web-frontend: true` in the `conf/fink-conf.yaml` file. +The dashboard listens at `http://localhost:8082`. + +The new web dashboard is work in progress. It starts an additional HTTP server (by default at port 8082) +that serves the new web pages and additional background requests. It also relies on the old HTTP server +for some requests still. + +**NOTE: Many values are placeholders still.** + + + +## Server Backend + +The server side of the dashboard is implemented using [Netty](http://netty.io) with +[Netty Router](https://github.com/sinetja/netty-router) for REST paths. +The framework has very lightweight dependencies. + +The code is regular Java code built via Maven. To add additional request handlers, follow the +example of the `org.apache.flink.runtime.webmonitor.handlers.JobSummaryHandler`. + + +## Dashboard Frontend + +The web dashboard is implemented using *angular.js*. The dashboard build infrastructure uses *node.js*. +The dashboard files are all pre-built, so one can try it out without building it. + + +### Preparing the Build Environment + +Depending on your version of Linux or MacOS, you may need to manually install *node.js* and *bower*. + + +#### Ubuntu Linux (12.04 and 14.04) + +Install *node.js* via +``` +sudo add-apt-repository ppa:chris-lea/node.js +sudo apt-get update +sudo apt-get -y install nodejs +``` +Verify that the installed version is at least *2.11.3*, via `npm -version`. + + +Install *bower* via +``` +sudo npm install -g bower +``` +Verify that the installed version is at least *1.4.1*, via `bower -version`. + + +Install *gulp* via +``` +sudo npm install -g gulp +``` +Verify that the installed version is at least *3.9.0*, via `gulp -version`. + + +### Building + +The build process downloads all requires libraries via the *node.js* package management tool (*npm*) +and the *bower* dependency management tool. The final build tool is *gulp*. + +``` +cd flink-runtime-web/web-dashboard +npm install +bower install +gulp +``` + +The dashboard code is under `/app`. The result of the build process is under `/web`. + +When building Flink with Maven (in particular the `flink-dist` project), the generated +files are copied into the build target, to the folder `resources/web-runtime-monitor`. + + +### Developing + +When developing the dashboard, every change needs to recompile the files and update the server: + +``` +cd flink-runtime-web/web-dashboard +gulp +cd ../../flink-dist +mvn -DskipTests clean package +``` + +To simplify continuous development, one can use a *standalone proxy server*, together with automatic +re-compilation: + +1. Edit the file `app/scripts/index.coffee`. Comment/uncomment the lines that define the `webServer`, `jobServer`, and `newServer` URLs. +2. Re-compile the files via `gulp`. By calling `gulp watch`, the build-tool autocompiles future changes. +3. Start the proxy server via `node server.js` +4. Access teh dashboardat [`http://localhost:3000`](http://localhost:3000) + http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/index.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/index.jade b/flink-runtime-web/web-dashboard/app/index.jade new file mode 100644 index 0000000..1a2faf5 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/index.jade @@ -0,0 +1,44 @@ +doctype html +html(lang='en') + head + meta(charset='utf-8') + meta(http-equiv='X-UA-Compatible', content='IE=edge') + meta(name='viewport', content='width=device-width, initial-scale=1') + + title Apache Flink Web Dashboard + + link(rel='stylesheet', href='css/vendor.css', type='text/css') + link(rel='stylesheet', href='css/index.css', type='text/css') + + script(src="js/vendor.js") + script(src="js/index.js") + + body(ng-app="flinkApp" ng-strict-di) + #sidebar(ng-class="{ 'sidebar-visible': sidebarVisible }") + nav.navbar.navbar-inverse.navbar-static-top + .navbar-header + a.navbar-brand(ui-sref="overview") + img.logo(alt="Apache Flink Dashboard" src="images/flink-logo.png") + a.navbar-brand.navbar-brand-text(ui-sref="overview") + | Apache Flink Dashboard + + .navbar.navbar-sidebar + ul.nav + li + a(ui-sref="overview" ui-sref-active='active') + i.fa.fa-dashboard.fa-fw + | + | Overview + li + a(ui-sref="running-jobs" ui-sref-active='active') + i.fa.fa-tasks.fa-fw + | + | Running Jobs + li + a(ui-sref="completed-jobs" ui-sref-active='active') + i.fa.fa-server.fa-fw + | + | Completed Jobs + + #content(ng-class="{ 'sidebar-visible': sidebarVisible }") + div(ui-view='main') http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/jobs/completed-jobs.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/completed-jobs.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/completed-jobs.jade new file mode 100644 index 0000000..fbc1791 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/jobs/completed-jobs.jade @@ -0,0 +1,30 @@ +nav.navbar.navbar-default.navbar-fixed-top.navbar-main + #fold-button.btn.btn-default.navbar-btn.pull-left(ng-click='showSidebar()') + i.fa.fa-navicon + + .navbar-title + | Completed Jobs + +#content-inner + table.table.table-hover.table-clickable + thead + tr + th Start Time + th End Time + th Duration + th Job Name + th Job ID + th Tasks + th Status + + tbody + tr(ng-repeat="job in jobs" ui-sref="single-job.plan({ jobid: job.jid })") + td {{job['start-time']}} + td {{job['end-time']}} + td {{job.duration}} + td {{job.name}} + td {{job.jid}} + td.label-group + bs-label(status="{{status}}" ng-repeat="(status, value) in job.operators") {{value}} + td + bs-label(status="{{job.state}}") {{job.state}} http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/jobs/job.exceptions.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/job.exceptions.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/job.exceptions.jade new file mode 100644 index 0000000..6e5fd42 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/jobs/job.exceptions.jade @@ -0,0 +1,52 @@ +pre + | org.apache.flink.client.program.ProgramInvocationException: The + | program execution failed: java.lang.Exception: The data preparation + | for task 'Reduce (SUM(2))' , caused an error: Error obtaining the + | sorted input: Thread 'SortMerger spilling thread' terminated due to an + | exception: The user-defined combiner failed in its 'open()' method. + | at org.apache.flink.runtime.operators.RegularPactTask.run(RegularPactTask.java:472) + | at org.apache.flink.runtime.operators.RegularPactTask.invoke(RegularPactTask.java:360) + | at org.apache.flink.runtime.execution.RuntimeEnvironment.run(RuntimeEnvironment.java:204) + | at java.lang.Thread.run(Thread.java:745) + | Caused by: java.lang.RuntimeException: Error obtaining the sorted + | input: Thread 'SortMerger spilling thread' terminated due to an + | exception: The user-defined combiner failed in its 'open()' method. + | at org.apache.flink.runtime.operators.sort.UnilateralSortMerger.getIterator(UnilateralSortMerger.java:607) + | at org.apache.flink.runtime.operators.RegularPactTask.getInput(RegularPactTask.java:1133) + | at org.apache.flink.runtime.operators.GroupReduceDriver.prepare(GroupReduceDriver.java:94) + | at org.apache.flink.runtime.operators.RegularPactTask.run(RegularPactTask.java:466) + | ... 3 more + | Caused by: java.io.IOException: Thread 'SortMerger spilling thread' + | terminated due to an exception: The user-defined combiner failed in + | its 'open()' method. + | at org.apache.flink.runtime.operators.sort.UnilateralSortMerger$ThreadBase.run(UnilateralSortMerger.java:785) + | Caused by: java.io.IOException: The user-defined combiner failed in + | its 'open()' method. + | at org.apache.flink.runtime.operators.sort.CombiningUnilateralSortMerger$CombiningSpillingThread.go(CombiningUnilateralSortMerger.java:264) + | at org.apache.flink.runtime.operators.sort.UnilateralSortMerger$ThreadBase.run(UnilateralSortMerger.java:781) + | Caused by: java.lang.IllegalStateException: The runtime context has + | not been initialized. + | at org.apache.flink.api.common.functions.AbstractRichFunction.getRuntimeContext(AbstractRichFunction.java:49) + | at org.apache.flink.api.scala.operators.ScalaAggregateOperator$AggregatingUdf.open(ScalaAggregateOperator.java:261) + | at org.apache.flink.api.common.functions.util.FunctionUtils.openFunction(FunctionUtils.java:33) + | at org.apache.flink.runtime.operators.sort.CombiningUnilateralSortMerger$CombiningSpillingThread.go(CombiningUnilateralSortMerger.java:261) + | ... 1 more + | + | at org.apache.flink.client.program.Client.run(Client.java:345) + | at org.apache.flink.client.program.Client.run(Client.java:304) + | at org.apache.flink.client.program.Client.run(Client.java:298) + | at org.apache.flink.client.program.ContextEnvironment.execute(ContextEnvironment.java:55) + | at org.apache.flink.api.scala.ExecutionEnvironment.execute(ExecutionEnvironment.scala:530) + | at com.gmp.MyJob$.main(MyJob.scala:33) + | at com.gmp.MyJob.main(MyJob.scala) + | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + | at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + | at java.lang.reflect.Method.invoke(Method.java:483) + | at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:437) + | at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:353) + | at org.apache.flink.client.program.Client.run(Client.java:250) + | at org.apache.flink.client.CliFrontend.executeProgram(CliFrontend.java:374) + | at org.apache.flink.client.CliFrontend.run(CliFrontend.java:347) + | at org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:1088) + | at org.apache.flink.client.CliFrontend.main(CliFrontend.java:1115) http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/jobs/job.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/job.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/job.jade new file mode 100644 index 0000000..b21393b --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/jobs/job.jade @@ -0,0 +1,39 @@ +nav.navbar.navbar-default.navbar-fixed-top.navbar-main(ng-if="job") + #fold-button.btn.btn-default.navbar-btn.pull-left(ng-click='showSidebar()') + i.fa.fa-navicon + + .navbar-title + indicator-primary(status="{{job.state}}") + | {{ job.name }} + + .navbar-info.first.last.hidden-xs.hidden-sm + | {{ job.jid }} + + .navbar-info.first.last + .label-group + bs-label(status="{{status}}" ng-repeat="(status, value) in job.operators") {{value}} + + .navbar-info.first.last.hidden-xs.hidden-sm + | {{ job['start-time'] }} + | - + | {{ job['end-time'] }} + + .navbar-info.last.first + | {{job.duration}} + +nav.navbar.navbar-default.navbar-fixed-top.navbar-main-additional(ng-if="job") + ul.nav.nav-tabs + li(ui-sref-active='active') + a(ui-sref=".plan") Plan + + li(ui-sref-active='active') + a(ui-sref=".statistics") Job Accumulators / Statistics + + li(ui-sref-active='active') + a(ui-sref=".timeline") Timeline + + li(ui-sref-active='active') + a(ui-sref=".exceptions") Exceptions + +#content-inner.has-navbar-main-additional + div(ui-view="details") http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.jade new file mode 100644 index 0000000..10e8ed9 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.jade @@ -0,0 +1,4 @@ +.canvas-wrapper + div.main-canvas(job-plan, plan="plan", jobid="{{jobid}}") + +div(ui-view="node") http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node.jade new file mode 100644 index 0000000..3496a2b --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node.jade @@ -0,0 +1,134 @@ +.panel.panel-default.panel-multi(ng-if="node") + .panel-heading.clearfix + .panel-title + | {{ node.pact }} + + .panel-info.first + | ID: {{ node.id }} + + .panel-info(ng-if="node.contents") + .label-group + bs-label(status="{{status}}" ng-repeat="(index, status) in stateList") {{node.vertex.groupvertex[status]}} + + .panel-heading.clearfix + .panel-info.first.last(ng-if="node.contents") + span {{ node.contents }} + + .panel-body + table.table.table-hover.table-clickable + thead + tr + th Name + th Status + + tbody + tr(colspan="2") + td + center + i WARNING! This is a sample + tr(ng-repeat="vertex in node.vertex.groupvertex.groupmembers") + td {{vertex.vertexname}} + td + bs-label(status="{{vertex.vertexstatus}}") {{vertex.vertexstatus}} + +//----------------------------------------------------------------- + +.row(ng-if="node") + .col-sm-6.col-md-4 + table.table.table-properties(ng-if="node.global_properties") + thead + tr + th(colspan="2") + | Global Data Properties + + tbody + tr(ng-repeat="property in node.global_properties") + td {{property.name}} + td(table-property value="property.value") + + table.table.table-properties(ng-if="node.local_properties") + thead + tr + th(colspan="2") + | Local Data Properties + + tbody + tr(ng-repeat="property in node.local_properties") + td {{property.name}} + td(table-property value="property.value") + + .visible-xs.visible-sm + table.table.table-properties + thead + tr + th(colspan="2") + | Pact Properties + + tbody + tr + td Operator + td(table-property value="node.driver_strategy") + + tr + td Parallelism + td(table-property value="node.parallelism") + + tr + td Subtasks-per-instance + td(table-property value="node.subtasks_per_instance") + + + .hidden-sm.col-md-4 + table.table.table-properties + thead + tr + th(colspan="2") + | Pact Properties + + tbody + tr + td Operator + td(table-property value="node.driver_strategy") + + tr + td Parallelism + td(table-property value="node.parallelism") + + tr + td Subtasks-per-instance + td(table-property value="node.subtasks_per_instance") + + table.table.table-properties(ng-if="node.estimates") + thead + tr + th(colspan="2") + | Size Estimates + + tbody + tr(ng-repeat="property in node.estimates") + td {{property.name}} + td(table-property value="property.value") + + .col-sm-6.col-md-4 + .visible-xs.visible-sm + table.table.table-properties(ng-if="node.estimates") + thead + tr + th(colspan="2") + | Size Estimates + + tbody + tr(ng-repeat="property in node.estimates") + td {{property.name}} + td(table-property value="property.value") + + table.table.table-properties(ng-if="node.costs") + thead + tr + th(colspan="2") + | Cost Estimates + + tbody + tr(ng-repeat="property in node.costs") + td {{property.name}} + td(table-property value="property.value") http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/jobs/job.statistics.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/job.statistics.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/job.statistics.jade new file mode 100644 index 0000000..d82893a --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/jobs/job.statistics.jade @@ -0,0 +1,18 @@ +table.table.table-properties + thead + tr + th(colspan="2") + | Some statistics + + tbody + tr + td Operator + td 1 + + tr + td Parallelism + td 2 + + tr + td Subtasks-per-instance + td 3 http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/jobs/job.timeline.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/job.timeline.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/job.timeline.jade new file mode 100644 index 0000000..91b1036 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/jobs/job.timeline.jade @@ -0,0 +1,4 @@ +.canvas-wrapper + div.timeline-canvas(timeline job="job") + +div(ui-view="vertex") http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/jobs/job.timeline.vertex.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/job.timeline.vertex.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/job.timeline.vertex.jade new file mode 100644 index 0000000..aaa83f7 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/jobs/job.timeline.vertex.jade @@ -0,0 +1,14 @@ +.panel.panel-default.panel-multi(ng-if="vertex") + .panel-heading.clearfix + .panel-title + | {{ vertex.groupvertex.groupvertexname }} + + //- .panel-info.first + //- | Vertex ID: {{ vertex.groupvertex.groupvertexid }} + + .panel-body + .canvas-wrapper + div.timeline-canvas(vertex data="vertex") + + + #timeline1 http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/jobs/running-jobs.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/running-jobs.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/running-jobs.jade new file mode 100644 index 0000000..21bc6a3 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/jobs/running-jobs.jade @@ -0,0 +1,30 @@ +nav.navbar.navbar-default.navbar-fixed-top.navbar-main + #fold-button.btn.btn-default.navbar-btn.pull-left(ng-click='showSidebar()') + i.fa.fa-navicon + + .navbar-title + | Running Jobs + +#content-inner + table.table.table-hover.table-clickable + thead + tr + th Start Time + th End Time + th Duration + th Job Name + th Job ID + th Tasks + th Status + + tbody + tr(ng-repeat="job in jobs" ui-sref="single-job.plan({ jobid: job.jid })") + td {{job['start-time']}} + td {{job['end-time']}} + td {{job.duration}} + td {{job.name}} + td {{job.jid}} + td.label-group + bs-label(status="{{status}}" ng-repeat="(status, value) in job.operators") {{value}} + td + bs-label(status="{{job.state}}") {{job.state}} http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/partials/overview.jade ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/partials/overview.jade b/flink-runtime-web/web-dashboard/app/partials/overview.jade new file mode 100644 index 0000000..eeae054 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/partials/overview.jade @@ -0,0 +1,118 @@ +nav.navbar.navbar-default.navbar-fixed-top.navbar-main + #fold-button.btn.btn-default.navbar-btn.pull-left(ng-click='showSidebar()') + i.fa.fa-navicon + + .navbar-title + | Overview + +#content-inner + .row + .col-md-6 + .panel.panel-default.panel-dashboard + .panel-heading + .row + .col-xs-3 + i.fa.fa-tasks.fa-3x + .col-xs-9.text-right + .huge 3 + div Task Managers + + .panel-heading + .row + .col-xs-3 + i.fa.fa-folder.fa-3x + .col-xs-9.text-right + .huge 5 + div Task Slots + + .panel-heading + .row + .col-xs-3 + i.fa.fa-folder-o.fa-3x + .col-xs-9.text-right + .huge 2 + div Available Task Slots + + .col-md-6 + .panel.panel-default.panel-lg + .panel-heading + | Total Jobs + .list-group + .list-group-item + .badge.badge-primary 3 + | Running + .list-group-item + .badge.badge-success 6 + | Finished + .list-group-item + .badge.badge-info 0 + | Canceled + .list-group-item + .badge.badge-danger 2 + | Failed + + //- .row + //- .col-sm-12 + //- .progress + //- .progress-bar.progress-bar-success(style='width: 35%') + //- span.sr-only 35% Complete (success) + //- .progress-bar.progress-bar-warning.progress-bar-striped(style='width: 20%') + //- span.sr-only 20% Complete (warning) + //- .progress-bar.progress-bar-danger(style='width: 10%') + //- span.sr-only 10% Complete (danger) + + .panel.panel-default + .panel-heading + h3.panel-title Running Jobs + .panel-body + + table.table.table-hover.table-clickable + thead + tr + th Start Time + th End Time + th Duration + th Job Name + th Job ID + th Tasks + th Status + + tbody + tr(ng-repeat="job in runningJobs" ui-sref="single-job.plan({ jobid: job.jid })") + td {{job['start-time']}} + td {{job['end-time']}} + td {{job.duration}} + td {{job.jobname}} + td {{job.jid}} + td.label-group + bs-label(status="{{status}}" ng-repeat="(status, value) in job.operators") {{value}} + td + bs-label(status="{{job.state}}") {{job.state}} + + .panel.panel-default + .panel-heading + h3.panel-title Completed Jobs + .panel-body + + table.table.table-hover.table-clickable + thead + tr + th Start Time + th End Time + th Duration + th Job Name + th Job ID + th Tasks + th Status + + tbody + tr(ng-repeat="job in finishedJobs" ui-sref="single-job.plan({ jobid: job.jid })") + td {{job['start-time']}} + td {{job['end-time']}} + td {{job.duration}} + td {{job.jobname}} + td {{job.jid}} + td.label-group + bs-label(status="{{status}}" ng-repeat="(status, value) in job.operators") {{value}} + td + bs-label(status="{{job.state}}") {{job.state}} http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/scripts/common/directives.coffee ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/scripts/common/directives.coffee b/flink-runtime-web/web-dashboard/app/scripts/common/directives.coffee new file mode 100644 index 0000000..ea658b9 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/scripts/common/directives.coffee @@ -0,0 +1,39 @@ +angular.module('flinkApp') + +# ---------------------------------------------- + +.directive 'bsLabel', (JobsService) -> + transclude: true + replace: true + scope: + getLabelClass: "&" + status: "@" + + template: "<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>" + + link: (scope, element, attrs) -> + scope.getLabelClass = -> + 'label label-' + JobsService.translateLabelState(attrs.status) + +# ---------------------------------------------- + +.directive 'indicatorPrimary', (JobsService) -> + replace: true + scope: + getLabelClass: "&" + status: '@' + + template: "<i title='{{status}}' ng-class='getLabelClass()' />" + + link: (scope, element, attrs) -> + scope.getLabelClass = -> + 'fa fa-circle indicator indicator-' + JobsService.translateLabelState(attrs.status) + +# ---------------------------------------------- + +.directive 'tableProperty', -> + replace: true + scope: + value: '=' + + template: "<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>" http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee b/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee new file mode 100644 index 0000000..ba75bb9 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee @@ -0,0 +1,11 @@ +angular.module('flinkApp') + +.filter "amDurationFormatExtended", (angularMomentConfig) -> + amDurationFormatExtendedFilter = (value, format, durationFormat) -> + return "" if typeof value is "undefined" or value is null + + moment.duration(value, format).format(durationFormat, { trim: false }) + + amDurationFormatExtendedFilter.$stateful = angularMomentConfig.statefulFilters + + amDurationFormatExtendedFilter \ No newline at end of file http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/scripts/index.coffee ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/scripts/index.coffee b/flink-runtime-web/web-dashboard/app/scripts/index.coffee new file mode 100644 index 0000000..b2bc8d5 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/scripts/index.coffee @@ -0,0 +1,104 @@ +angular.module('flinkApp', ['ui.router', 'angularMoment']) + +# -------------------------------------- + +.run ($rootScope) -> + $rootScope.sidebarVisible = false + $rootScope.showSidebar = -> + $rootScope.sidebarVisible = !$rootScope.sidebarVisible + $rootScope.sidebarClass = 'force-show' + +# -------------------------------------- + +.constant 'flinkConfig', { + webServer: 'http://localhost:8080' + jobServer: 'http://localhost:8081' + newServer: 'http://localhost:8082' +# webServer: 'http://localhost:3000/web-server' +# jobServer: 'http://localhost:3000/job-server' +# newServer: 'http://localhost:3000/new-server' + refreshInterval: 10000 +} + +# -------------------------------------- + +.run (JobsService, flinkConfig, $interval) -> + JobsService.listJobs() + + $interval -> + JobsService.listJobs() + , flinkConfig.refreshInterval + + +# -------------------------------------- + +.config ($stateProvider, $urlRouterProvider) -> + $stateProvider.state "overview", + url: "/overview" + views: + main: + templateUrl: "partials/overview.html" + controller: 'OverviewController' + + .state "running-jobs", + url: "/running-jobs" + views: + main: + templateUrl: "partials/jobs/running-jobs.html" + controller: 'RunningJobsController' + + .state "completed-jobs", + url: "/completed-jobs" + views: + main: + templateUrl: "partials/jobs/completed-jobs.html" + controller: 'CompletedJobsController' + + .state "single-job", + url: "/jobs/{jobid}" + abstract: true + views: + main: + templateUrl: "partials/jobs/job.html" + controller: 'SingleJobController' + + .state "single-job.plan", + url: "" + views: + details: + templateUrl: "partials/jobs/job.plan.html" + controller: 'JobPlanController' + + .state "single-job.plan.node", + url: "/{nodeid:int}" + views: + node: + templateUrl: "partials/jobs/job.plan.node.html" + controller: 'JobPlanNodeController' + + .state "single-job.timeline", + url: "/timeline" + views: + details: + templateUrl: "partials/jobs/job.timeline.html" + + .state "single-job.timeline.vertex", + url: "/{vertexId}" + views: + vertex: + templateUrl: "partials/jobs/job.timeline.vertex.html" + controller: 'JobTimelineVertexController' + + .state "single-job.statistics", + url: "/statistics" + views: + details: + templateUrl: "partials/jobs/job.statistics.html" + + .state "single-job.exceptions", + url: "/exceptions" + views: + details: + templateUrl: "partials/jobs/job.exceptions.html" + + $urlRouterProvider.otherwise "/overview" http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee new file mode 100644 index 0000000..4fcc5c6 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee @@ -0,0 +1,56 @@ +angular.module('flinkApp') + +.controller 'RunningJobsController', ($scope, $state, $stateParams, JobsService) -> + $scope.jobObserver = -> + $scope.jobs = JobsService.getJobs('running') + + JobsService.registerObserver($scope.jobObserver) + $scope.$on '$destroy', -> + JobsService.unRegisterObserver($scope.jobObserver) + + $scope.jobObserver() + +# -------------------------------------- + +.controller 'CompletedJobsController', ($scope, $state, $stateParams, JobsService) -> + $scope.jobObserver = -> + $scope.jobs = JobsService.getJobs('finished') + + JobsService.registerObserver($scope.jobObserver) + $scope.$on '$destroy', -> + JobsService.unRegisterObserver($scope.jobObserver) + + $scope.jobObserver() + +# -------------------------------------- + +.controller 'SingleJobController', ($scope, $state, $stateParams, JobsService, $rootScope) -> + $scope.jobid = $stateParams.jobid + $rootScope.job = null + + JobsService.loadJob($stateParams.jobid).then (data) -> + $rootScope.job = data + + $scope.$on '$destroy', -> + $rootScope.job = null + +# -------------------------------------- + +.controller 'JobPlanController', ($scope, $state, $stateParams, JobsService) -> + JobsService.loadPlan($stateParams.jobid).then (data) -> + $scope.plan = data + +# -------------------------------------- + +.controller 'JobPlanNodeController', ($scope, $state, $stateParams, JobsService) -> + $scope.nodeid = $stateParams.nodeid + $scope.stateList = JobsService.stateList() + + JobsService.getNode($scope.nodeid).then (data) -> + $scope.node = data + +# -------------------------------------- + +.controller 'JobTimelineVertexController', ($scope, $state, $stateParams, JobsService) -> + JobsService.getVertex($stateParams.jobid, $stateParams.vertexId).then (data) -> + $scope.vertex = data http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.dir.coffee ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.dir.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.dir.coffee new file mode 100644 index 0000000..276ffda --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.dir.coffee @@ -0,0 +1,441 @@ +angular.module('flinkApp') + +# ---------------------------------------------- + +.directive 'vertex', ($state) -> + template: "<svg class='timeline secondary' width='0' height='0'></svg>" + + scope: + data: "=" + + link: (scope, elem, attrs) -> + zoom = d3.behavior.zoom() + svgEl = elem.children()[0] + + containerW = elem.width() + angular.element(svgEl).attr('width', containerW - 16) + + analyzeTime = (data) -> + testData = [] + + angular.forEach data.groupvertex.groupmembers, (vertex, i) -> + vTime = data.verticetimes[vertex.vertexid] + + testData.push { + label: "#{vertex.vertexinstancename} (#{i})" + times: [ + { + label: "Scheduled" + color: "#666" + starting_time: vTime["SCHEDULED"] * 100 + ending_time: vTime["DEPLOYING"] * 100 + } + { + label: "Deploying" + color: "#aaa" + starting_time: vTime["DEPLOYING"] * 100 + ending_time: vTime["RUNNING"] * 100 + } + { + label: "Running" + color: "#ddd" + starting_time: vTime["RUNNING"] * 100 + ending_time: vTime["FINISHED"] * 100 + } + ] + } + + chart = d3.timeline().stack().tickFormat({ + format: d3.time.format("%S"), + # tickTime: d3.time.milliseconds, + tickInterval: 1, + tickSize: 1 + }).labelFormat((label) -> + label + ).margin({ left: 100, right: 0, top: 0, bottom: 0 }) + + svg = d3.select(svgEl) + .datum(testData) + .call(chart) + .call(zoom) + + svgG = svg.select("g") + + zoom.on("zoom", -> + ev = d3.event + + svgG.selectAll('rect').attr("transform", "translate(" + ev.translate[0] + ",0) scale(" + ev.scale + ",1)") + svgG.selectAll('text').attr("transform", "translate(" + ev.translate[0] + ",0) scale(" + ev.scale + ",1)") + ) + + bbox = svgG[0][0].getBBox() + svg.attr('height', bbox.height + 30) + + analyzeTime(scope.data) + + return + +# ---------------------------------------------- + +.directive 'timeline', ($state) -> + template: "<svg class='timeline' width='0' height='0'></svg>" + + scope: + job: "=" + + link: (scope, elem, attrs) -> + zoom = d3.behavior.zoom() + svgEl = elem.children()[0] + + containerW = elem.width() + angular.element(svgEl).attr('width', containerW - 16) + + analyzeTime = (data) -> + testData = [] + + angular.forEach data.oldV.groupvertices, (vertex) -> + vTime = data.oldV.groupverticetimes[vertex.groupvertexid] + + # console.log vTime, vertex.groupvertexid + + testData.push + times: [ + label: vertex.groupvertexname + color: "#3fb6d8" + starting_time: vTime["STARTED"] + ending_time: vTime["ENDED"] + link: vertex.groupvertexid + ] + + chart = d3.timeline().stack().click((d, i, datum) -> + $state.go "single-job.timeline.vertex", { jobid: data.jid, vertexId: d.link } + + ).tickFormat({ + format: d3.time.format("%S") + # tickTime: d3.time.milliseconds + tickInterval: 1 + tickSize: 1 + }).margin({ left: 0, right: 0, top: 0, bottom: 0 }) + + svg = d3.select(svgEl) + .datum(testData) + .call(chart) + .call(zoom) + + svgG = svg.select("g") + + zoom.on("zoom", -> + ev = d3.event + + svgG.selectAll('rect').attr("transform", "translate(" + ev.translate[0] + ",0) scale(" + ev.scale + ",1)") + svgG.selectAll('text').attr("transform", "translate(" + ev.translate[0] + ",0) scale(" + ev.scale + ",1)") + ) + + bbox = svgG[0][0].getBBox() + svg.attr('height', bbox.height + 30) + + scope.$watch attrs.job, (data) -> + analyzeTime(data) if data + + return + +# ---------------------------------------------- + +.directive 'jobPlan', ($timeout) -> + template: " + <svg class='graph' width='500' height='400'><g /></svg> + <svg class='tmp' width='1' height='1'><g /></svg> + <div class='btn-group zoom-buttons'> + <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> + <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> + </div>" + + scope: + plan: '=' + + link: (scope, elem, attrs) -> + mainZoom = d3.behavior.zoom() + subgraphs = [] + jobid = attrs.jobid + + mainSvgElement = elem.children()[0] + mainG = elem.children().children()[0] + mainTmpElement = elem.children()[1] + + d3mainSvg = d3.select(mainSvgElement) + d3mainSvgG = d3.select(mainG) + d3tmpSvg = d3.select(mainTmpElement) + + # angular.element(mainG).empty() + + containerW = elem.width() + angular.element(elem.children()[0]).width(containerW) + + scope.zoomIn = -> + if mainZoom.scale() < 2.99 + + # Calculate and store new values in zoom object + translate = mainZoom.translate() + v1 = translate[0] * (mainZoom.scale() + 0.1 / (mainZoom.scale())) + v2 = translate[1] * (mainZoom.scale() + 0.1 / (mainZoom.scale())) + mainZoom.scale mainZoom.scale() + 0.1 + mainZoom.translate [ v1, v2 ] + + # Transform svg + d3mainSvgG.attr "transform", "translate(" + v1 + "," + v2 + ") scale(" + mainZoom.scale() + ")" + + scope.zoomOut = -> + if mainZoom.scale() > 0.31 + + # Calculate and store new values in mainZoom object + mainZoom.scale mainZoom.scale() - 0.1 + translate = mainZoom.translate() + v1 = translate[0] * (mainZoom.scale() - 0.1 / (mainZoom.scale())) + v2 = translate[1] * (mainZoom.scale() - 0.1 / (mainZoom.scale())) + mainZoom.translate [ v1, v2 ] + + # Transform svg + d3mainSvgG.attr "transform", "translate(" + v1 + "," + v2 + ") scale(" + mainZoom.scale() + ")" + + #create a label of an edge + createLabelEdge = (el) -> + labelValue = "" + if el.ship_strategy? or el.local_strategy? + labelValue += "<div class='edge-label'>" + labelValue += el.ship_strategy if el.ship_strategy? + labelValue += " (" + el.temp_mode + ")" unless el.temp_mode is `undefined` + labelValue += ",<br>" + el.local_strategy unless el.local_strategy is `undefined` + labelValue += "</div>" + labelValue + + + # true, if the node is a special node from an iteration + isSpecialIterationNode = (info) -> + (info is "partialSolution" or info is "nextPartialSolution" or info is "workset" or info is "nextWorkset" or info is "solutionSet" or info is "solutionDelta") + + getNodeType = (el, info) -> + if info is "mirror" + 'node-mirror' + + else if isSpecialIterationNode(info) + 'node-iteration' + + else + if el.pact is "Data Source" + 'node-source' + else if el.pact is "Data Sink" + 'node-sink' + else + 'node-normal' + + # creates the label of a node, in info is stored, whether it is a special node (like a mirror in an iteration) + createLabelNode = (el, info, maxW, maxH) -> + labelValue = "<a href='#/jobs/" + jobid + "/" + el.id + "' class='node-label " + getNodeType(el, info) + "'>" + + # Nodename + if info is "mirror" + labelValue += "<h3 class='node-name'>Mirror of " + el.pact + "</h3>" + else + labelValue += "<h3 class='node-name'>" + el.pact + "</h3>" + if el.contents is "" + labelValue += "" + else + stepName = el.contents + + # clean stepName + stepName = shortenString(stepName) + labelValue += "<h4 class='step-name'>" + stepName + "</h4>" + + # If this node is an "iteration" we need a different panel-body + if el.step_function? + labelValue += extendLabelNodeForIteration(el.id, maxW, maxH) + else + + # Otherwise add infos + labelValue += "<h5>" + info + " Node</h5>" if isSpecialIterationNode(info) + labelValue += "<h5>Parallelism: " + el.parallelism + "</h5>" unless el.parallelism is "" + labelValue += "<h5>Driver Strategy: " + shortenString(el.driver_strategy) + "</h5" unless el.driver_strategy is `undefined` + + labelValue += "</a>" + labelValue + + # Extends the label of a node with an additional svg Element to present the iteration. + extendLabelNodeForIteration = (id, maxW, maxH) -> + svgID = "svg-" + id + + labelValue = "<svg class='" + svgID + "' width=" + maxW + " height=" + maxH + "><g /></svg>" + labelValue + + # Split a string into multiple lines so that each line has less than 30 letters. + shortenString = (s) -> + # make sure that name does not contain a < (because of html) + if s.charAt(0) is "<" + s = s.replace("<", "<") + s = s.replace(">", ">") + sbr = "" + while s.length > 30 + sbr = sbr + s.substring(0, 30) + "<br>" + s = s.substring(30, s.length) + sbr = sbr + s + sbr + + createNode = (g, data, el, isParent = false, maxW, maxH) -> + # create node, send additional informations about the node if it is a special one + if el.id is data.partial_solution + g.setNode el.id, + label: createLabelNode(el, "partialSolution", maxW, maxH) + labelType: 'html' + class: getNodeType(el, "partialSolution") + + else if el.id is data.next_partial_solution + g.setNode el.id, + label: createLabelNode(el, "nextPartialSolution", maxW, maxH) + labelType: 'html' + class: getNodeType(el, "nextPartialSolution") + + else if el.id is data.workset + g.setNode el.id, + label: createLabelNode(el, "workset", maxW, maxH) + labelType: 'html' + class: getNodeType(el, "workset") + + else if el.id is data.next_workset + g.setNode el.id, + label: createLabelNode(el, "nextWorkset", maxW, maxH) + labelType: 'html' + class: getNodeType(el, "nextWorkset") + + else if el.id is data.solution_set + g.setNode el.id, + label: createLabelNode(el, "solutionSet", maxW, maxH) + labelType: 'html' + class: getNodeType(el, "solutionSet") + + else if el.id is data.solution_delta + g.setNode el.id, + label: createLabelNode(el, "solutionDelta", maxW, maxH) + labelType: 'html' + class: getNodeType(el, "solutionDelta") + + else + g.setNode el.id, + label: createLabelNode(el, "", maxW, maxH) + labelType: 'html' + class: getNodeType(el, "") + + createEdge = (g, data, el, existingNodes, pred) -> + unless existingNodes.indexOf(pred.id) is -1 + g.setEdge pred.id, el.id, + label: createLabelEdge(pred) + labelType: 'html' + arrowhead: 'normal' + + else + missingNode = searchForNode(data, pred.id) + unless !missingNode or missingNode.alreadyAdded is true + missingNode.alreadyAdded = true + g.setNode missingNode.id, + label: createLabelNode(missingNode, "mirror") + labelType: 'html' + class: getNodeType(missingNode, 'mirror') + + g.setEdge missingNode.id, el.id, + label: createLabelEdge(missingNode) + labelType: 'html' + + loadJsonToDagre = (g, data) -> + existingNodes = [] + + if data.nodes? + # This is the normal json data + toIterate = data.nodes + + else + # This is an iteration, we now store special iteration nodes if possible + toIterate = data.step_function + isParent = true + + for el in toIterate + maxW = 0 + maxH = 0 + + if el.step_function + sg = new dagreD3.graphlib.Graph({ multigraph: true, compound: true }).setGraph({ + nodesep: 20 + edgesep: 0 + ranksep: 20 + rankdir: "LR" + marginx: 10 + marginy: 10 + }) + + subgraphs[el.id] = sg + + loadJsonToDagre(sg, el) + + r = new dagreD3.render() + d3tmpSvg.select('g').call(r, sg) + maxW = sg.graph().width + maxH = sg.graph().height + + angular.element(mainTmpElement).empty() + + createNode(g, data, el, isParent, maxW, maxH) + + existingNodes.push el.id + + # create edges from predecessors to current node + if el.predecessors? + for pred in el.predecessors + createEdge(g, data, el, existingNodes, pred) + + g + + # searches in the global JSONData for the node with the given id + searchForNode = (data, nodeID) -> + for i of data.nodes + el = data.nodes[i] + return el if el.id is nodeID + + # look for nodes that are in iterations + if el.step_function? + for j of el.step_function + return el.step_function[j] if el.step_function[j].id is nodeID + + drawGraph = (data) -> + g = new dagreD3.graphlib.Graph({ multigraph: true, compound: true }).setGraph({ + nodesep: 70 + edgesep: 0 + ranksep: 50 + rankdir: "LR" + marginx: 40 + marginy: 40 + }) + + loadJsonToDagre(g, data) + + renderer = new dagreD3.render() + d3mainSvgG.call(renderer, g) + + for i, sg of subgraphs + d3mainSvg.select('svg.svg-' + i + ' g').call(renderer, sg) + + newScale = 0.5 + + xCenterOffset = Math.floor((angular.element(mainSvgElement).width() - g.graph().width * newScale) / 2) + yCenterOffset = Math.floor((angular.element(mainSvgElement).height() - g.graph().height * newScale) / 2) + + mainZoom.scale(newScale).translate([xCenterOffset, yCenterOffset]) + + d3mainSvgG.attr("transform", "translate(" + xCenterOffset + ", " + yCenterOffset + ") scale(" + mainZoom.scale() + ")") + + mainZoom.on("zoom", -> + ev = d3.event + d3mainSvgG.attr "transform", "translate(" + ev.translate + ") scale(" + ev.scale + ")" + ) + mainZoom(d3mainSvg) + + scope.$watch attrs.plan, (newPlan) -> + drawGraph(newPlan) if newPlan + + return http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee new file mode 100644 index 0000000..d37c19c --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee @@ -0,0 +1,155 @@ +angular.module('flinkApp') + +.service 'JobsService', ($http, flinkConfig, $log, amMoment, $q, $timeout) -> + currentJob = null + currentPlan = null + deferreds = {} + jobs = { + running: [] + finished: [] + cancelled: [] + failed: [] + } + + jobObservers = [] + + notifyObservers = -> + angular.forEach jobObservers, (callback) -> + callback() + + @registerObserver = (callback) -> + jobObservers.push(callback) + + @unRegisterObserver = (callback) -> + index = jobObservers.indexOf(callback) + jobObservers.splice(index, 1) + + @stateList = -> + [ + # 'CREATED' + 'SCHEDULED' + 'DEPLOYING' + 'RUNNING' + 'FINISHED' + 'FAILED' + 'CANCELING' + 'CANCELED' + ] + + @translateLabelState = (state) -> + switch state.toLowerCase() + when 'finished' then 'success' + when 'failed' then 'danger' + when 'scheduled' then 'default' + when 'deploying' then 'info' + when 'running' then 'primary' + when 'canceling' then 'warning' + when 'pending' then 'info' + when 'total' then 'black' + else 'default' + + @listJobs = -> + deferred = $q.defer() + + $http.get flinkConfig.newServer + "/jobs" + .success (data, status, headers, config) -> + + angular.forEach data, (list, listKey) -> + + switch listKey + when 'jobs-running' then jobs.running = list + when 'jobs-finished' then jobs.finished = list + when 'jobs-cancelled' then jobs.cancelled = list + when 'jobs-failed' then jobs.failed = list + + angular.forEach list, (jobid, index) -> + $http.get flinkConfig.newServer + "/jobs/" + jobid + .success (details) -> + list[index] = details + + deferred.resolve(jobs) + notifyObservers() + + deferred.promise + + @getJobs = (type) -> + jobs[type] + + @getAllJobs = -> + jobs + + @loadJob = (jobid) -> + currentJob = null + deferreds.job = $q.defer() + + $http.get flinkConfig.newServer + "/jobs/" + jobid + .success (data, status, headers, config) -> + data.time = Date.now() + + $http.get flinkConfig.newServer + "/jobs/" + jobid + "/vertices" + .success (vertices) -> + data = angular.extend(data, vertices) + + $http.get flinkConfig.jobServer + "/jobsInfo?get=job&job=" + jobid + .success (oldVertices) -> + data.oldV = oldVertices[0] + + currentJob = data + deferreds.job.resolve(data) + + deferreds.job.promise + + @loadPlan = (jobid) -> + currentPlan = null + deferreds.plan = $q.defer() + + $http.get flinkConfig.newServer + "/jobs/" + jobid + "/plan" + .success (data) -> + currentPlan = data + + deferreds.plan.resolve(data) + + deferreds.plan.promise + + @getNode = (nodeid) -> + seekNode = (nodeid, data) -> + nodeid = parseInt(nodeid) + + for node in data + return node if node.id is nodeid + sub = seekNode(nodeid, node.step_function) if node.step_function + return sub if sub + + null + + deferred = $q.defer() + + # if currentPlan + # deferred.resolve(seekNode(nodeid, currentPlan.nodes)) + # else + # # deferreds.plan.promise.then (data) -> + # $q.all([deferreds.plan.promise, deferreds.job.promise]).then (data) -> + # console.log 'resolving getNode' + # deferred.resolve(seekNode(nodeid, currentPlan.nodes)) + + $q.all([deferreds.plan.promise, deferreds.job.promise]).then (data) => + foundNode = seekNode(nodeid, currentPlan.nodes) + + # TODO link to real vertex. for now there is no way to get the right one, so we are showing the first one - just for testing + @getVertex(currentJob.jid, currentJob.oldV.groupvertices[0].groupvertexid).then (vertex) -> + foundNode.vertex = vertex + deferred.resolve(foundNode) + + deferred.promise + + + @getVertex = (jobId, vertexId) -> + deferred = $q.defer() + + $http.get flinkConfig.jobServer + "/jobsInfo?get=groupvertex&job=" + jobId + "&groupvertex=" + vertexId + .success (data) -> + deferred.resolve(data) + + deferred.promise + + @ http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/scripts/modules/overview/overview.ctrl.coffee ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/overview/overview.ctrl.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/overview/overview.ctrl.coffee new file mode 100644 index 0000000..dc3ef3e --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/scripts/modules/overview/overview.ctrl.coffee @@ -0,0 +1,12 @@ +angular.module('flinkApp') + +.controller 'OverviewController', ($scope, OverviewService, JobsService) -> + $scope.jobObserver = -> + $scope.runningJobs = JobsService.getJobs('running') + $scope.finishedJobs = JobsService.getJobs('finished') + + JobsService.registerObserver($scope.jobObserver) + $scope.$on '$destroy', -> + JobsService.unRegisterObserver($scope.jobObserver) + + $scope.jobObserver() http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/scripts/modules/overview/overview.svc.coffee ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/overview/overview.svc.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/overview/overview.svc.coffee new file mode 100644 index 0000000..82cd9db --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/scripts/modules/overview/overview.svc.coffee @@ -0,0 +1,16 @@ +angular.module('flinkApp') + +.service 'OverviewService', ($http, flinkConfig, $log) -> + serverStatus = {} + + @loadServerStatus = -> + $http.get(flinkConfig.jobServer + "/monitor/status") + .success (data, status, headers, config) -> + $log data + + .error (data, status, headers, config) -> + return + + serverStatus + + @ http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/styles/bootstrap_custom.less ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/styles/bootstrap_custom.less b/flink-runtime-web/web-dashboard/app/styles/bootstrap_custom.less new file mode 100644 index 0000000..210111b --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/styles/bootstrap_custom.less @@ -0,0 +1,65 @@ +// Core variables and mixins +@import "../../bower_components/bootstrap/less/variables.less"; + +@font-size-h1: floor((@font-size-base * 2.0)); +@font-size-h2: floor((@font-size-base * 1.7)); +@font-size-h3: floor((@font-size-base * 1.5)); +@font-size-h4: floor((@font-size-base * 1.3)); + +@navbar-default-bg: #f5f5f5; +@brand-primary: #158cba; +@nav-tabs-link-hover-border-color: transparent; +@nav-tabs-active-link-hover-bg: transparent; +@nav-tabs-active-link-hover-border-color: transparent; +@nav-tabs-justified-link-border-color: transparent; +@nav-tabs-justified-active-link-border-color: transparent; + +@import "../../bower_components/bootstrap/less/mixins.less"; + +// Reset and dependencies +@import "../../bower_components/bootstrap/less/normalize.less"; +@import "../../bower_components/bootstrap/less/print.less"; +@import "../../bower_components/bootstrap/less/glyphicons.less"; + +// Core CSS +@import "../../bower_components/bootstrap/less/scaffolding.less"; +@import "../../bower_components/bootstrap/less/type.less"; +@import "../../bower_components/bootstrap/less/code.less"; +@import "../../bower_components/bootstrap/less/grid.less"; +@import "../../bower_components/bootstrap/less/tables.less"; +@import "../../bower_components/bootstrap/less/forms.less"; +@import "../../bower_components/bootstrap/less/buttons.less"; + +// Components +@import "../../bower_components/bootstrap/less/component-animations.less"; +@import "../../bower_components/bootstrap/less/dropdowns.less"; +@import "../../bower_components/bootstrap/less/button-groups.less"; +@import "../../bower_components/bootstrap/less/input-groups.less"; +@import "../../bower_components/bootstrap/less/navs.less"; +@import "../../bower_components/bootstrap/less/navbar.less"; +@import "../../bower_components/bootstrap/less/breadcrumbs.less"; +@import "../../bower_components/bootstrap/less/pagination.less"; +@import "../../bower_components/bootstrap/less/pager.less"; +@import "../../bower_components/bootstrap/less/labels.less"; +@import "../../bower_components/bootstrap/less/badges.less"; +@import "../../bower_components/bootstrap/less/jumbotron.less"; +@import "../../bower_components/bootstrap/less/thumbnails.less"; +@import "../../bower_components/bootstrap/less/alerts.less"; +@import "../../bower_components/bootstrap/less/progress-bars.less"; +@import "../../bower_components/bootstrap/less/media.less"; +@import "../../bower_components/bootstrap/less/list-group.less"; +@import "../../bower_components/bootstrap/less/panels.less"; +@import "../../bower_components/bootstrap/less/responsive-embed.less"; +@import "../../bower_components/bootstrap/less/wells.less"; +@import "../../bower_components/bootstrap/less/close.less"; + +// Components w/ JavaScript +@import "../../bower_components/bootstrap/less/modals.less"; +@import "../../bower_components/bootstrap/less/tooltip.less"; +@import "../../bower_components/bootstrap/less/popovers.less"; +@import "../../bower_components/bootstrap/less/carousel.less"; + +// Utility classes +@import "../../bower_components/bootstrap/less/utilities.less"; +@import "../../bower_components/bootstrap/less/responsive-utilities.less"; + http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/styles/graph.styl ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/styles/graph.styl b/flink-runtime-web/web-dashboard/app/styles/graph.styl new file mode 100644 index 0000000..072ff7f --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/styles/graph.styl @@ -0,0 +1,74 @@ +svg.graph + overflow hidden + + g + &.type-TK + & > rect + fill #00ffd0 + + text + font-weight 300 + font-size 14px + + .node + > rect + stroke #999 + stroke-width 5px + fill #fff + margin: 0 + + &[active] + > rect + fill #eeeeee + + &.node-mirror + > rect + stroke: #a8a8a8 + + &.node-iteration + > rect + stroke: #cd3333 + + &.node-source + > rect + stroke: #4ce199 + + &.node-sink + > rect + stroke: #e6ec8b + + &.node-normal + > rect + stroke: #3fb6d8 + + h4 + color: #000 + h5 + color: #999 + + .edgeLabel + rect + fill #fff + + .edgePath + path + stroke #333 + stroke-width 2px + fill #333 + + .label + color: #777 + margin: 0 + + .edge-label + padding: 5px + font-size: 14px + // border: 2px solid + + .node-label + display: block + // border-width: 4px + // border-style: solid + // padding: 10px + margin: 0 + text-decoration: none http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/styles/index.styl ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/styles/index.styl b/flink-runtime-web/web-dashboard/app/styles/index.styl new file mode 100644 index 0000000..4562776 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/styles/index.styl @@ -0,0 +1,253 @@ +@import 'nib' + +sidebar-width = 250px + +#sidebar + overflow: hidden + position: fixed + left: - sidebar-width + top: 0 + bottom: 0 + height: 100% + width: sidebar-width + background: rgba(21,21,21,1) + // background: linear-gradient(200deg, 0% rgba(31,31,31,1), 100% rgba(0,0,0,1)) + transition(400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all) + box-shadow(inset -10px 0px 10px rgba(0,0,0,0.2)) + + &.sidebar-visible + left: 0 + + .logo + width: auto + height: 22px + + .navbar-brand.navbar-brand-text + font-size: 14px + font-weight: bold + color: #ffffff + padding-left: 0 + + .nav > li > a + color: #aaaaaa + margin-bottom: 1px + + &:hover, &:focus + background-color: rgba(40, 40, 40, 0.5) + &.active + background-color: rgba(100, 100, 100, 0.5) + +#content + background-color: #ffffff + overflow: hidden + margin-left: 0 + padding-top: 70px + + transition(400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all) + + .navbar-main, .navbar-main-additional + transition(400ms cubic-bezier(0.250, 0.250, 0.750, 0.750) all) + + .navbar-main-additional + margin-top: 51px + border-bottom: none + padding: 0 20px + .nav-tabs + margin: 0 -20px + padding: 0 20px + + &.sidebar-visible + margin-left: sidebar-width + + .navbar-main, .navbar-main-additional + left: sidebar-width + + #fold-button + display: inline-block + margin-left: 20px + + #content-inner + padding: 0px 20px 20px 20px + &.has-navbar-main-additional + padding-top: 42px + +.page-header + margin: 0 0 20px 0 + +.nav + > li + > a, > a:hover, > a:focus + color: #aaaaaa + background-color: transparent + border-bottom: 2px solid transparent + + &.active + > a, > a:hover, > a:focus + color: #000000 + border-bottom: 2px solid #000000 + + &.nav-tabs + margin-bottom: 20px + +.table + th + font-weight: normal + color: #999999 + + &.table-clickable + tr + cursor: pointer + +.panel + &.panel-dashboard + .huge + font-size: 28px + + &.panel-lg + font-size: 16px + .badge + font-size: 14px + +.navbar-secondary + overflow: auto + +.navbar-main, .navbar-secondary, .navbar-main-additional, .panel.panel-multi + .navbar-title, .panel-title + float: left + font-size: 18px + padding: 12px 20px 13px 10px + color: #333333 + display: inline-block + + .navbar-info, .panel-info + float: left + font-size: 14px + padding: 15px 15px 15px 15px + color: #999999 + display: inline-block + border-right: 1px solid #e7e7e7 + overflow: hidden + + .overflow + position: absolute + display: block + text-overflow: ellipsis + overflow: hidden + height: 22px + line-height: 22px + vertical-align: middle + + &.first + border-left: 1px solid #e7e7e7 + &.last + border-right: none + +.panel.panel-multi + .panel-heading + padding: 0 + + .panel-body + padding: 10px + background-color: #fdfdfd + color: #999 + font-size: 13px + +.navbar-main-additional + min-height: 40px + background-color: #fdfdfd + + .navbar-info + font-size: 13px + padding: 10px 15px 10px 15px + +.nav-top-affix + &.affix + width: 100% + top: 50px + margin-left: -20px + padding-left: 20px + margin-right: -20px + padding-right: 20px + background-color: #fff + z-index: 1 + + +.badge-default[href]:hover, +.badge-default[href]:focus + background-color #808080 + +.badge-primary + background-color #428bca + +.badge-primary[href]:hover, +.badge-primary[href]:focus + background-color #3071a9 + +.badge-success + background-color #5cb85c + +.badge-success[href]:hover, +.badge-success[href]:focus + background-color #449d44 + +.badge-info + background-color #5bc0de + +.badge-info[href]:hover, +.badge-info[href]:focus + background-color #31b0d5 + +.badge-warning + background-color #f0ad4e + +.badge-warning[href]:hover, +.badge-warning[href]:focus + background-color #ec971f + +.badge-danger + background-color #d9534f + +.badge-danger[href]:hover, +.badge-danger[href]:focus + background-color #c9302c + + +.indicator + display: inline-block + margin-right: 15px + &.indicator-primary + color: #428bca + + &.indicator-success + color: #5cb85c + + &.indicator-info + color: #5bc0de + + &.indicator-warning + color: #f0ad4e + + &.indicator-danger + color: #d9534f + + +@import './job' +@import './graph' +@import './timeline' + +@media (min-width: 768px) + #sidebar + left: 0 + + #content + margin-left: sidebar-width + + #fold-button + display: none + + .navbar-main, .navbar-main-additional + left: sidebar-width + + .navbar-main, .navbar-secondary, .navbar-main-additional + .navbar-title + padding: 12px 20px 13px 20px http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/styles/job.styl ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/styles/job.styl b/flink-runtime-web/web-dashboard/app/styles/job.styl new file mode 100644 index 0000000..2570655 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/styles/job.styl @@ -0,0 +1,43 @@ +.canvas-wrapper + border: 1px solid #ddd + position: relative + margin-bottom: 20px + + .main-canvas + height: 400px + overflow: hidden + + .zoom-buttons + position: absolute + top: 10px + right: 10px + +.table + &.table-properties + table-layout: fixed + white-space: nowrap + td + width: 50% + white-space: nowrap + overflow: hidden + text-overflow: ellipsis + + +.label-group + .label + display: inline-block + width: 2em + padding-left: 0.1em + padding-right: 0.1em + margin: 0 + border-right: 1px solid #ffffff + border-radius(0) + + &.label-black + background-color: #000000 + + // &:first-child + // border-radius(4px 0 0 4px) + + // &:last-child + // border-radius(0 4px 4px 0) http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/app/styles/timeline.styl ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/app/styles/timeline.styl b/flink-runtime-web/web-dashboard/app/styles/timeline.styl new file mode 100644 index 0000000..d357e70 --- /dev/null +++ b/flink-runtime-web/web-dashboard/app/styles/timeline.styl @@ -0,0 +1,14 @@ +.timeline-canvas + overflow: hidden + padding: 10px + +svg.timeline + overflow hidden + + .timeline-insidelabel, .timeline-series + cursor: pointer + + &.secondary + .timeline-insidelabel, .timeline-series + cursor: auto + http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/assets/fonts/FontAwesome.otf ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/assets/fonts/FontAwesome.otf b/flink-runtime-web/web-dashboard/assets/fonts/FontAwesome.otf new file mode 100644 index 0000000..f7936cc Binary files /dev/null and b/flink-runtime-web/web-dashboard/assets/fonts/FontAwesome.otf differ http://git-wip-us.apache.org/repos/asf/flink/blob/d59cebd8/flink-runtime-web/web-dashboard/assets/fonts/fontawesome-webfont.eot ---------------------------------------------------------------------- diff --git a/flink-runtime-web/web-dashboard/assets/fonts/fontawesome-webfont.eot b/flink-runtime-web/web-dashboard/assets/fonts/fontawesome-webfont.eot new file mode 100644 index 0000000..33b2bb8 Binary files /dev/null and b/flink-runtime-web/web-dashboard/assets/fonts/fontawesome-webfont.eot differ
