Repository: incubator-griffin Updated Branches: refs/heads/master 2637ca885 -> 4370fd86e
[GRIFFIN-194] single call to fetch all tables of all dbs Significantly improves opening speed for measure dialogs. Previously taking 10+ seconds on huge schemas, with this call it takes fraction of second to load. Author: Nikolay Sokolov <chemika...@gmail.com> Closes #447 from chemikadze/GRIFFIN-194. Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/4370fd86 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/4370fd86 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/4370fd86 Branch: refs/heads/master Commit: 4370fd86eb7c7433f6f1d3e9c6ed5913d774ea5a Parents: 2637ca8 Author: Nikolay Sokolov <chemika...@gmail.com> Authored: Fri Oct 26 02:07:08 2018 -0700 Committer: Yuepeng <yue...@griffin-2454820.phx02.dev.ebayc3.com> Committed: Fri Oct 26 02:07:08 2018 -0700 ---------------------------------------------------------------------- .../metastore/hive/HiveMetaStoreController.java | 5 ++ .../metastore/hive/HiveMetaStoreService.java | 2 + .../hive/HiveMetaStoreServiceImpl.java | 14 +++++- .../measure/create-measure/ac/ac.component.ts | 51 ++++++++----------- .../create-measure/pr/step1/step1.component.ts | 52 ++++++++------------ ui/angular/src/app/service/service.service.ts | 1 + 6 files changed, 60 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java index 2371044..ba425ec 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java @@ -57,6 +57,11 @@ public class HiveMetaStoreController { return hiveMetaStoreService.getAllTable(); } + @RequestMapping(value = "/dbs/tables/names", method = RequestMethod.GET) + public Map<String, List<String>> getAllTableNames() { + return hiveMetaStoreService.getAllTableNames(); + } + @RequestMapping(value = "/table", method = RequestMethod.GET) public Table getTable(@RequestParam("db") String dbName, @RequestParam("table") String tableName) { http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java index 4ef7690..609a34e 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java @@ -30,6 +30,8 @@ public interface HiveMetaStoreService { Iterable<String> getAllTableNames(String dbName); + Map<String, List<String>> getAllTableNames(); + List<Table> getAllTable(String db); Map<String, List<Table>> getAllTable(); http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java index 72d8c56..48a78a4 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.google.common.collect.Lists; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Table; import org.slf4j.Logger; @@ -99,8 +100,17 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { public List<Table> getAllTable(String db) { return getTables(db); } - - + + @Override + @Cacheable(unless = "#result==null || #result.isEmpty()") + public Map<String, List<String>> getAllTableNames() { + Map<String, List<String>> result = new HashMap<>(); + for (String dbName: getAllDatabases()) { + result.put(dbName, Lists.newArrayList(getAllTableNames(dbName))); + } + return result; + } + @Override @Cacheable(unless = "#result==null") public Map<String, List<Table>> getAllTable() { http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/ui/angular/src/app/measure/create-measure/ac/ac.component.ts ---------------------------------------------------------------------- diff --git a/ui/angular/src/app/measure/create-measure/ac/ac.component.ts b/ui/angular/src/app/measure/create-measure/ac/ac.component.ts index 3007f34..72d0138 100644 --- a/ui/angular/src/app/measure/create-measure/ac/ac.component.ts +++ b/ui/angular/src/app/measure/create-measure/ac/ac.component.ts @@ -644,48 +644,37 @@ export class AcComponent implements OnInit, AfterViewChecked { } ngOnInit() { - let allDatabases = this.serviceService.config.uri.dblist; - let getTableNames = this.serviceService.config.uri.tablenames; + let getTableNames = this.serviceService.config.uri.dbtablenames; - this.http.get(allDatabases).subscribe((databases: Array<string>) => { + this.http.get(getTableNames).subscribe((databases) => { this.nodeList = new Array(); this.nodeListTarget = this.nodeList; // share same model instead of copying(?) let i = 1; - let pending = databases.length; - if (databases.length > 10) { - this.options.animateExpand = false; - this.updateTrees(); - } - for (let dbName of databases) { + for (let dbName in databases) { + if (!databases.hasOwnProperty(dbName)) { + continue; + } let dbNode = new node(); dbNode.name = dbName; dbNode.id = i++; dbNode.isExpanded = false; dbNode.children = new Array(); - let params = new HttpParams({fromString: "db="+dbName}); - this.http.get(getTableNames, {params: params}).subscribe((tables: Array<string>) => { - for (let tableName of tables) { - let tableNode = new node(); - tableNode.name = tableName; - dbNode.children.push(tableNode); - tableNode.isExpanded = true; - tableNode.location = null; - tableNode.parent = dbName; - tableNode.cols = null; - } - pending -= 1; - if (pending == 0) { - this.updateTrees(); - } - }, - () => { - pending -= 1; - if (pending == 0) { - this.updateTrees(); - } - }); + for (let tableName of databases[dbName]) { + let tableNode = new node(); + tableNode.name = tableName; + dbNode.children.push(tableNode); + tableNode.isExpanded = true; + tableNode.location = null; + tableNode.parent = dbName; + tableNode.cols = null; + } this.nodeList.push(dbNode); } + if (i >= 10) { + this.options.animateExpand = false; + this.targetOptions.animateExpand = false; + } + this.updateTrees(); }); this.src_size = "1day"; this.tgt_size = "1day"; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts ---------------------------------------------------------------------- diff --git a/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts b/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts index a5b052e..2f4c3cd 100644 --- a/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts +++ b/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts @@ -188,47 +188,35 @@ export class PrStep1Component implements AfterViewChecked, OnInit { ngOnInit() { if (this.step1.nodeList.length !== 0) return; - let allDatabases = this.serviceService.config.uri.dblist; - let getTableNames = this.serviceService.config.uri.tablenames; - - this.http.get(allDatabases).subscribe((databases: Array<string>) => { + let getTableNames = this.serviceService.config.uri.dbtablenames; + + this.http.get(getTableNames).subscribe((databases) => { this.step1.nodeList = new Array(); let i = 1; - let pending = databases.length; - if (databases.length > 10) { - this.options.animateExpand = false; - this.tree.treeModel.update(); - } - for (let dbName of databases) { + for (let dbName in databases) { + if (!databases.hasOwnProperty(dbName)) { + continue; + } let dbNode = new node(); dbNode.name = dbName; dbNode.id = i++; dbNode.isExpanded = false; dbNode.children = new Array(); - let params = new HttpParams({fromString: "db="+dbName}); - this.http.get(getTableNames, {params: params}).subscribe((tables: Array<string>) => { - for (let tableName of tables) { - let tableNode = new node(); - tableNode.name = tableName; - dbNode.children.push(tableNode); - tableNode.isExpanded = true; - tableNode.location = null; - tableNode.parent = dbName; - tableNode.cols = null; - } - pending -= 1; - if (pending == 0) { - this.tree.treeModel.update(); - } - }, - () => { - pending -= 1; - if (pending == 0) { - this.tree.treeModel.update(); - } - }); + for (let tableName of databases[dbName]) { + let tableNode = new node(); + tableNode.name = tableName; + dbNode.children.push(tableNode); + tableNode.isExpanded = true; + tableNode.location = null; + tableNode.parent = dbName; + tableNode.cols = null; + } this.step1.nodeList.push(dbNode); } + if (i >= 10) { + this.options.animateExpand = false; + } + this.tree.treeModel.update(); }); } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/ui/angular/src/app/service/service.service.ts ---------------------------------------------------------------------- diff --git a/ui/angular/src/app/service/service.service.ts b/ui/angular/src/app/service/service.service.ts index 6161af0..7d50b4b 100644 --- a/ui/angular/src/app/service/service.service.ts +++ b/ui/angular/src/app/service/service.service.ts @@ -38,6 +38,7 @@ export class ServiceService { dataassetlist: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/dbs/tables", dblist: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/dbs", tablenames: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/tables/names", // ?db=... + dbtablenames: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/dbs/tables/names", dbtable: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/table", // ?db=...&table=... getdataasset: this.BACKEND_SERVER + this.API_ROOT_PATH + "/dataassets",