This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new c132986 SUBMARINE-1002. Submarine model management database and
database sdk
c132986 is described below
commit c132986aec660c8bed2f7b8bd5af9bba0a22be7d
Author: KUAN-HSUN-LI <[email protected]>
AuthorDate: Sun Sep 12 18:39:29 2021 +0800
SUBMARINE-1002. Submarine model management database and database sdk
### What is this PR for?
1. Create submarine model management database including
`registered_models`, `registered_model_tags`, `model_versions` and
`model_version_tags` tables.
2. Move the `params` and `metrics` tables from `submarine.sql` to
`submarine-model.sql`
### What type of PR is it?
[Feature]
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-1002
### How should this be tested?
run the sdk unit tests `pytest --cov=submarine -vs -m "not e2e"`
TestModelVersion and TestRegisteredModel are two relevant unit tests
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: KUAN-HSUN-LI <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #731 from KUAN-HSUN-LI/SUBMARINE-1002 and squashes the following
commits:
ec551448 [KUAN-HSUN-LI] add model version on delete cascade
245126d0 [KUAN-HSUN-LI] fix conflict
09615f7d [KUAN-HSUN-LI] SUBMARINE-1002. Update comment
54d4242d [KUAN-HSUN-LI] fix conflict
82f80762 [KUAN-HSUN-LI] change datetime precision
aef97310 [KUAN-HSUN-LI] fix datetime default value
4a6998a3 [KUAN-HSUN-LI] replace database table with sigular name
0c504607 [KUAN-HSUN-LI] change bigint to datetime
9fd00a30 [KUAN-HSUN-LI] change table name from model_version_tags to
model_tags
8aa9ec45 [KUAN-HSUN-LI] SUBMARINE-1002. foreign key support
5b522e85 [KUAN-HSUN-LI] SUBMARINE-1002. foreign key for metrics and params
65608f71 [KUAN-HSUN-LI] fix lint
e58d7ddc [KUAN-HSUN-LI] SUBMARINE-1002. submarine-model-database sdk test
86254d96 [KUAN-HSUN-LI] handle tag
160dc049 [KUAN-HSUN-LI] change directory
4d536b80 [KUAN-HSUN-LI] fix
e0950bed [KUAN-HSUN-LI] SUBMARINE-1002. submarine model database sdk
337f038c [KUAN-HSUN-LI] fix database docker image
8f127208 [KUAN-HSUN-LI] SUBMARINE-1002. Create submarine model management
table
---
dev-support/database/README.md | 4 +-
dev-support/database/init-database.py | 2 +
dev-support/database/init-database.sh | 4 +-
dev-support/database/submarine-data.sql | 21 --
dev-support/database/submarine-model.sql | 79 +++++
dev-support/database/submarine.sql | 28 --
dev-support/docker-images/database/startup.sh | 6 +-
.../pysubmarine/submarine/entities/Metric.py | 2 +-
.../pysubmarine/submarine/entities/__init__.py | 2 +
.../pysubmarine/submarine/entities/experiment.py | 60 ++++
.../entities/{ => model_registry}/__init__.py | 12 +-
.../{__init__.py => model_registry/model_tag.py} | 20 +-
.../entities/model_registry/model_version.py | 106 ++++++
.../model_version_stages.py} | 11 +-
.../entities/model_registry/registered_model.py | 57 +++
.../registered_model_tag.py} | 20 +-
.../pysubmarine/submarine/store/database/models.py | 390 +++++++++++++++++++--
.../pysubmarine/submarine/tracking/fluent.py | 6 +-
.../pysubmarine/submarine/utils/validation.py | 7 +-
.../entities/model_registry/test_model_version.py | 125 +++++++
.../model_registry/test_registered_model.py | 74 ++++
.../pysubmarine/tests/entities/test_metrics.py | 4 +-
.../tests/store/test_sqlalchemy_store.py | 20 +-
.../pysubmarine/tests/tracking/test_tracking.py | 17 +-
.../server/workbench/database/entity/Metric.java | 10 +-
.../server/workbench/rest/MetricRestApi.java | 9 +-
.../submarine/database/mappers/MetricMapper.xml | 20 +-
.../submarine/database/mappers/ParamMapper.xml | 12 +-
.../database/service/MetricServiceTest.java | 38 +-
.../database/service/ParamServiceTest.java | 23 +-
30 files changed, 1040 insertions(+), 149 deletions(-)
diff --git a/dev-support/database/README.md b/dev-support/database/README.md
index f55f6a1..39047fa 100644
--- a/dev-support/database/README.md
+++ b/dev-support/database/README.md
@@ -92,10 +92,11 @@ Run [mysql docker container](https://hub.docker.com/_/mysql)
docker run --name some-mysql -e MYSQL_ROOT_PASSWORD=my-secret-pw -d mysql:tag
```
-Copy the files, submarine.sql, submarine-data.sql and metastore.sql to the
mysql docker.
+Copy the files, submarine.sql, submarine-model.sql, submarine-data.sql and
metastore.sql to the mysql docker.
```
docker cp ${SUBMARINE_HOME}/dev-support/database/submarine.sql ${DOCKER_ID}:/
+docker cp ${SUBMARINE_HOME}/dev-support/database/submarine-model.sql
${DOCKER_ID}:/
docker cp ${SUBMARINE_HOME}/dev-support/database/submarine-data.sql
${DOCKER_ID}:/
docker cp ${SUBMARINE_HOME}/dev-support/database/metastore.sql ${DOCKER_ID}:/
```
@@ -110,6 +111,7 @@ mysql> GRANT ALL PRIVILEGES ON * . * TO 'submarine'@'%';
mysql> CREATE DATABASE IF NOT EXISTS submarine CHARACTER SET utf8 COLLATE
utf8_general_ci;
mysql> use submarine;
mysql> source /submarine.sql;
+mysql> source /submarine-model.sql;
mysql> source /submarine-data.sql;
mysql> CREATE USER IF NOT EXISTS 'metastore'@'%' IDENTIFIED BY 'password';
mysql> GRANT ALL PRIVILEGES ON * . * TO 'metastore'@'%';
diff --git a/dev-support/database/init-database.py
b/dev-support/database/init-database.py
index aa1ac69..6df1131 100644
--- a/dev-support/database/init-database.py
+++ b/dev-support/database/init-database.py
@@ -52,6 +52,7 @@ commit("CREATE USER IF NOT EXISTS 'submarine_test'@'%'
IDENTIFIED BY 'password_t
commit("GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';")
commit("use submarine_test;")
commit_from_file("./dev-support/database/submarine.sql")
+commit_from_file("./dev-support/database/submarine-model.sql")
commit("show tables;")
@@ -68,6 +69,7 @@ commit("CREATE USER IF NOT EXISTS 'submarine'@'%' IDENTIFIED
BY 'password';")
commit("GRANT ALL PRIVILEGES ON *.* TO 'submarine'@'%';")
commit("use submarine;")
commit_from_file("./dev-support/database/submarine.sql")
+commit_from_file("./dev-support/database/submarine-model.sql")
commit_from_file("./dev-support/database/submarine-data.sql")
commit("show tables;")
diff --git a/dev-support/database/init-database.sh
b/dev-support/database/init-database.sh
index 0c3b37b..4288d36 100755
--- a/dev-support/database/init-database.sh
+++ b/dev-support/database/init-database.sh
@@ -23,6 +23,7 @@ mysql -e "CREATE DATABASE IF NOT EXISTS submarine_test;"
mysql -e "CREATE USER IF NOT EXISTS 'submarine_test'@'%' IDENTIFIED BY
'password_test';"
mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';"
mysql -e "use submarine_test; source ./submarine.sql; show tables;"
+mysql -e "use submarine_test; source ./submarine-model.sql; show tables;"
mysql -e "CREATE DATABASE IF NOT EXISTS metastore_test;"
mysql -e "CREATE USER IF NOT EXISTS 'metastore_test'@'%' IDENTIFIED BY
'password_test';"
@@ -32,7 +33,8 @@ mysql -e "use metastore_test; source ./metastore.sql; show
tables;"
mysql -e "CREATE DATABASE IF NOT EXISTS submarine;"
mysql -e "CREATE USER IF NOT EXISTS 'submarine'@'%' IDENTIFIED BY 'password';"
mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine'@'%';"
-mysql -e "use submarine; source ./submarine.sql; source ./submarine-data.sql;
show tables;"
+mysql -e "use submarine; source ./submarine.sql; source ./submarine-model.sql;"
+mysql -e "use submarine; source ./submarine-data.sql; show tables;"
mysql -e "CREATE DATABASE IF NOT EXISTS metastore;"
mysql -e "CREATE USER IF NOT EXISTS 'metastore'@'%' IDENTIFIED BY 'password';"
diff --git a/dev-support/database/submarine-data.sql
b/dev-support/database/submarine-data.sql
index 0084fca..a7a3743 100644
--- a/dev-support/database/submarine-data.sql
+++ b/dev-support/database/submarine-data.sql
@@ -61,27 +61,6 @@ INSERT INTO `sys_user` VALUES
('e9ca23d68d884d4ebb19d07889727dae', 'admin', 'adm
INSERT INTO `team` VALUES ('e9ca23d68d884d4ebb19d07889721234', 'admin',
'Submarine', 'admin', '2020-05-06 14:00:05', 'Jack', '2020-05-06 14:00:14');
-- ----------------------------
--- Records of metrics
--- ----------------------------
-INSERT INTO `metrics` (`id`, `key`, `value`, `worker_index`, `timestamp`,
`step`, `is_nan`) VALUES
-('application_123651651', 'score', 0.666667, 'worker-1', 1569139525097, 0, 0),
-('application_123651651', 'score', 0.666670, 'worker-1', 1569149139731, 1, 0),
-('experiment_1595332719154_0001', 'score', 0.666667, 'worker-1',
1569169376482, 0, 0),
-('experiment_1595332719154_0001', 'score', 0.666671, 'worker-1',
1569236290721, 0, 0),
-('experiment_1595332719154_0001', 'score', 0.666680, 'worker-1',
1569236466722, 0, 0);
-
--- ----------------------------
--- Records of params
--- ----------------------------
-INSERT INTO `params` (`id`, `key`, `value`, `worker_index`) VALUES
-('application_123651651', 'max_iter', '100', 'worker-1'),
-('application_123456898', 'n_jobs', '5', 'worker-1'),
-('application_123456789', 'alpha', '20', 'worker-1'),
-('experiment_1595332719154_0001', 'max_iter', '100', 'worker-1'),
-('experiment_1595332719154_0002', 'n_jobs', '5', 'worker-1'),
-('experiment_1595332719154_0003', 'alpha', '20', 'worker-1');
-
--- ----------------------------
-- Records of environment
-- ----------------------------
INSERT INTO `environment` VALUES
diff --git a/dev-support/database/submarine-model.sql
b/dev-support/database/submarine-model.sql
new file mode 100644
index 0000000..da124e9
--- /dev/null
+++ b/dev-support/database/submarine-model.sql
@@ -0,0 +1,79 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+-- http://www.apache.org/licenses/LICENSE-2.0
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS `registered_model`;
+CREATE TABLE `registered_model` (
+ `name` VARCHAR(256) NOT NULL,
+ `creation_time` DATETIME(3) COMMENT 'Millisecond precision',
+ `last_updated_time` DATETIME(3) COMMENT 'Millisecond precision',
+ `description` VARCHAR(5000),
+ CONSTRAINT `registered_model_pk` PRIMARY KEY (`name`),
+ UNIQUE (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+DROP TABLE IF EXISTS `registered_model_tag`;
+CREATE TABLE `registered_model_tag` (
+ `name` VARCHAR(256) NOT NULL,
+ `tag` VARCHAR(256) NOT NULL,
+ CONSTRAINT `registered_model_tag_pk` PRIMARY KEY (`name`, `tag`),
+ FOREIGN KEY(`name`) REFERENCES `registered_model` (`name`) ON UPDATE
CASCADE ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+DROP TABLE IF EXISTS `model_version`;
+CREATE TABLE `model_version` (
+ `name` VARCHAR(256) NOT NULL,
+ `version` INTEGER NOT NULL,
+ `user_id` VARCHAR(64) NOT NULL COMMENT 'Id of the created user',
+ `experiment_id` VARCHAR(64) NOT NULL,
+ `current_stage` VARCHAR(20) COMMENT 'Model stage ex: None,
production...',
+ `creation_time` DATETIME(3) COMMENT 'Millisecond precision',
+ `last_updated_time` DATETIME(3) COMMENT 'Millisecond precision',
+ `source` VARCHAR(512) COMMENT 'Model saved link',
+ `dataset` VARCHAR(256) COMMENT 'Which dataset is used',
+ `description` VARCHAR(5000),
+ CONSTRAINT `model_version_pk` PRIMARY KEY (`name`, `version`),
+ FOREIGN KEY(`name`) REFERENCES `registered_model` (`name`) ON UPDATE
CASCADE ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+DROP TABLE IF EXISTS `model_tag`;
+CREATE TABLE `model_tag` (
+ `name` VARCHAR(256) NOT NULL,
+ `version` INTEGER NOT NULL,
+ `tag` VARCHAR(256) NOT NULL,
+ CONSTRAINT `model_tag_pk` PRIMARY KEY (`name`, `version`, `tag`),
+ FOREIGN KEY(`name`, `version`) REFERENCES `model_version` (`name`,
`version`) ON UPDATE CASCADE ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+DROP TABLE IF EXISTS `metric`;
+CREATE TABLE `metric` (
+ `id` VARCHAR(64) NOT NULL COMMENT 'Id of the Experiment',
+ `key` VARCHAR(190) NOT NULL COMMENT 'Metric key: `String` (limit 190
characters). Part of *Primary Key* for ``metric`` table.',
+ `value` FLOAT NOT NULL COMMENT 'Metric value: `Float`. Defined as
*Non-null* in schema.',
+ `worker_index` VARCHAR(32) NOT NULL COMMENT 'Metric worker_index:
`String` (limit 32 characters). Part of *Primary Key* for\r\n ``metrics``
table.',
+ `timestamp` DATETIME(3) NOT NULL COMMENT 'Timestamp recorded for this
metric entry: `DATETIME` (millisecond precision).
+
Part of *Primary Key* for ``metrics`` table.',
+ `step` INTEGER NOT NULL COMMENT 'Step recorded for this metric entry:
`INTEGER`.',
+ `is_nan` BOOLEAN NOT NULL COMMENT 'True if the value is in fact NaN.',
+ CONSTRAINT `metric_pk` PRIMARY KEY (`id`, `key`, `timestamp`,
`worker_index`),
+ FOREIGN KEY(`id`) REFERENCES `experiment` (`id`) ON UPDATE CASCADE ON
DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+DROP TABLE IF EXISTS `param`;
+CREATE TABLE `param` (
+ `id` VARCHAR(64) NOT NULL COMMENT 'Id of the Experiment',
+ `key` VARCHAR(190) NOT NULL COMMENT '`String` (limit 190 characters).
Part of *Primary Key* for ``param`` table.',
+ `value` VARCHAR(32) NOT NULL COMMENT '`String` (limit 190 characters).
Defined as *Non-null* in schema.',
+ `worker_index` VARCHAR(32) NOT NULL COMMENT '`String` (limit 32
characters). Part of *Primary Key* for\r\n ``metric`` table.',
+ CONSTRAINT `param_pk` PRIMARY KEY (`id`, `key`, `worker_index`),
+ FOREIGN KEY(`id`) REFERENCES `experiment` (`id`) ON UPDATE CASCADE ON
DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
diff --git a/dev-support/database/submarine.sql
b/dev-support/database/submarine.sql
index 04f8b51..a2a89e6 100644
--- a/dev-support/database/submarine.sql
+++ b/dev-support/database/submarine.sql
@@ -265,34 +265,6 @@ CREATE TABLE `notebook` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- ----------------------------
--- Table structure for metric
--- ----------------------------
-DROP TABLE IF EXISTS `metrics`;
-CREATE TABLE `metrics` (
- `id` varchar(64) NOT NULL COMMENT 'Id of the Experiment',
- `key` varchar(190) NOT NULL COMMENT 'Metric key: `String` (limit 190
characters). Part of *Primary Key* for ``metrics`` table.',
- `value` float NOT NULL COMMENT 'Metric value: `Float`. Defined as *Non-null*
in schema.',
- `worker_index` varchar(32) NOT NULL COMMENT 'Metric worker_index: `String`
(limit 32 characters). Part of *Primary Key* for\r\n ``metrics`` table.',
- `timestamp` bigint(20) NOT NULL COMMENT 'Timestamp recorded for this metric
entry: `BigInteger`. Part of *Primary Key* for ``metrics`` table.',
- `step` bigint(11) NOT NULL COMMENT 'Step recorded for this metric entry:
`BigInteger`.',
- `is_nan` BOOLEAN NOT NULL COMMENT 'True if the value is in fact NaN.',
- PRIMARY KEY (`id`, `key`, `timestamp`, `worker_index`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
--- ----------------------------
--- Table structure for param
--- ----------------------------
-DROP TABLE IF EXISTS `params`;
-CREATE TABLE `params` (
- `id` varchar(64) NOT NULL COMMENT 'Id of the Experiment',
- `key` varchar(190) NOT NULL COMMENT '`String` (limit 190 characters). Part
of *Primary Key* for ``params`` table.',
- `value` varchar(32) NOT NULL COMMENT '`String` (limit 190 characters).
Defined as *Non-null* in schema.',
- `worker_index` varchar(32) NOT NULL COMMENT '`String` (limit 32 characters).
Part of *Primary Key* for\r\n ``metrics`` table.',
- PRIMARY KEY (`id`, `key`, `worker_index`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-
--- ----------------------------
-- Table structure for experiment_templates
-- ----------------------------
DROP TABLE IF EXISTS `experiment_template`;
diff --git a/dev-support/docker-images/database/startup.sh
b/dev-support/docker-images/database/startup.sh
index cce03a8..c52fc93 100755
--- a/dev-support/docker-images/database/startup.sh
+++ b/dev-support/docker-images/database/startup.sh
@@ -18,11 +18,13 @@ mysql -uroot -p$MYSQL_ROOT_PASSWORD <<EOF
CREATE DATABASE submarine_test;
CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test';
GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';
-use submarine_test; source /tmp/database/submarine.sql; source
/tmp/database/submarine-data.sql;
+use submarine_test; source /tmp/database/submarine.sql; source
/tmp/database/submarine-model.sql;
+source /tmp/database/submarine-data.sql;
CREATE DATABASE submarine;
CREATE USER 'submarine'@'%' IDENTIFIED BY 'password';
GRANT ALL PRIVILEGES ON *.* TO 'submarine'@'%';
-use submarine; source /tmp/database/submarine.sql; source
/tmp/database/submarine-data.sql;
+use submarine; source /tmp/database/submarine.sql; source
/tmp/database/submarine-model.sql;
+source /tmp/database/submarine-data.sql;
CREATE DATABASE metastore_test;
CREATE USER 'metastore_test'@'%' IDENTIFIED BY 'password_test';
GRANT ALL PRIVILEGES ON * . * TO 'metastore_test'@'%';
diff --git a/submarine-sdk/pysubmarine/submarine/entities/Metric.py
b/submarine-sdk/pysubmarine/submarine/entities/Metric.py
index ea69954..df2f00f 100644
--- a/submarine-sdk/pysubmarine/submarine/entities/Metric.py
+++ b/submarine-sdk/pysubmarine/submarine/entities/Metric.py
@@ -45,7 +45,7 @@ class Metric(_SubmarineObject):
@property
def timestamp(self):
- """Metric timestamp as an integer (milliseconds since the Unix
epoch)."""
+ """Metric timestamp as aa datetime object."""
return self._timestamp
@property
diff --git a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
b/submarine-sdk/pysubmarine/submarine/entities/__init__.py
index e2d8479..323e7e4 100644
--- a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
+++ b/submarine-sdk/pysubmarine/submarine/entities/__init__.py
@@ -13,10 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from submarine.entities.experiment import Experiment
from submarine.entities.Metric import Metric
from submarine.entities.Param import Param
__all__ = [
+ "Experiment",
"Metric",
"Param",
]
diff --git a/submarine-sdk/pysubmarine/submarine/entities/experiment.py
b/submarine-sdk/pysubmarine/submarine/entities/experiment.py
new file mode 100644
index 0000000..ffd89ec
--- /dev/null
+++ b/submarine-sdk/pysubmarine/submarine/entities/experiment.py
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from submarine.entities._submarine_object import _SubmarineObject
+
+
+class Experiment(_SubmarineObject):
+ """
+ Experiment object.
+ """
+
+ def __init__(self, id, experiment_spec, create_by, create_time, update_by,
update_time):
+ self._id = id
+ self._experiment_spec = experiment_spec
+ self._create_by = create_by
+ self._create_time = create_time
+ self._update_by = update_by
+ self._update_time = update_time
+
+ @property
+ def id(self):
+ """String ID of the experiment."""
+ return self._id
+
+ @property
+ def experiment_spec(self):
+ """String of the experiment spec."""
+ return self._experiment_spec
+
+ @property
+ def create_by(self):
+ """String name of created user id."""
+ return self.create_by
+
+ @property
+ def create_time(self):
+ """Datetime of create time."""
+ return self._create_time
+
+ @property
+ def update_by(self):
+ """String name of updated user id"."""
+ return self._update_by
+
+ @property
+ def update_time(self):
+ """Datetime of update time."""
+ return self._update_time
diff --git a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/__init__.py
similarity index 67%
copy from submarine-sdk/pysubmarine/submarine/entities/__init__.py
copy to submarine-sdk/pysubmarine/submarine/entities/model_registry/__init__.py
index e2d8479..7f537e8 100644
--- a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
+++ b/submarine-sdk/pysubmarine/submarine/entities/model_registry/__init__.py
@@ -13,10 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from submarine.entities.Metric import Metric
-from submarine.entities.Param import Param
+from submarine.entities.model_registry.model_tag import ModelTag
+from submarine.entities.model_registry.model_version import ModelVersion
+from submarine.entities.model_registry.registered_model import RegisteredModel
+from submarine.entities.model_registry.registered_model_tag import
RegisteredModelTag
__all__ = [
- "Metric",
- "Param",
+ "ModelVersion",
+ "ModelTag",
+ "RegisteredModel",
+ "RegisteredModelTag",
]
diff --git a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/model_tag.py
similarity index 71%
copy from submarine-sdk/pysubmarine/submarine/entities/__init__.py
copy to submarine-sdk/pysubmarine/submarine/entities/model_registry/model_tag.py
index e2d8479..5cbcae1 100644
--- a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
+++ b/submarine-sdk/pysubmarine/submarine/entities/model_registry/model_tag.py
@@ -13,10 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from submarine.entities.Metric import Metric
-from submarine.entities.Param import Param
+from submarine.entities._submarine_object import _SubmarineObject
-__all__ = [
- "Metric",
- "Param",
-]
+
+class ModelTag(_SubmarineObject):
+ """
+ Tag object associated with a model version.
+ """
+
+ def __init__(self, tag):
+ self._tag = tag
+
+ @property
+ def tag(self):
+ """String tag"""
+ return self._tag
diff --git
a/submarine-sdk/pysubmarine/submarine/entities/model_registry/model_version.py
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/model_version.py
new file mode 100644
index 0000000..88c9ff4
--- /dev/null
+++
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/model_version.py
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from submarine.entities._submarine_object import _SubmarineObject
+
+
+class ModelVersion(_SubmarineObject):
+ """
+ Model Version object.
+ """
+
+ def __init__(
+ self,
+ name,
+ version,
+ user_id,
+ experiment_id,
+ current_stage,
+ creation_time,
+ last_updated_time,
+ source,
+ dataset=None,
+ description=None,
+ tags=None,
+ ):
+ self._name = name
+ self._version = version
+ self._user_id = user_id
+ self._experiment_id = experiment_id
+ self._current_stage = current_stage
+ self._creation_time = creation_time
+ self._last_updated_time = last_updated_time
+ self._source = source
+ self._dataset = dataset
+ self._description = description
+ self._tags = [tag.tag for tag in (tags or [])]
+
+ @property
+ def name(self):
+ """String. Unique name within Model Registry."""
+ return self._name
+
+ @property
+ def version(self):
+ """String. version"""
+ return self._version
+
+ @property
+ def user_id(self):
+ """String. User ID that created this model version."""
+ return self._user_id
+
+ @property
+ def experiment_id(self):
+ """String. Experiment ID that created this model version."""
+ return self._experiment_id
+
+ @property
+ def creation_time(self):
+ """Datetime object. Model version creation timestamp."""
+ return self._creation_time
+
+ @property
+ def last_updated_time(self):
+ """Datetime object. Timestamp of last update for this model version."""
+ return self._last_updated_time
+
+ @property
+ def source(self):
+ """String. Source path for the model."""
+ return self._source
+
+ @property
+ def current_stage(self):
+ """String. Current stage of this model version."""
+ return self._current_stage
+
+ @property
+ def dataset(self):
+ """String. Dataset used by this model version"""
+ return self._dataset
+
+ @property
+ def description(self):
+ """String. Description"""
+ return self._description
+
+ @property
+ def tags(self):
+ """List of strings"""
+ return self._tags
+
+ def _add_tag(self, tag):
+ self._tags.append(tag)
diff --git a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/model_version_stages.py
similarity index 81%
copy from submarine-sdk/pysubmarine/submarine/entities/__init__.py
copy to
submarine-sdk/pysubmarine/submarine/entities/model_registry/model_version_stages.py
index e2d8479..a8fae8e 100644
--- a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
+++
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/model_version_stages.py
@@ -13,10 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from submarine.entities.Metric import Metric
-from submarine.entities.Param import Param
+STAGE_NONE = "None"
+STAGE_STAGING = "Staging"
+STAGE_PRODUCTION = "Production"
+STAGE_ARCHIVED = "Archived"
-__all__ = [
- "Metric",
- "Param",
-]
+ALL_STAGES = [STAGE_NONE, STAGE_STAGING, STAGE_PRODUCTION, STAGE_ARCHIVED]
diff --git
a/submarine-sdk/pysubmarine/submarine/entities/model_registry/registered_model.py
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/registered_model.py
new file mode 100644
index 0000000..7ff5e69
--- /dev/null
+++
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/registered_model.py
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from submarine.entities._submarine_object import _SubmarineObject
+
+
+class RegisteredModel(_SubmarineObject):
+ """
+ Registered Model object.
+ """
+
+ def __init__(self, name, creation_time, last_updated_time,
description=None, tags=None):
+ self._name = name
+ self._creation_time = creation_time
+ self._last_updated_time = last_updated_time
+ self._description = description
+ self._tags = [tag.tag for tag in (tags or [])]
+
+ @property
+ def name(self):
+ """String. Registered model name."""
+ return self._name
+
+ @property
+ def creation_time(self):
+ """Datetime object. Model version creation timestamp."""
+ return self._creation_time
+
+ @property
+ def last_updated_time(self):
+ """Datetime object. Timestamp of last update for this model version."""
+ return self._last_updated_time
+
+ @property
+ def description(self):
+ """String. Description"""
+ return self._description
+
+ @property
+ def tags(self):
+ """List of strings"""
+ return self._tags
+
+ def _add_tag(self, tag):
+ self._tags.append(tag)
diff --git a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/registered_model_tag.py
similarity index 71%
copy from submarine-sdk/pysubmarine/submarine/entities/__init__.py
copy to
submarine-sdk/pysubmarine/submarine/entities/model_registry/registered_model_tag.py
index e2d8479..22de3c9 100644
--- a/submarine-sdk/pysubmarine/submarine/entities/__init__.py
+++
b/submarine-sdk/pysubmarine/submarine/entities/model_registry/registered_model_tag.py
@@ -13,10 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from submarine.entities.Metric import Metric
-from submarine.entities.Param import Param
+from submarine.entities._submarine_object import _SubmarineObject
-__all__ = [
- "Metric",
- "Param",
-]
+
+class RegisteredModelTag(_SubmarineObject):
+ """
+ Tag object associated with a registered model.
+ """
+
+ def __init__(self, tag):
+ self._tag = tag
+
+ @property
+ def tag(self):
+ """String tag."""
+ return self._tag
diff --git a/submarine-sdk/pysubmarine/submarine/store/database/models.py
b/submarine-sdk/pysubmarine/submarine/store/database/models.py
index c4593d7..a1caffe 100644
--- a/submarine-sdk/pysubmarine/submarine/store/database/models.py
+++ b/submarine-sdk/pysubmarine/submarine/store/database/models.py
@@ -13,39 +13,380 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import time
+from datetime import datetime
from typing import Any
import sqlalchemy as sa
-from sqlalchemy import BigInteger, Boolean, Column, PrimaryKeyConstraint,
String
+from sqlalchemy import (
+ BigInteger,
+ Boolean,
+ Column,
+ ForeignKey,
+ ForeignKeyConstraint,
+ Integer,
+ PrimaryKeyConstraint,
+ String,
+ Text,
+)
+from sqlalchemy.dialects.mysql import DATETIME
from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import backref, relationship
-from submarine.entities import Metric, Param
+from submarine.entities import Experiment, Metric, Param
+from submarine.entities.model_registry import (
+ ModelTag,
+ ModelVersion,
+ RegisteredModel,
+ RegisteredModelTag,
+)
+from submarine.entities.model_registry.model_version_stages import STAGE_NONE
# Base class in sqlalchemy is a dynamic type
Base: Any = declarative_base()
+#
+---------------------+-------------------------+-------------------------+-------------+
+# | name | creation_time | last_updated_time |
description |
+#
+---------------------+-------------------------+-------------------------+-------------+
+# | image_classfication | 2021-08-31 11:11:11.111 | 2021-09-02 11:11:11.111 |
... |
+# | speech_recoginition | 2021-08-31 16:16:16.166 | 2021-08-31 20:20:20.200 |
... |
+#
+---------------------+-------------------------+-------------------------+-------------+
-#
+--------------------+-------+-------------------+--------------+---------------+------+--------+
-# | id | key | value | worker_index | timestamp
| step | is_nan |
-#
+--------------------+-------+-------------------+--------------+---------------+------+--------+
-# | application_123456 | score | 0.666666666666667 | worker-1 |
1595414873838 | 0 | 0 |
-# | application_123456 | score | 0.666666666666667 | worker-1 |
1595472286360 | 0 | 0 |
-# | application_123456 | score | 0.666666666666667 | worker-1 |
1595414632967 | 0 | 0 |
-# | application_123456 | score | 0.666666666666667 | worker-1 |
1595415075067 | 0 | 0 |
-#
+--------------------+-------+-------------------+--------------+---------------+------+--------+
+class SqlRegisteredModel(Base):
+ __tablename__ = "registered_model"
+
+ name = Column(String(256), unique=True, nullable=False)
+ """
+ Name for registered models: Part of *Primary Key* for ``registered_model``
table.
+ """
+
+ creation_time = Column(DATETIME(fsp=3), default=datetime.now())
+ """
+ Creation time of registered models: default current time in milliseconds
+ """
+
+ last_updated_time = Column(DATETIME(fsp=3), nullable=True, default=None)
+ """
+ Last updated time of registered model
+ """
+
+ description = Column(String(5000), nullable=True, default="")
+ """
+ Description for registered model
+ """
+
+ __table_args__ = (PrimaryKeyConstraint("name",
name="registered_model_pk"),)
+
+ def __repr__(self):
+ return "<SqlRegisteredModel ({}, {}, {}, {})>".format(
+ self.name, self.creation_time, self.last_updated_time,
self.description
+ )
+
+ def to_submarine_entity(self):
+ """
+ Convert DB model to corresponding Submarine entity.
+ :return: :py:class:`submarine.entities.RegisteredModel`.
+ """
+ return RegisteredModel(
+ name=self.name,
+ creation_time=self.creation_time,
+ last_updated_time=self.last_updated_time,
+ description=self.description,
+ tags=[tag.to_submarine_entity for tag in
self.registered_model_tag],
+ )
+
+
+# +---------------------+-------+
+# | name | tag |
+# +---------------------+-------+
+# | image_classfication | image |
+# | image_classfication | major |
+# | speech_recoginition | audio |
+# +---------------------+-------+
+
+
+class SqlRegisteredModelTag(Base):
+ __tablename__ = "registered_model_tag"
+
+ name = Column(
+ String(256), ForeignKey("registered_model.name", onupdate="cascade",
ondelete="cascade")
+ )
+ """
+ Name for registered models: Part of *Primary Key* for
``registered_model_tag`` table. Refer to
+ name of ``registered_model`` table.
+ """
+
+ tag = Column(String(256), nullable=False)
+ """
+ Registered model tag: `String` (limit 256 characters). Part of *Primary
Key* for
+ ``registered_model_tag`` table.
+ """
+
+ # linked entities
+ registered_model = relationship(
+ "SqlRegisteredModel", backref=backref("registered_model_tag",
cascade="all")
+ )
+
+ __table_args__ = (PrimaryKeyConstraint("name", "tag",
name="registered_model_tag_pk"),)
+
+ def __repr__(self):
+ return "<SqlRegisteredModelTag ({}, {})>".format(self.name, self.tag)
+
+ # entity mappers
+ def to_submarine_entity(self):
+ """
+ Convert DB model to corresponding submarine entity.
+ :return: :py:class:`submarine.entities.RegisteredModelTag`.
+ """
+ return RegisteredModelTag(self.tag)
+
+
+# +---------------------+---------+-----+-------------------------------+-----+
+# | name | version | ... | source | ... |
+# +---------------------+---------+-----+-------------------------------+-----+
+# | image_classfication | 1 | ... | s3://submarine/ResNet50/1/ | ... |
+# | image_classfication | 2 | ... | s3://submarine/DenseNet121/2/ | ... |
+# | speech_recoginition | 1 | ... | s3://submarine/ASR/1/ | ... |
+# +---------------------+---------+-----+-------------------------------+-----+
+
+
+class SqlModelVersion(Base):
+ __tablename__ = "model_version"
+
+ name = Column(
+ String(256), ForeignKey("registered_model.name", onupdate="cascade",
ondelete="cascade")
+ )
+ """
+ Name for registered models: Part of *Primary Key* for
``registered_model_tag`` table. Refer to
+ name of ``registered_model`` table.
+ """
+
+ version = Column(Integer, nullable=False)
+ """
+ Model version: Part of *Primary Key* for ``registered_model_tag`` table.
+ """
+
+ user_id = Column(String(64), nullable=False)
+ """
+ ID to whom this model is created
+ """
+
+ experiment_id = Column(String(64), nullable=False)
+ """
+ ID to which this model belongs to
+ """
+
+ current_stage = Column(String(20), default=STAGE_NONE)
+ """
+ Current stage of this model: it can be `None`, `Staging`, `Production` and
`Achieved`
+ """
+
+ creation_time = Column(DATETIME(fsp=3), default=datetime.now())
+ """
+ Creation time of this model version: default current time in milliseconds
+ """
+
+ last_updated_time = Column(DATETIME(fsp=3), nullable=True, default=None)
+ """
+ Last updated time of this model version
+ """
+
+ source = Column(String(512), nullable=True, default=None)
+ """
+ Source of model: database link refer to this model
+ """
+
+ dataset = Column(String(256), nullable=True, default=None)
+ """
+ Dataset used for this model.
+ """
+
+ description = Column(String(5000), nullable=True)
+ """
+ Description for model version.
+ """
+
+ # linked entities
+ registered_model = relationship(
+ "SqlRegisteredModel", backref=backref("model_version", cascade="all")
+ )
+
+ __table_args__ = (PrimaryKeyConstraint("name", "version",
name="model_version_pk"),)
+
+ def __repr__(self):
+ return "<SqlModelVersion ({}, {}, {}, {}, {}, {}, {}, {}, {},
{})>".format(
+ self.name,
+ self.version,
+ self.user_id,
+ self.experiment_id,
+ self.current_stage,
+ self.creation_time,
+ self.last_updated_time,
+ self.source,
+ self.dataset,
+ self.description,
+ )
+
+ def to_submarine_entity(self):
+ """
+ Convert DB model to corresponding Submarine entity.
+ :return: :py:class:`submarine.entities.RegisteredModel`.
+ """
+ return ModelVersion(
+ name=self.name,
+ version=self.version,
+ user_id=self.user_id,
+ experiment_id=self.experiment_id,
+ current_stage=self.current_stage,
+ creation_time=self.creation_time,
+ last_updated_time=self.last_updated_time,
+ source=self.source,
+ dataset=self.dataset,
+ description=self.description,
+ tags=[tag.to_submarine_entity for tag in self.model_tag],
+ )
+
+
+# +---------------------+---------+-----------------+
+# | name | version | tag |
+# +---------------------+---------+-----------------+
+# | image_classfication | 1 | best |
+# | image_classfication | 1 | anomaly_support |
+# | image_classfication | 2 | testing |
+# | speech_recoginition | 1 | best |
+# +---------------------+---------+-----------------+
+
+
+class SqlModelTag(Base):
+ __tablename__ = "model_tag"
+
+ name = Column(String(256), nullable=False)
+ """
+ Name for registered models: Part of *Foreign Key* for ``model_tag`` table.
Refer to
+ name of ``model_version`` table.
+ """
+
+ version = Column(Integer, nullable=False)
+ """
+ version of model: Part of *Foreign Key* for ``model_tag`` table. Refer to
+ version of ``model_version`` table.
+ """
+
+ tag = Column(String(256), nullable=False)
+ """
+ tag of model version: `String` (limit 256 characters). Part of *Primary
Key* for
+ ``model_tag`` table.
+ """
+
+ # linked entities
+ model_version = relationship(
+ "SqlModelVersion", foreign_keys=[name, version],
backref=backref("model_tag", cascade="all")
+ )
+
+ __table_args__ = (
+ PrimaryKeyConstraint("name", "tag", name="model_tag_pk"),
+ ForeignKeyConstraint(
+ ("name", "version"),
+ ("model_version.name", "model_version.version"),
+ onupdate="cascade",
+ ondelete="cascade",
+ ),
+ )
+
+ def __repr__(self):
+ return "<SqlRegisteredModelTag ({}, {}, {})>".format(self.name,
self.version, self.tag)
+
+ # entity mappers
+ def to_submarine_entity(self):
+ """
+ Convert DB model to corresponding submarine entity.
+ :return: :py:class:`submarine.entities.ModelTag`.
+ """
+ return ModelTag(self.tag)
-class SqlMetric(Base):
- __tablename__ = "metrics"
+
+#
+--------------------+-----------------+-----------+-------------------------+-----+
+# | id | experiment_spec | create_by | create_time
| ... |
+#
+--------------------+-----------------+-----------+-------------------------+-----+
+# | application_123456 | ... | root | 2021-08-30 10:10:10.100
| ... |
+#
+--------------------+-----------------+-----------+-------------------------+-----+
+
+
+class SqlExperiment(Base):
+ __tablename__ = "experiment"
id = Column(String(64))
"""
- ID to which this metric belongs to: Part of *Primary Key* for ``metrics``
table.
+ ID to which this metric belongs to: Part of *Primary Key* for
``experiment`` table.
+ """
+ experiment_spec = Column(Text)
+ """
+ The spec to create this experiment
+ """
+ create_by = Column(String(32))
+ """
+ This experiment is created by whom.
+ """
+ create_time = Column(DATETIME(fsp=3), default=datetime.now())
+ """
+ Datetime of this experiment be created
+ """
+ update_by = Column(String(32))
+ """
+ This experiment is created by whom.
+ """
+ update_time = Column(DATETIME(fsp=3))
+ """
+ Datetime of this experiment be updated
+ """
+
+ __table_args__ = (PrimaryKeyConstraint("id"),)
+
+ def __repr__(self):
+ return "<SqlMetric({}, {}, {}, {}, {}, {})>".format(
+ self.id,
+ self.experiment_spec,
+ self.create_by,
+ self.create_time,
+ self.update_by,
+ self.update_time,
+ )
+
+ def to_submarine_entity(self):
+ """
+ Convert DB model to corresponding Submarine entity.
+ :return: :py:class:`submarine.entities.Experiment`.
+ """
+ return Experiment(
+ id=self.id,
+ experiment_spec=self.experiment_spec,
+ create_by=self.create_by,
+ create_time=self.create_time,
+ update_by=self.update_by,
+ update_time=self.update_time,
+ )
+
+
+#
+--------------------+-------+-------------------+--------------+-------------------------+-----+
+# | id | key | value | worker_index | timestamp
| ... |
+#
+--------------------+-------+-------------------+--------------+-------------------------+-----+
+# | application_123456 | score | 0.666666666666667 | worker-1 | 2021-08-30
10:10:10.100 | ... |
+# | application_123456 | score | 0.666666668777777 | worker-1 | 2021-08-30
10:10:11.156 | ... |
+# | application_123456 | score | 0.666666670000001 | worker-1 | 2021-08-30
10:10:12.201 | ... |
+#
+--------------------+-------+-------------------+--------------+-------------------------+-----+
+
+
+class SqlMetric(Base):
+ __tablename__ = "metric"
+
+ id = Column(String(64), ForeignKey("experiment.id", onupdate="cascade",
ondelete="cascade"))
+ """
+ ID to which this metric belongs to: *Foreign Key* for ``experiment`` table.
+ Part of *Primary Key* for ``metric`` table.
"""
key = Column(String(190))
"""
- Metric key: `String` (limit 190 characters). Part of *Primary Key* for
``metrics`` table.
+ Metric key: `String` (limit 190 characters). Part of *Primary Key* for
``metric`` table.
"""
value = Column(sa.types.Float(precision=53), nullable=False)
"""
@@ -54,12 +395,12 @@ class SqlMetric(Base):
worker_index = Column(String(64))
"""
Metric worker_index: `String` (limit 32 characters). Part of *Primary Key*
for
- ``metrics`` table.
+ ``metric`` table.
"""
- timestamp = Column(BigInteger, default=lambda: int(time.time()))
+ timestamp = Column(DATETIME(fsp=3), default=datetime.now())
"""
- Timestamp recorded for this metric entry: `BigInteger`. Part of *Primary
Key* for
- ``metrics`` table.
+ Timestamp recorded for this metric entry: `DATETIME`. Part of *Primary
Key* for
+ ``metric`` table.
"""
step = Column(BigInteger, default=0, nullable=False)
"""
@@ -103,15 +444,16 @@ class SqlMetric(Base):
class SqlParam(Base):
- __tablename__ = "params"
+ __tablename__ = "param"
- id = Column(String(64))
+ id = Column(String(64), ForeignKey("experiment.id", onupdate="cascade",
ondelete="cascade"))
"""
- ID to which this parameter belongs to: Part of *Primary Key* for
``params`` table.
+ ID to which this parameter belongs to: *Foreign Key* for ``experiment``
table.
+ Part of *Primary Key* for ``param`` table.
"""
key = Column(String(190))
"""
- Param key: `String` (limit 190 characters). Part of *Primary Key* for
``params`` table.
+ Param key: `String` (limit 190 characters). Part of *Primary Key* for
``param`` table.
"""
value = Column(String(190), nullable=False)
"""
@@ -120,7 +462,7 @@ class SqlParam(Base):
worker_index = Column(String(32), nullable=False)
"""
Param worker_index: `String` (limit 32 characters). Part of *Primary Key*
for
- ``metrics`` table.
+ ``metric`` table.
"""
__table_args__ = (PrimaryKeyConstraint("id", "key", "worker_index",
name="param_pk"),)
diff --git a/submarine-sdk/pysubmarine/submarine/tracking/fluent.py
b/submarine-sdk/pysubmarine/submarine/tracking/fluent.py
index f7ee031..8406ce7 100644
--- a/submarine-sdk/pysubmarine/submarine/tracking/fluent.py
+++ b/submarine-sdk/pysubmarine/submarine/tracking/fluent.py
@@ -19,7 +19,7 @@ Submarine run. This module is exposed to users at the
top-level :py:mod:`submari
from __future__ import print_function
import logging
-import time
+from datetime import datetime
from submarine.tracking.client import SubmarineClient
from submarine.tracking.utils import get_job_id, get_worker_index
@@ -51,6 +51,4 @@ def log_metric(key, value, step=None):
"""
job_id = get_job_id()
worker_index = get_worker_index()
- SubmarineClient().log_metric(
- job_id, key, value, worker_index, int(time.time() * 1000), step or 0
- )
+ SubmarineClient().log_metric(job_id, key, value, worker_index,
datetime.now(), step or 0)
diff --git a/submarine-sdk/pysubmarine/submarine/utils/validation.py
b/submarine-sdk/pysubmarine/submarine/utils/validation.py
index 37e5766..60dd9b0 100644
--- a/submarine-sdk/pysubmarine/submarine/utils/validation.py
+++ b/submarine-sdk/pysubmarine/submarine/utils/validation.py
@@ -18,6 +18,7 @@ Utilities for validating user inputs such as metric names and
parameter names.
import numbers
import posixpath
import re
+from datetime import datetime
from submarine.exceptions import SubmarineException
from submarine.store.database.db_types import DATABASE_ENGINES
@@ -92,10 +93,10 @@ def validate_metric(key, value, timestamp, step):
"double (64-bit floating point)" % (value, key, timestamp),
)
- if not isinstance(timestamp, numbers.Number) or timestamp < 0:
+ if not isinstance(timestamp, datetime):
raise SubmarineException(
- "Got invalid timestamp %s for metric '%s' (value=%s). Timestamp
must be a nonnegative "
- "long (64-bit integer) " % (timestamp, key, value),
+ "Got invalid timestamp %s for metric '%s' (value=%s). Timestamp
must be a datetime "
+ "object." % (timestamp, key, value),
)
if not isinstance(step, numbers.Number):
diff --git
a/submarine-sdk/pysubmarine/tests/entities/model_registry/test_model_version.py
b/submarine-sdk/pysubmarine/tests/entities/model_registry/test_model_version.py
new file mode 100644
index 0000000..6611fb4
--- /dev/null
+++
b/submarine-sdk/pysubmarine/tests/entities/model_registry/test_model_version.py
@@ -0,0 +1,125 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+
+from submarine.entities.model_registry.model_tag import ModelTag
+from submarine.entities.model_registry.model_version import ModelVersion
+from submarine.entities.model_registry.model_version_stages import STAGE_NONE
+
+
+class TestModelVersion:
+ default_data = {
+ "name": "test",
+ "version": 1,
+ "user_id": "admin",
+ "experiment_id": "experiment_1",
+ "current_stage": STAGE_NONE,
+ "creation_time": datetime.now(),
+ "last_updated_time": datetime.now(),
+ "source": "path/to/source",
+ "dataset": "test",
+ "description": "registered model description",
+ "tags": [],
+ }
+
+ def _check(
+ self,
+ model_version,
+ name,
+ version,
+ user_id,
+ experiment_id,
+ current_stage,
+ creation_time,
+ last_updated_time,
+ source,
+ dataset,
+ description,
+ tags,
+ ):
+ isinstance(model_version, ModelVersion)
+ assert model_version.name == name
+ assert model_version.version == version
+ assert model_version.user_id == user_id
+ assert model_version.experiment_id == experiment_id
+ assert model_version.current_stage == current_stage
+ assert model_version.creation_time == creation_time
+ assert model_version.last_updated_time == last_updated_time
+ assert model_version.source == source
+ assert model_version.dataset == dataset
+ assert model_version.description == description
+ assert model_version.tags == tags
+
+ def test_creation_and_hydration(self):
+ mv = ModelVersion(
+ self.default_data["name"],
+ self.default_data["version"],
+ self.default_data["user_id"],
+ self.default_data["experiment_id"],
+ self.default_data["current_stage"],
+ self.default_data["creation_time"],
+ self.default_data["last_updated_time"],
+ self.default_data["source"],
+ self.default_data["dataset"],
+ self.default_data["description"],
+ self.default_data["tags"],
+ )
+ self._check(
+ mv,
+ self.default_data["name"],
+ self.default_data["version"],
+ self.default_data["user_id"],
+ self.default_data["experiment_id"],
+ self.default_data["current_stage"],
+ self.default_data["creation_time"],
+ self.default_data["last_updated_time"],
+ self.default_data["source"],
+ self.default_data["dataset"],
+ self.default_data["description"],
+ self.default_data["tags"],
+ )
+
+ def test_with_tags(self):
+ tag1 = ModelTag("tag1")
+ tag2 = ModelTag("tag2")
+ tags = [tag1, tag2]
+ mv = ModelVersion(
+ self.default_data["name"],
+ self.default_data["version"],
+ self.default_data["user_id"],
+ self.default_data["experiment_id"],
+ self.default_data["current_stage"],
+ self.default_data["creation_time"],
+ self.default_data["last_updated_time"],
+ self.default_data["source"],
+ self.default_data["dataset"],
+ self.default_data["description"],
+ tags,
+ )
+ self._check(
+ mv,
+ self.default_data["name"],
+ self.default_data["version"],
+ self.default_data["user_id"],
+ self.default_data["experiment_id"],
+ self.default_data["current_stage"],
+ self.default_data["creation_time"],
+ self.default_data["last_updated_time"],
+ self.default_data["source"],
+ self.default_data["dataset"],
+ self.default_data["description"],
+ [t.tag for t in tags],
+ )
diff --git
a/submarine-sdk/pysubmarine/tests/entities/model_registry/test_registered_model.py
b/submarine-sdk/pysubmarine/tests/entities/model_registry/test_registered_model.py
new file mode 100644
index 0000000..e1fa2af
--- /dev/null
+++
b/submarine-sdk/pysubmarine/tests/entities/model_registry/test_registered_model.py
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+
+from submarine.entities.model_registry.registered_model import RegisteredModel
+from submarine.entities.model_registry.registered_model_tag import
RegisteredModelTag
+
+
+class TestRegisteredModel:
+ default_data = {
+ "name": "test",
+ "creation_time": datetime.now(),
+ "last_updated_time": datetime.now(),
+ "description": "registered model description",
+ "tags": [],
+ }
+
+ def _check(self, registered_model, name, creation_time, last_updated_time,
description, tags):
+ isinstance(registered_model, RegisteredModel)
+ assert registered_model.name == name
+ assert registered_model.creation_time == creation_time
+ assert registered_model.last_updated_time == last_updated_time
+ assert registered_model.description == description
+ assert registered_model.tags == tags
+
+ def test_creation_and_hydration(self):
+ rm = RegisteredModel(
+ self.default_data["name"],
+ self.default_data["creation_time"],
+ self.default_data["last_updated_time"],
+ self.default_data["description"],
+ self.default_data["tags"],
+ )
+ self._check(
+ rm,
+ self.default_data["name"],
+ self.default_data["creation_time"],
+ self.default_data["last_updated_time"],
+ self.default_data["description"],
+ self.default_data["tags"],
+ )
+
+ def test_with_tags(self):
+ tag1 = RegisteredModelTag("tag1")
+ tag2 = RegisteredModelTag("tag2")
+ tags = [tag1, tag2]
+ rm = RegisteredModel(
+ self.default_data["name"],
+ self.default_data["creation_time"],
+ self.default_data["last_updated_time"],
+ self.default_data["description"],
+ tags,
+ )
+ self._check(
+ rm,
+ self.default_data["name"],
+ self.default_data["creation_time"],
+ self.default_data["last_updated_time"],
+ self.default_data["description"],
+ [t.tag for t in tags],
+ )
diff --git a/submarine-sdk/pysubmarine/tests/entities/test_metrics.py
b/submarine-sdk/pysubmarine/tests/entities/test_metrics.py
index c8a62e4..8e09de1 100644
--- a/submarine-sdk/pysubmarine/tests/entities/test_metrics.py
+++ b/submarine-sdk/pysubmarine/tests/entities/test_metrics.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import time
+from datetime import datetime
from submarine.entities import Metric
@@ -31,7 +31,7 @@ def test_creation_and_hydration():
key = "alpha"
value = 10000
worker_index = 1
- ts = int(time.time())
+ ts = datetime.now()
step = 0
metric = Metric(key, value, worker_index, ts, step)
diff --git a/submarine-sdk/pysubmarine/tests/store/test_sqlalchemy_store.py
b/submarine-sdk/pysubmarine/tests/store/test_sqlalchemy_store.py
index f30dcd7..d7597d8 100644
--- a/submarine-sdk/pysubmarine/tests/store/test_sqlalchemy_store.py
+++ b/submarine-sdk/pysubmarine/tests/store/test_sqlalchemy_store.py
@@ -13,15 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import time
import unittest
+from datetime import datetime
import pytest
import submarine
from submarine.entities import Metric, Param
from submarine.store.database import models
-from submarine.store.database.models import SqlMetric, SqlParam
+from submarine.store.database.models import SqlExperiment, SqlMetric, SqlParam
from submarine.tracking import utils
JOB_ID = "application_123456789"
@@ -35,6 +35,18 @@ class TestSqlAlchemyStore(unittest.TestCase):
)
self.tracking_uri = utils.get_tracking_uri()
self.store = utils.get_sqlalchemy_store(self.tracking_uri)
+ # TODO: use submarine.tracking.fluent to support experiment create
+ with self.store.ManagedSessionMaker() as session:
+ instance = SqlExperiment(
+ id=JOB_ID,
+ experiment_spec='{"value": 1}',
+ create_by="test",
+ create_time=datetime.now(),
+ update_by=None,
+ update_time=None,
+ )
+ session.add(instance)
+ session.commit()
def tearDown(self):
submarine.set_tracking_uri(None)
@@ -53,8 +65,8 @@ class TestSqlAlchemyStore(unittest.TestCase):
assert params[0].id == JOB_ID
def test_log_metric(self):
- metric1 = Metric("name_1", 5, "worker-1", int(time.time()), 0)
- metric2 = Metric("name_1", 6, "worker-2", int(time.time()), 0)
+ metric1 = Metric("name_1", 5, "worker-1", datetime.now(), 0)
+ metric2 = Metric("name_1", 6, "worker-2", datetime.now(), 0)
self.store.log_metric(JOB_ID, metric1)
self.store.log_metric(JOB_ID, metric2)
diff --git a/submarine-sdk/pysubmarine/tests/tracking/test_tracking.py
b/submarine-sdk/pysubmarine/tests/tracking/test_tracking.py
index 7058741..e167a1c 100644
--- a/submarine-sdk/pysubmarine/tests/tracking/test_tracking.py
+++ b/submarine-sdk/pysubmarine/tests/tracking/test_tracking.py
@@ -14,13 +14,14 @@
# limitations under the License.
import unittest
+from datetime import datetime
from os import environ
import pytest
import submarine
from submarine.store.database import models
-from submarine.store.database.models import SqlMetric, SqlParam
+from submarine.store.database.models import SqlExperiment, SqlMetric, SqlParam
from submarine.tracking import utils
JOB_ID = "application_123456789"
@@ -35,12 +36,24 @@ class TestTracking(unittest.TestCase):
)
self.tracking_uri = utils.get_tracking_uri()
self.store = utils.get_sqlalchemy_store(self.tracking_uri)
+ # TODO: use submarine.tracking.fluent to support experiment create
+ with self.store.ManagedSessionMaker() as session:
+ instance = SqlExperiment(
+ id=JOB_ID,
+ experiment_spec='{"value": 1}',
+ create_by="test",
+ create_time=datetime.now(),
+ update_by=None,
+ update_time=None,
+ )
+ session.add(instance)
+ session.commit()
def tearDown(self):
submarine.set_tracking_uri(None)
models.Base.metadata.drop_all(self.store.engine)
- def log_param(self):
+ def test_log_param(self):
submarine.log_param("name_1", "a")
# Validate params
with self.store.ManagedSessionMaker() as session:
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Metric.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Metric.java
index ae3112d..258e074 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Metric.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Metric.java
@@ -18,10 +18,10 @@
*/
package org.apache.submarine.server.workbench.database.entity;
-import java.math.BigInteger;
-
import org.apache.submarine.server.database.entity.BaseEntity;
+import java.sql.Timestamp;
+
/*
#
+--------------------+-------+-------------------+--------------+---------------+------+--------+
# | id | key | value | worker_index | timestamp
| step | is_nan |
@@ -38,7 +38,7 @@ public class Metric extends BaseEntity {
private String key;
private Float value;
private String workerIndex;
- private BigInteger timestamp;
+ private Timestamp timestamp;
private Integer step;
private Boolean isNan;
@@ -66,11 +66,11 @@ public class Metric extends BaseEntity {
this.workerIndex = workerIndex;
}
- public BigInteger getTimestamp() {
+ public Timestamp getTimestamp() {
return this.timestamp;
}
- public void setTimestamp(BigInteger timestamp) {
+ public void setTimestamp(Timestamp timestamp) {
this.timestamp = timestamp;
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/MetricRestApi.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/MetricRestApi.java
index 26cb50e..d3a60e2 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/MetricRestApi.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/MetricRestApi.java
@@ -18,16 +18,13 @@
*/
package org.apache.submarine.server.workbench.rest;
+import org.apache.submarine.server.response.JsonResponse;
import org.apache.submarine.server.workbench.annotation.SubmarineApi;
import org.apache.submarine.server.workbench.database.entity.Metric;
import org.apache.submarine.server.workbench.database.service.MetricService;
-import org.apache.submarine.server.response.JsonResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.math.BigInteger;
-import java.util.List;
-
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.DELETE;
@@ -39,6 +36,8 @@ import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
+import java.sql.Timestamp;
+import java.util.List;
@Path("/metric")
@Produces("application/json")
@@ -57,7 +56,7 @@ public class MetricRestApi {
public Response listMetric(@QueryParam("metricKey") String metricKey,
@QueryParam("value") Float value,
@QueryParam("workerIndex") String workerIndex,
- @QueryParam("timestamp") BigInteger timestamp,
+ @QueryParam("timestamp") Timestamp timestamp,
@QueryParam("step") Integer step,
@QueryParam("isNan") Boolean isNan,
@QueryParam("id") String id) {
diff --git
a/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/MetricMapper.xml
b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/MetricMapper.xml
index 9922019..5fc85fa 100644
---
a/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/MetricMapper.xml
+++
b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/MetricMapper.xml
@@ -27,7 +27,7 @@
<result column="key" jdbcType="VARCHAR" property="key" />
<result column="value" jdbcType="FLOAT" property="value" />
<result column="worker_index" jdbcType="VARCHAR" property="workerIndex" />
- <result column="timestamp" jdbcType="BIGINT" property="timestamp" />
+ <result column="timestamp" jdbcType="TIMESTAMP" property="timestamp" />
<result column="step" jdbcType="INTEGER" property="step" />
<result column="is_nan" jdbcType="BOOLEAN" property="isNan" />
</resultMap>
@@ -40,40 +40,40 @@
<select id="selectAll" parameterType="java.lang.String"
resultMap="resultMap">
select
<include refid="Base_Column_List" />
- from metrics
+ from metric
where 1 = 1
</select>
<select id="selectById" parameterType="java.lang.String"
resultMap="resultMap">
select
<include refid="Base_Column_List" />
- from metrics
+ from metric
where id = #{id,jdbcType=VARCHAR}
</select>
<delete id="deleteById" parameterType="java.lang.String">
- delete from metrics
+ delete from metric
where id = #{id,jdbcType=VARCHAR}
</delete>
<insert id="insert"
parameterType="org.apache.submarine.server.workbench.database.entity.Metric"
keyProperty="id, key, worker_index, timestamp">
- insert into metrics (id, `key`, value, worker_index, timestamp, step,
is_nan)
+ insert into metric (id, `key`, value, worker_index, timestamp, step,
is_nan)
values (#{id,jdbcType=VARCHAR},
#{key,jdbcType=VARCHAR},
#{value,jdbcType=FLOAT},
#{workerIndex,jdbcType=VARCHAR},
- #{timestamp,jdbcType=BIGINT},
+ #{timestamp,jdbcType=TIMESTAMP},
#{step,jdbcType=INTEGER},
#{isNan,jdbcType=BOOLEAN})
</insert>
<update id="update"
parameterType="org.apache.submarine.server.workbench.database.entity.Metric">
- update metrics
+ update metric
set `key` = #{key,jdbcType=VARCHAR},
value = #{value,jdbcType=FLOAT},
worker_index = #{workerIndex,jdbcType=VARCHAR},
- timestamp = #{timestamp,jdbcType=BIGINT},
+ timestamp = #{timestamp,jdbcType=TIMESTAMP},
step = #{step,jdbcType=INTEGER},
is_nan = #{isNan,jdbcType=BOOLEAN}
where id = #{id,jdbcType=VARCHAR}
@@ -82,7 +82,7 @@
<select id="selectByPrimaryKeySelective" parameterType="java.lang.String"
resultMap="resultMap">
select
<include refid="Base_Column_List" />
- from metrics
+ from metric
where 1 = 1
<if test="key != null">
AND `key` = #{key,jdbcType=VARCHAR}
@@ -91,7 +91,7 @@
AND worker_index = #{workerIndex,jdbcType=VARCHAR}
</if>
<if test="timestamp != null">
- AND timestamp = #{timestamp,jdbcType=BIGINT}
+ AND timestamp = #{timestamp,jdbcType=TIMESTAMP}
</if>
<if test="step != null">
AND step = #{step,jdbcType=INTEGER}
diff --git
a/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/ParamMapper.xml
b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/ParamMapper.xml
index 4e97759..dd76329 100644
---
a/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/ParamMapper.xml
+++
b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/ParamMapper.xml
@@ -37,30 +37,30 @@
<select id="selectAll" parameterType="java.lang.String"
resultMap="resultMap">
select
<include refid="Base_Column_List" />
- from params
+ from param
where 1 = 1
</select>
<select id="selectById" parameterType="java.lang.String"
resultMap="resultMap">
select
<include refid="Base_Column_List" />
- from params
+ from param
where id = #{id,jdbcType=VARCHAR}
</select>
<delete id="deleteById" parameterType="java.lang.String">
- delete from params
+ delete from param
where id = #{id,jdbcType=VARCHAR}
</delete>
<insert id="insert"
parameterType="org.apache.submarine.server.workbench.database.entity.Param"
keyProperty="id, key, worker_index">
- insert into params (id, `key`, value, worker_index)
+ insert into param (id, `key`, value, worker_index)
values (#{id,jdbcType=VARCHAR}, #{key,jdbcType=VARCHAR},
#{value,jdbcType=FLOAT}, #{workerIndex,jdbcType=VARCHAR})
</insert>
<update id="update"
parameterType="org.apache.submarine.server.workbench.database.entity.Param">
- update params
+ update param
set `key` = #{key,jdbcType=VARCHAR},
value = #{value,jdbcType=FLOAT},
worker_index = #{workerIndex,jdbcType=VARCHAR}
@@ -70,7 +70,7 @@
<select id="selectByPrimaryKeySelective" parameterType="java.lang.String"
resultMap="resultMap">
select
<include refid="Base_Column_List" />
- from params
+ from param
where 1 = 1
<if test="key != null">
AND `key` = #{key,jdbcType=VARCHAR}
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench/database/service/MetricServiceTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench/database/service/MetricServiceTest.java
index a919acf..9798151 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench/database/service/MetricServiceTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench/database/service/MetricServiceTest.java
@@ -18,13 +18,17 @@
*/
package org.apache.submarine.server.workbench.database.service;
+import org.apache.submarine.server.experiment.database.ExperimentEntity;
+import org.apache.submarine.server.experiment.database.ExperimentService;
import org.apache.submarine.server.workbench.database.entity.Metric;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.math.BigInteger;
+import java.sql.Timestamp;
+import java.util.Date;
import java.util.List;
import static junit.framework.TestCase.assertEquals;
@@ -33,6 +37,20 @@ import static org.junit.Assert.assertTrue;
public class MetricServiceTest {
private static final Logger LOG =
LoggerFactory.getLogger(MetricServiceTest.class);
MetricService metricService = new MetricService();
+ ExperimentService experimentService = new ExperimentService();
+
+ // Id of metric is a foreign key for experiment id so experiment must be
created before test.
+ @Before
+ public void createExperiment() {
+ ExperimentEntity entity = new ExperimentEntity();
+ String id = "test_application_1234";
+ String spec = "{\"value\": 1}";
+
+ entity.setId(id);
+ entity.setExperimentSpec(spec);
+
+ experimentService.insert(entity);
+ }
@After
public void removeAllRecord() throws Exception {
@@ -41,16 +59,20 @@ public class MetricServiceTest {
for (Metric metric : metricList) {
metricService.deleteById(metric.getId());
}
+
+ experimentService.selectAll().forEach(e ->
experimentService.delete(e.getId()));
}
@Test
public void testSelect() throws Exception {
+ Timestamp timestamp = new Timestamp(new Date().getTime());
+
Metric metric = new Metric();
metric.setId("test_application_1234");
metric.setKey("test_score");
metric.setValue((float) 0.666667);
metric.setWorkerIndex("test_worker-1");
- metric.setTimestamp(new BigInteger("1569139525097"));
+ metric.setTimestamp(timestamp);
metric.setStep(0);
metric.setIsNan(false);
boolean result = metricService.insert(metric);
@@ -68,22 +90,26 @@ public class MetricServiceTest {
@Test
public void testUpdate() throws Exception {
+ Timestamp timestamp = new Timestamp(new Date().getTime());
+
Metric metric = new Metric();
metric.setId("test_application_1234");
metric.setKey("test_score");
metric.setValue((float) 0.666667);
metric.setWorkerIndex("test_worker-2");
- metric.setTimestamp(new BigInteger("1569139525098"));
+ metric.setTimestamp(timestamp);
metric.setStep(0);
metric.setIsNan(false);
boolean result = metricService.insert(metric);
assertTrue(result);
+ Timestamp nextTimestamp = new Timestamp(new Date().getTime());
+
metric.setId("test_application_1234");
metric.setKey("test_scoreNew");
metric.setValue((float) 0.766667);
metric.setWorkerIndex("test_worker-New");
- metric.setTimestamp(new BigInteger("2569139525098"));
+ metric.setTimestamp(nextTimestamp);
metric.setStep(1);
metric.setIsNan(true);
@@ -96,12 +122,14 @@ public class MetricServiceTest {
@Test
public void testDelete() throws Exception {
+ Timestamp timestamp = new Timestamp(new Date().getTime());
+
Metric metric = new Metric();
metric.setId("test_application_1234");
metric.setKey("test_score");
metric.setValue((float) 0.666667);
metric.setWorkerIndex("test_worker-2");
- metric.setTimestamp(new BigInteger("1569139525098"));
+ metric.setTimestamp(timestamp);
metric.setStep(0);
metric.setIsNan(false);
boolean result = metricService.insert(metric);
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench/database/service/ParamServiceTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench/database/service/ParamServiceTest.java
index 2cbe1be..1904329 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench/database/service/ParamServiceTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench/database/service/ParamServiceTest.java
@@ -18,8 +18,11 @@
*/
package org.apache.submarine.server.workbench.database.service;
+import org.apache.submarine.server.experiment.database.ExperimentEntity;
+import org.apache.submarine.server.experiment.database.ExperimentService;
import org.apache.submarine.server.workbench.database.entity.Param;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +35,19 @@ import static org.junit.Assert.assertTrue;
public class ParamServiceTest {
private static final Logger LOG =
LoggerFactory.getLogger(ParamServiceTest.class);
ParamService paramService = new ParamService();
+ ExperimentService experimentService = new ExperimentService();
+ @Before
+ public void createExperiment() throws Exception {
+ ExperimentEntity entity = new ExperimentEntity();
+ String id = "test_application_12345";
+ String spec = "{\"value\": 1}";
+
+ entity.setId(id);
+ entity.setExperimentSpec(spec);
+
+ experimentService.insert(entity);
+ }
@After
public void removeAllRecord() throws Exception {
List<Param> paramList = paramService.selectAll();
@@ -40,12 +55,14 @@ public class ParamServiceTest {
for (Param param : paramList) {
paramService.deleteById(param.getId());
}
+
+ experimentService.selectAll().forEach(e ->
experimentService.delete(e.getId()));
}
@Test
public void testSelect() throws Exception {
Param param = new Param();
- param.setId("test_application_1234");
+ param.setId("test_application_12345");
param.setKey("test_score");
param.setValue("199");
param.setWorkerIndex("test_worker-1");
@@ -65,7 +82,7 @@ public class ParamServiceTest {
@Test
public void testUpdate() throws Exception {
Param param = new Param();
- param.setId("test_application_1234");
+ param.setId("test_application_12345");
param.setKey("test_score");
param.setValue("100");
param.setWorkerIndex("test_worker-2");
@@ -85,7 +102,7 @@ public class ParamServiceTest {
@Test
public void testDelete() throws Exception {
Param param = new Param();
- param.setId("test_application_1234");
+ param.setId("test_application_12345");
param.setKey("test_score");
param.setValue("100");
param.setWorkerIndex("test_worker-2");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
