Log Message
Rewrite 'pull-os-versions' script in _javascript_ to add support for reporting os revisions with sub commits. https://bugs.webkit.org/show_bug.cgi?id=169542
Reviewed by Ryosuke Niwa. Extend '/api/commits/<repository>/last-reported' to accept a range and return last reported commits in given range. Rewrite 'pull-os-versions' in _javascript_ and add unit tests for it. Instead of writing query manually while searching criteria contains null columns, use the methods provided in 'db.php'. Add '.gitignore' file to ommit files generated by while running tests/instances locally. * .gitignore: Added. * public/api/commits.php: * public/api/report-commits.php: * public/include/commit-log-fetcher.php: * public/include/db.php: 'null_columns' of prepare_params should be a reference. * public/include/report-processor.php: * server-tests/api-commits.js: (then): * server-tests/api-report-commits-tests.js: * server-tests/resources/mock-logger.js: Added. (MockLogger): (MockLogger.prototype.log): (MockLogger.prototype.error): * server-tests/resources/mock-subprocess.js: Added. (MockSubprocess.call): (MockSubprocess.waitingForInvocation): (MockSubprocess.inject): (MockSubprocess.reset): * server-tests/tools-buildbot-triggerable-tests.js: (MockLogger): Deleted. (MockLogger.prototype.log): Deleted. (MockLogger.prototype.error): Deleted. * server-tests/tools-os-build-fetcher-tests.js: Added. (beforeEach): (return.waitingForInvocationPromise.then): (then): (string_appeared_here.return.waitingForInvocationPromise.then): (return.addSlaveForReport.emptyReport.then): * tools/js/os-build-fetcher.js: Added. (OSBuildFetcher): (OSBuildFetcher.prototype._fetchAvailableBuilds): (OSBuildFetcher.prototype._computeOrder): (OSBuildFetcher.prototype._commitsForAvailableBuilds.return.this._subprocess.call.then.): (OSBuildFetcher.prototype._commitsForAvailableBuilds): (OSBuildFetcher.prototype._addSubCommitsForBuild): (OSBuildFetcher.prototype._submitCommits): (OSBuildFetcher.prototype.fetchAndReportNewBuilds): * tools/js/subprocess.js: Added. (const.childProcess.require.string_appeared_here.Subprocess.prototype.call): (const.childProcess.require.string_appeared_here.Subprocess): * tools/pull-os-versions.js: Added. (main): (syncLoop): * tools/sync-commits.py: (Repository.fetch_commits_and_submit):
Modified Paths
- trunk/Websites/perf.webkit.org/ChangeLog
- trunk/Websites/perf.webkit.org/public/api/commits.php
- trunk/Websites/perf.webkit.org/public/api/report-commits.php
- trunk/Websites/perf.webkit.org/public/include/commit-log-fetcher.php
- trunk/Websites/perf.webkit.org/public/include/db.php
- trunk/Websites/perf.webkit.org/public/include/report-processor.php
- trunk/Websites/perf.webkit.org/server-tests/api-commits.js
- trunk/Websites/perf.webkit.org/server-tests/api-report-commits-tests.js
- trunk/Websites/perf.webkit.org/server-tests/tools-buildbot-triggerable-tests.js
- trunk/Websites/perf.webkit.org/tools/sync-commits.py
Added Paths
- trunk/Websites/perf.webkit.org/.gitignore
- trunk/Websites/perf.webkit.org/server-tests/resources/mock-logger.js
- trunk/Websites/perf.webkit.org/server-tests/resources/mock-subprocess.js
- trunk/Websites/perf.webkit.org/server-tests/tools-os-build-fetcher-tests.js
- trunk/Websites/perf.webkit.org/tools/js/os-build-fetcher.js
- trunk/Websites/perf.webkit.org/tools/js/subprocess.js
- trunk/Websites/perf.webkit.org/tools/pull-os-versions.js
Diff
Added: trunk/Websites/perf.webkit.org/.gitignore (0 => 213976)
--- trunk/Websites/perf.webkit.org/.gitignore (rev 0)
+++ trunk/Websites/perf.webkit.org/.gitignore 2017-03-15 08:35:07 UTC (rev 213976)
@@ -0,0 +1,3 @@
+public/data
+node_modules
+server-tests/resources/test-server.log
Modified: trunk/Websites/perf.webkit.org/ChangeLog (213975 => 213976)
--- trunk/Websites/perf.webkit.org/ChangeLog 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/ChangeLog 2017-03-15 08:35:07 UTC (rev 213976)
@@ -1,3 +1,61 @@
+2017-03-15 Dewei Zhu <dewei_...@apple.com>
+
+ Rewrite 'pull-os-versions' script in _javascript_ to add support for reporting os revisions with sub commits.
+ https://bugs.webkit.org/show_bug.cgi?id=169542
+
+ Reviewed by Ryosuke Niwa.
+
+ Extend '/api/commits/<repository>/last-reported' to accept a range and return last reported commits in given range.
+ Rewrite 'pull-os-versions' in _javascript_ and add unit tests for it.
+ Instead of writing query manually while searching criteria contains null columns, use the methods provided in 'db.php'.
+ Add '.gitignore' file to ommit files generated by while running tests/instances locally.
+
+ * .gitignore: Added.
+ * public/api/commits.php:
+ * public/api/report-commits.php:
+ * public/include/commit-log-fetcher.php:
+ * public/include/db.php: 'null_columns' of prepare_params should be a reference.
+ * public/include/report-processor.php:
+ * server-tests/api-commits.js:
+ (then):
+ * server-tests/api-report-commits-tests.js:
+ * server-tests/resources/mock-logger.js: Added.
+ (MockLogger):
+ (MockLogger.prototype.log):
+ (MockLogger.prototype.error):
+ * server-tests/resources/mock-subprocess.js: Added.
+ (MockSubprocess.call):
+ (MockSubprocess.waitingForInvocation):
+ (MockSubprocess.inject):
+ (MockSubprocess.reset):
+ * server-tests/tools-buildbot-triggerable-tests.js:
+ (MockLogger): Deleted.
+ (MockLogger.prototype.log): Deleted.
+ (MockLogger.prototype.error): Deleted.
+ * server-tests/tools-os-build-fetcher-tests.js: Added.
+ (beforeEach):
+ (return.waitingForInvocationPromise.then):
+ (then):
+ (string_appeared_here.return.waitingForInvocationPromise.then):
+ (return.addSlaveForReport.emptyReport.then):
+ * tools/js/os-build-fetcher.js: Added.
+ (OSBuildFetcher):
+ (OSBuildFetcher.prototype._fetchAvailableBuilds):
+ (OSBuildFetcher.prototype._computeOrder):
+ (OSBuildFetcher.prototype._commitsForAvailableBuilds.return.this._subprocess.call.then.):
+ (OSBuildFetcher.prototype._commitsForAvailableBuilds):
+ (OSBuildFetcher.prototype._addSubCommitsForBuild):
+ (OSBuildFetcher.prototype._submitCommits):
+ (OSBuildFetcher.prototype.fetchAndReportNewBuilds):
+ * tools/js/subprocess.js: Added.
+ (const.childProcess.require.string_appeared_here.Subprocess.prototype.call):
+ (const.childProcess.require.string_appeared_here.Subprocess):
+ * tools/pull-os-versions.js: Added.
+ (main):
+ (syncLoop):
+ * tools/sync-commits.py:
+ (Repository.fetch_commits_and_submit):
+
2017-03-14 Ryosuke Niwa <rn...@webkit.org>
Make server tests return a promise instead of manually calling done
Modified: trunk/Websites/perf.webkit.org/public/api/commits.php (213975 => 213976)
--- trunk/Websites/perf.webkit.org/public/api/commits.php 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/public/api/commits.php 2017-03-15 08:35:07 UTC (rev 213976)
@@ -32,7 +32,12 @@
} else if ($filter == 'latest') {
$commits = $fetcher->fetch_latest($repository_id);
} else if ($filter == 'last-reported') {
- $commits = $fetcher->fetch_last_reported($repository_id);
+ $from = array_get($_GET, 'from');
+ $to = array_get($_GET, 'to');
+ if ($from && $to)
+ $commits = $fetcher->fetch_last_reported_between_orders($repository_id, $from, $to);
+ else
+ $commits = $fetcher->fetch_last_reported($repository_id);
} else if (ctype_alnum($filter)) {
$commits = $fetcher->fetch_revision($repository_id, $filter);
} else { // V2 UI compatibility.
Modified: trunk/Websites/perf.webkit.org/public/api/report-commits.php (213975 => 213976)
--- trunk/Websites/perf.webkit.org/public/api/report-commits.php 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/public/api/report-commits.php 2017-03-15 08:35:07 UTC (rev 213976)
@@ -26,7 +26,7 @@
$db->begin_transaction();
foreach ($commits as $commit_info) {
- $repository_id = $db->select_or_insert_repository_row($commit_info['repository'], NULL);
+ $repository_id = $db->select_or_insert_row('repositories', 'repository', array('name' => $commit_info['repository'], 'owner' => NULL));
if (!$repository_id) {
$db->rollback_transaction();
exit_with_error('FailedToInsertRepository', array('commit' => $commit_info));
@@ -40,7 +40,7 @@
$db->rollback_transaction();
exit_with_error('SubCommitShouldNotContainTimestamp', array('commit' => $sub_commit_info));
}
- $sub_commit_repository_id = $db->select_or_insert_repository_row($sub_commit_repository_name, $repository_id);
+ $sub_commit_repository_id = $db->select_or_insert_row('repositories', 'repository', array('name' => $sub_commit_repository_name, 'owner' => $repository_id));
if (!$sub_commit_repository_id) {
$db->rollback_transaction();
exit_with_error('FailedToInsertRepository', array('commit' => $sub_commit_info));
Modified: trunk/Websites/perf.webkit.org/public/include/commit-log-fetcher.php (213975 => 213976)
--- trunk/Websites/perf.webkit.org/public/include/commit-log-fetcher.php 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/public/include/commit-log-fetcher.php 2017-03-15 08:35:07 UTC (rev 213976)
@@ -27,13 +27,14 @@
function repository_id_from_name($name)
{
- $repository_row = $this->db->query_and_fetch_all('SELECT repository_id FROM repositories WHERE repository_name = $1 AND repository_owner is NULL', array($name));
+ $repository_row = $this->db->select_first_row('repositories', 'repository', array('name' => $name, 'owner' => NULL));
if (!$repository_row)
return NULL;
- return $repository_row[0]['repository_id'];
+ return $repository_row['repository_id'];
}
- function fetch_between($repository_id, $first, $second, $keyword = NULL) {
+ function fetch_between($repository_id, $first, $second, $keyword = NULL)
+ {
$statements = 'SELECT commit_id as "id",
commit_revision as "revision",
commit_previous_commit as "previousCommit",
@@ -83,6 +84,25 @@
return $commits;
}
+ # FIXME: this is not DRY. Ideally, $db should provide the ability to search with criteria that specifies a range.
+ function fetch_last_reported_between_orders($repository_id, $from, $to)
+ {
+ $statements = 'SELECT * FROM commits LEFT OUTER JOIN committers ON commit_committer = committer_id
+ WHERE commit_repository = $1 AND commit_reported = true';
+ $from = intval($from);
+ $to = intval($to);
+ $statements .= ' AND commit_order >= $2 AND commit_order <= $3 ORDER BY commit_order DESC LIMIT 1';
+
+ $commits = $this->db->query_and_fetch_all($statements, array($repository_id, $from, $to));
+ if (!is_array($commits))
+ return NULL;
+
+ foreach ($commits as &$commit)
+ $commit = $this->format_single_commit($commit)[0];
+
+ return $commits;
+ }
+
function fetch_oldest($repository_id) {
return $this->format_single_commit($this->db->select_first_row('commits', 'commit', array('repository' => $repository_id), array('time', 'order')));
}
Modified: trunk/Websites/perf.webkit.org/public/include/db.php (213975 => 213976)
--- trunk/Websites/perf.webkit.org/public/include/db.php 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/public/include/db.php 2017-03-15 08:35:07 UTC (rev 213976)
@@ -105,7 +105,7 @@
return $prefix ? $prefix . '_' . $column : $column;
}
- private function prepare_params($params, &$placeholders, &$values, $null_columns = NULL) {
+ private function prepare_params($params, &$placeholders, &$values, &$null_columns = NULL) {
$column_names = array();
$i = count($values) + 1;
@@ -207,26 +207,6 @@
return $rows ? ($returning == '*' ? $rows[0] : $rows[0][$returning_column_name]) : NULL;
}
- // FIXME: Should improve _select_update_or_insert_row to handle the NULL column case.
- function select_or_insert_repository_row($repository_name, $repository_owner_id)
- {
- $result = NULL;
- if ($repository_owner_id == NULL) {
- $result = $this->query_and_fetch_all('INSERT INTO repositories (repository_name) SELECT $1
- WHERE NOT EXISTS (SELECT repository_id FROM repositories WHERE repository_name = $2 AND repository_owner IS NULL) RETURNING repository_id',
- array($repository_name, $repository_name));
- if (!$result)
- $result = $this->query_and_fetch_all('SELECT repository_id FROM repositories WHERE repository_name = $1 AND repository_owner IS NULL', array($repository_name));
- } else {
- $result = $this->query_and_fetch_all('INSERT INTO repositories (repository_name, repository_owner) SELECT $1, $2
- WHERE NOT EXISTS (SELECT repository_id FROM repositories WHERE (repository_name, repository_owner) = ($3, $4)) RETURNING repository_id',
- array($repository_name, $repository_owner_id, $repository_name, $repository_owner_id));
- if (!$result)
- $result = $this->query_and_fetch_all('SELECT repository_id FROM repositories WHERE (repository_name, repository_owner) = ($1, $2)', array($repository_name, $repository_owner_id));
- }
- return $result ? $result[0]['repository_id'] : NULL;
- }
-
function select_first_row($table, $prefix, $params, $order_by = NULL) {
return $this->select_first_or_last_row($table, $prefix, $params, $order_by, FALSE);
}
Modified: trunk/Websites/perf.webkit.org/public/include/report-processor.php (213975 => 213976)
--- trunk/Websites/perf.webkit.org/public/include/report-processor.php 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/public/include/report-processor.php 2017-03-15 08:35:07 UTC (rev 213976)
@@ -150,7 +150,7 @@
foreach ($revisions as $repository_name => $revision_data) {
- $repository_id = $this->db->select_or_insert_repository_row($repository_name, NULL);
+ $repository_id = $this->db->select_or_insert_row('repositories', 'repository', array('name' => $repository_name, 'owner' => NULL));
if (!$repository_id)
$this->exit_with_error('FailedToInsertRepository', array('name' => $repository_name));
Modified: trunk/Websites/perf.webkit.org/server-tests/api-commits.js (213975 => 213976)
--- trunk/Websites/perf.webkit.org/server-tests/api-commits.js 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/server-tests/api-commits.js 2017-03-15 08:35:07 UTC (rev 213976)
@@ -301,6 +301,47 @@
});
});
+ describe('/api/commits/<repository>/last-reported?from=<start_order>&to=<end_order>', () => {
+ it("should return a list of commit in given valid order range", () => {
+ const db = TestServer.database();
+ return Promise.all([
+ db.insert('repositories', {'id': 1, 'name': 'OSX'}),
+ db.insert('commits', {'repository': 1, 'revision': 'Sierra16C67', 'order': 367, 'reported': true}),
+ db.insert('commits', {'repository': 1, 'revision': 'Sierra16C68', 'order': 368, 'reported': true}),
+ db.insert('commits', {'repository': 1, 'revision': 'Sierra16C69', 'order': 369, 'reported': false}),
+ db.insert('commits', {'repository': 1, 'revision': 'Sierra16D32', 'order': 432, 'reported': true})
+ ]).then(() => {
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=367&to=370');
+ }).then((response) => {
+ assert.equal(response['status'], 'OK');
+ const results = response['commits'];
+ assert.equal(results.length, 1);
+ const commit = results[0];
+ assert.equal(commit.revision, 'Sierra16C68');
+ }).then(() => {
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=370&to=367');
+ }).then((response) => {
+ assert.equal(response['status'], 'OK');
+ const results = response['commits'];
+ assert.equal(results.length, 0);
+ }).then(() => {
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=200&to=299');
+ }).then((response) => {
+ assert.equal(response['status'], 'OK');
+ const results = response['commits'];
+ assert.equal(results.length, 0);
+ }).then(() => {
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=369&to=432');
+ }).then((response) => {
+ assert.equal(response['status'], 'OK');
+ const results = response['commits'];
+ assert.equal(results.length, 1);
+ const commit = results[0];
+ assert.equal(commit.revision, 'Sierra16D32');
+ });
+ });
+ });
+
describe('/api/commits/<repository>/<commit>', () => {
it("should return RepositoryNotFound when there are no matching repository", () => {
return TestServer.remoteAPI().getJSON('/api/commits/WebKit/210949').then((response) => {
Modified: trunk/Websites/perf.webkit.org/server-tests/api-report-commits-tests.js (213975 => 213976)
--- trunk/Websites/perf.webkit.org/server-tests/api-report-commits-tests.js 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/server-tests/api-report-commits-tests.js 2017-03-15 08:35:07 UTC (rev 213976)
@@ -298,7 +298,7 @@
]
}
- it("should distinguish between repositories with the asme name but with a different owner.", () => {
+ it("should distinguish between repositories with the same name but with a different owner.", () => {
return addSlaveForReport(sameRepositoryNameInSubCommitAndMajorCommit).then(() => {
return TestServer.remoteAPI().postJSON('/api/report-commits/', sameRepositoryNameInSubCommitAndMajorCommit);
}).then((response) => {
Added: trunk/Websites/perf.webkit.org/server-tests/resources/mock-logger.js (0 => 213976)
--- trunk/Websites/perf.webkit.org/server-tests/resources/mock-logger.js (rev 0)
+++ trunk/Websites/perf.webkit.org/server-tests/resources/mock-logger.js 2017-03-15 08:35:07 UTC (rev 213976)
@@ -0,0 +1,14 @@
+'use strict';
+
+class MockLogger {
+ constructor()
+ {
+ this._logs = [];
+ }
+
+ log(text) { this._logs.push(text); }
+ error(text) { this._logs.push(text); }
+}
+
+if (typeof module != 'undefined')
+ module.exports.MockLogger = MockLogger;
Added: trunk/Websites/perf.webkit.org/server-tests/resources/mock-subprocess.js (0 => 213976)
--- trunk/Websites/perf.webkit.org/server-tests/resources/mock-subprocess.js (rev 0)
+++ trunk/Websites/perf.webkit.org/server-tests/resources/mock-subprocess.js 2017-03-15 08:35:07 UTC (rev 213976)
@@ -0,0 +1,57 @@
+var MockSubprocess = {
+ execute: function (command)
+ {
+ const invocation = {command};
+ invocation.promise = new Promise((resolve, reject) => {
+ invocation.resolve = resolve;
+ invocation.reject = reject;
+ });
+
+ if (this._waitingInvocation) {
+ this._waitingInvocationResolver();
+ this._waitingInvocation = null;
+ this._waitingInvocationResolver = null;
+ }
+
+ this.invocations.push(invocation);
+ return invocation.promise;
+ },
+ waitingForInvocation: function ()
+ {
+ if (!this._waitingInvocation) {
+ this._waitingInvocation = new Promise(function (resolve, reject) {
+ MockSubprocess._waitingInvocationResolver = resolve;
+ });
+ }
+ return this._waitingInvocation;
+ },
+ inject: function ()
+ {
+ const originalSubprocess = global.Subprocess;
+
+ beforeEach(function () {
+ MockSubprocess.reset();
+ originalSubprocess = global.Subprocess;
+ global.Subprocess = MockSubprocess;
+ });
+
+ afterEach(function () {
+ global.Subprocess = originalSubprocess;
+ });
+
+ return MockSubprocess.invocations ;
+ },
+ reset: function ()
+ {
+ MockSubprocess.invocations = [];
+ MockSubprocess._waitingInvocation = null;
+ MockSubprocess._waitingInvocationResolver = null;
+ },
+
+ _waitingInvocation: null,
+ _waitingInvocationResolver: null,
+ invocations: [],
+};
+
+if (typeof module != 'undefined')
+ module.exports.MockSubprocess = MockSubprocess;
Modified: trunk/Websites/perf.webkit.org/server-tests/tools-buildbot-triggerable-tests.js (213975 => 213976)
--- trunk/Websites/perf.webkit.org/server-tests/tools-buildbot-triggerable-tests.js 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/server-tests/tools-buildbot-triggerable-tests.js 2017-03-15 08:35:07 UTC (rev 213976)
@@ -7,17 +7,8 @@
const MockRemoteAPI = require('../unit-tests/resources/mock-remote-api.js').MockRemoteAPI;
const TestServer = require('./resources/test-server.js');
const prepareServerTest = require('./resources/common-operations.js').prepareServerTest;
+const MockLogger = require('./resources/mock-logger.js').MockLogger;
-class MockLogger {
- constructor()
- {
- this._logs = [];
- }
-
- log(text) { this._logs.push(text); }
- error(text) { this._logs.push(text); }
-}
-
describe('BuildbotTriggerable', function () {
prepareServerTest(this);
Added: trunk/Websites/perf.webkit.org/server-tests/tools-os-build-fetcher-tests.js (0 => 213976)
--- trunk/Websites/perf.webkit.org/server-tests/tools-os-build-fetcher-tests.js (rev 0)
+++ trunk/Websites/perf.webkit.org/server-tests/tools-os-build-fetcher-tests.js 2017-03-15 08:35:07 UTC (rev 213976)
@@ -0,0 +1,515 @@
+'use strict';
+
+const assert = require('assert');
+
+const OSBuildFetcher = require('../tools/js/os-build-fetcher.js').OSBuildFetcher;
+const MockRemoteAPI = require('../unit-tests/resources/mock-remote-api.js').MockRemoteAPI;
+const TestServer = require('./resources/test-server.js');
+const addSlaveForReport = require('./resources/common-operations.js').addSlaveForReport;
+const prepareServerTest = require('./resources/common-operations.js').prepareServerTest;
+const MockSubprocess = require('./resources/mock-subprocess.js').MockSubprocess;
+const MockLogger = require('./resources/mock-logger.js').MockLogger;
+
+
+describe('OSBuildFetcher', function() {
+ prepareServerTest(this);
+
+ beforeEach(function () {
+ MockRemoteAPI.reset('http://build.webkit.org');
+ MockSubprocess.reset();
+ });
+
+ const emptyReport = {
+ 'slaveName': 'someSlave',
+ 'slavePassword': 'somePassword',
+ };
+
+ const slaveAuth = {
+ 'name': 'someSlave',
+ 'password': 'somePassword'
+ };
+
+ const subCommitWithWebKit = {
+ 'WebKit': {'revision': '141978'}
+ };
+
+ const anotherSubCommitWithWebKit = {
+ 'WebKit': {'revision': '141999',}
+ };
+
+ const anotherSubCommitWithWebKitAndJavaScriptCore = {
+ 'WebKit': {'revision': '142000'},
+ '_javascript_Core': {'revision': '142000'}
+ };
+
+ const osxCommit = {
+ 'repository': 'OSX',
+ 'revision': 'Sierra16D32',
+ 'order': 1603003200
+ };
+
+ const anotherOSXCommit = {
+ 'repository': 'OSX',
+ 'revision': 'Sierra16E32',
+ 'order': 1603003200
+ };
+
+
+ const config = {
+ 'name': 'OSX',
+ 'customCommands': [
+ {
+ 'command': ['list', 'all osx 16Dxx builds'],
+ 'subCommitCommand': ['list', 'subCommit', 'for', 'revision'],
+ 'linesToIgnore': '^\\.*$',
+ 'minRevision': 'Sierra16D0',
+ 'maxRevision': 'Sierra16D999'
+ },
+ {
+ 'command': ['list', 'all osx 16Exx builds'],
+ 'subCommitCommand': ['list', 'subCommit', 'for', 'revision'],
+ 'linesToIgnore': '^\\.*$',
+ 'minRevision': 'Sierra16E0',
+ 'maxRevision': 'Sierra16E999'
+ }
+ ]
+ };
+
+
+ const configWithoutSubCommitCommand = {
+ 'name': 'OSX',
+ 'customCommands': [
+ {
+ 'command': ['list', 'all osx 16Dxx builds'],
+ 'linesToIgnore': '^\\.*$',
+ 'minRevision': 'Sierra16D0',
+ 'maxRevision': 'Sierra16D999'
+ },
+ {
+ 'command': ['list', 'all osx 16Exx builds'],
+ 'linesToIgnore': '^\\.*$',
+ 'minRevision': 'Sierra16E0',
+ 'maxRevision': 'Sierra16E999'
+ }
+ ]
+ };
+
+ describe('OSBuilderFetcher._computeOrder', () => {
+ it('should calculate the right order for a given valid revision', () => {
+ const fetcher = new OSBuildFetcher();
+ assert.equal(fetcher._computeOrder('Sierra16D32'), 1603003200);
+ assert.equal(fetcher._computeOrder('16D321'), 1603032100);
+ assert.equal(fetcher._computeOrder('16d321'), 1603032100);
+ assert.equal(fetcher._computeOrder('16D321z'), 1603032126);
+ assert.equal(fetcher._computeOrder('16d321Z'), 1603032126);
+ });
+
+ it('should throw assertion error when given a invalid revision', () => {
+ const fetcher = new OSBuildFetcher();
+ assert.throws(() => fetcher._computeOrder('invalid'), (error) => error.name == 'AssertionError');
+ assert.throws(() => fetcher._computeOrder(''), (error) => error.name == 'AssertionError');
+ assert.throws(() => fetcher._computeOrder('16'), (error) => error.name == 'AssertionError');
+ assert.throws(() => fetcher._computeOrder('16D'), (error) => error.name == 'AssertionError');
+ assert.throws(() => fetcher._computeOrder('123'), (error) => error.name == 'AssertionError');
+ assert.throws(() => fetcher._computeOrder('D123'), (error) => error.name == 'AssertionError');
+ assert.throws(() => fetcher._computeOrder('123z'), (error) => error.name == 'AssertionError');
+ assert.throws(() => fetcher._computeOrder('16[163'), (error) => error.name == 'AssertionError');
+ assert.throws(() => fetcher._computeOrder('16D163['), (error) => error.name == 'AssertionError');
+ })
+ });
+
+ describe('OSBuilderFetcher._commitsForAvailableBuilds', () => {
+ it('should only return commits whose orders are higher than specified order', () => {
+ const logger = new MockLogger;
+ const fetchter = new OSBuildFetcher(null, null, null, MockSubprocess, logger);
+ const waitingForInvocationPromise = MockSubprocess.waitingForInvocation();
+ const fetchCommitsPromise = fetchter._commitsForAvailableBuilds('OSX', ['list', 'build1'], '^\\.*$', 1604000000);
+
+ return waitingForInvocationPromise.then(() => {
+ assert.equal(MockSubprocess.invocations.length, 1);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'build1']);
+ MockSubprocess.invocations[0].resolve('16D321\n16E321z\n\n16F321');
+ return fetchCommitsPromise;
+ }).then((results) => {
+ assert.equal(results.length, 2);
+ assert.equal(results[0]['repository'], 'OSX');
+ assert.equal(results[0]['revision'], '16E321z');
+ assert.equal(results[0]['order'], 1604032126);
+ assert.equal(results[1]['repository'], 'OSX');
+ assert.equal(results[1]['revision'], '16F321');
+ assert.equal(results[1]['order'], 1605032100);
+ });
+ });
+ });
+
+ describe('OSBuildFetcher._addSubCommitsForBuild', () => {
+ it('should add sub-commit info for commits', () => {
+ const logger = new MockLogger;
+ const fetchter = new OSBuildFetcher(null, null, null, MockSubprocess, logger);
+ const waitingForInvocationPromise = MockSubprocess.waitingForInvocation();
+ const addSubCommitPromise = fetchter._addSubCommitsForBuild([osxCommit, anotherOSXCommit], ['subCommit', 'for', 'revision']);
+
+ return waitingForInvocationPromise.then(() => {
+ assert.equal(MockSubprocess.invocations.length, 1);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['subCommit', 'for', 'revision', 'Sierra16D32']);
+ MockSubprocess.invocations[0].resolve(JSON.stringify(subCommitWithWebKit));
+ MockSubprocess.reset();
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ assert.equal(MockSubprocess.invocations.length, 1);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['subCommit', 'for', 'revision', 'Sierra16E32']);
+ MockSubprocess.invocations[0].resolve(JSON.stringify(anotherSubCommitWithWebKit));
+ return addSubCommitPromise;
+ }).then((results) => {
+ assert.equal(results.length, 2);
+ assert.equal(results[0]['repository'], osxCommit['repository']);
+ assert.equal(results[0]['revision'], osxCommit['revision']);
+ assert.deepEqual(results[0]['subCommits'], subCommitWithWebKit);
+ assert.equal(results[1]['repository'], anotherOSXCommit['repository']);
+ assert.equal(results[1]['revision'], anotherOSXCommit['revision']);
+ assert.deepEqual(results[1]['subCommits'], anotherSubCommitWithWebKit);
+ });
+ });
+
+ it('should fail if the command to get sub-commit info fails', () => {
+ const logger = new MockLogger;
+ const fetchter = new OSBuildFetcher(null, null, null, MockSubprocess, logger);
+ const waitingForInvocationPromise = MockSubprocess.waitingForInvocation();
+ const addSubCommitPromise = fetchter._addSubCommitsForBuild([osxCommit], ['subCommit', 'for', 'revision'])
+
+ return waitingForInvocationPromise.then(() => {
+ assert.equal(MockSubprocess.invocations.length, 1);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['subCommit', 'for', 'revision', 'Sierra16D32']);
+ MockSubprocess.invocations[0].reject('Failed getting sub-commit');
+
+ return addSubCommitPromise.then(() => {
+ assert(false, 'should never be reached');
+ }, (error_output) => {
+ assert(error_output);
+ assert.equal(error_output, 'Failed getting sub-commit');
+ });
+ });
+ });
+
+
+ it('should fail if entries in sub-commits does not contain revision', () => {
+ const logger = new MockLogger;
+ const fetchter = new OSBuildFetcher(null, null, null, MockSubprocess, logger);
+ const waitingForInvocationPromise = MockSubprocess.waitingForInvocation();
+ const addSubCommitPromise = fetchter._addSubCommitsForBuild([osxCommit], ['subCommit', 'for', 'revision'])
+
+ return waitingForInvocationPromise.then(() => {
+ assert.equal(MockSubprocess.invocations.length, 1);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['subCommit', 'for', 'revision', 'Sierra16D32']);
+ MockSubprocess.invocations[0].resolve('{"WebKit":{"RandomKey": "RandomValue"}}');
+
+ return addSubCommitPromise.then(() => {
+ assert(false, 'should never be reached');
+ }, (error_output) => {
+ assert(error_output);
+ assert.equal(error_output.name, 'AssertionError');
+ });
+ });
+ })
+ })
+
+ describe('OSBuildFetcher.fetchAndReportNewBuilds', () => {
+ it('should report all build commits with sub-commits', () => {
+ const logger = new MockLogger;
+ const fetchter = new OSBuildFetcher(config, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger);
+ const db = TestServer.database();
+ let fetchAndReportPromise = null;
+ let fetchAvailableBuildsPromise = null;
+
+ return addSlaveForReport(emptyReport).then(() => {
+ return Promise.all([
+ db.insert('repositories', {'id': 10, 'name': 'OSX'}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D67', 'order': 1603006700, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D68', 'order': 1603006800, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D69', 'order': 1603006900, 'reported': false}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E32', 'order': 1604003200, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E33', 'order': 1604003300, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E33g', 'order': 1604003307, 'reported': true})]);
+ }).then(() => {
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1603000000&to=1603099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16D68');
+ assert.equal(result['commits'][0]['order'], 1603006800);
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1604000000&to=1604099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16E33g');
+ assert.equal(result['commits'][0]['order'], 1604003307);
+ const waitingForInvocationPromise = MockSubprocess.waitingForInvocation();
+ fetchAvailableBuildsPromise = fetchter._fetchAvailableBuilds();
+ return waitingForInvocationPromise;
+ }).then(() => {
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ MockSubprocess.invocations.sort((invocation, antoherInvocation) => invocation['command'] > antoherInvocation['command']);
+ assert.equal(MockSubprocess.invocations.length, 2);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'all osx 16Dxx builds']);
+ assert.deepEqual(MockSubprocess.invocations[1].command, ['list', 'all osx 16Exx builds']);
+ MockSubprocess.invocations[0].resolve('\n\nSierra16D68\nSierra16D69\n');
+ MockSubprocess.invocations[1].resolve('\n\nSierra16E32\nSierra16E33\nSierra16E33h\nSierra16E34');
+ MockSubprocess.reset();
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ MockSubprocess.invocations.sort((invocation, antoherInvocation) => invocation['command'] > antoherInvocation['command']);
+ assert.equal(MockSubprocess.invocations.length, 2);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'subCommit', 'for', 'revision', 'Sierra16D69']);
+ assert.deepEqual(MockSubprocess.invocations[1].command, ['list', 'subCommit', 'for', 'revision', 'Sierra16E33h']);
+
+ MockSubprocess.invocations[0].resolve(JSON.stringify(subCommitWithWebKit));
+ MockSubprocess.invocations[1].resolve(JSON.stringify(anotherSubCommitWithWebKit));
+ MockSubprocess.reset();
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ assert.equal(MockSubprocess.invocations.length, 1);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'subCommit', 'for', 'revision', 'Sierra16E34']);
+ MockSubprocess.invocations[0].resolve(JSON.stringify(anotherSubCommitWithWebKitAndJavaScriptCore));
+ return fetchAvailableBuildsPromise;
+ }).then((results) => {
+ assert.equal(results.length, 3);
+ MockSubprocess.reset();
+ fetchAndReportPromise = fetchter.fetchAndReportNewBuilds();
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ MockSubprocess.invocations.sort((invocation, antoherInvocation) => invocation['command'] > antoherInvocation['command']);
+ assert.equal(MockSubprocess.invocations.length, 2);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'all osx 16Dxx builds']);
+ assert.deepEqual(MockSubprocess.invocations[1].command, ['list', 'all osx 16Exx builds']);
+ MockSubprocess.invocations[0].resolve('\n\nSierra16D68\nSierra16D69\n');
+ MockSubprocess.invocations[1].resolve('\n\nSierra16E32\nSierra16E33\nSierra16E33h\nSierra16E34');
+ MockSubprocess.reset();
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ MockSubprocess.invocations.sort((invocation, antoherInvocation) => invocation['command'] > antoherInvocation['command']);
+ assert.equal(MockSubprocess.invocations.length, 2);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'subCommit', 'for', 'revision', 'Sierra16D69']);
+ assert.deepEqual(MockSubprocess.invocations[1].command, ['list', 'subCommit', 'for', 'revision', 'Sierra16E33h']);
+
+ MockSubprocess.invocations[0].resolve(JSON.stringify(subCommitWithWebKit));
+ MockSubprocess.invocations[1].resolve(JSON.stringify(anotherSubCommitWithWebKit));
+
+ MockSubprocess.reset();
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ assert.equal(MockSubprocess.invocations.length, 1);
+ MockSubprocess.invocations[0].resolve(JSON.stringify(anotherSubCommitWithWebKitAndJavaScriptCore));
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'subCommit', 'for', 'revision', 'Sierra16E34']);
+
+ return fetchAndReportPromise;
+ }).then((result) => {
+ assert.equal(result['status'], 'OK');
+ return Promise.all([
+ db.selectRows('repositories', {'name': 'WebKit'}),
+ db.selectRows('repositories', {'name': '_javascript_Core'}),
+ db.selectRows('commits', {'revision': 'Sierra16D69'}),
+ db.selectRows('commits', {'revision': 'Sierra16E33h'}),
+ db.selectRows('commits', {'revision': 'Sierra16E34'})]);
+ }).then((results) => {
+ const webkitRepository = results[0];
+ const jscRepository = results[1];
+ const osxCommit16D69 = results[2];
+ const osxCommit16E33h = results[3];
+ const osxCommit16E34 = results[4];
+
+ assert.equal(webkitRepository.length, 1);
+ assert.equal(webkitRepository[0]['owner'], 10);
+ assert.equal(jscRepository.length, 1)
+ assert.equal(jscRepository[0]['owner'], 10);
+
+ assert.equal(osxCommit16D69.length, 1);
+ assert.equal(osxCommit16D69[0]['repository'], 10);
+ assert.equal(osxCommit16D69[0]['order'], 1603006900);
+
+ assert.equal(osxCommit16E33h.length, 1);
+ assert.equal(osxCommit16E33h[0]['repository'], 10);
+ assert.equal(osxCommit16E33h[0]['order'], 1604003308);
+
+ assert.equal(osxCommit16E34.length, 1);
+ assert.equal(osxCommit16E34[0]['repository'], 10);
+ assert.equal(osxCommit16E34[0]['order'], 1604003400);
+
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1603000000&to=1603099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16D69');
+ assert.equal(result['commits'][0]['order'], 1603006900);
+
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1604000000&to=1604099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16E34');
+ assert.equal(result['commits'][0]['order'], 1604003400);
+ });
+ });
+
+ it('should report commits without sub-commits if "subCommitCommand" is not specified in config', () => {
+ const logger = new MockLogger;
+ const fetchter = new OSBuildFetcher(configWithoutSubCommitCommand, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger);
+ const db = TestServer.database();
+ let fetchAndReportPromise = null;
+ let fetchAvailableBuildsPromise = null;
+
+ return addSlaveForReport(emptyReport).then(() => {
+ return Promise.all([
+ db.insert('repositories', {'id': 10, 'name': 'OSX'}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D67', 'order': 1603006700, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D68', 'order': 1603006800, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D69', 'order': 1603006900, 'reported': false}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E32', 'order': 1604003200, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E33', 'order': 1604003300, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E33g', 'order': 1604003307, 'reported': true})]);
+ }).then(() => {
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1603000000&to=1603099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16D68');
+ assert.equal(result['commits'][0]['order'], 1603006800);
+
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1604000000&to=1604099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16E33g');
+ assert.equal(result['commits'][0]['order'], 1604003307);
+ const waitingForInvocationPromise = MockSubprocess.waitingForInvocation();
+ fetchAndReportPromise = fetchter.fetchAndReportNewBuilds();
+ return waitingForInvocationPromise;
+ }).then(() => {
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ MockSubprocess.invocations.sort((invocation, antoherInvocation) => invocation['command'] > antoherInvocation['command']);
+ assert.equal(MockSubprocess.invocations.length, 2);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'all osx 16Dxx builds']);
+ assert.deepEqual(MockSubprocess.invocations[1].command, ['list', 'all osx 16Exx builds']);
+ MockSubprocess.invocations[0].resolve('\n\nSierra16D68\nSierra16D69\n');
+ MockSubprocess.invocations[1].resolve('\n\nSierra16E32\nSierra16E33\nSierra16E33h\nSierra16E34');
+ return fetchAndReportPromise;
+ }).then((result) => {
+ assert.equal(result['status'], 'OK');
+ return Promise.all([
+ db.selectRows('repositories', {'name': 'WebKit'}),
+ db.selectRows('repositories', {'name': '_javascript_Core'}),
+ db.selectRows('commits', {'revision': 'Sierra16D69'}),
+ db.selectRows('commits', {'revision': 'Sierra16E33h'}),
+ db.selectRows('commits', {'revision': 'Sierra16E34'})]);
+ }).then((results) => {
+ const webkitRepository = results[0];
+ const jscRepository = results[1];
+ const osxCommit16D69 = results[2];
+ const osxCommit16E33h = results[3];
+ const osxCommit16E34 = results[4];
+
+ assert.equal(webkitRepository.length, 0);
+ assert.equal(jscRepository.length, 0)
+
+ assert.equal(osxCommit16D69.length, 1);
+ assert.equal(osxCommit16D69[0]['repository'], 10);
+ assert.equal(osxCommit16D69[0]['order'], 1603006900);
+
+ assert.equal(osxCommit16E33h.length, 1);
+ assert.equal(osxCommit16E33h[0]['repository'], 10);
+ assert.equal(osxCommit16E33h[0]['order'], 1604003308);
+
+ assert.equal(osxCommit16E34.length, 1);
+ assert.equal(osxCommit16E34[0]['repository'], 10);
+ assert.equal(osxCommit16E34[0]['order'], 1604003400);
+
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1603000000&to=1603099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16D69');
+ assert.equal(result['commits'][0]['order'], 1603006900);
+
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1604000000&to=1604099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16E34');
+ assert.equal(result['commits'][0]['order'], 1604003400);
+ });
+ });
+
+ it('should stop reporting if any custom command fails', () => {
+ const logger = new MockLogger;
+ const fetchter = new OSBuildFetcher(config, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger);
+ const db = TestServer.database();
+ let fetchAndReportPromise = null;
+
+ return addSlaveForReport(emptyReport).then(() => {
+ return Promise.all([
+ db.insert('repositories', {'id': 10, 'name': 'OSX'}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D67', 'order': 1603006700, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D68', 'order': 1603006800, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D69', 'order': 1603006900, 'reported': false}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E32', 'order': 1604003200, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E33', 'order': 1604003300, 'reported': true}),
+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E33g', 'order': 1604003307, 'reported': true})]);
+ }).then(() => {
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1603000000&to=1603099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16D68');
+ assert.equal(result['commits'][0]['order'], 1603006800);
+
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1604000000&to=1604099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16E33g');
+ assert.equal(result['commits'][0]['order'], 1604003307);
+
+ const waitingForInvocationPromise = MockSubprocess.waitingForInvocation();
+ fetchAndReportPromise = fetchter.fetchAndReportNewBuilds();
+ return waitingForInvocationPromise;
+ }).then(() => {
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ MockSubprocess.invocations.sort((invocation, antoherInvocation) => invocation['command'] > antoherInvocation['command']);
+ assert.equal(MockSubprocess.invocations.length, 2);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'all osx 16Dxx builds']);
+ assert.deepEqual(MockSubprocess.invocations[1].command, ['list', 'all osx 16Exx builds']);
+ MockSubprocess.invocations[0].resolve('\n\nSierra16D68\nSierra16D69\n');
+ MockSubprocess.invocations[1].resolve('\n\nSierra16E32\nSierra16E33\nSierra16E33h\nSierra16E34');
+ MockSubprocess.reset();
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ MockSubprocess.invocations.sort((invocation, antoherInvocation) => invocation['command'] > antoherInvocation['command']);
+ assert.equal(MockSubprocess.invocations.length, 2);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'subCommit', 'for', 'revision', 'Sierra16D69']);
+ assert.deepEqual(MockSubprocess.invocations[1].command, ['list', 'subCommit', 'for', 'revision', 'Sierra16E33h']);
+
+ MockSubprocess.invocations[0].resolve(JSON.stringify(subCommitWithWebKit));
+ MockSubprocess.invocations[1].resolve(JSON.stringify(anotherSubCommitWithWebKit));
+ MockSubprocess.reset();
+ return MockSubprocess.waitingForInvocation();
+ }).then(() => {
+ assert.equal(MockSubprocess.invocations.length, 1);
+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'subCommit', 'for', 'revision', 'Sierra16E34']);
+ MockSubprocess.invocations[0].reject('Command failed');
+
+ return fetchAndReportPromise.then(() => {
+ assert(false, 'should never be reached');
+ }, (error_output) => {
+ assert(error_output);
+ assert.equal(error_output, 'Command failed');
+ });
+ }).then(() => {
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1603000000&to=1603099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16D68');
+ assert.equal(result['commits'][0]['order'], 1603006800);
+
+ return TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1604000000&to=1604099900');
+ }).then((result) => {
+ assert.equal(result['commits'].length, 1);
+ assert.equal(result['commits'][0]['revision'], 'Sierra16E33g');
+ assert.equal(result['commits'][0]['order'], 1604003307);
+ });
+ })
+ })
+});
Added: trunk/Websites/perf.webkit.org/tools/js/os-build-fetcher.js (0 => 213976)
--- trunk/Websites/perf.webkit.org/tools/js/os-build-fetcher.js (rev 0)
+++ trunk/Websites/perf.webkit.org/tools/js/os-build-fetcher.js 2017-03-15 08:35:07 UTC (rev 213976)
@@ -0,0 +1,98 @@
+'use strict';
+
+let assert = require('assert');
+
+class OSBuildFetcher {
+
+ constructor(osConfig, remoteAPI, slaveAuth, subprocess, logger)
+ {
+ this._osConfig = osConfig;
+ this._reportedRevisions = new Set();
+ this._logger = logger;
+ this._slaveAuth = slaveAuth;
+ this._remoteAPI = remoteAPI;
+ this._subprocess = subprocess;
+ }
+
+ fetchAndReportNewBuilds()
+ {
+ return this._fetchAvailableBuilds().then((results) =>{
+ return this._submitCommits(results);
+ });
+ }
+
+ _fetchAvailableBuilds()
+ {
+ const config = this._osConfig;
+ const repositoryName = config['name'];
+ let customCommands = config['customCommands'];
+
+ return Promise.all(customCommands.map((command) => {
+ assert(command['minRevision']);
+ assert(command['maxRevision']);
+ const minRevisionOrder = this._computeOrder(command['minRevision']);
+ const maxRevisionOrder = this._computeOrder(command['maxRevision']);
+
+ let fetchCommitsPromise = this._remoteAPI.getJSONWithStatus(`/api/commits/${escape(repositoryName)}/last-reported?from=${minRevisionOrder}&to=${maxRevisionOrder}`).then((result) => {
+ const minOrder = result['commits'].length == 1 ? parseInt(result['commits'][0]['order']) : 0;
+ return this._commitsForAvailableBuilds(repositoryName, command['command'], command['linesToIgnore'], minOrder);
+ })
+
+ if ('subCommitCommand' in command)
+ fetchCommitsPromise = fetchCommitsPromise.then((commits) => this._addSubCommitsForBuild(commits, command['subCommitCommand']));
+
+ return fetchCommitsPromise;
+ })).then(results => {
+ return Array.prototype.concat.apply([], results);
+ });
+ }
+
+ _computeOrder(revision)
+ {
+ const buildNameRegex = /(\d+)([a-zA-Z])(\d+)([a-zA-Z]*)$/;
+ const match = buildNameRegex.exec(revision);
+ assert(match);
+ const major = parseInt(match[1]);
+ const kind = match[2].toUpperCase().charCodeAt(0) - "A".charCodeAt(0);
+ const minor = parseInt(match[3]);
+ const variant = match[4] ? match[4].toUpperCase().charCodeAt(0) - "A".charCodeAt(0) + 1 : 0;
+ return ((major * 100 + kind) * 10000 + minor) * 100 + variant;
+ }
+
+ _commitsForAvailableBuilds(repository, command, linesToIgnore, minOrder)
+ {
+ return this._subprocess.execute(command).then((output) => {
+ let lines = output.split('\n');
+ if (linesToIgnore){
+ const regex = new RegExp(linesToIgnore);
+ lines = lines.filter(function(line) {return !regex.exec(line);});
+ }
+ return lines.map(revision => ({repository, revision, 'order': this._computeOrder(revision)}))
+ .filter(commit => commit['order'] > minOrder);
+ });
+ }
+
+ _addSubCommitsForBuild(commits, command)
+ {
+ return commits.reduce((promise, commit) => {
+ return promise.then(() => {
+ return this._subprocess.execute(command.concat(commit['revision']));
+ }).then((subCommitOutput) => {
+ const subCommits = JSON.parse(subCommitOutput);
+ for (let repositoryName in subCommits) {
+ const subCommit = subCommits[repositoryName];
+ assert(subCommit['revision']);
+ }
+ commit['subCommits'] = subCommits;
+ });
+ }, Promise.resolve()).then(() => commits);
+ }
+
+ _submitCommits(commits)
+ {
+ const commitsToReport = {"slaveName": this._slaveAuth['name'], "slavePassword": this._slaveAuth['password'], 'commits': commits};
+ return this._remoteAPI.postJSONWithStatus('/api/report-commits/', commitsToReport);
+ }
+}
+if (typeof module != 'undefined')
+ module.exports.OSBuildFetcher = OSBuildFetcher;
Added: trunk/Websites/perf.webkit.org/tools/js/subprocess.js (0 => 213976)
--- trunk/Websites/perf.webkit.org/tools/js/subprocess.js (rev 0)
+++ trunk/Websites/perf.webkit.org/tools/js/subprocess.js 2017-03-15 08:35:07 UTC (rev 213976)
@@ -0,0 +1,18 @@
+'use strict';
+const childProcess = require('child_process').ChildProcess;
+
+class Subprocess {
+ execute(command) {
+ return new Promise((resolve, reject) => {
+ this._childProcess.execFile(command[0], command.slice(1), (error, stdout, stderr) => {
+ if (error)
+ reject(stderr);
+ else
+ resolve(stdout);
+ });
+ });
+ }
+};
+
+if (typeof module != 'undefined')
+ module.exports.Subprocess = Subprocess;
Added: trunk/Websites/perf.webkit.org/tools/pull-os-versions.js (0 => 213976)
--- trunk/Websites/perf.webkit.org/tools/pull-os-versions.js (rev 0)
+++ trunk/Websites/perf.webkit.org/tools/pull-os-versions.js 2017-03-15 08:35:07 UTC (rev 213976)
@@ -0,0 +1,43 @@
+#!/usr/local/bin/node
+'use strict';
+
+
+let OSBuildFetcher = require('./js/os-build-fetcher.js').OSBuildFetcher;
+let RemoteAPI = require('./js/remote.js').RemoteAPI;
+let Subprocess = require('./js/subprocess.js').Subprocess;
+let fs = require('fs');
+let parseArguments = require('./js/parse-arguments.js').parseArguments;
+
+function main(argv)
+{
+ let options = parseArguments(argv, [
+ {name: '--os-config-json', required: true},
+ {name: '--server-config-json', required: true},
+ {name: '--seconds-to-sleep', type: parseFloat, default: 43200},
+ ]);
+ if (!options)
+ return;
+
+ syncLoop(options);
+}
+
+function syncLoop(options)
+{
+ let osConfigList = JSON.parse(fs.readFileSync(options['--os-config-json'], 'utf8'));
+ let serverConfig = JSON.parse(fs.readFileSync(options['--server-config-json'], 'utf8'));
+
+ // v3 models use the global RemoteAPI to access the perf dashboard.
+ global.RemoteAPI = new RemoteAPI(serverConfig.server);
+
+ Promise.all(osConfigList.map(osConfig => new OSBuildFetcher(osConfig, global.RemoteAPI, new Subprocess, serverConfig.slave, console))).catch((error) => {
+ console.error(error);
+ if (typeof(error.stack) == 'string') {
+ for (let line of error.stack.split('\n'))
+ console.error(line);
+ }
+ }).then(function () {
+ setTimeout(syncLoop.bind(global, options), options['--seconds-to-sleep'] * 1000);
+ });
+}
+
+main(process.argv);
Modified: trunk/Websites/perf.webkit.org/tools/sync-commits.py (213975 => 213976)
--- trunk/Websites/perf.webkit.org/tools/sync-commits.py 2017-03-15 07:47:11 UTC (rev 213975)
+++ trunk/Websites/perf.webkit.org/tools/sync-commits.py 2017-03-15 08:35:07 UTC (rev 213976)
@@ -59,7 +59,7 @@
def fetch_commits_and_submit(self, server_config, max_fetch_count, max_ancestor_fetch_count):
if not self._last_fetched:
- print "Determining the stating revision for %s" % self._name
+ print "Determining the starting revision for %s" % self._name
self._last_fetched = self.determine_last_reported_revision(server_config)
pending_commits = []
_______________________________________________ webkit-changes mailing list webkit-changes@lists.webkit.org https://lists.webkit.org/mailman/listinfo/webkit-changes
