Copilot commented on code in PR #11473:
URL: https://github.com/apache/ozone/pull/11473#discussion_r4240375781
##########
hadoop-ozone/dev-support/checks/junit.sh:
##########
@@ -55,7 +55,18 @@ if [[ -f
hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh ]]; then
ozone_java_setup
fi
-mvn ${MAVEN_OPTIONS} clean
+if [[ "${OZONE_REPO_CACHED}" != "true" ]]; then
+ mvn ${MAVEN_OPTIONS} clean
+fi
+
+INTEGRATION_REACTOR_PL=()
+if [[ "${OZONE_REPO_CACHED}" == "true" ]] && [[ "${CHECK}" == "integration"
]]; then
+ args_str="$*"
+ if [[ "${args_str}" != *"-Ptest-filesystem"* ]] && [[ "${args_str}" !=
*"-Ptest-flaky"* ]] \
+ && [[ "${args_str}" != *"-Dtest="* ]]; then
+ INTEGRATION_REACTOR_PL=(-pl
'!:ozone-integration-test,!:ozone-integration-test-recon,!:ozone-integration-test-s3')
+ fi
Review Comment:
This selector removes the modules that contain the integration tests
selected by these profiles. For example, `test-om` includes
`org.apache.hadoop.ozone.om.**`, while those tests live under
`hadoop-ozone/integration-test`; cached dependency JARs do not cause Surefire
to execute their tests. The non-filesystem/non-flaky matrix legs can therefore
pass while om, hdds, recon, and other integration suites are skipped. Select
these modules instead of excluding them.
##########
dev-support/scripts/develocity_ci_summary.py:
##########
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Summarize Apache Ozone Main Develocity build scans for CI tuning."""
+
+from __future__ import annotations
+
+import argparse
+import configparser
+import json
+import os
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Dict, List, Optional
+
+DEVELOCITY_HOST = "https://develocity.apache.org";
+DEFAULT_CREDS = os.path.expanduser("~/.asf-jira-pat")
+ROOT_PROJECT = "Apache Ozone Main"
+
+
+def load_access_key(creds_path: str) -> str:
+ key = os.environ.get("DEVELOCITY_ACCESS_KEY", "").strip()
+ if key:
+ return key
+
+ if not os.path.isfile(creds_path):
+ raise SystemExit(
+ f"No DEVELOCITY_ACCESS_KEY in env and creds file missing:
{creds_path}")
+
+ parser = configparser.ConfigParser()
+ parser.read(creds_path)
+ if parser.has_section("develocity"):
+ key = parser.get("develocity", "DEVELOCITY_ACCESS_KEY",
fallback="").strip()
+ if key:
+ return key
+
+ raise SystemExit(
+ "Add a Develocity access key to ~/.asf-jira-pat:\n"
+ " [develocity]\n"
+ " DEVELOCITY_ACCESS_KEY=<key from develocity.apache.org My settings>\n"
+ "Or export DEVELOCITY_ACCESS_KEY.")
+
+
+def api_get(path: str, access_key: str, params: Optional[Dict[str, str]] =
None) -> Any:
+ query = urllib.parse.urlencode(params or {})
+ url = f"{DEVELOCITY_HOST}{path}"
+ if query:
+ url = f"{url}?{query}"
+ req = urllib.request.Request(
+ url,
+ headers={"Authorization": f"Bearer {access_key}", "Accept":
"application/json"},
+ )
+ with urllib.request.urlopen(req, timeout=120) as resp:
+ return json.loads(resp.read().decode("utf-8"))
+
+
+def verify_api_permission(access_key: str) -> None:
+ """Raise SystemExit if the key cannot access build data via the API."""
+ url = (
+ f"{DEVELOCITY_HOST}/api/auth/token"
+ "?permissions=accessBuildDataViaApi&expiresInHours=1"
+ )
+ req = urllib.request.Request(url, method="POST", headers={
+ "Authorization": f"Bearer {access_key}",
+ })
+ try:
+ urllib.request.urlopen(req, timeout=30)
+ except urllib.error.HTTPError as e:
+ body = e.read().decode("utf-8", errors="replace")
+ if e.code == 403 and "accessBuildDataViaApi" in body:
+ raise SystemExit(
+ "This access key can sign in to Develocity but lacks "
+ "'Access build data via the API' (API Client / exportData). "
+ "Create a new access key with that permission, or ask an admin to "
+ "assign the API Client or Developer role, then update "
+ "~/.asf-jira-pat [develocity] DEVELOCITY_ACCESS_KEY.") from e
+ raise SystemExit(f"Develocity token probe HTTP {e.code}: {body[:500]}")
from e
+
+
+def fetch_build_page(
+ access_key: str,
+ from_instant: int,
+ query: str,
+ max_builds: int,
+) -> List[Dict[str, Any]]:
+ params = {
+ "fromInstant": str(from_instant),
+ "reverse": "true",
Review Comment:
For the Develocity Builds API, `fromInstant` with `reverse=true` returns
builds processed before the supplied instant. With the default July 1
timestamp, this summarizes pre-July builds rather than the documented
July-and-later sample. Use forward traversal for a start timestamp.
##########
dev-support/scripts/develocity_ci_summary.py:
##########
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Summarize Apache Ozone Main Develocity build scans for CI tuning."""
+
+from __future__ import annotations
+
+import argparse
+import configparser
+import json
+import os
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Dict, List, Optional
+
+DEVELOCITY_HOST = "https://develocity.apache.org";
+DEFAULT_CREDS = os.path.expanduser("~/.asf-jira-pat")
+ROOT_PROJECT = "Apache Ozone Main"
+
+
+def load_access_key(creds_path: str) -> str:
+ key = os.environ.get("DEVELOCITY_ACCESS_KEY", "").strip()
+ if key:
+ return key
+
+ if not os.path.isfile(creds_path):
+ raise SystemExit(
+ f"No DEVELOCITY_ACCESS_KEY in env and creds file missing:
{creds_path}")
+
+ parser = configparser.ConfigParser()
+ parser.read(creds_path)
+ if parser.has_section("develocity"):
+ key = parser.get("develocity", "DEVELOCITY_ACCESS_KEY",
fallback="").strip()
+ if key:
+ return key
+
+ raise SystemExit(
+ "Add a Develocity access key to ~/.asf-jira-pat:\n"
+ " [develocity]\n"
+ " DEVELOCITY_ACCESS_KEY=<key from develocity.apache.org My settings>\n"
+ "Or export DEVELOCITY_ACCESS_KEY.")
+
+
+def api_get(path: str, access_key: str, params: Optional[Dict[str, str]] =
None) -> Any:
+ query = urllib.parse.urlencode(params or {})
+ url = f"{DEVELOCITY_HOST}{path}"
+ if query:
+ url = f"{url}?{query}"
+ req = urllib.request.Request(
+ url,
+ headers={"Authorization": f"Bearer {access_key}", "Accept":
"application/json"},
+ )
+ with urllib.request.urlopen(req, timeout=120) as resp:
+ return json.loads(resp.read().decode("utf-8"))
+
+
+def verify_api_permission(access_key: str) -> None:
+ """Raise SystemExit if the key cannot access build data via the API."""
+ url = (
+ f"{DEVELOCITY_HOST}/api/auth/token"
+ "?permissions=accessBuildDataViaApi&expiresInHours=1"
+ )
+ req = urllib.request.Request(url, method="POST", headers={
+ "Authorization": f"Bearer {access_key}",
+ })
+ try:
+ urllib.request.urlopen(req, timeout=30)
+ except urllib.error.HTTPError as e:
+ body = e.read().decode("utf-8", errors="replace")
+ if e.code == 403 and "accessBuildDataViaApi" in body:
+ raise SystemExit(
+ "This access key can sign in to Develocity but lacks "
+ "'Access build data via the API' (API Client / exportData). "
+ "Create a new access key with that permission, or ask an admin to "
+ "assign the API Client or Developer role, then update "
+ "~/.asf-jira-pat [develocity] DEVELOCITY_ACCESS_KEY.") from e
+ raise SystemExit(f"Develocity token probe HTTP {e.code}: {body[:500]}")
from e
+
+
+def fetch_build_page(
+ access_key: str,
+ from_instant: int,
+ query: str,
+ max_builds: int,
+) -> List[Dict[str, Any]]:
+ params = {
+ "fromInstant": str(from_instant),
+ "reverse": "true",
+ "maxBuilds": str(max_builds),
+ "maxWaitSecs": "30",
+ "query": query,
+ }
+ data = api_get("/api/builds", access_key, params)
+ if not isinstance(data, list):
+ raise SystemExit(f"Unexpected /api/builds response: {type(data)}")
+ return data
+
+
+def summarize(builds: List[Dict[str, Any]]) -> None:
+ total = len(builds)
+ failed = sum(1 for b in builds if b.get("buildOutcome") == "failed")
+ durations = [b.get("buildDuration") for b in builds if
b.get("buildDuration") is not None]
+ print(f"Build scans (sample): {total}")
+ print(f"Failed: {failed} ({100.0 * failed / total:.1f}%)" if total else
"Failed: 0")
+ if durations:
+ durations.sort()
+ mid = durations[len(durations) // 2]
Review Comment:
For an even number of durations, this selects the upper middle value instead
of the median, so the reported metric is biased upward. Average the two middle
values after sorting.
##########
hadoop-hdds/rocks-native/pom.xml:
##########
@@ -219,6 +219,8 @@
<url>https://github.com/facebook/rocksdb/archive/refs/tags/v${rocksdb.source.version}.tar.gz</url>
<outputFileName>rocksdb-v${rocksdb.source.version}.tar.gz</outputFileName>
<outputDirectory>${project.build.directory}/rocksdb</outputDirectory>
+ <retryCount>5</retryCount>
Review Comment:
`download-maven-plugin` 2.1.0 names this parameter `retries`; it has no
`retryCount` parameter. Maven therefore ignores this element and leaves the
download at the plugin's default retry setting, so the intended hardening is
not applied.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
