This is an automated email from the ASF dual-hosted git repository.
netudima pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-ccm.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6bf561e Download releases from dlcdn.apache.org before
archive.apache.org, and time out stalled downloads
6bf561e is described below
commit 6bf561eef441459d8c4700e940280ca768aa4864
Author: Dmitry Konstantinov <[email protected]>
AuthorDate: Sat Oct 24 17:40:23 2026 +0100
Download releases from dlcdn.apache.org before archive.apache.org, and time
out stalled downloads
Try dlcdn.apache.org, then downloads.apache.org, then archive.apache.org; a
404 moves on to the next mirror.
Use a 45s socket timeout and make up to 3 attempts per url on timeouts,
connection errors,
truncated responses and HTTP 5xx/408/429.
patch by Dmitry Konstantinov; reviewed by Michael Semb Wever for
CASSANDRA-21805
Assisted-By: Claude Code (Opus 5.5)
---
ccmlib/repository.py | 119 +++++++++++++++++++---------
tests/test_repository.py | 197 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 279 insertions(+), 37 deletions(-)
diff --git a/ccmlib/repository.py b/ccmlib/repository.py
index 62e9c0f..55af470 100644
--- a/ccmlib/repository.py
+++ b/ccmlib/repository.py
@@ -43,10 +43,15 @@ from ccmlib import common
from ccmlib.common import (ArgumentError, CCMError,
update_java_version, get_default_path,
get_jdk_version_int,
platform_binary, rmdirs, validate_install_dir)
-from six.moves import urllib
+from six.moves import http_client, urllib
ARCHIVE = "http://archive.apache.org/dist/cassandra";
+# dlcdn and downloads only keep the latest releases; archive keeps all of them
but is slow and throttled
+MIRRORS = ["https://dlcdn.apache.org/cassandra";,
"https://downloads.apache.org/cassandra";, ARCHIVE]
+DOWNLOAD_TIMEOUT = 45 # seconds a download may wait for data before the
attempt is abandoned
+DOWNLOAD_ATTEMPTS = 3 # attempts per url
+DOWNLOAD_RETRY_DELAY = 5 # seconds, multiplied by the attempt number
GIT_REPO = "https://github.com/apache/cassandra.git";
GITHUB_REPO = "https://github.com/apache/cassandra";
GITHUB_TAGS = "https://api.github.com/repos/apache/cassandra/git/refs/tags";
@@ -68,7 +73,7 @@ def setup(version, verbose=False):
download_version(version, verbose=verbose, binary=True)
cdir = version_directory(version)
except Exception as e:
- # If we failed to download from ARCHIVE,
+ # If we failed to download from the mirrors,
# then we build from source from the git repo,
# as it is more reliable.
# We don't do this if binary: or source: were
@@ -283,17 +288,18 @@ def download_version(version, url=None, verbose=False,
binary=False):
if binary == True, download precompiled tarball, otherwise build from
source tarball.
"""
- archive_url = ARCHIVE
+ repositories = MIRRORS
if CCM_CONFIG.has_option('repositories', 'cassandra'):
- archive_url = CCM_CONFIG.get('repositories', 'cassandra')
+ repositories = [CCM_CONFIG.get('repositories', 'cassandra')]
- if binary:
- archive_url = "%s/%s/apache-cassandra-%s-bin.tar.gz" % (archive_url,
version, version) if url is None else url
+ if url is not None:
+ urls = [url]
else:
- archive_url = "%s/%s/apache-cassandra-%s-src.tar.gz" % (archive_url,
version, version) if url is None else url
+ kind = 'bin' if binary else 'src'
+ urls = ["%s/%s/apache-cassandra-%s-%s.tar.gz" % (r, version, version,
kind) for r in repositories]
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
try:
- __download(archive_url, target, show_progress=verbose)
+ __download_from_first(urls, target, show_progress=verbose)
common.info("Extracting {} as version {} ...".format(target, version))
tar = tarfile.open(target)
dir = tar.next().name.split("/")[0] # pylint: disable=all
@@ -477,6 +483,21 @@ def get_tagged_version_numbers(series='stable'):
raise AssertionError("unknown release series:
{series}".format(series=series))
+def __download_from_first(urls, target, show_progress=False):
+ """Download the first of urls that is available, moving on to the next one
when a url is missing or keeps failing"""
+ for i, url in enumerate(urls):
+ try:
+ __download(url, target, show_progress=show_progress)
+ return
+ except (EnvironmentError, http_client.HTTPException) as e:
+ if i == len(urls) - 1:
+ raise
+ if isinstance(e, urllib.error.HTTPError) and e.code == 404:
+ common.debug("{} not found, trying {}".format(url, urls[i +
1]))
+ else:
+ common.warning("Downloading {} failed: {}, trying
{}".format(url, e, urls[i + 1]))
+
+
def __download(url, target, username=None, password=None, show_progress=False):
if username is not None:
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
@@ -485,36 +506,60 @@ def __download(url, target, username=None, password=None,
show_progress=False):
opener = urllib.request.build_opener(handler)
urllib.request.install_opener(opener) # pylint: disable=E1121
- u = urllib.request.urlopen(url)
- f = open(target, 'wb')
- meta = u.info()
- file_size = int(meta.get("Content-Length"))
- common.info("Downloading {} to {} ({:.3f}MB)".format(url, target,
float(file_size) / (1024 * 1024)))
-
- file_size_dl = 0
- block_sz = 8192
- status = None
- attempts = 0
- while file_size_dl < file_size:
- buffer = u.read(block_sz)
- if not buffer:
- attempts = attempts + 1
- if attempts >= 5:
- raise CCMError("Error downloading file (nothing read after {}
attempts, downloded only {} of {} bytes)".format(attempts, file_size_dl,
file_size))
- time.sleep(0.5 * attempts)
- continue
+ # local files (file://) are not retried
+ attempts = DOWNLOAD_ATTEMPTS if url.startswith(('http://', 'https://'))
else 1
+ for attempt in range(1, attempts + 1):
+ try:
+ __download_once(url, target, show_progress)
+ return
+ except urllib.error.HTTPError as e:
+ # a missing file or a refused request will not change on retry
+ if e.code < 500 and e.code not in (408, 429):
+ raise
+ error = e
+ except urllib.error.URLError as e:
+ # neither will a certificate that cannot be verified
+ if getattr(e.reason, 'reason', None) ==
'CERTIFICATE_VERIFY_FAILED':
+ raise
+ error = e
+ # EnvironmentError covers URLError, socket errors and timeouts on both
Python 2 and 3
+ except (EnvironmentError, http_client.HTTPException) as e:
+ error = e
+ if attempt == attempts:
+ raise error
+ common.warning("Downloading {} failed (attempt {} of {}):
{}".format(url, attempt, attempts, error))
+ time.sleep(DOWNLOAD_RETRY_DELAY * attempt)
+
+
+def __download_once(url, target, show_progress):
+ # the timeout bounds every socket operation, so a stalled connection fails
instead of blocking forever
+ u = urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT)
+ try:
+ content_length = u.info().get("Content-Length")
+ file_size = int(content_length) if content_length is not None else None
+ if file_size is not None:
+ common.info("Downloading {} to {} ({:.3f}MB)".format(url, target,
float(file_size) / (1024 * 1024)))
else:
- attempts = 0
-
- file_size_dl += len(buffer)
- f.write(buffer)
- if show_progress:
- status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. /
file_size)
- status = chr(8) * (len(status) + 1) + status
- print_(status, end='')
-
- f.close()
- u.close()
+ common.info("Downloading {} to {}".format(url, target))
+
+ file_size_dl = 0
+ block_sz = 8192
+ with open(target, 'wb') as f:
+ while True:
+ buffer = u.read(block_sz)
+ if not buffer:
+ break
+ file_size_dl += len(buffer)
+ f.write(buffer)
+ if show_progress and file_size:
+ status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl
* 100. / file_size)
+ status = chr(8) * (len(status) + 1) + status
+ print_(status, end='')
+ finally:
+ u.close()
+
+ if file_size is not None and file_size_dl < file_size:
+ raise IOError("Connection closed after {} of {}
bytes".format(file_size_dl, file_size))
def __get_dir():
diff --git a/tests/test_repository.py b/tests/test_repository.py
new file mode 100644
index 0000000..4b1031c
--- /dev/null
+++ b/tests/test_repository.py
@@ -0,0 +1,197 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import configparser
+import io
+import os
+import ssl
+import tarfile
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+from six.moves import urllib
+
+from ccmlib import repository
+
+PAYLOAD = bytes(range(256)) * 256
+STALL_SECONDS = 5
+SSL_DIR = os.path.join(os.path.dirname(__file__), '..', 'ssl')
+
+
+class _MirrorHandler(BaseHTTPRequestHandler):
+ """Serves the payload, misbehaving as the first path segment says"""
+
+ def do_GET(self):
+ requests = self.server.requests
+ requests.append(self.path)
+ behaviour = self.path.split('/')[1]
+ first = requests.count(self.path) == 1
+ payload = self.server.payload
+
+ if behaviour == 'missing':
+ self.send_error(404)
+ return
+ if behaviour == 'error-once' and first:
+ self.send_error(500)
+ return
+
+ self.send_response(200)
+ self.send_header('Content-Length', str(len(payload)))
+ self.end_headers()
+ if behaviour == 'stall-always' or (behaviour == 'stall-once' and
first):
+ self.wfile.write(payload[:len(payload) // 2])
+ self.wfile.flush()
+ time.sleep(STALL_SECONDS)
+ elif behaviour == 'truncate-once' and first:
+ # HTTP/1.0: the connection closes when the handler returns
+ self.wfile.write(payload[:len(payload) // 2])
+ else:
+ self.wfile.write(payload)
+
+ def log_message(self, *args):
+ pass
+
+
+class _MirrorServer(ThreadingHTTPServer):
+ block_on_close = False # do not wait for stalled handlers
+
+ def __init__(self, context=None):
+ ThreadingHTTPServer.__init__(self, ('127.0.0.1', 0), _MirrorHandler)
+ if context is not None:
+ self.socket = context.wrap_socket(self.socket, server_side=True)
+ self.connections = 0
+ self.requests = []
+ self.payload = PAYLOAD
+ self.base = '{}://127.0.0.1:{}'.format('https' if context else 'http',
self.server_address[1])
+
+ def get_request(self):
+ # counted before the TLS handshake, which never reaches the handler
when it fails
+ self.connections += 1
+ return ThreadingHTTPServer.get_request(self)
+
+
+def _serve(server):
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ yield server
+ server.shutdown()
+ server.server_close()
+
+
[email protected]
+def mirror():
+ yield from _serve(_MirrorServer())
+
+
[email protected]
+def untrusted_mirror():
+ # self-signed and long expired, so no client verifies it
+ context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ context.load_cert_chain(os.path.join(SSL_DIR, 'client_cert.pem'),
os.path.join(SSL_DIR, 'client_key.pem'))
+ yield from _serve(_MirrorServer(context))
+
+
[email protected](autouse=True)
+def fast_retries(monkeypatch):
+ monkeypatch.setattr(repository, 'DOWNLOAD_TIMEOUT', 0.5)
+ monkeypatch.setattr(repository, 'DOWNLOAD_RETRY_DELAY', 0)
+
+
+def _read(path):
+ with open(path, 'rb') as f:
+ return f.read()
+
+
+def test_missing_url_falls_through_to_next_without_retry(mirror, tmp_path):
+ target = str(tmp_path / 'download')
+ repository.__download_from_first([mirror.base + '/missing/f', mirror.base
+ '/ok/f'], target)
+
+ assert _read(target) == PAYLOAD
+ assert mirror.requests == ['/missing/f', '/ok/f']
+
+
+def test_stalled_connection_times_out_and_is_retried(mirror, tmp_path):
+ target = str(tmp_path / 'download')
+ start = time.time()
+ repository.__download_from_first([mirror.base + '/stall-once/f'], target)
+
+ assert _read(target) == PAYLOAD
+ assert mirror.requests == ['/stall-once/f'] * 2
+ assert time.time() - start < STALL_SECONDS
+
+
+def test_truncated_response_is_retried(mirror, tmp_path):
+ target = str(tmp_path / 'download')
+ repository.__download_from_first([mirror.base + '/truncate-once/f'],
target)
+
+ assert _read(target) == PAYLOAD
+ assert mirror.requests == ['/truncate-once/f'] * 2
+
+
+def test_server_error_is_retried(mirror, tmp_path):
+ target = str(tmp_path / 'download')
+ repository.__download_from_first([mirror.base + '/error-once/f'], target)
+
+ assert _read(target) == PAYLOAD
+ assert mirror.requests == ['/error-once/f'] * 2
+
+
+def test_url_that_keeps_failing_falls_through_to_next(mirror, tmp_path):
+ target = str(tmp_path / 'download')
+ repository.__download_from_first([mirror.base + '/stall-always/f',
mirror.base + '/ok/f'], target)
+
+ assert _read(target) == PAYLOAD
+ assert mirror.requests == ['/stall-always/f'] *
repository.DOWNLOAD_ATTEMPTS + ['/ok/f']
+
+
+def
test_untrusted_certificate_falls_through_to_next_without_retry(untrusted_mirror,
mirror, tmp_path):
+ target = str(tmp_path / 'download')
+ repository.__download_from_first([untrusted_mirror.base + '/ok/f',
mirror.base + '/ok/f'], target)
+
+ assert _read(target) == PAYLOAD
+ assert untrusted_mirror.connections == 1
+ assert mirror.requests == ['/ok/f']
+
+
+def test_last_error_is_raised_when_no_url_works(mirror, tmp_path):
+ target = str(tmp_path / 'download')
+ with pytest.raises(urllib.error.HTTPError) as e:
+ repository.__download_from_first([mirror.base + '/missing/a',
mirror.base + '/missing/b'], target)
+
+ assert e.value.code == 404
+ assert mirror.requests == ['/missing/a', '/missing/b']
+
+
+def test_download_version_tries_mirrors_in_order(mirror, tmp_path,
monkeypatch):
+ tarball = io.BytesIO()
+ with tarfile.open(fileobj=tarball, mode='w:gz') as tar:
+ script = tarfile.TarInfo('apache-cassandra-9.9.9/bin/cassandra')
+ script.size = len(PAYLOAD)
+ tar.addfile(script, io.BytesIO(PAYLOAD))
+ mirror.payload = tarball.getvalue()
+ monkeypatch.setattr(repository, 'MIRRORS', [mirror.base + '/missing',
mirror.base + '/ok'])
+ monkeypatch.setattr(repository, 'CCM_CONFIG',
configparser.RawConfigParser())
+ monkeypatch.setattr(repository, 'get_default_path', lambda: str(tmp_path))
+
+ repository.download_version('9.9.9', binary=True)
+
+ assert mirror.requests ==
['/missing/9.9.9/apache-cassandra-9.9.9-bin.tar.gz',
+ '/ok/9.9.9/apache-cassandra-9.9.9-bin.tar.gz']
+ installed = tmp_path / 'repository' / '9.9.9'
+ assert _read(str(installed / 'bin' / 'cassandra')) == PAYLOAD
+ assert _read(str(installed / '0.version.txt')) == b'9.9.9'
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
