Bobby Bruce has submitted this change. (
https://gem5-review.googlesource.com/c/public/gem5/+/58851 )
Change subject: stdlib: Add tar unpacking to downloader
......................................................................
stdlib: Add tar unpacking to downloader
With this commit gem5-resources can exist as tarballs to be unpacked
after download. This requires the field "is_tar_archive : true" to be
present in the resource description in resources.json.
Change-Id: Ia835c1777425a5aafe8ba7ee9c725edf6d45f68c
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/58851
Reviewed-by: Jason Lowe-Power <power...@gmail.com>
Maintainer: Jason Lowe-Power <power...@gmail.com>
Tested-by: kokoro <noreply+kok...@google.com>
---
M src/python/gem5/resources/downloader.py
1 file changed, 44 insertions(+), 2 deletions(-)
Approvals:
Jason Lowe-Power: Looks good to me, approved; Looks good to me, approved
kokoro: Regressions pass
diff --git a/src/python/gem5/resources/downloader.py
b/src/python/gem5/resources/downloader.py
index 56b27aa..c3f853d 100644
--- a/src/python/gem5/resources/downloader.py
+++ b/src/python/gem5/resources/downloader.py
@@ -35,6 +35,7 @@
import time
import random
from pathlib import Path
+import tarfile
from tempfile import gettempdir
from urllib.error import HTTPError
from typing import List, Dict
@@ -286,6 +287,7 @@
resource_name: str,
to_path: str,
unzip: bool = True,
+ untar: bool = True,
download_md5_mismatch: bool = True,
) -> None:
"""
@@ -300,6 +302,9 @@
:param unzip: If true, gzipped resources will be unzipped prior to
saving
to `to_path`. True by default.
+ :param untar: If true, tar achieve resource will be unpacked prior to
+ saving to `to_path`. True by default.
+
:param download_md5_mismatch: If a resource is present with an
incorrect
hash (e.g., an outdated version of the resource is present),
`get_resource`
will delete this local resource and re-download it if this parameter is
@@ -360,8 +365,16 @@
)
)
+ run_tar_extract = untar and "is_tar_archive" in resource_json and \
+ resource_json["is_tar_archive"]
+
+ tar_extension = ".tar"
+ if run_tar_extract:
+ download_dest += tar_extension
+
+ zip_extension = ".gz"
if run_unzip:
- download_dest += ".gz"
+ download_dest += zip_extension
# TODO: Might be nice to have some kind of download status bar
here.
# TODO: There might be a case where this should be silenced.
@@ -385,10 +398,22 @@
resource_name, download_dest
)
)
+ unzip_to = download_dest[:-len(zip_extension)]
with gzip.open(download_dest, "rb") as f:
- with open(to_path, "wb") as o:
+ with open(unzip_to, "wb") as o:
shutil.copyfileobj(f, o)
os.remove(download_dest)
+ download_dest = unzip_to
print(
"Finished decompressing
resource '{}'.".format(resource_name)
)
+
+ if run_tar_extract:
+ print(
+ f"Unpacking the the resource '{resource_name}' "
+ f"('{download_dest}')"
+ )
+ unpack_to = download_dest[:-len(tar_extension)]
+ with tarfile.open(download_dest) as f:
+ f.extractall(unpack_to)
+ os.remove(download_dest)
--
To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/58851
To unsubscribe, or for help writing mail filters, visit
https://gem5-review.googlesource.com/settings
Gerrit-Project: public/gem5
Gerrit-Branch: develop
Gerrit-Change-Id: Ia835c1777425a5aafe8ba7ee9c725edf6d45f68c
Gerrit-Change-Number: 58851
Gerrit-PatchSet: 2
Gerrit-Owner: Bobby Bruce <bbr...@ucdavis.edu>
Gerrit-Reviewer: Bobby Bruce <bbr...@ucdavis.edu>
Gerrit-Reviewer: Jason Lowe-Power <power...@gmail.com>
Gerrit-Reviewer: kokoro <noreply+kok...@google.com>
Gerrit-MessageType: merged
_______________________________________________
gem5-dev mailing list -- gem5-dev@gem5.org
To unsubscribe send an email to gem5-dev-le...@gem5.org
%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s
