>From 57b633a93bd91b7b1aa21cce5ac7997b958ca917 Mon Sep 17 00:00:00 2001
From: Devendra Tewari <devendra.tew...@gmail.com>
Date: Thu, 1 Apr 2021 16:07:25 -0300
Subject: [PATCH] Use shutil.move when os.rename fails
Incremental build in Docker fails with
OSError: [Errno 18] Invalid cross-device link
When source and destination are on different overlay filesystems.
This change handles error with os.rename and retries with shutil.move.
The reason os.rename is still used is because shutil.move is too slow
for speed sensitive sections of code.
[Yocto #14301]
---
meta/classes/sstate.bbclass | 22 ++++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 8e8efd18d5..60f7a94285 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -384,6 +384,7 @@ def sstate_installpkg(ss, d):
def sstate_installpkgdir(ss, d):
import oe.path
import subprocess
+ import shutil
sstateinst = d.getVar("SSTATE_INSTDIR")
d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
@@ -401,7 +402,10 @@ def sstate_installpkgdir(ss, d):
for state in ss['dirs']:
prepdir(state[1])
- os.rename(sstateinst + state[0], state[1])
+ try:
+ os.rename(sstateinst + state[0], state[1])
+ except OSError:
+ shutil.move(sstateinst + state[0], state[1])
sstate_install(ss, d)
for plain in ss['plaindirs']:
@@ -413,7 +417,10 @@ def sstate_installpkgdir(ss, d):
dest = plain
bb.utils.mkdirhier(src)
prepdir(dest)
- os.rename(src, dest)
+ try:
+ os.rename(src, dest)
+ except OSError:
+ shutil.move(src, dest)
return True
@@ -638,6 +645,7 @@ python sstate_hardcode_path () {
def sstate_package(ss, d):
import oe.path
+ import shutil
tmpdir = d.getVar('TMPDIR')
@@ -664,7 +672,10 @@ def sstate_package(ss, d):
continue
bb.error("sstate found an absolute path symlink %s
pointing at %s. Please replace this with a relative link." % (srcpath,
link))
bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1],
sstatebuild + state[0]))
- os.rename(state[1], sstatebuild + state[0])
+ try:
+ os.rename(state[1], sstatebuild + state[0])
+ except OSError:
+ shutil.move(state[1], sstatebuild + state[0])
workdir = d.getVar('WORKDIR')
sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
@@ -674,7 +685,10 @@ def sstate_package(ss, d):
pdir = plain.replace(sharedworkdir, sstatebuild)
bb.utils.mkdirhier(plain)
bb.utils.mkdirhier(pdir)
- os.rename(plain, pdir)
+ try:
+ os.rename(plain, pdir)
+ except OSError:
+ shutil.move(plain, pdir)
d.setVar('SSTATE_BUILDDIR', sstatebuild)
d.setVar('SSTATE_INSTDIR', sstatebuild)
--
2.29.2
On Tue, Mar 30, 2021 at 7:39 PM Devendra Tewari <devendra.tew...@gmail.com>
wrote:
> Here's the correct link
> https://bugzilla.yoctoproject.org/show_bug.cgi?id=14301. I'll resubmit
> the patch referencing the bug in the commit message. Thanks.
>
> Em 30 de mar. de 2021, à(s) 18:59, Denys Dmytriyenko <de...@denix.org>
> escreveu:
>
> The link is for a 10-year old bug, probably not what you wanted.
> Also, if a patch fixes existing bug in bugzilla, it needs to reference it
> in
> commit message as well - [YOCTO#1234567]
>
>
> On Mon, Mar 29, 2021 at 12:37:45PM -0300, Devendra Tewari wrote:
>
> Also, this is due to
> https://bugzilla.yoctoproject.org/show_bug.cgi?id=1430.
>
>
> On 29 Mar 2021, at 12:35, Devendra Tewari <devendra.tew...@gmail.com>
> wrote:
>
>
> Sure.
>
>
> Would the following commit message be sufficient?
>
>
> Use shutil.move when os.rename fails
>
>
> Incremental build in Docker fails with
>
>
> OSError: [Errno 18] Invalid cross-device link
>
>
> When source and destination are on different overlay filesystems.
>
>
> This change handles the error with os.rename and retries with
> shutil.move.
>
>
> Thanks,
>
> Devendra
>
>
> On 29 Mar 2021, at 12:23, Konrad Weihmann <kweihm...@outlook.com> wrote:
>
>
> Yes please quote a bit from the python manpage [1] - I certainly see the
> difference (and the edge cases where os.rename might fail), but that should
> be documented as part of the commit message
>
>
> [1] https://docs.python.org/3/library/shutil.html#shutil.move
>
>
> On 29.03.21 17:21, Bruce Ashfield wrote:
>
> Can you document the cases that os.rename() is failing ? And also why
>
> would we expect the shutil.move() to work in those cases ?
>
> If a change like this cases issues in the future, we need that extra
>
> information in the commit head for proper triage.
>
> Bruce
>
> On Mon, Mar 29, 2021 at 11:16 AM Devendra Tewari
>
> <devendra.tew...@gmail.com> wrote:
>
>
> ---
>
> meta/classes/sstate.bbclass | 26 ++++++++++++++++++++++----
>
> 1 file changed, 22 insertions(+), 4 deletions(-)
>
>
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
>
> index f579168162..f94aa96d70 100644
>
> --- a/meta/classes/sstate.bbclass
>
> +++ b/meta/classes/sstate.bbclass
>
> @@ -384,6 +384,7 @@ def sstate_installpkg(ss, d):
>
> def sstate_installpkgdir(ss, d):
>
> import oe.path
>
> import subprocess
>
> + import shutil
>
>
> sstateinst = d.getVar("SSTATE_INSTDIR")
>
> d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
>
> @@ -401,7 +402,11 @@ def sstate_installpkgdir(ss, d):
>
>
> for state in ss['dirs']:
>
> prepdir(state[1])
>
> - os.rename(sstateinst + state[0], state[1])
>
> + try:
>
> + os.rename(sstateinst + state[0], state[1])
>
> + break
>
> + except OSError:
>
> + shutil.move(sstateinst + state[0], state[1])
>
> sstate_install(ss, d)
>
>
> for plain in ss['plaindirs']:
>
> @@ -413,7 +418,11 @@ def sstate_installpkgdir(ss, d):
>
> dest = plain
>
> bb.utils.mkdirhier(src)
>
> prepdir(dest)
>
> - os.rename(src, dest)
>
> + try:
>
> + os.rename(src, dest)
>
> + break
>
> + except OSError:
>
> + shutil.move(src, dest)
>
>
> return True
>
>
> @@ -638,6 +647,7 @@ python sstate_hardcode_path () {
>
>
> def sstate_package(ss, d):
>
> import oe.path
>
> + import shutil
>
>
> tmpdir = d.getVar('TMPDIR')
>
>
> @@ -664,7 +674,11 @@ def sstate_package(ss, d):
>
> continue
>
> bb.error("sstate found an absolute path symlink %s pointing
> at %s. Please replace this with a relative link." % (srcpath, link))
>
> bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1],
> sstatebuild + state[0]))
>
> - os.rename(state[1], sstatebuild + state[0])
>
> + try:
>
> + os.rename(state[1], sstatebuild + state[0])
>
> + break
>
> + except OSError:
>
> + shutil.move(state[1], sstatebuild + state[0])
>
>
> workdir = d.getVar('WORKDIR')
>
> sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
>
> @@ -674,7 +688,11 @@ def sstate_package(ss, d):
>
> pdir = plain.replace(sharedworkdir, sstatebuild)
>
> bb.utils.mkdirhier(plain)
>
> bb.utils.mkdirhier(pdir)
>
> - os.rename(plain, pdir)
>
> + try:
>
> + os.rename(plain, pdir)
>
> + break
>
> + except OSError:
>
> + shutil.move(plain, pdir)
>
>
> d.setVar('SSTATE_BUILDDIR', sstatebuild)
>
> d.setVar('SSTATE_INSTDIR', sstatebuild)
>
> --
>
> 2.29.2
>
>
>
>
>
>
>
>
>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#150169):
https://lists.openembedded.org/g/openembedded-core/message/150169
Mute This Topic: https://lists.openembedded.org/mt/81698791/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
