summaryrefslogtreecommitdiffstats
path: root/meta/classes/sstate.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'meta/classes/sstate.bbclass')
-rw-r--r--meta/classes/sstate.bbclass98
1 files changed, 78 insertions, 20 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 66a96a7603..1058778980 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -20,7 +20,7 @@ def generate_sstatefn(spec, hash, taskname, siginfo, d):
components = spec.split(":")
# Fields 0,5,6 are mandatory, 1 is most useful, 2,3,4 are just for information
# 7 is for the separators
- avail = (254 - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3
+ avail = (limit - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3
components[2] = components[2][:avail]
components[3] = components[3][:avail]
components[4] = components[4][:avail]
@@ -72,6 +72,7 @@ BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
SSTATE_ARCHS = " \
${BUILD_ARCH} \
+ ${BUILD_ARCH}_${ORIGNATIVELSBSTRING} \
${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
${BUILD_ARCH}_${TARGET_ARCH} \
${SDK_ARCH}_${SDK_OS} \
@@ -80,6 +81,7 @@ SSTATE_ARCHS = " \
${PACKAGE_ARCH} \
${PACKAGE_EXTRA_ARCHS} \
${MACHINE_ARCH}"
+SSTATE_ARCHS[vardepsexclude] = "ORIGNATIVELSBSTRING"
SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
@@ -315,6 +317,8 @@ def sstate_install(ss, d):
if os.path.exists(i):
with open(i, "r") as f:
manifests = f.readlines()
+ # We append new entries, we don't remove older entries which may have the same
+ # manifest name but different versions from stamp/workdir. See below.
if filedata not in manifests:
with open(i, "a+") as f:
f.write(filedata)
@@ -477,7 +481,7 @@ def sstate_clean_cachefiles(d):
ss = sstate_state_fromvars(ld, task)
sstate_clean_cachefile(ss, ld)
-def sstate_clean_manifest(manifest, d, prefix=None):
+def sstate_clean_manifest(manifest, d, canrace=False, prefix=None):
import oe.path
mfile = open(manifest)
@@ -495,7 +499,9 @@ def sstate_clean_manifest(manifest, d, prefix=None):
if entry.endswith("/"):
if os.path.islink(entry[:-1]):
os.remove(entry[:-1])
- elif os.path.exists(entry) and len(os.listdir(entry)) == 0:
+ elif os.path.exists(entry) and len(os.listdir(entry)) == 0 and not canrace:
+ # Removing directories whilst builds are in progress exposes a race. Only
+ # do it in contexts where it is safe to do so.
os.rmdir(entry[:-1])
else:
os.remove(entry)
@@ -533,7 +539,7 @@ def sstate_clean(ss, d):
for lock in ss['lockfiles']:
locks.append(bb.utils.lockfile(lock))
- sstate_clean_manifest(manifest, d)
+ sstate_clean_manifest(manifest, d, canrace=True)
for lock in locks:
bb.utils.unlockfile(lock)
@@ -634,10 +640,21 @@ python sstate_hardcode_path () {
def sstate_package(ss, d):
import oe.path
+ import time
tmpdir = d.getVar('TMPDIR')
+ fixtime = False
+ if ss['task'] == "package":
+ fixtime = True
+
+ def fixtimestamp(root, path):
+ f = os.path.join(root, path)
+ if os.lstat(f).st_mtime > sde:
+ os.utime(f, (sde, sde), follow_symlinks=False)
+
sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
+ sde = int(d.getVar("SOURCE_DATE_EPOCH") or time.time())
d.setVar("SSTATE_CURRTASK", ss['task'])
bb.utils.remove(sstatebuild, recurse=True)
bb.utils.mkdirhier(sstatebuild)
@@ -650,6 +667,8 @@ def sstate_package(ss, d):
# to sstate tasks but there aren't many of these so better just avoid them entirely.
for walkroot, dirs, files in os.walk(state[1]):
for file in files + dirs:
+ if fixtime:
+ fixtimestamp(walkroot, file)
srcpath = os.path.join(walkroot, file)
if not os.path.islink(srcpath):
continue
@@ -671,6 +690,11 @@ def sstate_package(ss, d):
bb.utils.mkdirhier(plain)
bb.utils.mkdirhier(pdir)
os.rename(plain, pdir)
+ if fixtime:
+ fixtimestamp(pdir, "")
+ for walkroot, dirs, files in os.walk(pdir):
+ for file in files + dirs:
+ fixtimestamp(walkroot, file)
d.setVar('SSTATE_BUILDDIR', sstatebuild)
d.setVar('SSTATE_INSTDIR', sstatebuild)
@@ -697,9 +721,16 @@ def sstate_package(ss, d):
os.utime(siginfo, None)
except PermissionError:
pass
+ except OSError as e:
+ # Handle read-only file systems gracefully
+ import errno
+ if e.errno != errno.EROFS:
+ raise e
return
+sstate_package[vardepsexclude] += "SSTATE_SIG_KEY"
+
def pstaging_fetch(sstatefetch, d):
import bb.fetch2
@@ -783,7 +814,7 @@ sstate_task_postfunc[dirs] = "${WORKDIR}"
sstate_create_package () {
# Exit early if it already exists
if [ -e ${SSTATE_PKG} ]; then
- [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
+ touch ${SSTATE_PKG} 2>/dev/null || true
return
fi
@@ -810,14 +841,18 @@ sstate_create_package () {
fi
chmod 0664 $TFILE
# Skip if it was already created by some other process
- if [ ! -e ${SSTATE_PKG} ]; then
+ if [ -h ${SSTATE_PKG} ] && [ ! -e ${SSTATE_PKG} ]; then
+ # There is a symbolic link, but it links to nothing.
+ # Forcefully replace it with the new file.
+ ln -f $TFILE ${SSTATE_PKG} || true
+ elif [ ! -e ${SSTATE_PKG} ]; then
# Move into place using ln to attempt an atomic op.
# Abort if it already exists
- ln $TFILE ${SSTATE_PKG} && rm $TFILE
+ ln $TFILE ${SSTATE_PKG} || true
else
- rm $TFILE
+ touch ${SSTATE_PKG} 2>/dev/null || true
fi
- [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
+ rm $TFILE
}
python sstate_sign_package () {
@@ -846,12 +881,12 @@ python sstate_report_unihash() {
#
sstate_unpack_package () {
tar -xvzf ${SSTATE_PKG}
- # update .siginfo atime on local/NFS mirror
- [ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
- # Use "! -w ||" to return true for read only files
- [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG}
- [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig
- [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo
+ # update .siginfo atime on local/NFS mirror if it is a symbolic link
+ [ ! -h ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch -a ${SSTATE_PKG}.siginfo 2>/dev/null || true
+ # update each symbolic link instead of any referenced file
+ touch --no-dereference ${SSTATE_PKG} 2>/dev/null || true
+ [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig 2>/dev/null || true
+ [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo 2>/dev/null || true
}
BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
@@ -926,7 +961,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
localdata2 = bb.data.createCopy(localdata)
srcuri = "file://" + sstatefile
- localdata.setVar('SRC_URI', srcuri)
+ localdata2.setVar('SRC_URI', srcuri)
bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
try:
@@ -937,10 +972,11 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
found.add(tid)
if tid in missed:
missed.remove(tid)
- except:
+ except bb.fetch2.FetchError as e:
missed.add(tid)
- bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
- pass
+ bb.debug(2, "SState: Unsuccessful fetch test for %s (%s)" % (srcuri, e))
+ except Exception as e:
+ bb.error("SState: cannot test %s: %s" % (srcuri, e))
if len(tasklist) >= min_tasks:
bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
@@ -1002,6 +1038,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
bb.parse.siggen.checkhashes(sq_data, missed, found, d)
return found
+setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT"
BB_SETSCENE_DEPVALID = "setscene_depvalid"
@@ -1027,6 +1064,10 @@ def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
if taskdependees[task][1] == "do_populate_lic":
return True
+ # We only need to trigger deploy_source_date_epoch through direct dependencies
+ if taskdependees[task][1] == "do_deploy_source_date_epoch":
+ return True
+
# stash_locale and gcc_stash_builddir are never needed as a dependency for built objects
if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir":
return True
@@ -1133,6 +1174,11 @@ python sstate_eventhandler() {
os.utime(siginfo, None)
except PermissionError:
pass
+ except OSError as e:
+ # Handle read-only file systems gracefully
+ import errno
+ if e.errno != errno.EROFS:
+ raise e
}
@@ -1171,11 +1217,21 @@ python sstate_eventhandler2() {
i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
if not os.path.exists(i):
continue
+ manseen = set()
+ ignore = []
with open(i, "r") as f:
lines = f.readlines()
- for l in lines:
+ for l in reversed(lines):
try:
(stamp, manifest, workdir) = l.split()
+ # The index may have multiple entries for the same manifest as the code above only appends
+ # new entries and there may be an entry with matching manifest but differing version in stamp/workdir.
+ # The last entry in the list is the valid one, any earlier entries with matching manifests
+ # should be ignored.
+ if manifest in manseen:
+ ignore.append(l)
+ continue
+ manseen.add(manifest)
if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
toremove.append(l)
if stamp not in seen:
@@ -1206,6 +1262,8 @@ python sstate_eventhandler2() {
with open(i, "w") as f:
for l in lines:
+ if l in ignore:
+ continue
f.write(l)
machineindex |= set(stamps)
with open(mi, "w") as f:
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-06-19 06:49:42 +0000