summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRoss Burton <ross.burton@intel.com>2019-11-07 23:57:33 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-11-21 23:09:03 +0000
commit9d2fd91844fe387186b780b5457d0c02bea998e7 (patch)
treeaa7f445c6ba6dc168050a234fa8b360514115dcf /bitbake/lib/bb
parent1d8a0f4f00e4dda80b9e1c6dc05ac5d99c5a653c (diff)
downloadpoky-9d2fd91844fe387186b780b5457d0c02bea998e7.tar.gz
poky-9d2fd91844fe387186b780b5457d0c02bea998e7.tar.bz2
poky-9d2fd91844fe387186b780b5457d0c02bea998e7.zip
bitbake: utils: also use mmap for SHA256 and SHA1, for performance
md5_file() uses a mmap() window to improve performance when hashing files, so refactor the code and do the same for SHA1 and SHA256. (Bitbake rev: ecf87437ff796e17c3e4f210b5803b0136a9e8a4) Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/utils.py34
1 files changed, 14 insertions, 20 deletions
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index d035949b3d..8d40bcdf83 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -520,22 +520,26 @@ def unlockfile(lf):
fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
lf.close()
-def md5_file(filename):
- """
- Return the hex string representation of the MD5 checksum of filename.
- """
- import hashlib, mmap
+def _hasher(method, filename):
+ import mmap
with open(filename, "rb") as f:
- m = hashlib.md5()
try:
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
for chunk in iter(lambda: mm.read(8192), b''):
- m.update(chunk)
+ method.update(chunk)
except ValueError:
# You can't mmap() an empty file so silence this exception
pass
- return m.hexdigest()
+ return method.hexdigest()
+
+
+def md5_file(filename):
+ """
+ Return the hex string representation of the MD5 checksum of filename.
+ """
+ import hashlib
+ return _hasher(hashlib.md5(), filename)
def sha256_file(filename):
"""
@@ -543,24 +547,14 @@ def sha256_file(filename):
filename.
"""
import hashlib
-
- s = hashlib.sha256()
- with open(filename, "rb") as f:
- for line in f:
- s.update(line)
- return s.hexdigest()
+ return _hasher(hashlib.sha256(), filename)
def sha1_file(filename):
"""
Return the hex string representation of the SHA1 checksum of the filename
"""
import hashlib
-
- s = hashlib.sha1()
- with open(filename, "rb") as f:
- for line in f:
- s.update(line)
- return s.hexdigest()
+ return _hasher(hashlib.sha1(), filename)
def preserved_envvars_exported():
"""Variables which are taken from the environment and placed in and exported