diff options
Diffstat (limited to 'bitbake/lib/bb/tests/fetch.py')
| -rw-r--r-- | bitbake/lib/bb/tests/fetch.py | 737 |
1 files changed, 649 insertions, 88 deletions
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 1152e89c0d5..85c1f79ff32 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py @@ -6,11 +6,13 @@ # SPDX-License-Identifier: GPL-2.0-only # +import contextlib import unittest import hashlib import tempfile import collections import os +import signal import tarfile from bb.fetch2 import URI from bb.fetch2 import FetchMethod @@ -22,6 +24,25 @@ def skipIfNoNetwork(): return unittest.skip("network test") return lambda f: f +class TestTimeout(Exception): + # Indicate to pytest that this is not a test suite + __test__ = False + +class Timeout(): + + def __init__(self, seconds): + self.seconds = seconds + + def handle_timeout(self, signum, frame): + raise TestTimeout("Test failed: timeout reached") + + def __enter__(self): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, exc_type, exc_val, exc_tb): + signal.alarm(0) + class URITest(unittest.TestCase): test_uris = { "http://www.google.com/index.html" : { @@ -287,6 +308,21 @@ class URITest(unittest.TestCase): 'params': {"someparam" : "1"}, 'query': {}, 'relative': True + }, + "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": { + 'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip', + 'scheme': 'https', + 'hostname': 'www.innodisk.com', + 'port': None, + 'hostport': 'www.innodisk.com', + 'path': '/Download_file', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {"downloadfilename" : "EGPL-T101.zip"}, + 'query': {"9BE0BF6657": None}, + 'relative': False } } @@ -396,18 +432,28 @@ class FetcherTest(unittest.TestCase): def git(self, cmd, cwd=None): if isinstance(cmd, str): - cmd = 'git ' + cmd + cmd = 'git -c safe.bareRepository=all ' + cmd else: - cmd = ['git'] + cmd + cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd if cwd is None: cwd = self.gitdir return bb.process.run(cmd, cwd=cwd)[0] def git_init(self, cwd=None): self.git('init', cwd=cwd) - if not self.git(['config', 'user.email'], cwd=cwd): + # Explicitly set initial branch to master as + # a common setup is to use other default + # branch than master. + self.git(['checkout', '-b', 'master'], cwd=cwd) + + try: + self.git(['config', 'user.email'], cwd=cwd) + except bb.process.ExecutionError: self.git(['config', 'user.email', 'you@example.com'], cwd=cwd) - if not self.git(['config', 'user.name'], cwd=cwd): + + try: + self.git(['config', 'user.name'], cwd=cwd) + except bb.process.ExecutionError: self.git(['config', 'user.name', 'Your Name'], cwd=cwd) class MirrorUriTest(FetcherTest): @@ -468,6 +514,7 @@ class MirrorUriTest(FetcherTest): "http://.*/.* file:///someotherpath/downloads/" def test_urireplace(self): + self.d.setVar("FILESPATH", ".") for k, v in self.replaceuris.items(): ud = bb.fetch.FetchData(k[0], self.d) ud.setup_localpath(self.d) @@ -515,7 +562,7 @@ class MirrorUriTest(FetcherTest): class GitDownloadDirectoryNamingTest(FetcherTest): def setUp(self): super(GitDownloadDirectoryNamingTest, self).setUp() - self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" + self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" self.recipe_dir = "git.openembedded.org.bitbake" self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" self.mirror_dir = "github.com.openembedded.bitbake.git" @@ -563,7 +610,7 @@ class GitDownloadDirectoryNamingTest(FetcherTest): class TarballNamingTest(FetcherTest): def setUp(self): super(TarballNamingTest, self).setUp() - self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" + self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" @@ -597,7 +644,7 @@ class TarballNamingTest(FetcherTest): class GitShallowTarballNamingTest(FetcherTest): def setUp(self): super(GitShallowTarballNamingTest, self).setUp() - self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" + self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" @@ -632,7 +679,7 @@ class GitShallowTarballNamingTest(FetcherTest): class CleanTarballTest(FetcherTest): def setUp(self): super(CleanTarballTest, self).setUp() - self.recipe_url = "git://git.openembedded.org/bitbake" + self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https" self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') @@ -653,11 +700,13 @@ class CleanTarballTest(FetcherTest): archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball)) self.assertNotEqual(len(archive.members), 0) for member in archive.members: - self.assertEqual(member.uname, 'oe') - self.assertEqual(member.uid, 0) - self.assertEqual(member.gname, 'oe') - self.assertEqual(member.gid, 0) - self.assertEqual(member.mtime, mtime) + if member.name == ".": + continue + self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name) + self.assertEqual(member.uid, 0, "uid for %s differs" % member.name) + self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name) + self.assertEqual(member.gid, 0, "gid for %s differs" % member.name) + self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name) class FetcherLocalTest(FetcherTest): @@ -693,6 +742,11 @@ class FetcherLocalTest(FetcherTest): flst.sort() return flst + def test_local_checksum_fails_no_file(self): + self.d.setVar("SRC_URI", "file://404") + with self.assertRaises(bb.BBHandledException): + bb.fetch.get_checksum_file_list(self.d) + def test_local(self): tree = self.fetchUnpack(['file://a', 'file://dir/c']) self.assertEqual(tree, ['a', 'dir/c']) @@ -760,7 +814,7 @@ class FetcherLocalTest(FetcherTest): # Fetch and check revision self.d.setVar("SRCREV", "AUTOINC") - self.d.setVar("__BBSEENSRCREV", "1") + self.d.setVar("__BBSRCREV_SEEN", "1") url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() @@ -920,6 +974,7 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_fetch_file_mirror_of_mirror(self): + self.d.setVar("FILESPATH", ".") self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) os.mkdir(self.dldir + "/some2where") @@ -985,25 +1040,25 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_gitfetch(self): - url1 = url2 = "git://git.openembedded.org/bitbake;branch=master" + url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https" self.gitfetcher(url1, url2) @skipIfNoNetwork() def test_gitfetch_goodsrcrev(self): # SRCREV is set but matches rev= parameter - url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master" + url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" self.gitfetcher(url1, url2) @skipIfNoNetwork() def test_gitfetch_badsrcrev(self): # SRCREV is set but does not match rev= parameter - url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master" + url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) @skipIfNoNetwork() def test_gitfetch_tagandrev(self): # SRCREV is set but does not match rev= parameter - url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5" + url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5;protocol=https" self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) @skipIfNoNetwork() @@ -1012,7 +1067,7 @@ class FetcherNetworkTest(FetcherTest): # `usehead=1' and instead fetch the specified SRCREV. See # test_local_gitfetch_usehead() for a positive use of the usehead # feature. - url = "git://git.openembedded.org/bitbake;usehead=1;branch=master" + url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https" self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) @skipIfNoNetwork() @@ -1021,26 +1076,26 @@ class FetcherNetworkTest(FetcherTest): # `usehead=1' and instead fetch the specified SRCREV. See # test_local_gitfetch_usehead() for a positive use of the usehead # feature. - url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master" + url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https" self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) @skipIfNoNetwork() def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): - recipeurl = "git://git.openembedded.org/bitbake;branch=master" - mirrorurl = "git://someserver.org/bitbake;branch=master" + recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https" + mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https" self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") self.gitfetcher(recipeurl, mirrorurl) @skipIfNoNetwork() def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): - recipeurl = "git://someserver.org/bitbake;branch=master" + recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https" self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") self.gitfetcher(recipeurl, recipeurl) @skipIfNoNetwork() def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): - realurl = "git://git.openembedded.org/bitbake" - recipeurl = "git://someserver.org/bitbake" + realurl = "https://git.openembedded.org/bitbake" + recipeurl = "git://someserver.org/bitbake;protocol=https" self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") os.chdir(self.tempdir) self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) @@ -1050,9 +1105,9 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_git_submodule(self): # URL with ssh submodules - url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master" + url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https" # Original URL (comment this if you have ssh access to git.yoctoproject.org) - url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master" + url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -1069,6 +1124,25 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') @skipIfNoNetwork() + def test_git_submodule_restricted_network_premirrors(self): + # this test is to ensure that premirrors will be tried in restricted network + # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses + url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231" + # create a download directory to be used as premirror later + tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") + dl_premirror = os.path.join(tempdir, "download-premirror") + os.mkdir(dl_premirror) + self.d.setVar("DL_DIR", dl_premirror) + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + # now use the premirror in restricted network + self.d.setVar("DL_DIR", self.dldir) + self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror) + self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain") + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + + @skipIfNoNetwork() def test_git_submodule_dbus_broker(self): # The following external repositories have show failures in fetch and unpack operations # We want to avoid regressions! @@ -1165,6 +1239,15 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') + @skipIfNoNetwork() + def test_git_submodule_reference_to_parent(self): + self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master" + self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1") + with Timeout(60): + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + with self.assertRaises(bb.fetch2.FetchError): + fetcher.download() + class SVNTest(FetcherTest): def skipIfNoSvn(): import shutil @@ -1199,8 +1282,9 @@ class SVNTest(FetcherTest): cwd=repo_dir) bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) - # Github will emulate SVN. Use this to check if we're downloding... - bb.process.run("svn propset svn:externals 'bitbake https://github.com/PhilipHazel/pcre2.git' .", + # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/) + # Use still accessible svn repo (only trunk to avoid longer downloads) + bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .", cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) bb.process.run("svn commit --non-interactive -m 'Add external'", cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) @@ -1228,8 +1312,8 @@ class SVNTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") - self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist") - self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit") + self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist") + self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit") @skipIfNoSvn() def test_external_svn(self): @@ -1242,8 +1326,8 @@ class SVNTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") - self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist") - self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit") + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist") + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit") class TrustedNetworksTest(FetcherTest): def test_trusted_network(self): @@ -1294,14 +1378,17 @@ class URLHandle(unittest.TestCase): "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), - "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}), + "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}), "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), + "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}), + r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main;protocol=https': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main', 'protocol' : 'https'}), } # we require a pathname to encodeurl but users can still pass such urls to # decodeurl and we need to handle them decodedata = datatable.copy() decodedata.update({ "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), + "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}), }) def test_decodeurl(self): @@ -1318,37 +1405,39 @@ class FetchLatestVersionTest(FetcherTest): test_git_uris = { # version pattern "X.Y.Z" - ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") + ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "") : "1.99.4", # version pattern "vX.Y" # mirror of git.infradead.org since network issues interfered with testing - ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") + ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "") : "1.5.0", # version pattern "pkg_name-X.Y" # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing - ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") + ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "") : "1.0", # version pattern "pkg_name-vX.Y.Z" - ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "") + ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "") : "1.4.0", # combination version pattern - ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "") + ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "") : "1.2.0", - ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "") + ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "") : "2014.01", # version pattern "yyyymmdd" - ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "") + ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "") : "20120614", # packages with a valid UPSTREAM_CHECK_GITTAGREGEX # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing - ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))") + ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "") : "0.4.3", - ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))") + ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "") : "11.0.0", - ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))") + ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "") : "1.3.59", - ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))") + ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "") : "3.82+dbg0.9", + ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0") + : "0.28.0", } test_wget_uris = { @@ -1364,6 +1453,9 @@ class FetchLatestVersionTest(FetcherTest): # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") : "2.8.12.1", + # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz + ("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz", "", "") + : "2.10.3", # # packages with versions only in current directory # @@ -1413,6 +1505,9 @@ class FetchLatestVersionTest(FetcherTest): self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) r = bb.utils.vercmp_string(v, verstring) self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) + if k[4]: + r = bb.utils.vercmp_string(verstring, k[4]) + self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4])) def test_wget_latest_versionstring(self): testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" @@ -1614,7 +1709,7 @@ class GitShallowTest(FetcherTest): self.d.setVar('BB_GIT_SHALLOW', '1') self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') - self.d.setVar("__BBSEENSRCREV", "1") + self.d.setVar("__BBSRCREV_SEEN", "1") def assertRefs(self, expected_refs, cwd=None): if cwd is None: @@ -1834,7 +1929,7 @@ class GitShallowTest(FetcherTest): self.add_empty_file('bsub', cwd=smdir) self.git('submodule init', cwd=self.srcdir) - self.git('submodule add file://%s' % smdir, cwd=self.srcdir) + self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) self.git('submodule update', cwd=self.srcdir) self.git('commit -m submodule -a', cwd=self.srcdir) @@ -1864,7 +1959,7 @@ class GitShallowTest(FetcherTest): self.add_empty_file('bsub', cwd=smdir) self.git('submodule init', cwd=self.srcdir) - self.git('submodule add file://%s' % smdir, cwd=self.srcdir) + self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) self.git('submodule update', cwd=self.srcdir) self.git('commit -m submodule -a', cwd=self.srcdir) @@ -2149,7 +2244,7 @@ class GitShallowTest(FetcherTest): self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0') self.d.setVar('BB_GIT_SHALLOW', '1') self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') - fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master"], self.d) + fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master;protocol=https"], self.d) fetcher.download() bb.utils.remove(self.dldir + "/*.tar.gz") @@ -2159,6 +2254,12 @@ class GitShallowTest(FetcherTest): self.assertIn("fstests.doap", dir) class GitLfsTest(FetcherTest): + def skipIfNoGitLFS(): + import shutil + if not shutil.which('git-lfs'): + return unittest.skip('git-lfs not installed') + return lambda f: f + def setUp(self): FetcherTest.setUp(self) @@ -2172,14 +2273,18 @@ class GitLfsTest(FetcherTest): self.d.setVar('SRCREV', '${AUTOREV}') self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') - self.d.setVar("__BBSEENSRCREV", "1") + self.d.setVar("__BBSRCREV_SEEN", "1") bb.utils.mkdirhier(self.srcdir) self.git_init(cwd=self.srcdir) - with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: - attrs.write('*.mp3 filter=lfs -text') - self.git(['add', '.gitattributes'], cwd=self.srcdir) - self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) + self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') + + def commit_file(self, filename, content): + with open(os.path.join(self.srcdir, filename), "w") as f: + f.write(content) + self.git(["add", filename], cwd=self.srcdir) + self.git(["commit", "-m", "Change"], cwd=self.srcdir) + return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip() def fetch(self, uri=None, download=True): uris = self.d.getVar('SRC_URI').split() @@ -2192,55 +2297,148 @@ class GitLfsTest(FetcherTest): ud = fetcher.ud[uri] return fetcher, ud + def get_real_git_lfs_file(self): + self.d.setVar('PATH', os.environ.get('PATH')) + fetcher, ud = self.fetch() + fetcher.unpack(self.d.getVar('WORKDIR')) + unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") + return unpacked_lfs_file + + @skipIfNoGitLFS() + def test_fetch_lfs_on_srcrev_change(self): + """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" + self.git(["lfs", "install", "--local"], cwd=self.srcdir) + + @contextlib.contextmanager + def hide_upstream_repository(): + """Hide the upstream repository to make sure that git lfs cannot pull from it""" + temp_name = self.srcdir + ".bak" + os.rename(self.srcdir, temp_name) + try: + yield + finally: + os.rename(temp_name, self.srcdir) + + def fetch_and_verify(revision, filename, content): + self.d.setVar('SRCREV', revision) + fetcher, ud = self.fetch() + + with hide_upstream_repository(): + workdir = self.d.getVar('WORKDIR') + fetcher.unpack(workdir) + + with open(os.path.join(workdir, "git", filename)) as f: + self.assertEqual(f.read(), content) + + commit_1 = self.commit_file("a.mp3", "version 1") + commit_2 = self.commit_file("a.mp3", "version 2") + + self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir) + + # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are + # available even when the upstream repository disappears. + fetch_and_verify(commit_2, "a.mp3", "version 2") + # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download + # folder. + fetch_and_verify(commit_1, "a.mp3", "version 1") + + @skipIfNoGitLFS() + @skipIfNoNetwork() + def test_real_git_lfs_repo_succeeds_without_lfs_param(self): + self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") + f = self.get_real_git_lfs_file() + self.assertTrue(os.path.exists(f)) + self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) + + @skipIfNoGitLFS() + @skipIfNoNetwork() + def test_real_git_lfs_repo_succeeds(self): + self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1") + f = self.get_real_git_lfs_file() + self.assertTrue(os.path.exists(f)) + self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) + + @skipIfNoGitLFS() + @skipIfNoNetwork() + def test_real_git_lfs_repo_skips(self): + self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0") + f = self.get_real_git_lfs_file() + # This is the actual non-smudged placeholder file on the repo if git-lfs does not run + lfs_file = ( + 'version https://git-lfs.github.com/spec/v1\n' + 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n' + 'size 11423554\n' + ) + + with open(f) as fh: + self.assertEqual(lfs_file, fh.read()) + + @skipIfNoGitLFS() def test_lfs_enabled(self): import shutil uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir self.d.setVar('SRC_URI', uri) - # Careful: suppress initial attempt at downloading until - # we know whether git-lfs is installed. - fetcher, ud = self.fetch(uri=None, download=False) - self.assertIsNotNone(ud.method._find_git_lfs) - - # If git-lfs can be found, the unpack should be successful. Only - # attempt this with the real live copy of git-lfs installed. - if ud.method._find_git_lfs(self.d): - fetcher.download() - shutil.rmtree(self.gitdir, ignore_errors=True) - fetcher.unpack(self.d.getVar('WORKDIR')) - - # If git-lfs cannot be found, the unpack should throw an error - with self.assertRaises(bb.fetch2.FetchError): - fetcher.download() - ud.method._find_git_lfs = lambda d: False - shutil.rmtree(self.gitdir, ignore_errors=True) - fetcher.unpack(self.d.getVar('WORKDIR')) + # With git-lfs installed, test that we can fetch and unpack + fetcher, ud = self.fetch() + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + @skipIfNoGitLFS() def test_lfs_disabled(self): import shutil uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir self.d.setVar('SRC_URI', uri) - # In contrast to test_lfs_enabled(), allow the implicit download - # done by self.fetch() to occur here. The point of this test case - # is to verify that the fetcher can survive even if the source + # Verify that the fetcher can survive even if the source # repository has Git LFS usage configured. fetcher, ud = self.fetch() - self.assertIsNotNone(ud.method._find_git_lfs) - - # If git-lfs can be found, the unpack should be successful. A - # live copy of git-lfs is not required for this case, so - # unconditionally forge its presence. - ud.method._find_git_lfs = lambda d: True - shutil.rmtree(self.gitdir, ignore_errors=True) fetcher.unpack(self.d.getVar('WORKDIR')) - # If git-lfs cannot be found, the unpack should be successful - ud.method._find_git_lfs = lambda d: False - shutil.rmtree(self.gitdir, ignore_errors=True) - fetcher.unpack(self.d.getVar('WORKDIR')) + def test_lfs_enabled_not_installed(self): + import shutil + + uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir + self.d.setVar('SRC_URI', uri) + + # Careful: suppress initial attempt at downloading + fetcher, ud = self.fetch(uri=None, download=False) + + # Artificially assert that git-lfs is not installed, so + # we can verify a failure to unpack in it's absence. + old_find_git_lfs = ud.method._find_git_lfs + try: + # If git-lfs cannot be found, the unpack should throw an error + with self.assertRaises(bb.fetch2.FetchError): + fetcher.download() + ud.method._find_git_lfs = lambda d: False + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + finally: + ud.method._find_git_lfs = old_find_git_lfs + + def test_lfs_disabled_not_installed(self): + import shutil + + uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir + self.d.setVar('SRC_URI', uri) + + # Careful: suppress initial attempt at downloading + fetcher, ud = self.fetch(uri=None, download=False) + + # Artificially assert that git-lfs is not installed, so + # we can verify a failure to unpack in it's absence. + old_find_git_lfs = ud.method._find_git_lfs + try: + # Even if git-lfs cannot be found, the unpack should be successful + fetcher.download() + ud.method._find_git_lfs = lambda d: False + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + finally: + ud.method._find_git_lfs = old_find_git_lfs class GitURLWithSpacesTest(FetcherTest): test_git_urls = { @@ -2285,6 +2483,13 @@ class CrateTest(FetcherTest): d = self.d fetcher = bb.fetch2.Fetch(uris, self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertIn("name", ud.parm) + self.assertEqual(ud.parm["name"], "glob-0.2.11") + self.assertIn("downloadfilename", ud.parm) + self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") + fetcher.download() fetcher.unpack(self.tempdir) self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) @@ -2293,6 +2498,55 @@ class CrateTest(FetcherTest): self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) @skipIfNoNetwork() + def test_crate_url_matching_recipe(self): + + self.d.setVar('BP', 'glob-0.2.11') + + uri = "crate://crates.io/glob/0.2.11" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertIn("name", ud.parm) + self.assertEqual(ud.parm["name"], "glob-0.2.11") + self.assertIn("downloadfilename", ud.parm) + self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") + + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) + self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs")) + + @skipIfNoNetwork() + def test_crate_url_params(self): + + uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertIn("name", ud.parm) + self.assertEqual(ud.parm["name"], "aho-corasick-renamed") + self.assertIn("downloadfilename", ud.parm) + self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate") + + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done']) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json")) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs")) + + @skipIfNoNetwork() def test_crate_url_multi(self): uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" @@ -2302,6 +2556,19 @@ class CrateTest(FetcherTest): d = self.d fetcher = bb.fetch2.Fetch(uris, self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertIn("name", ud.parm) + self.assertEqual(ud.parm["name"], "glob-0.2.11") + self.assertIn("downloadfilename", ud.parm) + self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") + + ud = fetcher.ud[fetcher.urls[1]] + self.assertIn("name", ud.parm) + self.assertEqual(ud.parm["name"], "time-0.1.35") + self.assertIn("downloadfilename", ud.parm) + self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate") + fetcher.download() fetcher.unpack(self.tempdir) self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) @@ -2311,6 +2578,18 @@ class CrateTest(FetcherTest): self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) + @skipIfNoNetwork() + def test_crate_incorrect_cksum(self): + uri = "crate://crates.io/aho-corasick/0.7.20" + self.d.setVar('SRC_URI', uri) + self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest()) + + uris = self.d.getVar('SRC_URI').split() + + fetcher = bb.fetch2.Fetch(uris, self.d) + with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"): + fetcher.download() + class NPMTest(FetcherTest): def skipIfNoNpm(): import shutil @@ -2574,6 +2853,45 @@ class NPMTest(FetcherTest): @skipIfNoNpm() @skipIfNoNetwork() + def test_npmsw_git(self): + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'cookie': { + 'version': 'github:jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', + 'from': 'github:jshttp/cookie.git' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) + + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'cookie': { + 'version': 'jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', + 'from': 'jshttp/cookie.git' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) + + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'nodejs': { + 'version': 'gitlab:gitlab-examples/nodejs.git#892a1f16725e56cc3a2cb0d677be42935c8fc262', + 'from': 'gitlab:gitlab-examples/nodejs' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'gitlab.com.gitlab-examples.nodejs.git'))) + + @skipIfNoNpm() + @skipIfNoNetwork() def test_npmsw_dev(self): swfile = self.create_shrinkwrap_file({ 'dependencies': { @@ -2779,9 +3097,9 @@ class NPMTest(FetcherTest): class GitSharedTest(FetcherTest): def setUp(self): super(GitSharedTest, self).setUp() - self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" + self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') - self.d.setVar("__BBSEENSRCREV", "1") + self.d.setVar("__BBSRCREV_SEEN", "1") @skipIfNoNetwork() def test_shared_unpack(self): @@ -2802,3 +3120,246 @@ class GitSharedTest(FetcherTest): fetcher.unpack(self.unpackdir) alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') self.assertFalse(os.path.exists(alt)) + + +class FetchPremirroronlyLocalTest(FetcherTest): + + def setUp(self): + super(FetchPremirroronlyLocalTest, self).setUp() + self.mirrordir = os.path.join(self.tempdir, "mirrors") + os.mkdir(self.mirrordir) + self.reponame = "bitbake" + self.gitdir = os.path.join(self.tempdir, "git", self.reponame) + self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https" + self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") + self.d.setVar("BB_NO_NETWORK", "1") + self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") + self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" + self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname) + self.testfilename = "bitbake-fetch.test" + + def make_git_repo(self): + recipeurl = "git:/git.fake.repo/bitbake" + os.makedirs(self.gitdir) + self.git_init(cwd=self.gitdir) + for i in range(0): + self.git_new_commit() + bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) + + def git_new_commit(self): + import random + os.unlink(os.path.join(self.mirrordir, self.mirrorname)) + branch = self.git("branch --show-current", self.gitdir).split() + with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile: + testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random())) + self.git("add {}".format(self.testfilename), self.gitdir) + self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir) + bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) + return self.git("rev-parse HEAD", self.gitdir).strip() + + def git_new_branch(self, name): + self.git_new_commit() + head = self.git("rev-parse HEAD", self.gitdir).strip() + self.git("checkout -b {}".format(name), self.gitdir) + newrev = self.git_new_commit() + self.git("checkout {}".format(head), self.gitdir) + return newrev + + def test_mirror_multiple_fetches(self): + self.make_git_repo() + self.d.setVar("SRCREV", self.git_new_commit()) + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + ## New commit in premirror. it's not in the download_dir + self.d.setVar("SRCREV", self.git_new_commit()) + fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d) + fetcher2.download() + fetcher2.unpack(self.unpackdir) + ## New commit in premirror. it's not in the download_dir + self.d.setVar("SRCREV", self.git_new_commit()) + fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d) + fetcher3.download() + fetcher3.unpack(self.unpackdir) + + + def test_mirror_commit_nonexistent(self): + self.make_git_repo() + self.d.setVar("SRCREV", "0"*40) + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + def test_mirror_commit_exists(self): + self.make_git_repo() + self.d.setVar("SRCREV", self.git_new_commit()) + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + def test_mirror_tarball_nonexistent(self): + self.d.setVar("SRCREV", "0"*40) + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + def test_mirror_tarball_multiple_branches(self): + """ + test if PREMIRRORS can handle multiple name/branches correctly + both branches have required revisions + """ + self.make_git_repo() + branch1rev = self.git_new_branch("testbranch1") + branch2rev = self.git_new_branch("testbranch2") + self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" + self.d.setVar("SRCREV_branch1", branch1rev) + self.d.setVar("SRCREV_branch2", branch2rev) + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") + fetcher.download() + fetcher.unpack(os.path.join(self.tempdir, "unpacked")) + unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename) + self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!") + with open(unpacked, 'r') as f: + content = f.read() + ## We expect to see testbranch1 in the file, not master, not testbranch2 + self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!") + + def test_mirror_tarball_multiple_branches_nobranch(self): + """ + test if PREMIRRORS can handle multiple name/branches correctly + Unbalanced name/branches raises ParameterError + """ + self.make_git_repo() + branch1rev = self.git_new_branch("testbranch1") + branch2rev = self.git_new_branch("testbranch2") + self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2" + self.d.setVar("SRCREV_branch1", branch1rev) + self.d.setVar("SRCREV_branch2", branch2rev) + with self.assertRaises(bb.fetch2.ParameterError): + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + + def test_mirror_tarball_multiple_branches_norev(self): + """ + test if PREMIRRORS can handle multiple name/branches correctly + one of the branches specifies non existing SRCREV + """ + self.make_git_repo() + branch1rev = self.git_new_branch("testbranch1") + branch2rev = self.git_new_branch("testbranch2") + self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" + self.d.setVar("SRCREV_branch1", branch1rev) + self.d.setVar("SRCREV_branch2", "0"*40) + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + +class FetchPremirroronlyNetworkTest(FetcherTest): + + def setUp(self): + super(FetchPremirroronlyNetworkTest, self).setUp() + self.mirrordir = os.path.join(self.tempdir, "mirrors") + os.mkdir(self.mirrordir) + self.reponame = "fstests" + self.clonedir = os.path.join(self.tempdir, "git") + self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) + self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https" + self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") + self.d.setVar("BB_NO_NETWORK", "0") + self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") + + def make_git_repo(self): + import shutil + self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" + os.makedirs(self.clonedir) + self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir) + bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) + shutil.rmtree(self.clonedir) + + @skipIfNoNetwork() + def test_mirror_tarball_updated(self): + self.make_git_repo() + ## Upstream commit is in the mirror + self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec") + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + fetcher.download() + + @skipIfNoNetwork() + def test_mirror_tarball_outdated(self): + self.make_git_repo() + ## Upstream commit not in the mirror + self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f") + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + +class FetchPremirroronlyMercurialTest(FetcherTest): + """ Test for premirrors with mercurial repos + the test covers also basic hg:// clone (see fetch_and_create_tarball + """ + def skipIfNoHg(): + import shutil + if not shutil.which('hg'): + return unittest.skip('Mercurial not installed') + return lambda f: f + + def setUp(self): + super(FetchPremirroronlyMercurialTest, self).setUp() + self.mirrordir = os.path.join(self.tempdir, "mirrors") + os.mkdir(self.mirrordir) + self.reponame = "libgnt" + self.clonedir = os.path.join(self.tempdir, "hg") + self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt" + self.d.setVar("SRCREV", "53e8b422faaf") + self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz" + + def fetch_and_create_tarball(self): + """ + Ask bitbake to download repo and prepare mirror tarball for us + """ + self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + fetcher.download() + mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname) + self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile)) + ## moving tarball to mirror directory + os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname)) + self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0") + + + @skipIfNoNetwork() + @skipIfNoHg() + def test_premirror_mercurial(self): + self.fetch_and_create_tarball() + self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") + self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") + self.d.setVar("BB_NO_NETWORK", "1") + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + fetcher.download() + +class FetchPremirroronlyBrokenTarball(FetcherTest): + + def setUp(self): + super(FetchPremirroronlyBrokenTarball, self).setUp() + self.mirrordir = os.path.join(self.tempdir, "mirrors") + os.mkdir(self.mirrordir) + self.reponame = "bitbake" + self.gitdir = os.path.join(self.tempdir, "git", self.reponame) + self.recipe_url = "git://git.fake.repo/bitbake;protocol=https" + self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") + self.d.setVar("BB_NO_NETWORK", "1") + self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") + self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" + with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz: + targz.write("This is not tar.gz file!") + + def test_mirror_broken_download(self): + import sys + self.d.setVar("SRCREV", "0"*40) + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs: + fetcher.download() + output = "".join(logs.output) + self.assertFalse(" not a git repository (or any parent up to mount point /)" in output) |