diff options
Diffstat (limited to 'meta/lib/oeqa')
46 files changed, 1186 insertions, 178 deletions
diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index 0db6cb80a9..0b7e7dc42d 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py @@ -10,25 +10,29 @@ import os, re, mmap import unittest import inspect - - -def loadTests(tc): - - # set the context object passed from the test class - setattr(oeTest, "tc", tc) - # set ps command to use - setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps") - # prepare test suite, loader and runner - suite = unittest.TestSuite() +import subprocess +from oeqa.utils.decorators import LogResults + +def loadTests(tc, type="runtime"): + if type == "runtime": + # set the context object passed from the test class + setattr(oeTest, "tc", tc) + # set ps command to use + setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps") + # prepare test suite, loader and runner + suite = unittest.TestSuite() + elif type == "sdk": + # set the context object passed from the test class + setattr(oeTest, "tc", tc) testloader = unittest.TestLoader() testloader.sortTestMethodsUsing = None suite = testloader.loadTestsFromNames(tc.testslist) return suite -def runTests(tc): +def runTests(tc, type="runtime"): - suite = loadTests(tc) + suite = loadTests(tc, type) print("Test modules %s" % tc.testslist) print("Found %s tests" % suite.countTestCases()) runner = unittest.TextTestRunner(verbosity=2) @@ -36,24 +40,10 @@ def runTests(tc): return result - +@LogResults class oeTest(unittest.TestCase): longMessage = True - testFailures = [] - testSkipped = [] - testErrors = [] - - def run(self, result=None): - super(oeTest, self).run(result) - - # we add to our own lists the results, we use those for decorators - if len(result.failures) > len(oeTest.testFailures): - oeTest.testFailures.append(str(result.failures[-1][0]).split()[0]) - if len(result.skipped) > len(oeTest.testSkipped): - oeTest.testSkipped.append(str(result.skipped[-1][0]).split()[0]) - if len(result.errors) > len(oeTest.testErrors): - oeTest.testErrors.append(str(result.errors[-1][0]).split()[0]) @classmethod def hasPackage(self, pkg): @@ -71,13 +61,22 @@ class oeTest(unittest.TestCase): else: return False - class oeRuntimeTest(oeTest): - def __init__(self, methodName='runTest'): self.target = oeRuntimeTest.tc.target super(oeRuntimeTest, self).__init__(methodName) + #TODO: use package_manager.py to install packages on any type of image + def install_packages(self, packagelist): + for package in packagelist: + (status, result) = self.target.run("smart install -y "+package) + if status != 0: + return status + +class oeSDKTest(oeTest): + def __init__(self, methodName='runTest'): + self.sdktestdir = oeSDKTest.tc.sdktestdir + super(oeSDKTest, self).__init__(methodName) def getmodule(pos=2): # stack returns a list of tuples containg frame information diff --git a/meta/lib/oeqa/runtime/_ptest.py b/meta/lib/oeqa/runtime/_ptest.py new file mode 100644 index 0000000000..4c58dc1d7f --- /dev/null +++ b/meta/lib/oeqa/runtime/_ptest.py @@ -0,0 +1,124 @@ +import unittest, os, shutil +from oeqa.oetest import oeRuntimeTest, skipModule +from oeqa.utils.decorators import * +from oeqa.utils.logparser import * +from oeqa.utils.httpserver import HTTPService +import bb +import glob +from oe.package_manager import RpmPkgsList +import subprocess + +def setUpModule(): + if not oeRuntimeTest.hasFeature("package-management"): + skipModule("Image doesn't have package management feature") + if not oeRuntimeTest.hasPackage("smart"): + skipModule("Image doesn't have smart installed") + if "package_rpm" != oeRuntimeTest.tc.d.getVar("PACKAGE_CLASSES", True).split()[0]: + skipModule("Rpm is not the primary package manager") + +class PtestRunnerTest(oeRuntimeTest): + + # a ptest log parser + def parse_ptest(self, logfile): + parser = Lparser(test_0_pass_regex="^PASS:(.+)", test_0_fail_regex="^FAIL:(.+)", section_0_begin_regex="^BEGIN: .*/(.+)/ptest", section_0_end_regex="^END: .*/(.+)/ptest") + parser.init() + result = Result() + + with open(logfile) as f: + for line in f: + result_tuple = parser.parse_line(line) + if not result_tuple: + continue + result_tuple = line_type, category, status, name = parser.parse_line(line) + + if line_type == 'section' and status == 'begin': + current_section = name + continue + + if line_type == 'section' and status == 'end': + current_section = None + continue + + if line_type == 'test' and status == 'pass': + result.store(current_section, name, status) + continue + + if line_type == 'test' and status == 'fail': + result.store(current_section, name, status) + continue + + result.sort_tests() + return result + + @classmethod + def setUpClass(self): + #note the existing channels that are on the board before creating new ones +# self.existingchannels = set() +# (status, result) = oeRuntimeTest.tc.target.run('smart channel --show | grep "\["', 0) +# for x in result.split("\n"): +# self.existingchannels.add(x) + self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.target.server_ip) + self.repo_server.start() + + @classmethod + def tearDownClass(self): + self.repo_server.stop() + #remove created channels to be able to repeat the tests on same image +# (status, result) = oeRuntimeTest.tc.target.run('smart channel --show | grep "\["', 0) +# for x in result.split("\n"): +# if x not in self.existingchannels: +# oeRuntimeTest.tc.target.run('smart channel --remove '+x[1:-1]+' -y', 0) + + def add_smart_channel(self): + image_pkgtype = self.tc.d.getVar('IMAGE_PKGTYPE', True) + deploy_url = 'http://%s:%s/%s' %(self.target.server_ip, self.repo_server.port, image_pkgtype) + pkgarchs = self.tc.d.getVar('PACKAGE_ARCHS', True).replace("-","_").split() + for arch in os.listdir('%s/%s' % (self.repo_server.root_dir, image_pkgtype)): + if arch in pkgarchs: + self.target.run('smart channel -y --add {a} type=rpm-md baseurl={u}/{a}'.format(a=arch, u=deploy_url), 0) + self.target.run('smart update', 0) + + def install_complementary(self, globs=None): + installed_pkgs_file = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), + "installed_pkgs.txt") + self.pkgs_list = RpmPkgsList(oeRuntimeTest.tc.d, oeRuntimeTest.tc.d.getVar('IMAGE_ROOTFS', True), oeRuntimeTest.tc.d.getVar('arch_var', True), oeRuntimeTest.tc.d.getVar('os_var', True)) + with open(installed_pkgs_file, "w+") as installed_pkgs: + installed_pkgs.write(self.pkgs_list.list("arch")) + + cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), + "glob", oeRuntimeTest.tc.d.getVar('PKGDATA_DIR', True), installed_pkgs_file, + globs] + try: + bb.note("Installing complementary packages ...") + complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.fatal("Could not compute complementary packages list. Command " + "'%s' returned %d:\n%s" % + (' '.join(cmd), e.returncode, e.output)) + + return complementary_pkgs.split() + + def setUp(self): + self.buildhist_dir = oeRuntimeTest.tc.d.getVar("BUILDHISTORY_DIR_IMAGE", True) + self.assertTrue(os.path.exists(self.buildhist_dir)) + self.ptest_log = os.path.join(oeRuntimeTest.tc.d.getVar("TEST_LOG_DIR",True), "ptest-%s.log" % oeRuntimeTest.tc.d.getVar('DATETIME', True)) + + @skipUnlessPassed('test_ssh') + def test_ptestrunner(self): + self.add_smart_channel() + cond = oeRuntimeTest.hasPackage("ptest-runner") and oeRuntimeTest.hasFeature("ptest") and oeRuntimeTest.hasPackage("-ptest") + if not cond: + self.install_packages(self.install_complementary("*-ptest")) + self.install_packages(['ptest-runner']) + + self.target.run('/usr/bin/ptest-runner > /tmp/ptest.log 2>&1', 0) + self.target.copy_from('/tmp/ptest.log', self.ptest_log) + shutil.copyfile(self.ptest_log, os.path.join(self.buildhist_dir, "ptest.log")) + + result = self.parse_ptest(os.path.join(self.buildhist_dir, "ptest.log")) + log_results_to_location = "./results" + if os.path.exists(log_results_to_location): + shutil.rmtree(log_results_to_location) + os.makedirs(log_results_to_location) + + result.log_as_files(log_results_to_location, test_status = ['fail']) diff --git a/meta/lib/oeqa/runtime/buildcvs.py b/meta/lib/oeqa/runtime/buildcvs.py index f1fbf19c1f..fe6cbfbcd5 100644 --- a/meta/lib/oeqa/runtime/buildcvs.py +++ b/meta/lib/oeqa/runtime/buildcvs.py @@ -1,4 +1,4 @@ -from oeqa.oetest import oeRuntimeTest +from oeqa.oetest import oeRuntimeTest, skipModule from oeqa.utils.decorators import * from oeqa.utils.targetbuild import TargetBuildProject @@ -14,6 +14,7 @@ class BuildCvsTest(oeRuntimeTest): "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2") self.project.download_archive() + @testcase(205) @skipUnlessPassed("test_ssh") def test_cvs(self): self.assertEqual(self.project.run_configure(), 0, diff --git a/meta/lib/oeqa/runtime/buildiptables.py b/meta/lib/oeqa/runtime/buildiptables.py index f6061a7f98..09e252df8c 100644 --- a/meta/lib/oeqa/runtime/buildiptables.py +++ b/meta/lib/oeqa/runtime/buildiptables.py @@ -1,4 +1,4 @@ -from oeqa.oetest import oeRuntimeTest +from oeqa.oetest import oeRuntimeTest, skipModule from oeqa.utils.decorators import * from oeqa.utils.targetbuild import TargetBuildProject @@ -14,6 +14,7 @@ class BuildIptablesTest(oeRuntimeTest): "http://netfilter.org/projects/iptables/files/iptables-1.4.13.tar.bz2") self.project.download_archive() + @testcase(206) @skipUnlessPassed("test_ssh") def test_iptables(self): self.assertEqual(self.project.run_configure(), 0, diff --git a/meta/lib/oeqa/runtime/buildsudoku.py b/meta/lib/oeqa/runtime/buildsudoku.py index a754f1d9ea..802b060010 100644 --- a/meta/lib/oeqa/runtime/buildsudoku.py +++ b/meta/lib/oeqa/runtime/buildsudoku.py @@ -1,4 +1,4 @@ -from oeqa.oetest import oeRuntimeTest +from oeqa.oetest import oeRuntimeTest, skipModule from oeqa.utils.decorators import * from oeqa.utils.targetbuild import TargetBuildProject @@ -14,6 +14,7 @@ class SudokuTest(oeRuntimeTest): "http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2") self.project.download_archive() + @testcase(207) @skipUnlessPassed("test_ssh") def test_sudoku(self): self.assertEqual(self.project.run_configure(), 0, diff --git a/meta/lib/oeqa/runtime/connman.py b/meta/lib/oeqa/runtime/connman.py index c03688206f..cc537f7766 100644 --- a/meta/lib/oeqa/runtime/connman.py +++ b/meta/lib/oeqa/runtime/connman.py @@ -21,7 +21,7 @@ class ConnmanTest(oeRuntimeTest): (status, output) = self.target.run('/usr/sbin/connmand --help') self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) - + @testcase(221) @skipUnlessPassed('test_connmand_help') def test_connmand_running(self): (status, output) = self.target.run(oeRuntimeTest.pscmd + ' | grep [c]onnmand') diff --git a/meta/lib/oeqa/runtime/date.py b/meta/lib/oeqa/runtime/date.py index a208e29ada..97e8ee42ad 100644 --- a/meta/lib/oeqa/runtime/date.py +++ b/meta/lib/oeqa/runtime/date.py @@ -4,6 +4,7 @@ import re class DateTest(oeRuntimeTest): + @testcase(211) @skipUnlessPassed("test_ssh") def test_date(self): (status, output) = self.target.run('date +"%Y-%m-%d %T"') diff --git a/meta/lib/oeqa/runtime/df.py b/meta/lib/oeqa/runtime/df.py index b6da35027c..09569d5ff6 100644 --- a/meta/lib/oeqa/runtime/df.py +++ b/meta/lib/oeqa/runtime/df.py @@ -5,6 +5,7 @@ from oeqa.utils.decorators import * class DfTest(oeRuntimeTest): + @testcase(234) @skipUnlessPassed("test_ssh") def test_df(self): (status,output) = self.target.run("df / | sed -n '2p' | awk '{print $4}'") diff --git a/meta/lib/oeqa/runtime/dmesg.py b/meta/lib/oeqa/runtime/dmesg.py index 64247ea704..5831471e50 100644 --- a/meta/lib/oeqa/runtime/dmesg.py +++ b/meta/lib/oeqa/runtime/dmesg.py @@ -5,7 +5,8 @@ from oeqa.utils.decorators import * class DmesgTest(oeRuntimeTest): + @testcase(215) @skipUnlessPassed('test_ssh') def test_dmesg(self): - (status, output) = self.target.run('dmesg | grep -v mmci-pl18x | grep -v "error changing net interface name" | grep -iv "dma timeout" | grep -i error') + (status, output) = self.target.run('dmesg | grep -v mmci-pl18x | grep -v "error changing net interface name" | grep -iv "dma timeout" | grep -v usbhid | grep -i error') self.assertEqual(status, 1, msg = "Error messages in dmesg log: %s" % output) diff --git a/meta/lib/oeqa/runtime/files/test.cpp b/meta/lib/oeqa/runtime/files/test.cpp new file mode 100644 index 0000000000..9e1a76473d --- /dev/null +++ b/meta/lib/oeqa/runtime/files/test.cpp @@ -0,0 +1,3 @@ +#include <limits> + +int main() {}
\ No newline at end of file diff --git a/meta/lib/oeqa/runtime/gcc.py b/meta/lib/oeqa/runtime/gcc.py index b63badd3e4..a7f62e1758 100644 --- a/meta/lib/oeqa/runtime/gcc.py +++ b/meta/lib/oeqa/runtime/gcc.py @@ -14,19 +14,29 @@ class GccCompileTest(oeRuntimeTest): def setUpClass(self): oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.c"), "/tmp/test.c") oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "testmakefile"), "/tmp/testmakefile") + oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.cpp"), "/tmp/test.cpp") + @testcase(203) def test_gcc_compile(self): (status, output) = self.target.run('gcc /tmp/test.c -o /tmp/test -lm') self.assertEqual(status, 0, msg="gcc compile failed, output: %s" % output) (status, output) = self.target.run('/tmp/test') self.assertEqual(status, 0, msg="running compiled file failed, output %s" % output) + @testcase(200) def test_gpp_compile(self): (status, output) = self.target.run('g++ /tmp/test.c -o /tmp/test -lm') self.assertEqual(status, 0, msg="g++ compile failed, output: %s" % output) (status, output) = self.target.run('/tmp/test') self.assertEqual(status, 0, msg="running compiled file failed, output %s" % output) + def test_gpp2_compile(self): + (status, output) = self.target.run('g++ /tmp/test.cpp -o /tmp/test -lm') + self.assertEqual(status, 0, msg="g++ compile failed, output: %s" % output) + (status, output) = self.target.run('/tmp/test') + self.assertEqual(status, 0, msg="running compiled file failed, output %s" % output) + + @testcase(204) def test_make(self): (status, output) = self.target.run('cd /tmp; make -f testmakefile') self.assertEqual(status, 0, msg="running make failed, output %s" % output) diff --git a/meta/lib/oeqa/runtime/kernelmodule.py b/meta/lib/oeqa/runtime/kernelmodule.py index cbc5742eff..2e81720327 100644 --- a/meta/lib/oeqa/runtime/kernelmodule.py +++ b/meta/lib/oeqa/runtime/kernelmodule.py @@ -14,6 +14,7 @@ class KernelModuleTest(oeRuntimeTest): self.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "hellomod.c"), "/tmp/hellomod.c") self.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "hellomod_makefile"), "/tmp/Makefile") + @testcase('316') @skipUnlessPassed('test_ssh') @skipUnlessPassed('test_gcc_compile') def test_kernel_module(self): diff --git a/meta/lib/oeqa/runtime/ldd.py b/meta/lib/oeqa/runtime/ldd.py index 4374530fc4..bce56c4270 100644 --- a/meta/lib/oeqa/runtime/ldd.py +++ b/meta/lib/oeqa/runtime/ldd.py @@ -1,5 +1,5 @@ import unittest -from oeqa.oetest import oeRuntimeTest +from oeqa.oetest import oeRuntimeTest, skipModule from oeqa.utils.decorators import * def setUpModule(): @@ -13,6 +13,7 @@ class LddTest(oeRuntimeTest): (status, output) = self.target.run('which ldd') self.assertEqual(status, 0, msg = "ldd does not exist in PATH: which ldd: %s" % output) + @testcase(239) @skipUnlessPassed('test_ldd_exists') def test_ldd_rtldlist_check(self): (status, output) = self.target.run('for i in $(which ldd | xargs cat | grep "^RTLDLIST"|cut -d\'=\' -f2|tr -d \'"\'); do test -f $i && echo $i && break; done') diff --git a/meta/lib/oeqa/runtime/logrotate.py b/meta/lib/oeqa/runtime/logrotate.py index 80489a3267..86d791c300 100644 --- a/meta/lib/oeqa/runtime/logrotate.py +++ b/meta/lib/oeqa/runtime/logrotate.py @@ -19,6 +19,7 @@ class LogrotateTest(oeRuntimeTest): (status, output) = self.target.run("sed -i 's#wtmp {#wtmp {\\n olddir /home/root/logrotate_dir#' /etc/logrotate.conf") self.assertEqual(status, 0, msg = "Could not write to logrotate.conf file. Status and output: %s and %s)" % (status, output)) + @testcase(289) @skipUnlessPassed("test_1_logrotate_setup") def test_2_logrotate(self): (status, output) = self.target.run('logrotate -f /etc/logrotate.conf') diff --git a/meta/lib/oeqa/runtime/multilib.py b/meta/lib/oeqa/runtime/multilib.py index 13a3b54b18..ab0a6ccd69 100644 --- a/meta/lib/oeqa/runtime/multilib.py +++ b/meta/lib/oeqa/runtime/multilib.py @@ -10,6 +10,7 @@ def setUpModule(): class MultilibTest(oeRuntimeTest): + @testcase('279') @skipUnlessPassed('test_ssh') def test_file_connman(self): self.assertTrue(oeRuntimeTest.hasPackage('connman-gnome'), msg="This test assumes connman-gnome is installed") diff --git a/meta/lib/oeqa/runtime/pam.py b/meta/lib/oeqa/runtime/pam.py index 52e1eb88e6..c8205c9abc 100644 --- a/meta/lib/oeqa/runtime/pam.py +++ b/meta/lib/oeqa/runtime/pam.py @@ -2,7 +2,7 @@ # Note that the image under test must have "pam" in DISTRO_FEATURES import unittest -from oeqa.oetest import oeRuntimeTest +from oeqa.oetest import oeRuntimeTest, skipModule from oeqa.utils.decorators import * def setUpModule(): @@ -12,13 +12,14 @@ def setUpModule(): class PamBasicTest(oeRuntimeTest): + @testcase(287) @skipUnlessPassed('test_ssh') def test_pam(self): (status, output) = self.target.run('login --help') self.assertEqual(status, 1, msg = "login command does not work as expected. Status and output:%s and %s" %(status, output)) (status, output) = self.target.run('passwd --help') - self.assertEqual(status, 6, msg = "passwd command does not work as expected. Status and output:%s and %s" %(status, output)) + self.assertEqual(status, 0, msg = "passwd command does not work as expected. Status and output:%s and %s" %(status, output)) (status, output) = self.target.run('su --help') - self.assertEqual(status, 2, msg = "su command does not work as expected. Status and output:%s and %s" %(status, output)) + self.assertEqual(status, 0, msg = "su command does not work as expected. Status and output:%s and %s" %(status, output)) (status, output) = self.target.run('useradd --help') - self.assertEqual(status, 2, msg = "useradd command does not work as expected. Status and output:%s and %s" %(status, output)) + self.assertEqual(status, 0, msg = "useradd command does not work as expected. Status and output:%s and %s" %(status, output)) diff --git a/meta/lib/oeqa/runtime/parselogs.py b/meta/lib/oeqa/runtime/parselogs.py new file mode 100644 index 0000000000..2953742f23 --- /dev/null +++ b/meta/lib/oeqa/runtime/parselogs.py @@ -0,0 +1,202 @@ +import os +import unittest +from oeqa.oetest import oeRuntimeTest +from oeqa.utils.decorators import * + +#in the future these lists could be moved outside of module +errors = ["error", "cannot", "can\'t", "failed"] + +common_errors = [ + "(WW) warning, (EE) error, (NI) not implemented, (??) unknown.", + "dma timeout", + "can\'t add hid device:", + "usbhid: probe of ", + "_OSC failed (AE_ERROR)", + "_OSC failed (AE_SUPPORT)", + "AE_ALREADY_EXISTS" + "ACPI _OSC request failed (AE_SUPPORT)" + "can\'t disable ASPM", + "Failed to load module \"vesa\"", + "Failed to load module vesa", + "Failed to load module \"modesetting\"", + "Failed to load module modesetting", + "Failed to load module \"glx\"", + "Failed to load module glx", + "[drm] Cannot find any crtc or sizes - going 1024x768" + ] + +x86_common = [ + '[drm:psb_do_init] *ERROR* Debug is', + 'wrong ELF class', + 'Could not enable PowerButton event', + 'probe of LNXPWRBN:00 failed with error -22', +] + common_errors + +qemux86_common = [ + 'Fast TSC calibration', + '_OSC failed (AE_NOT_FOUND); disabling ASPM', + 'Open ACPI failed (/var/run/acpid.socket) (No such file or directory)', + 'wrong ELF class', +] + common_errors + +ignore_errors = { + 'default' : common_errors, + 'qemux86' : [ + 'Failed to access perfctr msr (MSR c1 is 0)', + "fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.", + ] + qemux86_common, + 'qemux86-64' : qemux86_common, + 'qemumips' : [ + 'Failed to load module "glx"', + ] + common_errors, + 'qemuppc' : [ + 'PCI 0000:00 Cannot reserve Legacy IO [io 0x0000-0x0fff]', + 'host side 80-wire cable detection failed, limiting max speed', + 'mode "640x480" test failed', + 'Failed to load module "glx"', + ] + common_errors, + 'qemuarm' : [ + 'mmci-pl18x: probe of fpga:05 failed with error -22', + 'mmci-pl18x: probe of fpga:0b failed with error -22', + 'Failed to load module "glx"' + ] + common_errors, + 'emenlow' : x86_common, + 'crownbay' : x86_common, + 'genericx86' : x86_common, + 'genericx86-64' : x86_common, + 'edgerouter' : [ + 'Fatal server error:', + ] + common_errors, + 'minnow' : [ + 'netlink init failed', + 'NETLINK INITIALIZATION FAILED', + ] + common_errors, + 'jasperforest' : [ + 'Activated service \'org.bluez\' failed:', + 'Unable to find NFC netlink family', + 'netlink init failed', + 'NETLINK INITIALIZATION FAILED', + ] + common_errors, +} + +log_locations = ["/var/log/","/var/log/dmesg", "/tmp/dmesg_output.log"] + +class ParseLogsTest(oeRuntimeTest): + + @classmethod + def setUpClass(self): + self.errors = errors + self.ignore_errors = ignore_errors + self.log_locations = log_locations + self.msg = "" + + def getMachine(self): + (status, output) = self.target.run("uname -n") + return output + + #get some information on the CPU of the machine to display at the beginning of the output. This info might be useful in some cases. + def getHardwareInfo(self): + hwi = "" + (status, cpu_name) = self.target.run("cat /proc/cpuinfo | grep \"model name\" | head -n1 | awk 'BEGIN{FS=\":\"}{print $2}'") + (status, cpu_physical_cores) = self.target.run("cat /proc/cpuinfo | grep \"cpu cores\" | head -n1 | awk {'print $4'}") + (status, cpu_logical_cores) = self.target.run("cat /proc/cpuinfo | grep \"processor\" | wc -l") + (status, cpu_arch) = self.target.run("uname -m") + hwi += "Machine information: \n" + hwi += "*******************************\n" + hwi += "Machine name: "+self.getMachine()+"\n" + hwi += "CPU: "+str(cpu_name)+"\n" + hwi += "Arch: "+str(cpu_arch)+"\n" + hwi += "Physical cores: "+str(cpu_physical_cores)+"\n" + hwi += "Logical cores: "+str(cpu_logical_cores)+"\n" + hwi += "*******************************\n" + return hwi + + #go through the log locations provided and if it's a folder create a list with all the .log files in it, if it's a file just add + #it to that list + def getLogList(self, log_locations): + logs = [] + for location in log_locations: + (status, output) = self.target.run("test -f "+str(location)) + if (status == 0): + logs.append(str(location)) + else: + (status, output) = self.target.run("test -d "+str(location)) + if (status == 0): + (status, output) = self.target.run("find "+str(location)+"/*.log -maxdepth 1 -type f") + if (status == 0): + output = output.splitlines() + for logfile in output: + logs.append(os.path.join(location,str(logfile))) + return logs + + #build the grep command to be used with filters and exclusions + def build_grepcmd(self, errors, ignore_errors, log): + grepcmd = "grep " + grepcmd +="-Ei \"" + for error in errors: + grepcmd += error+"|" + grepcmd = grepcmd[:-1] + grepcmd += "\" "+str(log)+" | grep -Eiv \'" + try: + errorlist = ignore_errors[self.getMachine()] + except KeyError: + self.msg += "No ignore list found for this machine, using default\n" + errorlist = ignore_errors['default'] + for ignore_error in errorlist: + ignore_error = ignore_error.replace("(", "\(") + ignore_error = ignore_error.replace(")", "\)") + ignore_error = ignore_error.replace("'", ".") + ignore_error = ignore_error.replace("?", "\?") + ignore_error = ignore_error.replace("[", "\[") + ignore_error = ignore_error.replace("]", "\]") + ignore_error = ignore_error.replace("*", "\*") + grepcmd += ignore_error+"|" + grepcmd = grepcmd[:-1] + grepcmd += "\'" + return grepcmd + + #grep only the errors so that their context could be collected. Default context is 10 lines before and after the error itself + def parse_logs(self, errors, ignore_errors, logs, lines_before = 10, lines_after = 10): + results = {} + rez = [] + for log in logs: + thegrep = self.build_grepcmd(errors, ignore_errors, log) + try: + (status, result) = self.target.run(thegrep) + except: + pass + if result: + results[log] = {} + rez = result.splitlines() + for xrez in rez: + command = "grep \"\\"+str(xrez)+"\" -B "+str(lines_before)+" -A "+str(lines_after)+" "+str(log) + try: + (status, yrez) = self.target.run(command) + except: + pass + results[log][xrez]=yrez + return results + + #get the output of dmesg and write it in a file. This file is added to log_locations. + def write_dmesg(self): + (status, dmesg) = self.target.run("dmesg") + (status, dmesg2) = self.target.run("echo \""+str(dmesg)+"\" > /tmp/dmesg_output.log") + + @skipUnlessPassed('test_ssh') + def test_parselogs(self): + self.write_dmesg() + log_list = self.getLogList(self.log_locations) + result = self.parse_logs(self.errors, self.ignore_errors, log_list) + print self.getHardwareInfo() + errcount = 0 + for log in result: + self.msg += "Log: "+log+"\n" + self.msg += "-----------------------\n" + for error in result[log]: + errcount += 1 + self.msg += "Central error: "+str(error)+"\n" + self.msg += "***********************\n" + self.msg += result[str(log)][str(error)]+"\n" + self.msg += "***********************\n" + self.msg += "%s errors found in logs." % errcount + self.assertEqual(errcount, 0, msg=self.msg) diff --git a/meta/lib/oeqa/runtime/perl.py b/meta/lib/oeqa/runtime/perl.py index c9bb684c11..65da028d4b 100644 --- a/meta/lib/oeqa/runtime/perl.py +++ b/meta/lib/oeqa/runtime/perl.py @@ -18,6 +18,7 @@ class PerlTest(oeRuntimeTest): (status, output) = self.target.run('which perl') self.assertEqual(status, 0, msg="Perl binary not in PATH or not on target.") + @testcase(208) def test_perl_works(self): (status, output) = self.target.run('perl /tmp/test.pl') self.assertEqual(status, 0, msg="Exit status was not 0. Output: %s" % output) diff --git a/meta/lib/oeqa/runtime/python.py b/meta/lib/oeqa/runtime/python.py index c037ab2c18..0387b9a03e 100644 --- a/meta/lib/oeqa/runtime/python.py +++ b/meta/lib/oeqa/runtime/python.py @@ -18,6 +18,7 @@ class PythonTest(oeRuntimeTest): (status, output) = self.target.run('which python') self.assertEqual(status, 0, msg="Python binary not in PATH or not on target.") + @testcase(965) def test_python_stdout(self): (status, output) = self.target.run('python /tmp/test.py') self.assertEqual(status, 0, msg="Exit status was not 0. Output: %s" % output) diff --git a/meta/lib/oeqa/runtime/rpm.py b/meta/lib/oeqa/runtime/rpm.py index 084d22f96b..b17e8b46a8 100644 --- a/meta/lib/oeqa/runtime/rpm.py +++ b/meta/lib/oeqa/runtime/rpm.py @@ -18,6 +18,7 @@ class RpmBasicTest(oeRuntimeTest): (status, output) = self.target.run('rpm --help') self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) + @testcase(191) @skipUnlessPassed('test_rpm_help') def test_rpm_query(self): (status, output) = self.target.run('rpm -q rpm') @@ -34,11 +35,13 @@ class RpmInstallRemoveTest(oeRuntimeTest): testrpmfile = f oeRuntimeTest.tc.target.copy_to(os.path.join(rpmdir,testrpmfile), "/tmp/rpm-doc.rpm") + @testcase(192) @skipUnlessPassed('test_rpm_help') def test_rpm_install(self): (status, output) = self.target.run('rpm -ivh /tmp/rpm-doc.rpm') self.assertEqual(status, 0, msg="Failed to install rpm-doc package: %s" % output) + @testcase(194) @skipUnlessPassed('test_rpm_install') def test_rpm_remove(self): (status,output) = self.target.run('rpm -e rpm-doc') diff --git a/meta/lib/oeqa/runtime/scanelf.py b/meta/lib/oeqa/runtime/scanelf.py index b9abf24640..43a024ab9a 100644 --- a/meta/lib/oeqa/runtime/scanelf.py +++ b/meta/lib/oeqa/runtime/scanelf.py @@ -11,6 +11,7 @@ class ScanelfTest(oeRuntimeTest): def setUp(self): self.scancmd = 'scanelf --quiet --recursive --mount --ldpath --path' + @testcase(966) @skipUnlessPassed('test_ssh') def test_scanelf_textrel(self): # print TEXTREL information @@ -18,6 +19,7 @@ class ScanelfTest(oeRuntimeTest): (status, output) = self.target.run(self.scancmd) self.assertEqual(output.strip(), "", "\n".join([self.scancmd, output])) + @testcase(967) @skipUnlessPassed('test_ssh') def test_scanelf_rpath(self): # print RPATH information diff --git a/meta/lib/oeqa/runtime/scp.py b/meta/lib/oeqa/runtime/scp.py index 03095bf966..48e87d2d0b 100644 --- a/meta/lib/oeqa/runtime/scp.py +++ b/meta/lib/oeqa/runtime/scp.py @@ -1,6 +1,6 @@ import os from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import skipUnlessPassed +from oeqa.utils.decorators import skipUnlessPassed, testcase def setUpModule(): if not (oeRuntimeTest.hasPackage("dropbear") or oeRuntimeTest.hasPackage("openssh-sshd")): @@ -8,6 +8,7 @@ def setUpModule(): class ScpTest(oeRuntimeTest): + @testcase(220) @skipUnlessPassed('test_ssh') def test_scp_file(self): test_log_dir = oeRuntimeTest.tc.d.getVar("TEST_LOG_DIR", True) diff --git a/meta/lib/oeqa/runtime/skeletoninit.py b/meta/lib/oeqa/runtime/skeletoninit.py index 557e715a3e..7c7f402e5d 100644 --- a/meta/lib/oeqa/runtime/skeletoninit.py +++ b/meta/lib/oeqa/runtime/skeletoninit.py @@ -2,7 +2,7 @@ # Note that the image under test must have meta-skeleton layer in bblayers and IMAGE_INSTALL_append = " service" in local.conf import unittest -from oeqa.oetest import oeRuntimeTest +from oeqa.oetest import oeRuntimeTest, skipModule from oeqa.utils.decorators import * def setUpModule(): @@ -20,6 +20,7 @@ class SkeletonBasicTest(oeRuntimeTest): (status, output) = self.target.run('ls /usr/sbin/skeleton-test') self.assertEqual(status, 0, msg = "skeleton-test not found. Output:\n%s" % output) + @testcase(284) @skipUnlessPassed('test_skeleton_availability') @unittest.skipIf("systemd" == oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager"), "Not appropiate for systemd image") def test_skeleton_script(self): diff --git a/meta/lib/oeqa/runtime/smart.py b/meta/lib/oeqa/runtime/smart.py index 195f1170c6..3b49314df7 100644 --- a/meta/lib/oeqa/runtime/smart.py +++ b/meta/lib/oeqa/runtime/smart.py @@ -1,6 +1,6 @@ import unittest import re -from oeqa.oetest import oeRuntimeTest +from oeqa.oetest import oeRuntimeTest, skipModule from oeqa.utils.decorators import * from oeqa.utils.httpserver import HTTPService @@ -25,6 +25,7 @@ class SmartTest(oeRuntimeTest): class SmartBasicTest(SmartTest): + @testcase(716) @skipUnlessPassed('test_ssh') def test_smart_help(self): self.smart('--help') @@ -32,15 +33,19 @@ class SmartBasicTest(SmartTest): def test_smart_version(self): self.smart('--version') + @testcase(721) def test_smart_info(self): self.smart('info python-smartpm') + @testcase(421) def test_smart_query(self): self.smart('query python-smartpm') + @testcase(720) def test_smart_search(self): self.smart('search python-smartpm') + @testcase(722) def test_smart_stats(self): self.smart('stats') @@ -58,6 +63,7 @@ class SmartRepoTest(SmartTest): def test_smart_channel(self): self.smart('channel', 1) + @testcase(719) def test_smart_channel_add(self): image_pkgtype = self.tc.d.getVar('IMAGE_PKGTYPE', True) deploy_url = 'http://%s:%s/%s' %(self.target.server_ip, self.repo_server.port, image_pkgtype) @@ -76,6 +82,7 @@ class SmartRepoTest(SmartTest): def test_smart_channel_show(self): self.smart('channel --show') + @testcase(717) def test_smart_channel_rpmsys(self): self.smart('channel --show rpmsys') self.smart('channel --disable rpmsys') @@ -86,17 +93,20 @@ class SmartRepoTest(SmartTest): self.smart('remove -y psplash-default') self.smart('install -y psplash-default') + @testcase(728) @skipUnlessPassed('test_smart_install') def test_smart_install_dependency(self): self.smart('remove -y psplash') self.smart('install -y psplash-default') + @testcase(723) @skipUnlessPassed('test_smart_channel_add') def test_smart_install_from_disk(self): self.smart('remove -y psplash-default') self.smart('download psplash-default') self.smart('install -y ./psplash-default*') + @testcase(725) @skipUnlessPassed('test_smart_channel_add') def test_smart_install_from_http(self): output = self.smart('download --urls psplash-default') @@ -105,6 +115,7 @@ class SmartRepoTest(SmartTest): self.smart('remove -y psplash-default') self.smart('install -y %s' % url.group(0)) + @testcase(729) @skipUnlessPassed('test_smart_install') def test_smart_reinstall(self): self.smart('reinstall -y psplash-default') diff --git a/meta/lib/oeqa/runtime/ssh.py b/meta/lib/oeqa/runtime/ssh.py index e64866019f..0e76d5d512 100644 --- a/meta/lib/oeqa/runtime/ssh.py +++ b/meta/lib/oeqa/runtime/ssh.py @@ -10,6 +10,7 @@ def setUpModule(): class SshTest(oeRuntimeTest): + @testcase(224) @skipUnlessPassed('test_ping') def test_ssh(self): (status, output) = self.target.run('uname -a') diff --git a/meta/lib/oeqa/runtime/syslog.py b/meta/lib/oeqa/runtime/syslog.py index b95b36175a..7fa018e97f 100644 --- a/meta/lib/oeqa/runtime/syslog.py +++ b/meta/lib/oeqa/runtime/syslog.py @@ -13,6 +13,7 @@ class SyslogTest(oeRuntimeTest): (status,output) = self.target.run('/sbin/syslogd --help') self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) + @testcase(201) @skipUnlessPassed("test_syslog_help") def test_syslog_running(self): (status,output) = self.target.run(oeRuntimeTest.pscmd + ' | grep -i [s]yslogd') @@ -33,6 +34,7 @@ class SyslogTestConfig(oeRuntimeTest): else: (status,output) = self.target.run('systemctl restart syslog.service') + @testcase(202) @skipUnlessPassed("test_syslog_restart") @skipUnlessPassed("test_syslog_logger") @unittest.skipIf("systemd" == oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager"), "Not appropiate for systemd image") diff --git a/meta/lib/oeqa/runtime/systemd.py b/meta/lib/oeqa/runtime/systemd.py index 6de84f891b..1451698bb3 100644 --- a/meta/lib/oeqa/runtime/systemd.py +++ b/meta/lib/oeqa/runtime/systemd.py @@ -28,6 +28,7 @@ class SystemdBasicTests(SystemdTest): def test_systemd_basic(self): self.systemctl('--version') + @testcase(551) @skipUnlessPassed('test_system_basic') def test_systemd_list(self): self.systemctl('list-unit-files') @@ -51,6 +52,7 @@ class SystemdBasicTests(SystemdTest): return (False, output) time.sleep(10) + @testcase(550) @skipUnlessPassed('test_systemd_basic') def test_systemd_failed(self): settled, output = self.settle() @@ -69,6 +71,7 @@ class SystemdServiceTests(SystemdTest): def test_systemd_status(self): self.systemctl('status --full', 'avahi-daemon.service') + @testcase(695) @skipUnlessPassed('test_systemd_status') def test_systemd_stop_start(self): self.systemctl('stop', 'avahi-daemon.service') @@ -76,6 +79,7 @@ class SystemdServiceTests(SystemdTest): self.systemctl('start','avahi-daemon.service') self.systemctl('is-active', 'avahi-daemon.service', verbose=True) + @testcase(696) @skipUnlessPassed('test_systemd_basic') def test_systemd_disable_enable(self): self.systemctl('disable', 'avahi-daemon.service') diff --git a/meta/lib/oeqa/runtime/vnc.py b/meta/lib/oeqa/runtime/vnc.py index 5ed10727bc..f31deff306 100644 --- a/meta/lib/oeqa/runtime/vnc.py +++ b/meta/lib/oeqa/runtime/vnc.py @@ -1,4 +1,4 @@ -from oeqa.oetest import oeRuntimeTest +from oeqa.oetest import oeRuntimeTest, skipModuleUnless from oeqa.utils.decorators import * import re @@ -7,6 +7,7 @@ def setUpModule(): class VNCTest(oeRuntimeTest): + @testcase(213) @skipUnlessPassed('test_ssh') def test_vnc(self): (status, output) = self.target.run('x11vnc -display :0 -bg -o x11vnc.log') diff --git a/meta/lib/oeqa/runtime/x32lib.py b/meta/lib/oeqa/runtime/x32lib.py index 6bad201b12..ce5e214035 100644 --- a/meta/lib/oeqa/runtime/x32lib.py +++ b/meta/lib/oeqa/runtime/x32lib.py @@ -10,6 +10,7 @@ def setUpModule(): class X32libTest(oeRuntimeTest): + @testcase(281) @skipUnlessPassed("test_ssh") def test_x32_file(self): status1 = self.target.run("readelf -h /bin/ls | grep Class | grep ELF32")[0] diff --git a/meta/lib/oeqa/runtime/xorg.py b/meta/lib/oeqa/runtime/xorg.py index 12dccd8198..a07031e5c8 100644 --- a/meta/lib/oeqa/runtime/xorg.py +++ b/meta/lib/oeqa/runtime/xorg.py @@ -14,8 +14,4 @@ class XorgTest(oeRuntimeTest): (status, output) = self.target.run(oeRuntimeTest.pscmd + ' | grep -v xinit | grep [X]org') self.assertEqual(status, 0, msg="Xorg does not appear to be running %s" % self.target.run(oeRuntimeTest.pscmd)[1]) - @skipUnlessPassed('test_ssh') - def test_xorg_error(self): - (status, output) = self.target.run('cat /var/log/Xorg.0.log | grep -v "(EE) error," | grep -v "PreInit" | grep -v "evdev:" | grep -v "glx" | grep "(EE)"') - self.assertEqual(status, 1, msg="Errors in Xorg log: %s" % output) diff --git a/meta/lib/oeqa/sdk/__init__.py b/meta/lib/oeqa/sdk/__init__.py new file mode 100644 index 0000000000..4cf3fa76b6 --- /dev/null +++ b/meta/lib/oeqa/sdk/__init__.py @@ -0,0 +1,3 @@ +# Enable other layers to have tests in the same named directory +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/meta/lib/oeqa/sdk/buildcvs.py b/meta/lib/oeqa/sdk/buildcvs.py new file mode 100644 index 0000000000..c7146fa4af --- /dev/null +++ b/meta/lib/oeqa/sdk/buildcvs.py @@ -0,0 +1,25 @@ +from oeqa.oetest import oeSDKTest, skipModule +from oeqa.utils.decorators import * +from oeqa.utils.targetbuild import SDKBuildProject + +class BuildCvsTest(oeSDKTest): + + @classmethod + def setUpClass(self): + self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/cvs/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d, + "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2") + self.project.download_archive() + + def test_cvs(self): + self.assertEqual(self.project.run_configure(), 0, + msg="Running configure failed") + + self.assertEqual(self.project.run_make(), 0, + msg="Running make failed") + + self.assertEqual(self.project.run_install(), 0, + msg="Running make install failed") + + @classmethod + def tearDownClass(self): + self.project.clean() diff --git a/meta/lib/oeqa/sdk/buildiptables.py b/meta/lib/oeqa/sdk/buildiptables.py new file mode 100644 index 0000000000..062e5316e7 --- /dev/null +++ b/meta/lib/oeqa/sdk/buildiptables.py @@ -0,0 +1,26 @@ +from oeqa.oetest import oeSDKTest +from oeqa.utils.decorators import * +from oeqa.utils.targetbuild import SDKBuildProject + + +class BuildIptablesTest(oeSDKTest): + + @classmethod + def setUpClass(self): + self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/iptables/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d, + "http://netfilter.org/projects/iptables/files/iptables-1.4.13.tar.bz2") + self.project.download_archive() + + def test_iptables(self): + self.assertEqual(self.project.run_configure(), 0, + msg="Running configure failed") + + self.assertEqual(self.project.run_make(), 0, + msg="Running make failed") + + self.assertEqual(self.project.run_install(), 0, + msg="Running make install failed") + + @classmethod + def tearDownClass(self): + self.project.clean() diff --git a/meta/lib/oeqa/sdk/buildsudoku.py b/meta/lib/oeqa/sdk/buildsudoku.py new file mode 100644 index 0000000000..dea77c6599 --- /dev/null +++ b/meta/lib/oeqa/sdk/buildsudoku.py @@ -0,0 +1,26 @@ +from oeqa.oetest import oeSDKTest, skipModule +from oeqa.utils.decorators import * +from oeqa.utils.targetbuild import SDKBuildProject + +def setUpModule(): + if not oeSDKTest.hasPackage("gtk\+"): + skipModule("Image doesn't have gtk+ in manifest") + +class SudokuTest(oeSDKTest): + + @classmethod + def setUpClass(self): + self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/sudoku/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d, + "http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2") + self.project.download_archive() + + def test_sudoku(self): + self.assertEqual(self.project.run_configure(), 0, + msg="Running configure failed") + + self.assertEqual(self.project.run_make(), 0, + msg="Running make failed") + + @classmethod + def tearDownClass(self): + self.project.clean() diff --git a/meta/lib/oeqa/selftest/_toaster.py b/meta/lib/oeqa/selftest/_toaster.py index 5a42e937bc..1cf28a0144 100644 --- a/meta/lib/oeqa/selftest/_toaster.py +++ b/meta/lib/oeqa/selftest/_toaster.py @@ -14,6 +14,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toastermain.settings") import toastermain.settings from django.db.models import Q from orm.models import * +from oeqa.utils.decorators import testcase class ToasterSetup(oeSelfTest): @@ -31,19 +32,22 @@ class ToasterSetup(oeSelfTest): class Toaster_DB_Tests(ToasterSetup): # Check if build name is unique - tc_id=795 - def test_Build_Unique_Name_TC795(self): + @testcase(795) + def test_Build_Unique_Name(self): all_builds = Build.objects.all().count() distinct_builds = Build.objects.values('id').distinct().count() self.assertEqual(distinct_builds, all_builds, msg = 'Build name is not unique') # Check if build coocker log path is unique - tc_id=819 - def test_Build_Unique_Cooker_Log_Path_TC819(self): + @testcase(819) + def test_Build_Unique_Cooker_Log_Path(self): distinct_path = Build.objects.values('cooker_log_path').distinct().count() total_builds = Build.objects.values('id').count() self.assertEqual(distinct_path, total_builds, msg = 'Build coocker log path is not unique') # Check if the number of errors matches the number of orm_logmessage.level entries with value 2 - tc_id=820 - def test_Build_Errors_No_TC820(self): + @testcase(820) + def test_Build_Errors_No(self): builds = Build.objects.values('id', 'errors_no') cnt_err = [] for build in builds: @@ -53,7 +57,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for build id: %s' % cnt_err) # Check if the number of warnings matches the number of orm_logmessage.level entries with value 1 - tc=821 - def test_Build_Warnings_No_TC821(self): + @testcase(821) + def test_Build_Warnings_No(self): builds = Build.objects.values('id', 'warnings_no') cnt_err = [] for build in builds: @@ -63,7 +68,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for build id: %s' % cnt_err) # Check if the build succeeded then the errors_no is 0 - tc_id=822 - def test_Build_Suceeded_Errors_No_TC822(self): + @testcase(822) + def test_Build_Suceeded_Errors_No(self): builds = Build.objects.filter(outcome = 0).values('id', 'errors_no') cnt_err = [] for build in builds: @@ -71,8 +77,9 @@ class Toaster_DB_Tests(ToasterSetup): cnt_err.append(build['id']) self.assertEqual(len(cnt_err), 0, msg = 'Errors for build id: %s' % cnt_err) - # Check if task order is unique for one build - tc=824 - def test_Task_Unique_Order_TC824(self): + # Check if task order is unique for one build - tc=824 + @testcase(824) + def test_Task_Unique_Order(self): builds = Build.objects.values('id') cnt_err = [] for build in builds: @@ -83,7 +90,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for build id: %s' % cnt_err) # Check task order sequence for one build - tc=825 - def test_Task_Order_Sequence_TC825(self): + @testcase(825) + def test_Task_Order_Sequence(self): builds = builds = Build.objects.values('id') cnt_err = [] for build in builds: @@ -100,7 +108,8 @@ class Toaster_DB_Tests(ToasterSetup): #def test_Task_Disk_IO_TC828(self): # Check if outcome = 2 (SSTATE) then sstate_result must be 3 (RESTORED) - tc=832 - def test_Task_If_Outcome_2_Sstate_Result_Must_Be_3_TC832(self): + @testcase(832) + def test_Task_If_Outcome_2_Sstate_Result_Must_Be_3(self): tasks = Task.objects.filter(outcome = 2).values('id', 'sstate_result') cnt_err = [] for task in tasks: @@ -109,7 +118,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) # Check if outcome = 1 (COVERED) or 3 (EXISTING) then sstate_result must be 0 (SSTATE_NA) - tc=833 - def test_Task_If_Outcome_1_3_Sstate_Result_Must_Be_0_TC833(self): + @testcase(833) + def test_Task_If_Outcome_1_3_Sstate_Result_Must_Be_0(self): tasks = Task.objects.filter(outcome__in = (1, 3)).values('id', 'sstate_result') cnt_err = [] for task in tasks: @@ -118,7 +128,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) # Check if outcome is 0 (SUCCESS) or 4 (FAILED) then sstate_result must be 0 (NA), 1 (MISS) or 2 (FAILED) - tc=834 - def test_Task_If_Outcome_0_4_Sstate_Result_Must_Be_0_1_2_TC834(self): + @testcase(834) + def test_Task_If_Outcome_0_4_Sstate_Result_Must_Be_0_1_2(self): tasks = Task.objects.filter(outcome__in = (0, 4)).values('id', 'sstate_result') cnt_err = [] for task in tasks: @@ -127,7 +138,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) # Check if task_executed = TRUE (1), script_type must be 0 (CODING_NA), 2 (CODING_PYTHON), 3 (CODING_SHELL) - tc=891 - def test_Task_If_Task_Executed_True_Script_Type_0_2_3_TC891(self): + @testcase(891) + def test_Task_If_Task_Executed_True_Script_Type_0_2_3(self): tasks = Task.objects.filter(task_executed = 1).values('id', 'script_type') cnt_err = [] for task in tasks: @@ -136,7 +148,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) # Check if task_executed = TRUE (1), outcome must be 0 (SUCCESS) or 4 (FAILED) - tc=836 - def test_Task_If_Task_Executed_True_Outcome_0_4_TC836(self): + @testcase(836) + def test_Task_If_Task_Executed_True_Outcome_0_4(self): tasks = Task.objects.filter(task_executed = 1).values('id', 'outcome') cnt_err = [] for task in tasks: @@ -145,7 +158,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) # Check if task_executed = FALSE (0), script_type must be 0 - tc=890 - def test_Task_If_Task_Executed_False_Script_Type_0_TC890(self): + @testcase(890) + def test_Task_If_Task_Executed_False_Script_Type_0(self): tasks = Task.objects.filter(task_executed = 0).values('id', 'script_type') cnt_err = [] for task in tasks: @@ -154,7 +168,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) # Check if task_executed = FALSE (0) and build outcome = SUCCEEDED (0), task outcome must be 1 (COVERED), 2 (CACHED), 3 (PREBUILT), 5 (EMPTY) - tc=837 - def test_Task_If_Task_Executed_False_Outcome_1_2_3_5_TC837(self): + @testcase(837) + def test_Task_If_Task_Executed_False_Outcome_1_2_3_5(self): builds = Build.objects.filter(outcome = 0).values('id') cnt_err = [] for build in builds: @@ -165,7 +180,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) # Key verification - tc=888 - def test_Target_Installed_Package_TC888(self): + @testcase(888) + def test_Target_Installed_Package(self): rows = Target_Installed_Package.objects.values('id', 'target_id', 'package_id') cnt_err = [] for row in rows: @@ -176,7 +192,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for target installed package id: %s' % cnt_err) # Key verification - tc=889 - def test_Task_Dependency_TC889(self): + @testcase(889) + def test_Task_Dependency(self): rows = Task_Dependency.objects.values('id', 'task_id', 'depends_on_id') cnt_err = [] for row in rows: @@ -188,6 +205,7 @@ class Toaster_DB_Tests(ToasterSetup): # Check if build target file_name is populated only if is_image=true AND orm_build.outcome=0 then if the file exists and its size matches the file_size value ### Need to add the tc in the test run + @testcase(1037) def test_Target_File_Name_Populated(self): builds = Build.objects.filter(outcome = 0).values('id') for build in builds: @@ -210,7 +228,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for target image file id: %s' % cnt_err) # Key verification - tc=884 - def test_Package_Dependency_TC884(self): + @testcase(884) + def test_Package_Dependency(self): cnt_err = [] deps = Package_Dependency.objects.values('id', 'package_id', 'depends_on_id') for dep in deps: @@ -219,7 +238,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for package dependency id: %s' % cnt_err) # Check if recipe name does not start with a number (0-9) - tc=838 - def test_Recipe_Name_TC838(self): + @testcase(838) + def test_Recipe_Name(self): recipes = Recipe.objects.values('id', 'name') cnt_err = [] for recipe in recipes: @@ -228,7 +248,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) # Check if recipe section matches the content of the SECTION variable (if set) in file_path - tc=839 - def test_Recipe_DB_Section_Match_Recipe_File_Section_TC839(self): + @testcase(839) + def test_Recipe_DB_Section_Match_Recipe_File_Section(self): recipes = Recipe.objects.values('id', 'section', 'file_path') cnt_err = [] for recipe in recipes: @@ -245,7 +266,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) # Check if recipe license matches the content of the LICENSE variable (if set) in file_path - tc=840 - def test_Recipe_DB_License_Match_Recipe_File_License_TC840(self): + @testcase(840) + def test_Recipe_DB_License_Match_Recipe_File_License(self): recipes = Recipe.objects.values('id', 'license', 'file_path') cnt_err = [] for recipe in recipes: @@ -262,7 +284,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) # Check if recipe homepage matches the content of the HOMEPAGE variable (if set) in file_path - tc=841 - def test_Recipe_DB_Homepage_Match_Recipe_File_Homepage_TC841(self): + @testcase(841) + def test_Recipe_DB_Homepage_Match_Recipe_File_Homepage(self): recipes = Recipe.objects.values('id', 'homepage', 'file_path') cnt_err = [] for recipe in recipes: @@ -279,7 +302,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) # Check if recipe bugtracker matches the content of the BUGTRACKER variable (if set) in file_path - tc=842 - def test_Recipe_DB_Bugtracker_Match_Recipe_File_Bugtracker_TC842(self): + @testcase(842) + def test_Recipe_DB_Bugtracker_Match_Recipe_File_Bugtracker(self): recipes = Recipe.objects.values('id', 'bugtracker', 'file_path') cnt_err = [] for recipe in recipes: @@ -296,7 +320,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) # Recipe key verification, recipe name does not depends on a recipe having the same name - tc=883 - def test_Recipe_Dependency_TC883(self): + @testcase(883) + def test_Recipe_Dependency(self): deps = Recipe_Dependency.objects.values('id', 'recipe_id', 'depends_on_id') cnt_err = [] for dep in deps: @@ -310,7 +335,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe dependency id: %s' % cnt_err) # Check if package name does not start with a number (0-9) - tc=846 - def test_Package_Name_For_Number_TC846(self): + @testcase(846) + def test_Package_Name_For_Number(self): packages = Package.objects.filter(~Q(size = -1)).values('id', 'name') cnt_err = [] for package in packages: @@ -319,7 +345,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) # Check if package version starts with a number (0-9) - tc=847 - def test_Package_Version_Starts_With_Number_TC847(self): + @testcase(847) + def test_Package_Version_Starts_With_Number(self): packages = Package.objects.filter(~Q(size = -1)).values('id', 'version') cnt_err = [] for package in packages: @@ -328,7 +355,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) # Check if package revision starts with 'r' - tc=848 - def test_Package_Revision_Starts_With_r_TC848(self): + @testcase(848) + def test_Package_Revision_Starts_With_r(self): packages = Package.objects.filter(~Q(size = -1)).values('id', 'revision') cnt_err = [] for package in packages: @@ -338,6 +366,7 @@ class Toaster_DB_Tests(ToasterSetup): # Check the validity of the package build_id ### TC must be added in test run + @testcase(1038) def test_Package_Build_Id(self): packages = Package.objects.filter(~Q(size = -1)).values('id', 'build_id') cnt_err = [] @@ -349,6 +378,7 @@ class Toaster_DB_Tests(ToasterSetup): # Check the validity of package recipe_id ### TC must be added in test run + @testcase(1039) def test_Package_Recipe_Id(self): packages = Package.objects.filter(~Q(size = -1)).values('id', 'recipe_id') cnt_err = [] @@ -360,6 +390,7 @@ class Toaster_DB_Tests(ToasterSetup): # Check if package installed_size field is not null ### TC must be aded in test run + @testcase(1040) def test_Package_Installed_Size_Not_NULL(self): packages = Package.objects.filter(installed_size__isnull = True).values('id') cnt_err = [] @@ -368,7 +399,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) # Check if all layers requests return exit code is 200 - tc=843 - def test_Layers_Requests_Exit_Code_TC843(self): + @testcase(843) + def test_Layers_Requests_Exit_Code(self): layers = Layer.objects.values('id', 'layer_index_url') cnt_err = [] for layer in layers: @@ -378,7 +410,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for layer id: %s' % cnt_err) # Check if the output of bitbake-layers show_layers matches the info from database - tc=895 - def test_Layers_Show_Layers_TC895(self): + @testcase(895) + def test_Layers_Show_Layers(self): layers = Layer.objects.values('id', 'name', 'local_path') cmd = commands.getoutput('bitbake-layers show_layers') cnt_err = [] @@ -388,7 +421,8 @@ class Toaster_DB_Tests(ToasterSetup): self.assertEqual(len(cnt_err), 0, msg = 'Errors for layer id: %s' % cnt_err) # Check if django server starts regardless of the timezone set on the machine - tc=905 - def test_Start_Django_Timezone_TC905(self): + @testcase(905) + def test_Start_Django_Timezone(self): current_path = os.getcwd() zonefilelist = [] ZONEINFOPATH = '/usr/share/zoneinfo/' diff --git a/meta/lib/oeqa/selftest/bbtests.py b/meta/lib/oeqa/selftest/bbtests.py index d730bfd755..5708d3dc9b 100644 --- a/meta/lib/oeqa/selftest/bbtests.py +++ b/meta/lib/oeqa/selftest/bbtests.py @@ -7,28 +7,33 @@ import shutil import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.decorators import testcase class BitbakeTests(oeSelfTest): + @testcase(789) def test_run_bitbake_from_dir_1(self): os.chdir(os.path.join(self.builddir, 'conf')) bitbake('-e') + @testcase(790) def test_run_bitbake_from_dir_2(self): my_env = os.environ.copy() my_env['BBPATH'] = my_env['BUILDDIR'] os.chdir(os.path.dirname(os.environ['BUILDDIR'])) bitbake('-e', env=my_env) + @testcase(806) def test_event_handler(self): self.write_config("INHERIT += \"test_events\"") result = bitbake('m4-native') - find_build_started = re.search("NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Preparing runqueue", result.output) + find_build_started = re.search("NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Preparing RunQueue", result.output) find_build_completed = re.search("Tasks Summary:.*(\n.*)*NOTE: Test for bb\.event\.BuildCompleted", result.output) self.assertTrue(find_build_started, msg = "Match failed in:\n%s" % result.output) self.assertTrue(find_build_completed, msg = "Match failed in:\n%s" % result.output) self.assertFalse('Test for bb.event.InvalidEvent' in result.output) + @testcase(103) def test_local_sstate(self): bitbake('m4-native -ccleansstate') bitbake('m4-native') @@ -37,14 +42,17 @@ class BitbakeTests(oeSelfTest): find_setscene = re.search("m4-native.*do_.*_setscene", result.output) self.assertTrue(find_setscene) + @testcase(105) def test_bitbake_invalid_recipe(self): result = bitbake('-b asdf', ignore_status=True) self.assertTrue("ERROR: Unable to find any recipe file matching 'asdf'" in result.output) + @testcase(107) def test_bitbake_invalid_target(self): result = bitbake('asdf', ignore_status=True) self.assertTrue("ERROR: Nothing PROVIDES 'asdf'" in result.output) + @testcase(106) def test_warnings_errors(self): result = bitbake('-b asdf', ignore_status=True) find_warnings = re.search("Summary: There w.{2,3}? [1-9][0-9]* WARNING messages* shown", result.output) @@ -52,6 +60,7 @@ class BitbakeTests(oeSelfTest): self.assertTrue(find_warnings, msg="Did not find the mumber of warnings at the end of the build:\n" + result.output) self.assertTrue(find_errors, msg="Did not find the mumber of errors at the end of the build:\n" + result.output) + @testcase(108) def test_invalid_patch(self): self.write_recipeinc('man', 'SRC_URI += "file://man-1.5h1-make.patch"') result = bitbake('man -c patch', ignore_status=True) @@ -59,14 +68,17 @@ class BitbakeTests(oeSelfTest): bitbake('-cclean man') self.assertTrue("ERROR: Function failed: patch_do_patch" in result.output) + @testcase(163) def test_force_task(self): bitbake('m4-native') + self.add_command_to_tearDown('bitbake -c clean m4-native') result = bitbake('-C compile m4-native') look_for_tasks = ['do_compile', 'do_install', 'do_populate_sysroot'] for task in look_for_tasks: find_task = re.search("m4-native.*%s" % task, result.output) self.assertTrue(find_task) + @testcase(167) def test_bitbake_g(self): result = bitbake('-g core-image-full-cmdline') self.assertTrue('NOTE: PN build list saved to \'pn-buildlist\'' in result.output) @@ -74,6 +86,7 @@ class BitbakeTests(oeSelfTest): for f in ['pn-buildlist', 'pn-depends.dot', 'package-depends.dot', 'task-depends.dot']: os.remove(f) + @testcase(899) def test_image_manifest(self): bitbake('core-image-minimal') deploydir = get_bb_var("DEPLOY_DIR_IMAGE", target="core-image-minimal") @@ -81,6 +94,7 @@ class BitbakeTests(oeSelfTest): manifest = os.path.join(deploydir, imagename + ".manifest") self.assertTrue(os.path.islink(manifest), msg="No manifest file created for image") + @testcase(168) def test_invalid_recipe_src_uri(self): data = 'SRC_URI = "file://invalid"' self.write_recipeinc('man', data) @@ -89,9 +103,10 @@ class BitbakeTests(oeSelfTest): bitbake('-ccleanall man') self.delete_recipeinc('man') self.assertEqual(result.status, 1, msg='Command succeded when it should have failed') - self.assertTrue('ERROR: Fetcher failure: Unable to find file file://invalid anywhere. The paths that were searched were:' in result.output) + self.assertTrue('Fetcher failure: Unable to find file file://invalid anywhere. The paths that were searched were:' in result.output) self.assertTrue('ERROR: Function failed: Fetcher failure for URL: \'file://invalid\'. Unable to fetch URL from any source.' in result.output) + @testcase(171) def test_rename_downloaded_file(self): data = 'SRC_URI_append = ";downloadfilename=test-aspell.tar.gz"' self.write_recipeinc('aspell', data) @@ -103,25 +118,30 @@ class BitbakeTests(oeSelfTest): self.assertTrue(os.path.isfile(os.path.join(get_bb_var("DL_DIR"), 'test-aspell.tar.gz.done'))) bitbake('-ccleanall aspell') + @testcase(1028) def test_environment(self): self.append_config("TEST_ENV=\"localconf\"") result = runCmd('bitbake -e | grep TEST_ENV=') self.assertTrue('localconf' in result.output) self.remove_config("TEST_ENV=\"localconf\"") + @testcase(1029) def test_dry_run(self): result = runCmd('bitbake -n m4-native') self.assertEqual(0, result.status) + @testcase(1030) def test_just_parse(self): result = runCmd('bitbake -p') self.assertEqual(0, result.status) + @testcase(1031) def test_version(self): result = runCmd('bitbake -s | grep wget') find = re.search("wget *:([0-9a-zA-Z\.\-]+)", result.output) self.assertTrue(find) + @testcase(1032) def test_prefile(self): preconf = os.path.join(self.builddir, 'conf/prefile.conf') self.track_for_cleanup(preconf) @@ -133,6 +153,7 @@ class BitbakeTests(oeSelfTest): self.assertTrue('localconf' in result.output) self.remove_config("TEST_PREFILE=\"localconf\"") + @testcase(1033) def test_postfile(self): postconf = os.path.join(self.builddir, 'conf/postfile.conf') self.track_for_cleanup(postconf) @@ -142,10 +163,12 @@ class BitbakeTests(oeSelfTest): self.assertTrue('postfile' in result.output) self.remove_config("TEST_POSTFILE=\"localconf\"") + @testcase(1034) def test_checkuri(self): result = runCmd('bitbake -c checkuri m4') self.assertEqual(0, result.status) + @testcase(1035) def test_continue(self): self.write_recipeinc('man',"\ndo_fail_task () {\nexit 1 \n}\n\naddtask do_fail_task before do_fetch\n" ) runCmd('bitbake -c cleanall man xcursor-transparent-theme') diff --git a/meta/lib/oeqa/selftest/buildoptions.py b/meta/lib/oeqa/selftest/buildoptions.py index 27fc452e72..926ffe9993 100644 --- a/meta/lib/oeqa/selftest/buildoptions.py +++ b/meta/lib/oeqa/selftest/buildoptions.py @@ -7,21 +7,30 @@ from oeqa.selftest.base import oeSelfTest from oeqa.selftest.buildhistory import BuildhistoryBase from oeqa.utils.commands import runCmd, bitbake, get_bb_var import oeqa.utils.ftools as ftools +from oeqa.utils.decorators import testcase class ImageOptionsTests(oeSelfTest): + @testcase(761) def test_incremental_image_generation(self): + image_pkgtype = get_bb_var("IMAGE_PKGTYPE") + if image_pkgtype != 'rpm': + self.skipTest('Not using RPM as main package format') bitbake("-c cleanall core-image-minimal") self.write_config('INC_RPM_IMAGE_GEN = "1"') self.append_config('IMAGE_FEATURES += "ssh-server-openssh"') bitbake("core-image-minimal") - res = runCmd("grep 'Installing openssh-sshd' %s" % (os.path.join(get_bb_var("WORKDIR", "core-image-minimal"), "temp/log.do_rootfs")), ignore_status=True) + log_data_file = os.path.join(get_bb_var("WORKDIR", "core-image-minimal"), "temp/log.do_rootfs") + log_data_created = ftools.read_file(log_data_file) + incremental_created = re.search("NOTE: load old install solution for incremental install\nNOTE: old install solution not exist\nNOTE: creating new install solution for incremental install(\n.*)*NOTE: Installing the following packages:.*packagegroup-core-ssh-openssh", log_data_created) self.remove_config('IMAGE_FEATURES += "ssh-server-openssh"') - self.assertEqual(0, res.status, msg="No match for openssh-sshd in log.do_rootfs") + self.assertTrue(incremental_created, msg = "Match failed in:\n%s" % log_data_created) bitbake("core-image-minimal") - res = runCmd("grep 'Removing openssh-sshd' %s" %(os.path.join(get_bb_var("WORKDIR", "core-image-minimal"), "temp/log.do_rootfs")),ignore_status=True) - self.assertEqual(0, res.status, msg="openssh-sshd was not removed from image") + log_data_removed = ftools.read_file(log_data_file) + incremental_removed = re.search("NOTE: load old install solution for incremental install\nNOTE: creating new install solution for incremental install(\n.*)*NOTE: incremental removed:.*openssh-sshd-.*", log_data_removed) + self.assertTrue(incremental_removed, msg = "Match failed in:\n%s" % log_data_removed) + @testcase(925) def test_rm_old_image(self): bitbake("core-image-minimal") deploydir = get_bb_var("DEPLOY_DIR_IMAGE", target="core-image-minimal") @@ -37,6 +46,7 @@ class ImageOptionsTests(oeSelfTest): remaining_not_expected = [path for path in track_original_files if os.path.basename(path) in deploydir_files] self.assertFalse(remaining_not_expected, msg="\nThe following image files ware not removed: %s" % ', '.join(map(str, remaining_not_expected))) + @testcase(286) def test_ccache_tool(self): bitbake("ccache-native") self.assertTrue(os.path.isfile(os.path.join(get_bb_var('STAGING_BINDIR_NATIVE', 'ccache-native'), "ccache"))) @@ -50,6 +60,7 @@ class ImageOptionsTests(oeSelfTest): class DiskMonTest(oeSelfTest): + @testcase(277) def test_stoptask_behavior(self): self.write_config('BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},100000G,100K"') res = bitbake("m4", ignore_status = True) @@ -65,6 +76,7 @@ class DiskMonTest(oeSelfTest): class SanityOptionsTest(oeSelfTest): + @testcase(927) def test_options_warnqa_errorqa_switch(self): bitbake("xcursor-transparent-theme -ccleansstate") @@ -74,16 +86,17 @@ class SanityOptionsTest(oeSelfTest): self.write_recipeinc('xcursor-transparent-theme', 'PACKAGES += \"${PN}-dbg\"') res = bitbake("xcursor-transparent-theme", ignore_status=True) self.delete_recipeinc('xcursor-transparent-theme') - self.assertTrue("ERROR: QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors." in res.output) + self.assertTrue("ERROR: QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors." in res.output, msg=res.output) self.assertEqual(res.status, 1) self.write_recipeinc('xcursor-transparent-theme', 'PACKAGES += \"${PN}-dbg\"') self.append_config('ERROR_QA_remove = "packages-list"') self.append_config('WARN_QA_append = " packages-list"') - res = bitbake("xcursor-transparent-theme") bitbake("xcursor-transparent-theme -ccleansstate") + res = bitbake("xcursor-transparent-theme") self.delete_recipeinc('xcursor-transparent-theme') - self.assertTrue("WARNING: QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors." in res.output) + self.assertTrue("WARNING: QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors." in res.output, msg=res.output) + @testcase(278) def test_sanity_userspace_dependency(self): self.append_config('WARN_QA_append = " unsafe-references-in-binaries unsafe-references-in-scripts"') bitbake("-ccleansstate gzip nfs-utils") @@ -93,10 +106,12 @@ class SanityOptionsTest(oeSelfTest): class BuildhistoryTests(BuildhistoryBase): + @testcase(293) def test_buildhistory_basic(self): self.run_buildhistory_operation('xcursor-transparent-theme') self.assertTrue(os.path.isdir(get_bb_var('BUILDHISTORY_DIR'))) + @testcase(294) def test_buildhistory_buildtime_pr_backwards(self): self.add_command_to_tearDown('cleanup-workdir') target = 'xcursor-transparent-theme' diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py new file mode 100644 index 0000000000..74fb325803 --- /dev/null +++ b/meta/lib/oeqa/selftest/devtool.py @@ -0,0 +1,241 @@ +import unittest +import os +import logging +import re +import shutil +import tempfile +import glob + +import oeqa.utils.ftools as ftools +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.decorators import testcase + +class DevtoolTests(oeSelfTest): + + def test_create_workspace(self): + # Check preconditions + workspacedir = os.path.join(self.builddir, 'workspace') + self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') + result = runCmd('bitbake-layers show-layers') + self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf') + # Try creating a workspace layer with a specific path + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + result = runCmd('devtool create-workspace %s' % tempdir) + self.assertTrue(os.path.isfile(os.path.join(tempdir, 'conf', 'layer.conf'))) + result = runCmd('bitbake-layers show-layers') + self.assertIn(tempdir, result.output) + # Try creating a workspace layer with the default path + self.track_for_cleanup(workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool create-workspace') + self.assertTrue(os.path.isfile(os.path.join(workspacedir, 'conf', 'layer.conf'))) + result = runCmd('bitbake-layers show-layers') + self.assertNotIn(tempdir, result.output) + self.assertIn(workspacedir, result.output) + + def test_recipetool_create(self): + # Try adding a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + tempsrc = os.path.join(tempdir, 'srctree') + os.makedirs(tempsrc) + recipefile = os.path.join(tempdir, 'logrotate_3.8.7.bb') + srcuri = 'https://fedorahosted.org/releases/l/o/logrotate/logrotate-3.8.7.tar.gz' + result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = 'GPLv2' + checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=18810669f13b87348459e611d31ab760' + checkvars['SRC_URI'] = 'https://fedorahosted.org/releases/l/o/logrotate/logrotate-${PV}.tar.gz' + checkvars['SRC_URI[md5sum]'] = '99e08503ef24c3e2e3ff74cc5f3be213' + checkvars['SRC_URI[sha256sum]'] = 'f6ba691f40e30e640efa2752c1f9499a3f9738257660994de70a45fe00d12b64' + with open(recipefile, 'r') as f: + for line in f: + if '=' in line: + splitline = line.split('=', 1) + var = splitline[0].rstrip() + value = splitline[1].strip().strip('"') + if var in checkvars: + needvalue = checkvars.pop(var) + self.assertEqual(value, needvalue) + if line.startswith('inherit '): + inherits = line.split()[1:] + + self.assertEqual(checkvars, {}, 'Some variables not found') + + def test_recipetool_create_git(self): + # Ensure we have the right data in shlibs/pkgdata + bitbake('libpng pango libx11 libxext') + # Try adding a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + tempsrc = os.path.join(tempdir, 'srctree') + os.makedirs(tempsrc) + recipefile = os.path.join(tempdir, 'libmatchbox.bb') + srcuri = 'git://git.yoctoproject.org/libmatchbox' + result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc)) + self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) + checkvars = {} + checkvars['LICENSE'] = 'LGPLv2.1' + checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34' + checkvars['S'] = '${WORKDIR}/git' + checkvars['PV'] = '1.0+git${SRCPV}' + checkvars['SRC_URI'] = srcuri + checkvars['DEPENDS'] = 'libpng pango libx11 libxext' + inherits = [] + with open(recipefile, 'r') as f: + for line in f: + if '=' in line: + splitline = line.split('=', 1) + var = splitline[0].rstrip() + value = splitline[1].strip().strip('"') + if var in checkvars: + needvalue = checkvars.pop(var) + self.assertEqual(value, needvalue) + if line.startswith('inherit '): + inherits = line.split()[1:] + + self.assertEqual(checkvars, {}, 'Some variables not found') + + self.assertIn('autotools', inherits, 'Missing inherit of autotools') + self.assertIn('pkgconfig', inherits, 'Missing inherit of pkgconfig') + + def test_devtool_add(self): + # Check preconditions + workspacedir = os.path.join(self.builddir, 'workspace') + self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Fetch source + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + url = 'http://www.ivarch.com/programs/sources/pv-1.5.3.tar.bz2' + result = runCmd('wget %s' % url, cwd=tempdir) + result = runCmd('tar xfv pv-1.5.3.tar.bz2', cwd=tempdir) + srcdir = os.path.join(tempdir, 'pv-1.5.3') + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory') + # Test devtool add + self.track_for_cleanup(workspacedir) + self.add_command_to_tearDown('bitbake -c cleansstate pv') + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add pv %s' % srcdir) + self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn('pv', result.output) + self.assertIn(srcdir, result.output) + # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then) + bitbake('pv -c cleansstate') + # Test devtool build + result = runCmd('devtool build pv') + installdir = get_bb_var('D', 'pv') + self.assertTrue(installdir, 'Could not query installdir variable') + bindir = get_bb_var('bindir', 'pv') + self.assertTrue(bindir, 'Could not query bindir variable') + if bindir[0] == '/': + bindir = bindir[1:] + self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D') + + def test_devtool_modify(self): + # Check preconditions + workspacedir = os.path.join(self.builddir, 'workspace') + self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('mdadm -c cleansstate') + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean mdadm') + result = runCmd('devtool modify mdadm -x %s' % tempdir) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found') + self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found') + self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + matches = glob.glob(os.path.join(workspacedir, 'appends', 'mdadm_*.bbappend')) + self.assertTrue(matches, 'bbappend not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn('mdadm', result.output) + self.assertIn(tempdir, result.output) + # Check git repo + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertEqual(result.output.strip(), "", 'Created git repo is not clean') + result = runCmd('git symbolic-ref HEAD', cwd=tempdir) + self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo') + # Try building + bitbake('mdadm') + # Try making (minor) modifications to the source + result = runCmd("sed -i 's!^\.TH.*!.TH MDADM 8 \"\" v9.999-custom!' %s" % os.path.join(tempdir, 'mdadm.8.in')) + bitbake('mdadm -c package') + pkgd = get_bb_var('PKGD', 'mdadm') + self.assertTrue(pkgd, 'Could not query PKGD variable') + mandir = get_bb_var('mandir', 'mdadm') + self.assertTrue(mandir, 'Could not query mandir variable') + if mandir[0] == '/': + mandir = mandir[1:] + with open(os.path.join(pkgd, mandir, 'man8', 'mdadm.8'), 'r') as f: + for line in f: + if line.startswith('.TH'): + self.assertEqual(line.rstrip(), '.TH MDADM 8 "" v9.999-custom', 'man file not modified') + # Test devtool reset + result = runCmd('devtool reset mdadm') + result = runCmd('devtool status') + self.assertNotIn('mdadm', result.output) + + def test_devtool_update_recipe(self): + # Check preconditions + workspacedir = os.path.join(self.builddir, 'workspace') + self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') + testrecipe = 'minicom' + recipefile = get_bb_var('FILE', testrecipe) + result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) + self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe) + # First, modify a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + # Check git repo + self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found') + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertEqual(result.output.strip(), "", 'Created git repo is not clean') + result = runCmd('git symbolic-ref HEAD', cwd=tempdir) + self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo') + # Add a couple of commits + # FIXME: this only tests adding, need to also test update and remove + result = runCmd('echo "Additional line" >> README', cwd=tempdir) + result = runCmd('git commit -a -m "Change the README"', cwd=tempdir) + result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir) + result = runCmd('git add devtool-new-file', cwd=tempdir) + result = runCmd('git commit -m "Add a new file"', cwd=tempdir) + self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe %s' % testrecipe) + result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) + self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe) + status = result.output.splitlines() + self.assertEqual(len(status), 3, 'Less/more files modified than expected. Entire status:\n%s' % result.output) + for line in status: + if line.endswith('0001-Change-the-README.patch'): + self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line) + elif line.endswith('0002-Add-a-new-file.patch'): + self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line) + elif re.search('minicom_[^_]*.bb$', line): + self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line) + else: + raise AssertionError('Unexpected modified file in status: %s' % line) + + def test_devtool_extract(self): + # Check preconditions + workspacedir = os.path.join(self.builddir, 'workspace') + self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + # Try devtool extract + self.track_for_cleanup(tempdir) + self.track_for_cleanup(workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool extract remake %s' % tempdir) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found') diff --git a/meta/lib/oeqa/selftest/oescripts.py b/meta/lib/oeqa/selftest/oescripts.py index 4aab2ed095..31cd50809c 100644 --- a/meta/lib/oeqa/selftest/oescripts.py +++ b/meta/lib/oeqa/selftest/oescripts.py @@ -8,9 +8,11 @@ import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest from oeqa.selftest.buildhistory import BuildhistoryBase from oeqa.utils.commands import Command, runCmd, bitbake, get_bb_var, get_test_layer +from oeqa.utils.decorators import testcase class TestScripts(oeSelfTest): + @testcase(300) def test_cleanup_workdir(self): path = os.path.dirname(get_bb_var('WORKDIR', 'gzip')) old_version_recipe = os.path.join(get_bb_var('COREBASE'), 'meta/recipes-extended/gzip/gzip_1.3.12.bb') @@ -41,6 +43,7 @@ class TestScripts(oeSelfTest): class BuildhistoryDiffTests(BuildhistoryBase): + @testcase(295) def test_buildhistory_diff(self): self.add_command_to_tearDown('cleanup-workdir') target = 'xcursor-transparent-theme' @@ -49,12 +52,3 @@ class BuildhistoryDiffTests(BuildhistoryBase): result = runCmd("buildhistory-diff -p %s" % get_bb_var('BUILDHISTORY_DIR')) expected_output = 'PR changed from "r1" to "r0"' self.assertTrue(expected_output in result.output, msg="Did not find expected output: %s" % result.output) - - - - - - - - - diff --git a/meta/lib/oeqa/selftest/prservice.py b/meta/lib/oeqa/selftest/prservice.py index 789c05f1e5..fb6d68d3bf 100644 --- a/meta/lib/oeqa/selftest/prservice.py +++ b/meta/lib/oeqa/selftest/prservice.py @@ -8,6 +8,7 @@ import datetime import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.decorators import testcase class BitbakePrTests(oeSelfTest): @@ -87,27 +88,34 @@ class BitbakePrTests(oeSelfTest): bitbake("-ccleansstate %s" % package_name) self.assertTrue(pr_2 - pr_1 == 1) - + @testcase(930) def test_import_export_replace_db(self): self.run_test_pr_export_import('m4') + @testcase(931) def test_import_export_override_db(self): self.run_test_pr_export_import('m4', replace_current_db=False) + @testcase(932) def test_pr_service_rpm_arch_dep(self): self.run_test_pr_service('m4', 'rpm', 'do_package') + @testcase(934) def test_pr_service_deb_arch_dep(self): self.run_test_pr_service('m4', 'deb', 'do_package') + @testcase(933) def test_pr_service_ipk_arch_dep(self): self.run_test_pr_service('m4', 'ipk', 'do_package') + @testcase(935) def test_pr_service_rpm_arch_indep(self): self.run_test_pr_service('xcursor-transparent-theme', 'rpm', 'do_package') + @testcase(937) def test_pr_service_deb_arch_indep(self): self.run_test_pr_service('xcursor-transparent-theme', 'deb', 'do_package') + @testcase(936) def test_pr_service_ipk_arch_indep(self): self.run_test_pr_service('xcursor-transparent-theme', 'ipk', 'do_package') diff --git a/meta/lib/oeqa/selftest/sstatetests.py b/meta/lib/oeqa/selftest/sstatetests.py index 44dcea847f..d578ddd489 100644 --- a/meta/lib/oeqa/selftest/sstatetests.py +++ b/meta/lib/oeqa/selftest/sstatetests.py @@ -8,7 +8,7 @@ import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer from oeqa.selftest.sstate import SStateBase - +from oeqa.utils.decorators import testcase class SStateTests(SStateBase): @@ -28,19 +28,23 @@ class SStateTests(SStateBase): else: self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s" % ', '.join(map(str, targets))) + @testcase(975) def test_sstate_creation_distro_specific_pass(self): targetarch = get_bb_var('TUNE_ARCH') self.run_test_sstate_creation(['binutils-cross-'+ targetarch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) + @testcase(975) def test_sstate_creation_distro_specific_fail(self): targetarch = get_bb_var('TUNE_ARCH') self.run_test_sstate_creation(['binutils-cross-'+ targetarch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) + @testcase(976) def test_sstate_creation_distro_nonspecific_pass(self): - self.run_test_sstate_creation(['eglibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + self.run_test_sstate_creation(['glibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + @testcase(976) def test_sstate_creation_distro_nonspecific_fail(self): - self.run_test_sstate_creation(['eglibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) + self.run_test_sstate_creation(['glibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) # Test the sstate files deletion part of the do_cleansstate task @@ -60,16 +64,19 @@ class SStateTests(SStateBase): tgz_removed = self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) self.assertTrue(not tgz_removed, msg="do_cleansstate didn't remove .tgz sstate files for: %s" % ', '.join(map(str, targets))) + @testcase(977) def test_cleansstate_task_distro_specific_nonspecific(self): targetarch = get_bb_var('TUNE_ARCH') - self.run_test_cleansstate_task(['binutils-cross-' + targetarch, 'binutils-native', 'eglibc-initial'], distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) + self.run_test_cleansstate_task(['binutils-cross-' + targetarch, 'binutils-native', 'glibc-initial'], distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) + @testcase(977) def test_cleansstate_task_distro_nonspecific(self): - self.run_test_cleansstate_task(['eglibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + self.run_test_cleansstate_task(['glibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + @testcase(977) def test_cleansstate_task_distro_specific(self): targetarch = get_bb_var('TUNE_ARCH') - self.run_test_cleansstate_task(['binutils-cross-'+ targetarch, 'binutils-native', 'eglibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) + self.run_test_cleansstate_task(['binutils-cross-'+ targetarch, 'binutils-native', 'glibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) # Test rebuilding of distro-specific sstate files @@ -98,14 +105,17 @@ class SStateTests(SStateBase): created_once = [x for x in file_tracker_2 if x not in file_tracker_1] self.assertTrue(created_once == [], msg="The following sstate files ware created only in the second run: %s" % ', '.join(map(str, created_once))) + @testcase(175) def test_rebuild_distro_specific_sstate_cross_native_targets(self): targetarch = get_bb_var('TUNE_ARCH') self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + targetarch, 'binutils-native'], temp_sstate_location=True) + @testcase(175) def test_rebuild_distro_specific_sstate_cross_target(self): targetarch = get_bb_var('TUNE_ARCH') self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + targetarch], temp_sstate_location=True) + @testcase(175) def test_rebuild_distro_specific_sstate_native_target(self): self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) @@ -153,7 +163,7 @@ class SStateTests(SStateBase): expected_not_actual = [x for x in expected_remaining_sstate if x not in actual_remaining_sstate] self.assertFalse(expected_not_actual, msg="Extra files ware removed: %s" ', '.join(map(str, expected_not_actual))) - + @testcase(973) def test_sstate_cache_management_script_using_pr_1(self): global_config = [] target_config = [] @@ -161,6 +171,7 @@ class SStateTests(SStateBase): target_config.append('PR = "0"') self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) + @testcase(978) def test_sstate_cache_management_script_using_pr_2(self): global_config = [] target_config = [] @@ -170,6 +181,7 @@ class SStateTests(SStateBase): target_config.append('PR = "1"') self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) + @testcase(979) def test_sstate_cache_management_script_using_pr_3(self): global_config = [] target_config = [] @@ -181,6 +193,7 @@ class SStateTests(SStateBase): target_config.append('PR = "1"') self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) + @testcase(974) def test_sstate_cache_management_script_using_machine(self): global_config = [] target_config = [] @@ -189,11 +202,3 @@ class SStateTests(SStateBase): global_config.append('MACHINE = "qemux86"') target_config.append('') self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) - - - - - - - - diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index 802bc2f208..5b601d9806 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -110,11 +110,11 @@ def runCmd(command, ignore_status=False, timeout=None, assert_error=True, **opti def bitbake(command, ignore_status=False, timeout=None, postconfig=None, **options): if postconfig: - postconfig_file = os.path.join(os.environ.get('BUILDDIR'), 'oeqa-post.conf') - ftools.write_file(postconfig_file, postconfig) - extra_args = "-R %s" % postconfig_file + postconfig_file = os.path.join(os.environ.get('BUILDDIR'), 'oeqa-post.conf') + ftools.write_file(postconfig_file, postconfig) + extra_args = "-R %s" % postconfig_file else: - extra_args = "" + extra_args = "" if isinstance(command, basestring): cmd = "bitbake " + extra_args + " " + command @@ -122,7 +122,7 @@ def bitbake(command, ignore_status=False, timeout=None, postconfig=None, **optio cmd = [ "bitbake" ] + [a for a in (command + extra_args.split(" ")) if a not in [""]] try: - return runCmd(cmd, ignore_status, timeout, **options) + return runCmd(cmd, ignore_status, timeout, **options) finally: if postconfig: os.remove(postconfig_file) @@ -138,9 +138,9 @@ def get_bb_var(var, target=None, postconfig=None): val = None bbenv = get_bb_env(target, postconfig=postconfig) for line in bbenv.splitlines(): - if line.startswith(var + "="): + if line.startswith(var + "=") or line.startswith("export " + var + "="): val = line.split('=')[1] - val = val.replace('\"','') + val = val.strip('\"') break return val diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py index a0d94e6d24..ff5f278bc1 100644 --- a/meta/lib/oeqa/utils/decorators.py +++ b/meta/lib/oeqa/utils/decorators.py @@ -6,9 +6,42 @@ # Most useful is skipUnlessPassed which can be used for # creating dependecies between two test methods. -from oeqa.oetest import * +import os import logging import sys +import unittest + +#get the "result" object from one of the upper frames provided that one of these upper frames is a unittest.case frame +class getResults(object): + def __init__(self): + #dynamically determine the unittest.case frame and use it to get the name of the test method + upperf = sys._current_frames().values()[0] + while (upperf.f_globals['__name__'] != 'unittest.case'): + upperf = upperf.f_back + + def handleList(items): + ret = [] + # items is a list of tuples, (test, failure) or (_ErrorHandler(), Exception()) + for i in items: + s = i[0].id() + #Handle the _ErrorHolder objects from skipModule failures + if "setUpModule (" in s: + ret.append(s.replace("setUpModule (", "").replace(")","")) + else: + ret.append(s) + return ret + self.faillist = handleList(upperf.f_locals['result'].failures) + self.errorlist = handleList(upperf.f_locals['result'].errors) + self.skiplist = handleList(upperf.f_locals['result'].skipped) + + def getFailList(self): + return self.faillist + + def getErrorList(self): + return self.errorlist + + def getSkipList(self): + return self.skiplist class skipIfFailure(object): @@ -17,7 +50,8 @@ class skipIfFailure(object): def __call__(self,f): def wrapped_f(*args): - if self.testcase in (oeTest.testFailures or oeTest.testErrors): + res = getResults() + if self.testcase in (res.getFailList() or res.getErrorList()): raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) return f(*args) wrapped_f.__name__ = f.__name__ @@ -30,7 +64,8 @@ class skipIfSkipped(object): def __call__(self,f): def wrapped_f(*args): - if self.testcase in oeTest.testSkipped: + res = getResults() + if self.testcase in res.getSkipList(): raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) return f(*args) wrapped_f.__name__ = f.__name__ @@ -43,9 +78,10 @@ class skipUnlessPassed(object): def __call__(self,f): def wrapped_f(*args): - if self.testcase in oeTest.testSkipped or \ - self.testcase in oeTest.testFailures or \ - self.testcase in oeTest.testErrors: + res = getResults() + if self.testcase in res.getSkipList() or \ + self.testcase in res.getFailList() or \ + self.testcase in res.getErrorList(): raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) return f(*args) wrapped_f.__name__ = f.__name__ @@ -57,10 +93,15 @@ class testcase(object): self.test_case = test_case def __call__(self, func): - def wrapped_f(*args): - return func(*args) - wrapped_f.test_case = self.test_case - return wrapped_f + def wrapped_f(*args): + return func(*args) + wrapped_f.test_case = self.test_case + wrapped_f.__name__ = func.__name__ + return wrapped_f + +class NoParsingFilter(logging.Filter): + def filter(self, record): + return record.levelno == 100 def LogResults(original_class): orig_method = original_class.run @@ -68,49 +109,51 @@ def LogResults(original_class): #rewrite the run method of unittest.TestCase to add testcase logging def run(self, result, *args, **kws): orig_method(self, result, *args, **kws) - passed = True - testMethod = getattr(self, self._testMethodName) - - #if test case is decorated then use it's number, else use it's name - try: - test_case = testMethod.test_case - except AttributeError: - test_case = self._testMethodName - - #create custom logging level for filtering. - custom_log_level = 100 - logging.addLevelName(custom_log_level, 'RESULTS') - caller = os.path.basename(sys.argv[0]) - - def results(self, message, *args, **kws): - if self.isEnabledFor(custom_log_level): - self.log(custom_log_level, message, *args, **kws) - logging.Logger.results = results - - logging.basicConfig(filename=os.path.join(os.getcwd(),'results-'+caller+'.log'), + passed = True + testMethod = getattr(self, self._testMethodName) + + #if test case is decorated then use it's number, else use it's name + try: + test_case = testMethod.test_case + except AttributeError: + test_case = self._testMethodName + + #create custom logging level for filtering. + custom_log_level = 100 + logging.addLevelName(custom_log_level, 'RESULTS') + caller = os.path.basename(sys.argv[0]) + + def results(self, message, *args, **kws): + if self.isEnabledFor(custom_log_level): + self.log(custom_log_level, message, *args, **kws) + logging.Logger.results = results + + logging.basicConfig(filename=os.path.join(os.getcwd(),'results-'+caller+'.log'), filemode='w', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt='%H:%M:%S', level=custom_log_level) - local_log = logging.getLogger(caller) + for handler in logging.root.handlers: + handler.addFilter(NoParsingFilter()) + local_log = logging.getLogger(caller) - #check status of tests and record it + #check status of tests and record it for (name, msg) in result.errors: - if self._testMethodName == str(name).split(' ')[0]: - local_log.results("Testcase "+str(test_case)+": ERROR") - local_log.results("Testcase "+str(test_case)+":\n"+msg) - passed = False + if self._testMethodName == str(name).split(' ')[0]: + local_log.results("Testcase "+str(test_case)+": ERROR") + local_log.results("Testcase "+str(test_case)+":\n"+msg) + passed = False for (name, msg) in result.failures: - if self._testMethodName == str(name).split(' ')[0]: - local_log.results("Testcase "+str(test_case)+": FAILED") - local_log.results("Testcase "+str(test_case)+":\n"+msg) - passed = False + if self._testMethodName == str(name).split(' ')[0]: + local_log.results("Testcase "+str(test_case)+": FAILED") + local_log.results("Testcase "+str(test_case)+":\n"+msg) + passed = False for (name, msg) in result.skipped: - if self._testMethodName == str(name).split(' ')[0]: - local_log.results("Testcase "+str(test_case)+": SKIPPED") - passed = False - if passed: - local_log.results("Testcase "+str(test_case)+": PASSED") + if self._testMethodName == str(name).split(' ')[0]: + local_log.results("Testcase "+str(test_case)+": SKIPPED") + passed = False + if passed: + local_log.results("Testcase "+str(test_case)+": PASSED") original_class.run = run return original_class diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py index f161a1bddd..76518d8ef9 100644 --- a/meta/lib/oeqa/utils/httpserver.py +++ b/meta/lib/oeqa/utils/httpserver.py @@ -5,6 +5,8 @@ import os class HTTPServer(SimpleHTTPServer.BaseHTTPServer.HTTPServer): def server_start(self, root_dir): + import signal + signal.signal(signal.SIGTERM, signal.SIG_DFL) os.chdir(root_dir) self.serve_forever() diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py new file mode 100644 index 0000000000..87b50354cd --- /dev/null +++ b/meta/lib/oeqa/utils/logparser.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python + +import sys +import os +import re +import ftools + + +# A parser that can be used to identify weather a line is a test result or a section statement. +class Lparser(object): + + def __init__(self, test_0_pass_regex, test_0_fail_regex, section_0_begin_regex=None, section_0_end_regex=None, **kwargs): + # Initialize the arguments dictionary + if kwargs: + self.args = kwargs + else: + self.args = {} + + # Add the default args to the dictionary + self.args['test_0_pass_regex'] = test_0_pass_regex + self.args['test_0_fail_regex'] = test_0_fail_regex + if section_0_begin_regex: + self.args['section_0_begin_regex'] = section_0_begin_regex + if section_0_end_regex: + self.args['section_0_end_regex'] = section_0_end_regex + + self.test_possible_status = ['pass', 'fail', 'error'] + self.section_possible_status = ['begin', 'end'] + + self.initialized = False + + + # Initialize the parser with the current configuration + def init(self): + + # extra arguments can be added by the user to define new test and section categories. They must follow a pre-defined pattern: <type>_<category_name>_<status>_regex + self.test_argument_pattern = "^test_(.+?)_(%s)_regex" % '|'.join(map(str, self.test_possible_status)) + self.section_argument_pattern = "^section_(.+?)_(%s)_regex" % '|'.join(map(str, self.section_possible_status)) + + # Initialize the test and section regex dictionaries + self.test_regex = {} + self.section_regex ={} + + for arg, value in self.args.items(): + if not value: + raise Exception('The value of provided argument %s is %s. Should have a valid value.' % (key, value)) + is_test = re.search(self.test_argument_pattern, arg) + is_section = re.search(self.section_argument_pattern, arg) + if is_test: + if not is_test.group(1) in self.test_regex: + self.test_regex[is_test.group(1)] = {} + self.test_regex[is_test.group(1)][is_test.group(2)] = re.compile(value) + elif is_section: + if not is_section.group(1) in self.section_regex: + self.section_regex[is_section.group(1)] = {} + self.section_regex[is_section.group(1)][is_section.group(2)] = re.compile(value) + else: + # TODO: Make these call a traceback instead of a simple exception.. + raise Exception("The provided argument name does not correspond to any valid type. Please give one of the following types:\nfor tests: %s\nfor sections: %s" % (self.test_argument_pattern, self.section_argument_pattern)) + + self.initialized = True + + # Parse a line and return a tuple containing the type of result (test/section) and its category, status and name + def parse_line(self, line): + if not self.initialized: + raise Exception("The parser is not initialized..") + + for test_category, test_status_list in self.test_regex.items(): + for test_status, status_regex in test_status_list.items(): + test_name = status_regex.search(line) + if test_name: + return ['test', test_category, test_status, test_name.group(1)] + + for section_category, section_status_list in self.section_regex.items(): + for section_status, status_regex in section_status_list.items(): + section_name = status_regex.search(line) + if section_name: + return ['section', section_category, section_status, section_name.group(1)] + return None + + +class Result(object): + + def __init__(self): + self.result_dict = {} + + def store(self, section, test, status): + if not section in self.result_dict: + self.result_dict[section] = [] + + self.result_dict[section].append((test, status)) + + # sort tests by the test name(the first element of the tuple), for each section. This can be helpful when using git to diff for changes by making sure they are always in the same order. + def sort_tests(self): + for package in self.result_dict: + sorted_results = sorted(self.result_dict[package], key=lambda tup: tup[0]) + self.result_dict[package] = sorted_results + + # Log the results as files. The file name is the section name and the contents are the tests in that section. + def log_as_files(self, target_dir, test_status): + status_regex = re.compile('|'.join(map(str, test_status))) + if not type(test_status) == type([]): + raise Exception("test_status should be a list. Got " + str(test_status) + " instead.") + if not os.path.exists(target_dir): + raise Exception("Target directory does not exist: %s" % target_dir) + + for section, test_results in self.result_dict.items(): + prefix = '' + for x in test_status: + prefix +=x+'.' + if (section != ''): + prefix += section + section_file = os.path.join(target_dir, prefix) + # purge the file contents if it exists + open(section_file, 'w').close() + for test_result in test_results: + (test_name, status) = test_result + # we log only the tests with status in the test_status list + match_status = status_regex.search(status) + if match_status: + ftools.append_file(section_file, status + ": " + test_name) + + # Not yet implemented! + def log_to_lava(self): + pass diff --git a/meta/lib/oeqa/utils/targetbuild.py b/meta/lib/oeqa/utils/targetbuild.py index 32296762c0..eeb08ba716 100644 --- a/meta/lib/oeqa/utils/targetbuild.py +++ b/meta/lib/oeqa/utils/targetbuild.py @@ -6,23 +6,25 @@ import os import re +import bb.utils import subprocess +from abc import ABCMeta, abstractmethod +class BuildProject(): -class TargetBuildProject(): + __metaclass__ = ABCMeta - def __init__(self, target, d, uri, foldername=None): - self.target = target + def __init__(self, d, uri, foldername=None, tmpdir="/tmp/"): self.d = d self.uri = uri - self.targetdir = "~/" self.archive = os.path.basename(uri) - self.localarchive = "/tmp/" + self.archive + self.localarchive = os.path.join(tmpdir,self.archive) self.fname = re.sub(r'.tar.bz2|tar.gz$', '', self.archive) if foldername: self.fname = foldername - def download_archive(self): + # Download self.archive to self.localarchive + def _download_archive(self): exportvars = ['HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY', 'https_proxy', @@ -41,6 +43,38 @@ class TargetBuildProject(): cmd = cmd + "wget -O %s %s" % (self.localarchive, self.uri) subprocess.check_call(cmd, shell=True) + # This method should provide a way to run a command in the desired environment. + @abstractmethod + def _run(self, cmd): + pass + + # The timeout parameter of target.run is set to 0 to make the ssh command + # run with no timeout. + def run_configure(self, configure_args=''): + return self._run('cd %s; ./configure %s' % (self.targetdir, configure_args)) + + def run_make(self, make_args=''): + return self._run('cd %s; make %s' % (self.targetdir, make_args)) + + def run_install(self, install_args=''): + return self._run('cd %s; make install %s' % (self.targetdir, install_args)) + + def clean(self): + self._run('rm -rf %s' % self.targetdir) + subprocess.call('rm -f %s' % self.localarchive, shell=True) + pass + +class TargetBuildProject(BuildProject): + + def __init__(self, target, d, uri, foldername=None): + self.target = target + self.targetdir = "~/" + BuildProject.__init__(self, d, uri, foldername, tmpdir="/tmp") + + def download_archive(self): + + self._download_archive() + (status, output) = self.target.copy_to(self.localarchive, self.targetdir) if status != 0: raise Exception("Failed to copy archive to target, output: %s" % output) @@ -54,15 +88,45 @@ class TargetBuildProject(): # The timeout parameter of target.run is set to 0 to make the ssh command # run with no timeout. - def run_configure(self): - return self.target.run('cd %s; ./configure' % self.targetdir, 0)[0] + def _run(self, cmd): + return self.target.run(cmd, 0)[0] - def run_make(self): - return self.target.run('cd %s; make' % self.targetdir, 0)[0] - def run_install(self): - return self.target.run('cd %s; make install' % self.targetdir, 0)[0] +class SDKBuildProject(BuildProject): + + def __init__(self, testpath, sdkenv, d, uri, foldername=None): + self.sdkenv = sdkenv + self.testdir = testpath + self.targetdir = testpath + bb.utils.mkdirhier(testpath) + self.datetime = d.getVar('DATETIME', True) + self.testlogdir = d.getVar("TEST_LOG_DIR", True) + bb.utils.mkdirhier(self.testlogdir) + self.logfile = os.path.join(self.testlogdir, "sdk_target_log.%s" % self.datetime) + BuildProject.__init__(self, d, uri, foldername, tmpdir=testpath) + + def download_archive(self): + + self._download_archive() + + cmd = 'tar xf %s%s -C %s' % (self.targetdir, self.archive, self.targetdir) + subprocess.check_call(cmd, shell=True) + + #Change targetdir to project folder + self.targetdir = self.targetdir + self.fname + + def run_configure(self, configure_args=''): + return super(SDKBuildProject, self).run_configure(configure_args=(configure_args or '$CONFIGURE_FLAGS')) + + def run_install(self, install_args=''): + return super(SDKBuildProject, self).run_install(install_args=(install_args or "DESTDIR=%s/../install" % self.targetdir)) + + def log(self, msg): + if self.logfile: + with open(self.logfile, "a") as f: + f.write("%s\n" % msg) + + def _run(self, cmd): + self.log("Running source %s; " % self.sdkenv + cmd) + return subprocess.call("source %s; " % self.sdkenv + cmd, shell=True) - def clean(self): - self.target.run('rm -rf %s' % self.targetdir) - subprocess.call('rm -f %s' % self.localarchive, shell=True) |