Docker/CI: Update to "focal" and latest build

Move us up to being based on Ubuntu 20.04 "focal" and the latest tag
from Ubuntu for this release.  For this, we make sure that "python" is
now python3 but still include python2.7 for the rx51 qemu build as that
is very old and does not support python3.

Signed-off-by: Tom Rini <trini@konsulko.com>

.azure-pipelines.yml

@@ -2,7 +2,7 @@ variables:
   windows_vm: vs2017-win2016
   ubuntu_vm: ubuntu-18.04
   macos_vm: macOS-10.15
-  ci_runner_image: trini/u-boot-gitlab-ci-runner:bionic-20200807-02Sep2020
+  ci_runner_image: trini/u-boot-gitlab-ci-runner:focal-20210609-01Jul2021
   # Add '-u 0' options for Azure pipelines, otherwise we get "permission
   # denied" error when it tries to "useradd -m -u 1001 vsts_azpcontainer",
   # since our $(ci_runner_image) user is not root.
@@ -318,7 +318,22 @@ jobs:
           # as sandbox testing need create files like spi flash images, etc.
           # (TODO: clean up this in the future)
           chmod 777 .
-          docker run -v $PWD:$(work_dir) $(ci_runner_image) /bin/bash $(work_dir)/test.sh
+          # Filesystem tests need extra docker args to run
+          set --
+          if [[ "${TEST_PY_BD}" == "sandbox" ]]; then
+              # mount -o loop needs the loop devices
+              if modprobe loop; then
+                  for d in $(find /dev -maxdepth 1 -name 'loop*'); do
+                      set -- "$@" --device $d:$d
+                  done
+              fi
+              # Needed for mount syscall (for guestmount as well)
+              set -- "$@" --cap-add SYS_ADMIN
+              # Default apparmor profile denies mounts
+              set -- "$@" --security-opt apparmor=unconfined
+          fi
+          # Some tests using libguestfs-tools need the fuse device to run
+          docker run "$@" --device /dev/fuse:/dev/fuse -v $PWD:$(work_dir) $(ci_runner_image) /bin/bash $(work_dir)/test.sh
 
   - job: build_the_world
     displayName: 'Build the World'

.gitlab-ci.yml

@@ -2,7 +2,7 @@
 
 # Grab our configured image.  The source for this is found at:
 # https://source.denx.de/u-boot/gitlab-ci-runner
-image: trini/u-boot-gitlab-ci-runner:bionic-20200807-02Sep2020
+image: trini/u-boot-gitlab-ci-runner:focal-20210609-01Jul2021
 
 # We run some tests in different order, to catch some failures quicker.
 stages:

test/nokia_rx51_test.sh

@@ -56,7 +56,7 @@ if ! test -f qemu-system-arm; then
 	test -d qemu-linaro || git clone https://git.linaro.org/qemu/qemu-linaro.git
 	cd qemu-linaro
 	git checkout 8f8d8e0796efe1a6f34cdd83fb798f3c41217ec1
-	./configure --enable-system --target-list=arm-softmmu --disable-sdl --disable-gtk --disable-curses --audio-drv-list= --audio-card-list= --disable-werror --disable-xen --disable-xen-pci-passthrough --disable-brlapi --disable-vnc --disable-curl --disable-slirp --disable-kvm --disable-user --disable-linux-user --disable-bsd-user --disable-guest-base --disable-uuid --disable-vde --disable-linux-aio --disable-cap-ng --disable-attr --disable-blobs --disable-docs --disable-spice --disable-libiscsi --disable-smartcard-nss --disable-usb-redir --disable-guest-agent --disable-seccomp --disable-glusterfs --disable-nptl --disable-fdt
+	./configure --enable-system --target-list=arm-softmmu --python=/usr/bin/python2.7 --disable-sdl --disable-gtk --disable-curses --audio-drv-list= --audio-card-list= --disable-werror --disable-xen --disable-xen-pci-passthrough --disable-brlapi --disable-vnc --disable-curl --disable-slirp --disable-kvm --disable-user --disable-linux-user --disable-bsd-user --disable-guest-base --disable-uuid --disable-vde --disable-linux-aio --disable-cap-ng --disable-attr --disable-blobs --disable-docs --disable-spice --disable-libiscsi --disable-smartcard-nss --disable-usb-redir --disable-guest-agent --disable-seccomp --disable-glusterfs --disable-nptl --disable-fdt
 	make -j4
 	cd ..
 	ln -s qemu-linaro/arm-softmmu/qemu-system-arm .

test/py/tests/test_fs/conftest.py

@@ -8,6 +8,7 @@ import pytest
 import re
 from subprocess import call, check_call, check_output, CalledProcessError
 from fstest_defs import *
+import u_boot_utils as util
 
 supported_fs_basic = ['fat16', 'fat32', 'ext4']
 supported_fs_ext = ['fat16', 'fat32']
@@ -209,24 +210,23 @@ def mount_fs(fs_type, device, mount_point):
     """
     global fuse_mounted
 
-    fuse_mounted = False
     try:
-        if tool_is_in_path('guestmount'):
-            fuse_mounted = True
-            check_call('guestmount -a %s -m /dev/sda %s'
-                % (device, mount_point), shell=True)
-        else:
-            mount_opt = 'loop,rw'
-            if re.match('fat', fs_type):
-                mount_opt += ',umask=0000'
-
-            check_call('sudo mount -o %s %s %s'
-                % (mount_opt, device, mount_point), shell=True)
-
-            # may not be effective for some file systems
-            check_call('sudo chmod a+rw %s' % mount_point, shell=True)
+        check_call('guestmount --pid-file guestmount.pid -a %s -m /dev/sda %s'
+            % (device, mount_point), shell=True)
+        fuse_mounted = True
+        return
     except CalledProcessError:
-        raise
+        fuse_mounted = False
+
+    mount_opt = 'loop,rw'
+    if re.match('fat', fs_type):
+        mount_opt += ',umask=0000'
+
+    check_call('sudo mount -o %s %s %s'
+        % (mount_opt, device, mount_point), shell=True)
+
+    # may not be effective for some file systems
+    check_call('sudo chmod a+rw %s' % mount_point, shell=True)
 
 def umount_fs(mount_point):
     """Unmount a volume.
@@ -240,6 +240,16 @@ def umount_fs(mount_point):
     if fuse_mounted:
         call('sync')
         call('guestunmount %s' % mount_point, shell=True)
+
+        try:
+            with open("guestmount.pid", "r") as pidfile:
+                pid = int(pidfile.read())
+            util.waitpid(pid, kill=True)
+            os.remove("guestmount.pid")
+
+        except FileNotFoundError:
+            pass
+
     else:
         call('sudo umount %s' % mount_point, shell=True)
 

test/py/tests/test_fs/test_squashfs/sqfs_common.py

@@ -3,74 +3,203 @@
 # Author: Joao Marcos Costa <joaomarcos.costa@bootlin.com>
 
 import os
-import random
-import string
+import shutil
 import subprocess
 
-def sqfs_get_random_letters(size):
-    letters = []
-    for i in range(0, size):
-            letters.append(random.choice(string.ascii_letters))
+""" standard test images table: Each table item is a key:value pair
+representing the output image name and its respective mksquashfs options.
+This table should be modified only when adding support for new compression
+algorithms. The 'default' case takes no options but the input and output
+names, so it must be assigned with an empty string.
+"""
+STANDARD_TABLE = {
+        'default' : '',
+        'lzo_comp_frag' : '',
+        'lzo_frag' : '',
+        'lzo_no_frag' : '',
+        'zstd_comp_frag' : '',
+        'zstd_frag' : '',
+        'zstd_no_frag' : '',
+        'gzip_comp_frag' : '',
+        'gzip_frag' : '',
+        'gzip_no_frag' : ''
+}
 
-    return ''.join(letters)
+""" EXTRA_TABLE: Set this table's keys and values if you want to make squashfs
+images with your own customized options.
+"""
+EXTRA_TABLE = {}
 
-def sqfs_generate_file(path, size):
-    content = sqfs_get_random_letters(size)
-    file = open(path, "w")
+# path to source directory used to make squashfs test images
+SQFS_SRC_DIR = 'sqfs_src_dir'
+
+def get_opts_list():
+    """ Combines fragmentation and compression options into a list of strings.
+
+    opts_list's firts item is an empty string as STANDARD_TABLE's first item is
+    the 'default' case.
+
+    Returns:
+        A list of strings whose items are formed by a compression and a
+        fragmentation option joined by a whitespace.
+    """
+    # supported compression options only
+    comp_opts = ['-comp lzo', '-comp zstd', '-comp gzip']
+    # file fragmentation options
+    frag_opts = ['-always-use-fragments', '-always-use-fragments -noF', '-no-fragments']
+
+    opts_list = [' ']
+    for comp_opt in comp_opts:
+        for frag_opt in frag_opts:
+            opts_list.append(' '.join([comp_opt, frag_opt]))
+
+    return opts_list
+
+def init_standard_table():
+    """ Initializes STANDARD_TABLE values.
+
+    STANDARD_TABLE's keys are pre-defined, and init_standard_table() assigns
+    the right value for each one of them.
+    """
+    opts_list = get_opts_list()
+
+    for key, value in zip(STANDARD_TABLE.keys(), opts_list):
+        STANDARD_TABLE[key] = value
+
+def generate_file(file_name, file_size):
+    """ Generates a file filled with 'x'.
+
+    Args:
+        file_name: the file's name.
+        file_size: the content's length and therefore the file size.
+    """
+    content = 'x' * file_size
+
+    file = open(file_name, 'w')
     file.write(content)
     file.close()
 
-class Compression:
-    def __init__(self, name, files, sizes, block_size = 4096):
-        self.name = name
-        self.files = files
-        self.sizes = sizes
-        self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name
+def generate_sqfs_src_dir(build_dir):
+    """ Generates the source directory used to make the SquashFS images.
 
-    def add_opt(self, opt):
-        self.mksquashfs_opts += " " + opt
+    The source directory is generated at build_dir, and it has the following
+    structure:
+    sqfs_src_dir/
+    ├── empty-dir/
+    ├── f1000
+    ├── f4096
+    ├── f5096
+    ├── subdir/
+    │   └── subdir-file
+    └── sym -> subdir
 
-    def gen_image(self, build_dir):
-        src = os.path.join(build_dir, "sqfs_src/")
-        os.mkdir(src)
-        for (f, s) in zip(self.files, self.sizes):
-            sqfs_generate_file(src + f, s)
+    3 directories, 4 files
 
-        # the symbolic link always targets the first file
-        os.symlink(self.files[0], src + "sym")
+    The files in the root dir. are prefixed with an 'f' followed by its size.
 
-        sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
-        i_o = src + " " + sqfs_img
-        opts = self.mksquashfs_opts
-        try:
-            subprocess.run(["mksquashfs " + i_o + opts], shell = True, check = True)
-        except:
-            print("mksquashfs error. Compression type: " + self.name)
-            raise RuntimeError
+    Args:
+        build_dir: u-boot's build-sandbox directory.
+    """
 
-    def clean_source(self, build_dir):
-        src = os.path.join(build_dir, "sqfs_src/")
-        for f in self.files:
-            os.remove(src + f)
-        os.remove(src + "sym")
-        os.rmdir(src)
+    root = os.path.join(build_dir, SQFS_SRC_DIR)
+    # make root directory
+    os.makedirs(root)
 
-    def cleanup(self, build_dir):
-        self.clean_source(build_dir)
-        sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
-        os.remove(sqfs_img)
+    # 4096: minimum block size
+    file_name = 'f4096'
+    generate_file(os.path.join(root, file_name), 4096)
 
-files = ["blks_only", "blks_frag", "frag_only"]
-sizes = [4096, 5100, 100]
-gzip = Compression("gzip", files, sizes)
-zstd = Compression("zstd", files, sizes)
-lzo = Compression("lzo", files, sizes)
+    # 5096: minimum block size + 1000 chars (fragment)
+    file_name = 'f5096'
+    generate_file(os.path.join(root, file_name), 5096)
 
-# use fragment blocks for files larger than block_size
-gzip.add_opt("-always-use-fragments")
-zstd.add_opt("-always-use-fragments")
+    # 1000: less than minimum block size (fragment only)
+    file_name = 'f1000'
+    generate_file(os.path.join(root, file_name), 1000)
 
-# avoid fragments if lzo is used
-lzo.add_opt("-no-fragments")
+    # sub-directory with a single file inside
+    subdir_path = os.path.join(root, 'subdir')
+    os.makedirs(subdir_path)
+    generate_file(os.path.join(subdir_path, 'subdir-file'), 100)
 
-comp_opts = [gzip, zstd, lzo]
+    # symlink (target: sub-directory)
+    os.symlink('subdir', os.path.join(root, 'sym'))
+
+    # empty directory
+    os.makedirs(os.path.join(root, 'empty-dir'))
+
+def mksquashfs(args):
+    """ Runs mksquashfs command.
+
+    Args:
+        args: mksquashfs options (e.g.: compression and fragmentation).
+    """
+    subprocess.run(['mksquashfs ' + args], shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
+
+def get_mksquashfs_version():
+    """ Parses the output of mksquashfs -version.
+
+    Returns:
+        mksquashfs's version as a float.
+    """
+    out = subprocess.run(['mksquashfs -version'], shell=True, check=True,
+                         capture_output=True, text=True)
+    # 'out' is: mksquashfs version X (yyyy/mm/dd) ...
+    return float(out.stdout.split()[2])
+
+def check_mksquashfs_version():
+    """ Checks if mksquashfs meets the required version. """
+
+    required_version = 4.4
+    if get_mksquashfs_version() < required_version:
+        print('Error: mksquashfs is too old.')
+        print('Required version: {}'.format(required_version))
+        raise AssertionError
+
+def make_all_images(build_dir):
+    """ Makes the SquashFS images used in the test suite.
+
+    The image names and respective mksquashfs options are defined in STANDARD_TABLE
+    and EXTRA_TABLE. The destination is defined by 'build_dir'.
+
+    Args:
+        build_dir: u-boot's build-sandbox directory.
+    """
+
+    init_standard_table()
+    input_path = os.path.join(build_dir, SQFS_SRC_DIR)
+
+    # make squashfs images according to STANDARD_TABLE
+    for out, opts in zip(STANDARD_TABLE.keys(), STANDARD_TABLE.values()):
+        output_path = os.path.join(build_dir, out)
+        mksquashfs(' '.join([input_path, output_path, opts]))
+
+    # make squashfs images according to EXTRA_TABLE
+    for out, opts in zip(EXTRA_TABLE.keys(), EXTRA_TABLE.values()):
+        output_path = os.path.join(build_dir, out)
+        mksquashfs(' '.join([input_path, output_path, opts]))
+
+def clean_all_images(build_dir):
+    """ Deletes the SquashFS images at build_dir.
+
+    Args:
+        build_dir: u-boot's build-sandbox directory.
+    """
+
+    for image_name in STANDARD_TABLE:
+        image_path = os.path.join(build_dir, image_name)
+        os.remove(image_path)
+
+    for image_name in EXTRA_TABLE:
+        image_path = os.path.join(build_dir, image_name)
+        os.remove(image_path)
+
+def clean_sqfs_src_dir(build_dir):
+    """ Deletes the source directory at build_dir.
+
+    Args:
+        build_dir: u-boot's build-sandbox directory.
+    """
+    path = os.path.join(build_dir, SQFS_SRC_DIR)
+    shutil.rmtree(path)

test/py/tests/test_fs/test_squashfs/test_sqfs_load.py

@@ -3,8 +3,118 @@
 # Author: Joao Marcos Costa <joaomarcos.costa@bootlin.com>
 
 import os
+import subprocess
 import pytest
-from sqfs_common import *
+
+from sqfs_common import SQFS_SRC_DIR, STANDARD_TABLE
+from sqfs_common import generate_sqfs_src_dir, make_all_images
+from sqfs_common import clean_sqfs_src_dir, clean_all_images
+from sqfs_common import check_mksquashfs_version
+
+@pytest.mark.requiredtool('md5sum')
+def original_md5sum(path):
+    """ Runs md5sum command.
+
+    Args:
+        path: path to original file.
+    Returns:
+        The original file's checksum as a string.
+    """
+
+    out = subprocess.run(['md5sum ' + path], shell=True, check=True,
+                         capture_output=True, text=True)
+    checksum = out.stdout.split()[0]
+
+    return checksum
+
+def uboot_md5sum(u_boot_console, address, count):
+    """ Runs U-Boot's md5sum command.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+        address: address where the file was loaded (e.g.: $kernel_addr_r).
+        count: file's size. It was named 'count' to match md5sum's respective
+        argument name.
+    Returns:
+        The checksum of the file loaded with sqfsload as a string.
+    """
+
+    out = u_boot_console.run_command('md5sum {} {}'.format(address, count))
+    checksum = out.split()[-1]
+
+    return checksum
+
+def sqfs_load_files(u_boot_console, files, sizes, address):
+    """ Loads files and asserts their checksums.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+        files: list of files to be loaded.
+        sizes: the sizes of each file.
+        address: the address where the files should be loaded.
+    """
+    build_dir = u_boot_console.config.build_dir
+    for (file, size) in zip(files, sizes):
+        out = u_boot_console.run_command('sqfsload host 0 {} {}'.format(address, file))
+
+        # check if the right amount of bytes was read
+        assert size in out
+
+        # compare original file's checksum against u-boot's
+        u_boot_checksum = uboot_md5sum(u_boot_console, address, hex(int(size)))
+        original_file_path = os.path.join(build_dir, SQFS_SRC_DIR + '/' + file)
+        original_checksum = original_md5sum(original_file_path)
+        assert u_boot_checksum == original_checksum
+
+def sqfs_load_files_at_root(u_boot_console):
+    """ Calls sqfs_load_files passing the files at the SquashFS image's root.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+
+    files = ['f4096', 'f5096', 'f1000']
+    sizes = ['4096', '5096', '1000']
+    address = '$kernel_addr_r'
+    sqfs_load_files(u_boot_console, files, sizes, address)
+
+def sqfs_load_files_at_subdir(u_boot_console):
+    """ Calls sqfs_load_files passing the files at the SquashFS image's subdir.
+
+    This test checks if the path resolution works, since the file is not at the
+    root directory.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+    files = ['subdir/subdir-file']
+    sizes = ['100']
+    address = '$kernel_addr_r'
+    sqfs_load_files(u_boot_console, files, sizes, address)
+
+def sqfs_load_non_existent_file(u_boot_console):
+    """ Calls sqfs_load_files passing an non-existent file to raise an error.
+
+    This test checks if the SquashFS support won't crash if it doesn't find the
+    specified file.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+    address = '$kernel_addr_r'
+    file = 'non-existent'
+    out = u_boot_console.run_command('sqfsload host 0 {} {}'.format(address, file))
+    assert 'Failed to load' in out
+
+def sqfs_run_all_load_tests(u_boot_console):
+    """ Runs all the previously defined test cases.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+    sqfs_load_files_at_root(u_boot_console)
+    sqfs_load_files_at_subdir(u_boot_console)
+    sqfs_load_non_existent_file(u_boot_console)
 
 @pytest.mark.boardspec('sandbox')
 @pytest.mark.buildconfigspec('cmd_fs_generic')
@@ -12,35 +122,33 @@ from sqfs_common import *
 @pytest.mark.buildconfigspec('fs_squashfs')
 @pytest.mark.requiredtool('mksquashfs')
 def test_sqfs_load(u_boot_console):
+    """ Executes the sqfsload test suite.
+
+    First, it generates the SquashFS images, then it runs the test cases and
+    finally cleans the workspace. If an exception is raised, the workspace is
+    cleaned before exiting.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
     build_dir = u_boot_console.config.build_dir
-    command = "sqfsload host 0 $kernel_addr_r "
 
-    for opt in comp_opts:
-        # generate and load the squashfs image
+    # setup test environment
+    check_mksquashfs_version()
+    generate_sqfs_src_dir(build_dir)
+    make_all_images(build_dir)
+
+    # run all tests for each image
+    for image in STANDARD_TABLE:
         try:
-            opt.gen_image(build_dir)
-        except RuntimeError:
-            opt.clean_source(build_dir)
-            # skip unsupported compression types
-            continue
+            image_path = os.path.join(build_dir, image)
+            u_boot_console.run_command('host bind 0 {}'.format(image_path))
+            sqfs_run_all_load_tests(u_boot_console)
+        except:
+            clean_all_images(build_dir)
+            clean_sqfs_src_dir(build_dir)
+            raise AssertionError
 
-        path = os.path.join(build_dir, "sqfs-" + opt.name)
-        output = u_boot_console.run_command("host bind 0 " + path)
-
-        output = u_boot_console.run_command(command + "xxx")
-        assert "File not found." in output
-
-        for (f, s) in zip(opt.files, opt.sizes):
-            try:
-                output = u_boot_console.run_command(command + f)
-                assert str(s) in output
-            except:
-                assert False
-                opt.cleanup(build_dir)
-
-        # test symbolic link
-        output = u_boot_console.run_command(command + "sym")
-        assert str(opt.sizes[0]) in output
-
-        # remove generated files
-        opt.cleanup(build_dir)
+    # clean test environment
+    clean_all_images(build_dir)
+    clean_sqfs_src_dir(build_dir)

test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py

@@ -4,7 +4,101 @@
 
 import os
 import pytest
-from sqfs_common import *
+
+from sqfs_common import STANDARD_TABLE
+from sqfs_common import generate_sqfs_src_dir, make_all_images
+from sqfs_common import clean_sqfs_src_dir, clean_all_images
+from sqfs_common import check_mksquashfs_version
+
+def sqfs_ls_at_root(u_boot_console):
+    """ Runs sqfsls at image's root.
+
+    This test checks if all the present files and directories were listed. Also,
+    it checks if passing the slash or not changes the output, which it shouldn't.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+
+    no_slash = u_boot_console.run_command('sqfsls host 0')
+    slash = u_boot_console.run_command('sqfsls host 0 /')
+    assert no_slash == slash
+
+    expected_lines = ['empty-dir/', '1000   f1000', '4096   f4096', '5096   f5096',
+                      'subdir/', '<SYM>   sym', '4 file(s), 2 dir(s)']
+
+    output = u_boot_console.run_command('sqfsls host 0')
+    for line in expected_lines:
+        assert line in output
+
+def sqfs_ls_at_empty_dir(u_boot_console):
+    """ Runs sqfsls at an empty directory.
+
+    This tests checks if sqfsls will print anything other than the 'Empty directory'
+    message.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+    assert u_boot_console.run_command('sqfsls host 0 empty-dir') == 'Empty directory.'
+
+def sqfs_ls_at_subdir(u_boot_console):
+    """ Runs sqfsls at the SquashFS image's subdir.
+
+    This test checks if the path resolution works, since the directory is not the
+    root.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+    expected_lines = ['100   subdir-file', '1 file(s), 0 dir(s)']
+    output = u_boot_console.run_command('sqfsls host 0 subdir')
+    for line in expected_lines:
+        assert line in output
+
+def sqfs_ls_at_symlink(u_boot_console):
+    """ Runs sqfsls at a SquashFS image's symbolic link.
+
+    This test checks if the symbolic link's target resolution works.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+    # since sym -> subdir, the following outputs must be equal
+    output = u_boot_console.run_command('sqfsls host 0 sym')
+    output_subdir = u_boot_console.run_command('sqfsls host 0 subdir')
+    assert output == output_subdir
+
+    expected_lines = ['100   subdir-file', '1 file(s), 0 dir(s)']
+    for line in expected_lines:
+        assert line in output
+
+def sqfs_ls_at_non_existent_dir(u_boot_console):
+    """ Runs sqfsls at a file and at a non-existent directory.
+
+    This test checks if the SquashFS support won't crash if it doesn't find the
+    specified directory or if it takes a file as an input instead of an actual
+    directory. In both cases, the output should be the same.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+    out_non_existent = u_boot_console.run_command('sqfsls host 0 fff')
+    out_not_dir = u_boot_console.run_command('sqfsls host 0 f1000')
+    assert out_non_existent == out_not_dir
+    assert '** Cannot find directory. **' in out_non_existent
+
+def sqfs_run_all_ls_tests(u_boot_console):
+    """ Runs all the previously defined test cases.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+    sqfs_ls_at_root(u_boot_console)
+    sqfs_ls_at_empty_dir(u_boot_console)
+    sqfs_ls_at_subdir(u_boot_console)
+    sqfs_ls_at_symlink(u_boot_console)
+    sqfs_ls_at_non_existent_dir(u_boot_console)
 
 @pytest.mark.boardspec('sandbox')
 @pytest.mark.buildconfigspec('cmd_fs_generic')
@@ -12,25 +106,33 @@ from sqfs_common import *
 @pytest.mark.buildconfigspec('fs_squashfs')
 @pytest.mark.requiredtool('mksquashfs')
 def test_sqfs_ls(u_boot_console):
-    build_dir = u_boot_console.config.build_dir
-    for opt in comp_opts:
-        try:
-            opt.gen_image(build_dir)
-        except RuntimeError:
-            opt.clean_source(build_dir)
-            # skip unsupported compression types
-            continue
-        path = os.path.join(build_dir, "sqfs-" + opt.name)
-        output = u_boot_console.run_command("host bind 0 " + path)
+    """ Executes the sqfsls test suite.
 
+    First, it generates the SquashFS images, then it runs the test cases and
+    finally cleans the workspace. If an exception is raised, the workspace is
+    cleaned before exiting.
+
+    Args:
+        u_boot_console: provides the means to interact with U-Boot's console.
+    """
+    build_dir = u_boot_console.config.build_dir
+
+    # setup test environment
+    check_mksquashfs_version()
+    generate_sqfs_src_dir(build_dir)
+    make_all_images(build_dir)
+
+    # run all tests for each image
+    for image in STANDARD_TABLE:
         try:
-            # list files in root directory
-            output = u_boot_console.run_command("sqfsls host 0")
-            assert str(len(opt.files) + 1) + " file(s), 0 dir(s)" in output
-            assert "<SYM>   sym" in output
-            output = u_boot_console.run_command("sqfsls host 0 xxx")
-            assert "** Cannot find directory. **" in output
+            image_path = os.path.join(build_dir, image)
+            u_boot_console.run_command('host bind 0 {}'.format(image_path))
+            sqfs_run_all_ls_tests(u_boot_console)
         except:
-            opt.cleanup(build_dir)
-            assert False
-        opt.cleanup(build_dir)
+            clean_all_images(build_dir)
+            clean_sqfs_src_dir(build_dir)
+            raise AssertionError
+
+    # clean test environment
+    clean_all_images(build_dir)
+    clean_sqfs_src_dir(build_dir)

test/py/u_boot_utils.py

@@ -8,6 +8,7 @@ import inspect
 import os
 import os.path
 import pytest
+import signal
 import sys
 import time
 import re
@@ -339,3 +340,38 @@ def crc32(u_boot_console, address, count):
     assert m, 'CRC32 operation failed.'
 
     return m.group(1)
+
+def waitpid(pid, timeout=60, kill=False):
+    """Wait a process to terminate by its PID
+
+    This is an alternative to a os.waitpid(pid, 0) call that works on
+    processes that aren't children of the python process.
+
+    Args:
+        pid: PID of a running process.
+        timeout: Time in seconds to wait.
+        kill: Whether to forcibly kill the process after timeout.
+
+    Returns:
+        True, if the process ended on its own.
+        False, if the process was killed by this function.
+
+    Raises:
+        TimeoutError, if the process is still running after timeout.
+    """
+    try:
+        for _ in range(timeout):
+            os.kill(pid, 0)
+            time.sleep(1)
+
+        if kill:
+            os.kill(pid, signal.SIGKILL)
+            return False
+
+    except ProcessLookupError:
+        return True
+
+    raise TimeoutError(
+        "Process with PID {} did not terminate after {} seconds."
+        .format(pid, timeout)
+    )

tools/docker/Dockerfile

@@ -2,7 +2,7 @@
 # This Dockerfile is used to build an image containing basic stuff to be used
 # to build U-Boot and run our test suites.
 
-FROM ubuntu:bionic-20200807
+FROM ubuntu:focal-20210609
 MAINTAINER Tom Rini <trini@konsulko.com>
 LABEL Description=" This image is for building U-Boot inside a container"
 
@@ -12,7 +12,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 # Add LLVM repository
 RUN apt-get update && apt-get install -y gnupg2 wget xz-utils && rm -rf /var/lib/apt/lists/*
 RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-RUN echo deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main | tee /etc/apt/sources.list.d/llvm.list
+RUN echo deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main | tee /etc/apt/sources.list.d/llvm.list
 
 # Manually install the kernel.org "Crosstool" based toolchains for gcc-7.3
 RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/9.2.0/x86_64-gcc-9.2.0-nolibc-aarch64-linux.tar.xz | tar -C /opt -xJ
@@ -61,16 +61,17 @@ RUN apt-get update && apt-get install -y \
 	iasl \
 	imagemagick \
 	iputils-ping \
+	libgit2-dev \
 	libguestfs-tools \
-	libisl15 \
 	liblz4-tool \
 	libpixman-1-dev \
-	libpython-dev \
+	libpython3-dev \
 	libsdl1.2-dev \
 	libsdl2-dev \
 	libssl-dev \
 	libudev-dev \
 	libusb-1.0-0-dev \
+	linux-image-kvm \
 	lzma-alone \
 	lzop \
 	mount \
@@ -80,12 +81,13 @@ RUN apt-get update && apt-get install -y \
 	picocom \
 	parted \
 	pkg-config \
-	python \
-	python-dev \
-	python-pip \
-	python-virtualenv \
+	python-is-python3 \
+	python2.7 \
+	python3 \
+	python3-dev \
 	python3-pip \
 	python3-sphinx \
+	python3-virtualenv \
 	rpm2cpio \
 	sbsigntool \
 	sloccount \
@@ -99,12 +101,12 @@ RUN apt-get update && apt-get install -y \
 	zip \
 	&& rm -rf /var/lib/apt/lists/*
 
+# Make kernels readable for libguestfs tools to work correctly
+RUN chmod +r /boot/vmlinu*
+
 # Manually install libmpfr4 for the toolchains
 RUN wget http://mirrors.kernel.org/ubuntu/pool/main/m/mpfr4/libmpfr4_3.1.4-1_amd64.deb && dpkg -i libmpfr4_3.1.4-1_amd64.deb && rm libmpfr4_3.1.4-1_amd64.deb
 
-# Manually install a new enough version of efitools (must be v1.5.2 or later)
-RUN wget http://mirrors.kernel.org/ubuntu/pool/universe/e/efitools/efitools_1.8.1-0ubuntu2_amd64.deb && sudo dpkg -i efitools_1.8.1-0ubuntu2_amd64.deb && rm efitools_1.8.1-0ubuntu2_amd64.deb
-
 # Manually install a new enough version of sbsigntools (must be v0.9.4 or later)
 RUN git clone https://git.kernel.org/pub/scm/linux/kernel/git/jejb/sbsigntools.git /tmp/sbsigntools && \
 	cd /tmp/sbsigntools && \