From b1793a531e5934ea5453b7e24495e2fcddd9c493 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 11 May 2019 12:46:39 -0600 Subject: [PATCH 01/74] patman: Update cros_subprocess to use bytes At present this function uses lists and strings. This does not work so well with Python 3, and testing against '' does not work for a bytearray. Update the code to fix these issues. Signed-off-by: Simon Glass --- tools/patman/cros_subprocess.py | 47 +++++++++++++++++++-------------- tools/patman/gitutil.py | 2 ++ 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/tools/patman/cros_subprocess.py b/tools/patman/cros_subprocess.py index ebd4300dfd..0f0d60dfb7 100644 --- a/tools/patman/cros_subprocess.py +++ b/tools/patman/cros_subprocess.py @@ -100,6 +100,19 @@ class Popen(subprocess.Popen): if kwargs: raise ValueError("Unit tests do not test extra args - please add tests") + def ConvertData(self, data): + """Convert stdout/stderr data to the correct format for output + + Args: + data: Data to convert, or None for '' + + Returns: + Converted data, as bytes + """ + if data is None: + return b'' + return data + def CommunicateFilter(self, output): """Interact with process: Read data from stdout and stderr. @@ -156,11 +169,11 @@ class Popen(subprocess.Popen): self.stdin.close() if self.stdout: read_set.append(self.stdout) - stdout = [] + stdout = b'' if self.stderr and self.stderr != self.stdout: read_set.append(self.stderr) - stderr = [] - combined = [] + stderr = b'' + combined = b'' input_offset = 0 while read_set or write_set: @@ -186,46 +199,40 @@ class Popen(subprocess.Popen): write_set.remove(self.stdin) if self.stdout in rlist: - data = "" + data = b'' # We will get an error on read if the pty is closed try: data = os.read(self.stdout.fileno(), 1024) except OSError: pass - if data == "": + if not len(data): self.stdout.close() read_set.remove(self.stdout) else: - stdout.append(data) - combined.append(data) + stdout += data + combined += data if output: output(sys.stdout, data) if self.stderr in rlist: - data = "" + data = b'' # We will get an error on read if the pty is closed try: data = os.read(self.stderr.fileno(), 1024) except OSError: pass - if data == "": + if not len(data): self.stderr.close() read_set.remove(self.stderr) else: - stderr.append(data) - combined.append(data) + stderr += data + combined += data if output: output(sys.stderr, data) # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = ''.join(stdout) - else: - stdout = '' - if stderr is not None: - stderr = ''.join(stderr) - else: - stderr = '' - combined = ''.join(combined) + stdout = self.ConvertData(stdout) + stderr = self.ConvertData(stderr) + combined = self.ConvertData(combined) # Translate newlines, if requested. We cannot let the file # object do the translation: It is based on stdio, which is diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 9905bb0bbd..7650b51bd5 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -326,6 +326,8 @@ def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True): result = [] for item in raw: if not item in result: + if type(item) == unicode: + item = item.encode('utf-8') result.append(item) if tag: return ['%s %s%s%s' % (tag, quote, email, quote) for email in result] From 5a1af1dafeab06f0ef3bb749ec0ce59784bf8925 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:36 -0600 Subject: [PATCH 02/74] patman: Convert print statements to Python 3 Update all print statements to be functions, as required by Python 3. Signed-off-by: Simon Glass --- tools/patman/func_test.py | 3 --- tools/patman/gitutil.py | 10 +++++----- tools/patman/test_util.py | 16 +++++++++------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index d79e716074..31481157fc 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -159,7 +159,6 @@ class TestFunctional(unittest.TestCase): os.remove(cc_file) lines = out[0].splitlines() - #print '\n'.join(lines) self.assertEqual('Cleaned %s patches' % len(series.commits), lines[0]) self.assertEqual('Change log missing for v2', lines[1]) self.assertEqual('Change log missing for v3', lines[2]) @@ -223,7 +222,6 @@ Simon Glass (2): ''' lines = open(cover_fname).read().splitlines() - #print '\n'.join(lines) self.assertEqual( 'Subject: [RFC PATCH v3 0/2] test: A test patch series', lines[3]) @@ -231,7 +229,6 @@ Simon Glass (2): for i, fname in enumerate(args): lines = open(fname).read().splitlines() - #print '\n'.join(lines) subject = [line for line in lines if line.startswith('Subject')] self.assertEqual('Subject: [RFC %d/%d]' % (i + 1, count), subject[0][:18]) diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 7650b51bd5..11aeb73b74 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -397,11 +397,11 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, git_config_to = command.Output('git', 'config', 'sendemail.to', raise_on_error=False) if not git_config_to: - print ("No recipient.\n" - "Please add something like this to a commit\n" - "Series-to: Fred Bloggs \n" - "Or do something like this\n" - "git config sendemail.to u-boot@lists.denx.de") + print("No recipient.\n" + "Please add something like this to a commit\n" + "Series-to: Fred Bloggs \n" + "Or do something like this\n" + "git config sendemail.to u-boot@lists.denx.de") return cc = BuildEmailList(list(set(series.get('cc')) - set(series.get('to'))), '--cc', alias, raise_on_error) diff --git a/tools/patman/test_util.py b/tools/patman/test_util.py index 687d40704a..e462ec8f72 100644 --- a/tools/patman/test_util.py +++ b/tools/patman/test_util.py @@ -3,6 +3,8 @@ # Copyright (c) 2016 Google, Inc # +from __future__ import print_function + from contextlib import contextmanager import glob import os @@ -54,18 +56,18 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): missing_list = required missing_list.difference_update(test_set) if missing_list: - print 'Missing tests for %s' % (', '.join(missing_list)) - print stdout + print('Missing tests for %s' % (', '.join(missing_list))) + print(stdout) ok = False coverage = lines[-1].split(' ')[-1] ok = True - print coverage + print(coverage) if coverage != '100%': - print stdout - print ("Type 'python-coverage html' to get a report in " - 'htmlcov/index.html') - print 'Coverage error: %s, but should be 100%%' % coverage + print(stdout) + print("Type 'python-coverage html' to get a report in " + 'htmlcov/index.html') + print('Coverage error: %s, but should be 100%%' % coverage) ok = False if not ok: raise ValueError('Test coverage failure') From 2ca8468026b678c8630ad99290cc5d2428eaeb55 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:37 -0600 Subject: [PATCH 03/74] binman: Convert print statements to Python 3 Update all print statements to be functions, as required by Python 3. Signed-off-by: Simon Glass --- tools/binman/binman.py | 14 ++++++++------ tools/binman/control.py | 6 ++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tools/binman/binman.py b/tools/binman/binman.py index 439908e665..4206d2b0e3 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -9,6 +9,8 @@ """See README for more information""" +from __future__ import print_function + import glob import multiprocessing import os @@ -85,13 +87,13 @@ def RunTests(debug, processes, args): else: suite.run(result) - print result + print(result) for test, err in result.errors: - print test.id(), err + print(test.id(), err) for test, err in result.failures: - print err, result.failures + print(err, result.failures) if result.errors or result.failures: - print 'binman tests FAILED' + print('binman tests FAILED') return 1 return 0 @@ -143,9 +145,9 @@ def RunBinman(options, args): try: ret_code = control.Binman(options, args) except Exception as e: - print 'binman: %s' % e + print('binman: %s' % e) if options.debug: - print + print() traceback.print_exc() ret_code = 1 return ret_code diff --git a/tools/binman/control.py b/tools/binman/control.py index b32e4e1996..8f7f906860 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -5,6 +5,8 @@ # Creates binary images from input files controlled by a description # +from __future__ import print_function + from collections import OrderedDict import os import sys @@ -134,7 +136,7 @@ def Binman(options, args): del images[name] skip.append(name) if skip and options.verbosity >= 2: - print 'Skipping images: %s' % ', '.join(skip) + print('Skipping images: %s' % ', '.join(skip)) state.Prepare(images, dtb) @@ -170,7 +172,7 @@ def Binman(options, args): except Exception as e: if options.map: fname = image.WriteMap() - print "Wrote map file '%s' to show errors" % fname + print("Wrote map file '%s' to show errors" % fname) raise image.SetImagePos() if options.update_fdt: From 35343dc41493666261e37a8a559b78eb2841a528 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:38 -0600 Subject: [PATCH 04/74] binman: Don't show errors for failed tests An unfortunate new feature in Python 3.5 causes binman to print errors for non-existent tests, when the test filter is used. Work around this by detecting the unwanted tests and removing them from the result. Signed-off-by: Simon Glass --- tools/binman/binman.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/binman/binman.py b/tools/binman/binman.py index 4206d2b0e3..aad2e9c8bc 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -87,6 +87,18 @@ def RunTests(debug, processes, args): else: suite.run(result) + # Remove errors which just indicate a missing test. Since Python v3.5 If an + # ImportError or AttributeError occurs while traversing name then a + # synthetic test that raises that error when run will be returned. These + # errors are included in the errors accumulated by result.errors. + if test_name: + errors = [] + for test, err in result.errors: + if ("has no attribute '%s'" % test_name) not in err: + errors.append((test, err)) + result.testsRun -= 1 + result.errors = errors + print(result) for test, err in result.errors: print(test.id(), err) From d141f6c1cacc754a691507db736a21b8eda2a630 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:39 -0600 Subject: [PATCH 05/74] binman: Remove use of Set() A new built-in set() is used in both Python 2 and 3 now. Move it to use that instead of Set(). Signed-off-by: Simon Glass --- tools/binman/bsection.py | 3 +-- tools/binman/entry.py | 5 ++--- tools/binman/state.py | 7 +++---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tools/binman/bsection.py b/tools/binman/bsection.py index 0ba542ee98..be67ff957e 100644 --- a/tools/binman/bsection.py +++ b/tools/binman/bsection.py @@ -8,7 +8,6 @@ from __future__ import print_function from collections import OrderedDict -from sets import Set import sys import fdt_util @@ -109,7 +108,7 @@ class Section(object): def GetFdtSet(self): """Get the set of device tree files used by this image""" - fdt_set = Set() + fdt_set = set() for entry in self._entries.values(): fdt_set.update(entry.GetFdtSet()) return fdt_set diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 648cfd241f..d842d89dd6 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -18,7 +18,6 @@ except: have_importlib = False import os -from sets import Set import sys import fdt_util @@ -178,8 +177,8 @@ class Entry(object): # It would be better to use isinstance(self, Entry_blob_dtb) here but # we cannot access Entry_blob_dtb if fname and fname.endswith('.dtb'): - return Set([fname]) - return Set() + return set([fname]) + return set() def ExpandEntries(self): pass diff --git a/tools/binman/state.py b/tools/binman/state.py index d945e4bf65..af9678649c 100644 --- a/tools/binman/state.py +++ b/tools/binman/state.py @@ -7,7 +7,6 @@ import hashlib import re -from sets import Set import os import tools @@ -24,10 +23,10 @@ entry_args = {} use_fake_dtb = False # Set of all device tree files references by images -fdt_set = Set() +fdt_set = set() # Same as above, but excluding the main one -fdt_subset = Set() +fdt_subset = set() # The DTB which contains the full image information main_dtb = None @@ -136,7 +135,7 @@ def Prepare(images, dtb): main_dtb = dtb fdt_files.clear() fdt_files['u-boot.dtb'] = dtb - fdt_subset = Set() + fdt_subset = set() if not use_fake_dtb: for image in images.values(): fdt_subset.update(image.GetFdtSet()) From 4a4c5dd43f02f60717ef121aa4bb89a85465ff3c Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:40 -0600 Subject: [PATCH 06/74] patman: Use items() instead of iteritems() Python 3 requires this, and Python 2 allows it. Convert the code over to ensure compatibility with Python 3. Signed-off-by: Simon Glass --- tools/patman/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/patman/settings.py b/tools/patman/settings.py index ea2bc74f75..07bf6a6ea4 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -163,7 +163,7 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser): item_dict = dict(top_items) item_dict.update(project_items) return {(self._to_unicode(item), self._to_unicode(val)) - for item, val in item_dict.iteritems()} + for item, val in item_dict.items()} def ReadGitAliases(fname): """Read a git alias file. This is in the form used by git: From 509791542816ce984ac4716827205de49b82b282 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:41 -0600 Subject: [PATCH 07/74] binman: Use items() instead of iteritems() Python 3 requires this, and Python 2 allows it. Convert the code over to ensure compatibility with Python 3. Signed-off-by: Simon Glass --- tools/binman/bsection.py | 2 +- tools/binman/control.py | 2 +- tools/binman/elf.py | 4 ++-- tools/binman/etype/gbb.py | 2 +- tools/binman/ftest.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/binman/bsection.py b/tools/binman/bsection.py index be67ff957e..3ca0592fe1 100644 --- a/tools/binman/bsection.py +++ b/tools/binman/bsection.py @@ -253,7 +253,7 @@ class Section(object): """ for entry in self._entries.values(): offset_dict = entry.GetOffsets() - for name, info in offset_dict.iteritems(): + for name, info in offset_dict.items(): self._SetEntryOffsetSize(name, *info) def PackEntries(self): diff --git a/tools/binman/control.py b/tools/binman/control.py index 8f7f906860..ce25eb5485 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -131,7 +131,7 @@ def Binman(options, args): if options.image: skip = [] - for name, image in images.iteritems(): + for name, image in images.items(): if name not in options.image: del images[name] skip.append(name) diff --git a/tools/binman/elf.py b/tools/binman/elf.py index 97df8e32c5..828681d76d 100644 --- a/tools/binman/elf.py +++ b/tools/binman/elf.py @@ -59,7 +59,7 @@ def GetSymbols(fname, patterns): flags[1] == 'w') # Sort dict by address - return OrderedDict(sorted(syms.iteritems(), key=lambda x: x[1].address)) + return OrderedDict(sorted(syms.items(), key=lambda x: x[1].address)) def GetSymbolAddress(fname, sym_name): """Get a value of a symbol from an ELF file @@ -98,7 +98,7 @@ def LookupAndWriteSymbols(elf_fname, entry, section): base = syms.get('__image_copy_start') if not base: return - for name, sym in syms.iteritems(): + for name, sym in syms.items(): if name.startswith('_binman'): msg = ("Section '%s': Symbol '%s'\n in entry '%s'" % (section.GetPath(), name, entry.GetPath())) diff --git a/tools/binman/etype/gbb.py b/tools/binman/etype/gbb.py index 8fe10f4713..a94c0fca9d 100644 --- a/tools/binman/etype/gbb.py +++ b/tools/binman/etype/gbb.py @@ -64,7 +64,7 @@ class Entry_gbb(Entry): self.gbb_flags = 0 flags_node = node.FindNode('flags') if flags_node: - for flag, value in gbb_flag_properties.iteritems(): + for flag, value in gbb_flag_properties.items(): if fdt_util.GetBool(flags_node, flag): self.gbb_flags |= value diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index daea1ea138..7cf17526a7 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -214,7 +214,7 @@ class TestFunctional(unittest.TestCase): if verbosity is not None: args.append('-v%d' % verbosity) if entry_args: - for arg, value in entry_args.iteritems(): + for arg, value in entry_args.items(): args.append('-a%s=%s' % (arg, value)) if images: for image in images: From 1d0ebf7d636ea923cafb45d2b0f61200b26e4198 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:42 -0600 Subject: [PATCH 08/74] tools: binman: Open all binary files in binary mode At present some files are opened in text mode despite containing binary data. This works on Python 2 but not always on Python 3, due to unicode problems. BC&D are not my favourite people. Adjust the affected open() statements to use binary mode. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 7cf17526a7..1111852a40 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -119,11 +119,11 @@ class TestFunctional(unittest.TestCase): TestFunctional._MakeInputFile('refcode.bin', REFCODE_DATA) # ELF file with a '_dt_ucode_base_size' symbol - with open(self.TestFile('u_boot_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('u-boot', fd.read()) # Intel flash descriptor file - with open(self.TestFile('descriptor.bin')) as fd: + with open(self.TestFile('descriptor.bin'), 'rb') as fd: TestFunctional._MakeInputFile('descriptor.bin', fd.read()) shutil.copytree(self.TestFile('files'), @@ -236,7 +236,7 @@ class TestFunctional(unittest.TestCase): """ tools.PrepareOutputDir(None) dtb = fdt_util.EnsureCompiled(self.TestFile(fname)) - with open(dtb) as fd: + with open(dtb, 'rb') as fd: data = fd.read() TestFunctional._MakeInputFile(outfile, data) tools.FinaliseOutputDir() @@ -317,7 +317,7 @@ class TestFunctional(unittest.TestCase): map_data = fd.read() else: map_data = None - with open(image_fname) as fd: + with open(image_fname, 'rb') as fd: return fd.read(), dtb_data, map_data, out_dtb_fname finally: # Put the test file back @@ -379,7 +379,7 @@ class TestFunctional(unittest.TestCase): Args: Filename of ELF file to use as SPL """ - with open(self.TestFile(src_fname)) as fd: + with open(self.TestFile(src_fname), 'rb') as fd: TestFunctional._MakeInputFile('spl/u-boot-spl', fd.read()) @classmethod @@ -541,7 +541,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(len(U_BOOT_DATA), image._size) fname = tools.GetOutputFilename('image1.bin') self.assertTrue(os.path.exists(fname)) - with open(fname) as fd: + with open(fname, 'rb') as fd: data = fd.read() self.assertEqual(U_BOOT_DATA, data) @@ -549,7 +549,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(3 + len(U_BOOT_DATA) + 5, image._size) fname = tools.GetOutputFilename('image2.bin') self.assertTrue(os.path.exists(fname)) - with open(fname) as fd: + with open(fname, 'rb') as fd: data = fd.read() self.assertEqual(U_BOOT_DATA, data[3:7]) self.assertEqual(chr(0) * 3, data[:3]) @@ -970,7 +970,7 @@ class TestFunctional(unittest.TestCase): """Test that a U-Boot binary without the microcode symbol is detected""" # ELF file without a '_dt_ucode_base_size' symbol try: - with open(self.TestFile('u_boot_no_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_no_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('u-boot', fd.read()) with self.assertRaises(ValueError) as e: @@ -980,7 +980,7 @@ class TestFunctional(unittest.TestCase): finally: # Put the original file back - with open(self.TestFile('u_boot_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('u-boot', fd.read()) def testMicrocodeNotInImage(self): @@ -993,7 +993,7 @@ class TestFunctional(unittest.TestCase): def testWithoutMicrocode(self): """Test that we can cope with an image without microcode (e.g. qemu)""" - with open(self.TestFile('u_boot_no_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_no_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('u-boot', fd.read()) data, dtb, _, _ = self._DoReadFileDtb('044_x86_optional_ucode.dts', True) @@ -1357,7 +1357,7 @@ class TestFunctional(unittest.TestCase): fname = pipe_list[0][-1] # Append our GBB data to the file, which will happen every time the # futility command is called. - with open(fname, 'a') as fd: + with open(fname, 'ab') as fd: fd.write(GBB_DATA) return command.CommandResult() @@ -1431,7 +1431,7 @@ class TestFunctional(unittest.TestCase): def testTpl(self): """Test that an image with TPL and ots device tree can be created""" # ELF file with a '__bss_size' symbol - with open(self.TestFile('bss_data')) as fd: + with open(self.TestFile('bss_data'), 'rb') as fd: TestFunctional._MakeInputFile('tpl/u-boot-tpl', fd.read()) data = self._DoReadFile('078_u_boot_tpl.dts') self.assertEqual(U_BOOT_TPL_DATA + U_BOOT_TPL_DTB_DATA, data) @@ -1693,7 +1693,7 @@ class TestFunctional(unittest.TestCase): u-boot-tpl.dtb with the microcode removed the microcode """ - with open(self.TestFile('u_boot_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('tpl/u-boot-tpl', fd.read()) first, pos_and_size = self._RunMicrocodeTest('093_x86_tpl_ucode.dts', U_BOOT_TPL_NODTB_DATA) @@ -1748,14 +1748,14 @@ class TestFunctional(unittest.TestCase): def testElf(self): """Basic test of ELF entries""" self._SetupSplElf() - with open(self.TestFile('bss_data')) as fd: + with open(self.TestFile('bss_data'), 'rb') as fd: TestFunctional._MakeInputFile('-boot', fd.read()) data = self._DoReadFile('096_elf.dts') def testElfStripg(self): """Basic test of ELF entries""" self._SetupSplElf() - with open(self.TestFile('bss_data')) as fd: + with open(self.TestFile('bss_data'), 'rb') as fd: TestFunctional._MakeInputFile('-boot', fd.read()) data = self._DoReadFile('097_elf_strip.dts') From 3e4b51e8d6dc86c0ea80552a1d2e19756b19e4a1 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:43 -0600 Subject: [PATCH 09/74] tools: dtoc: Open all binary files in binary mode Fix an open() statement to use binary mode, so that it works as expected with Python 3. Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 9ad72f89ec..031b3a0084 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -469,7 +469,7 @@ class Fdt: if self._fname: self._fname = fdt_util.EnsureCompiled(self._fname) - with open(self._fname) as fd: + with open(self._fname, 'rb') as fd: self._fdt_obj = libfdt.Fdt(fd.read()) @staticmethod From a92939a4aa65700c3cdfa230fa986d40c508c5e6 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:44 -0600 Subject: [PATCH 10/74] patman: Provide a way to get program output in binary mode At present cros_subprocess and the tools library use a string to obtain stdout from a program. This works fine on Python 2. With Python 3 we end up with unicode errors in some cases. Fix this by providing a binary mode, which returns the data as bytes() instead of a string. Signed-off-by: Simon Glass --- tools/patman/cros_subprocess.py | 3 ++- tools/patman/tools.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/patman/cros_subprocess.py b/tools/patman/cros_subprocess.py index 0f0d60dfb7..06be64cc2c 100644 --- a/tools/patman/cros_subprocess.py +++ b/tools/patman/cros_subprocess.py @@ -54,7 +54,7 @@ class Popen(subprocess.Popen): """ def __init__(self, args, stdin=None, stdout=PIPE_PTY, stderr=PIPE_PTY, - shell=False, cwd=None, env=None, **kwargs): + shell=False, cwd=None, env=None, binary=False, **kwargs): """Cut-down constructor Args: @@ -72,6 +72,7 @@ class Popen(subprocess.Popen): """ stdout_pty = None stderr_pty = None + self.binary = binary if stdout == PIPE_PTY: stdout_pty = pty.openpty() diff --git a/tools/patman/tools.py b/tools/patman/tools.py index bf099798e6..1df8f2ecd2 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -167,9 +167,9 @@ def PathHasFile(fname): return True return False -def Run(name, *args): +def Run(name, *args, **kwargs): try: - return command.Run(name, *args, cwd=outdir, capture=True) + return command.Run(name, *args, cwd=outdir, capture=True, **kwargs) except: if not PathHasFile(name): msg = "Plesae install tool '%s'" % name From b8f0876f1eccc17a1275401ff7bc846235fd3edd Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:45 -0600 Subject: [PATCH 11/74] binman: Use binary mode when compressing data The lz4 utility inserts binary data in its output which cannot always be converted to unicode (nor should it be). Fix this by using the new binary mode for program output. Signed-off-by: Simon Glass --- tools/binman/etype/blob.py | 2 +- tools/binman/ftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index ae80bbee53..f56a1f8768 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -60,7 +60,7 @@ class Entry_blob(Entry): except AttributeError: data = lz4.compress(data) ''' - data = tools.Run('lz4', '-c', self._pathname) + data = tools.Run('lz4', '-c', self._pathname, binary=True) self.SetContents(data) return True diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 1111852a40..4d96933cb4 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -1557,7 +1557,7 @@ class TestFunctional(unittest.TestCase): out = os.path.join(self._indir, 'lz4.tmp') with open(out, 'wb') as fd: fd.write(data) - return tools.Run('lz4', '-dc', out) + return tools.Run('lz4', '-dc', out, binary=True) ''' try: orig = lz4.frame.decompress(data) From 6d1d6418645229d5de2f8c52bf6a9ce6a345c2a1 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:46 -0600 Subject: [PATCH 12/74] binman: Drop an unused input file Drop this line which is not used or needed. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 4d96933cb4..e1fc9e8e9e 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -291,7 +291,6 @@ class TestFunctional(unittest.TestCase): # Use the compiled test file as the u-boot-dtb input if use_real_dtb: dtb_data = self._SetupDtb(fname) - infile = os.path.join(self._indir, 'u-boot.dtb') # For testing purposes, make a copy of the DT for SPL and TPL. Add # a node indicating which it is, so aid verification. From e6d85ff9f239a338748871b93ed11f066609e869 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:47 -0600 Subject: [PATCH 13/74] binman: Handle repeated bytes for Python 3 The method of multiplying a character by a number works well for creating a repeated string in Python 2. But in Python 3 we need to use bytes() instead, to avoid unicode problems, since 'bytes' is no-longer just an alias of 'str'. Create a function to handle this detail and call it from the relevant places in binman. Signed-off-by: Simon Glass --- tools/binman/bsection.py | 2 +- tools/binman/elf_test.py | 2 +- tools/binman/etype/fill.py | 4 +-- tools/binman/etype/u_boot_spl_bss_pad.py | 2 +- tools/binman/ftest.py | 46 +++++++++++++----------- tools/patman/tools.py | 19 ++++++++++ 6 files changed, 50 insertions(+), 25 deletions(-) diff --git a/tools/binman/bsection.py b/tools/binman/bsection.py index 3ca0592fe1..03dfa2f805 100644 --- a/tools/binman/bsection.py +++ b/tools/binman/bsection.py @@ -332,7 +332,7 @@ class Section(object): def GetData(self): """Get the contents of the section""" - section_data = chr(self._pad_byte) * self._size + section_data = tools.GetBytes(self._pad_byte, self._size) for entry in self._entries.values(): data = entry.GetData() diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index b68530c19b..3d55cb1946 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -122,7 +122,7 @@ class TestElf(unittest.TestCase): section = FakeSection(sym_value=None) elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms') syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) - self.assertEqual(chr(255) * 16 + 'a' * 4, entry.data) + self.assertEqual(tools.GetBytes(255, 16) + 'a' * 4, entry.data) def testDebug(self): """Check that enabling debug in the elf module produced debug output""" diff --git a/tools/binman/etype/fill.py b/tools/binman/etype/fill.py index dcfe978a5b..68efe42ec0 100644 --- a/tools/binman/etype/fill.py +++ b/tools/binman/etype/fill.py @@ -5,7 +5,7 @@ from entry import Entry import fdt_util - +import tools class Entry_fill(Entry): """An entry which is filled to a particular byte value @@ -28,5 +28,5 @@ class Entry_fill(Entry): self.fill_value = fdt_util.GetByte(self._node, 'fill-byte', 0) def ObtainContents(self): - self.SetContents(chr(self.fill_value) * self.size) + self.SetContents(tools.GetBytes(self.fill_value, self.size)) return True diff --git a/tools/binman/etype/u_boot_spl_bss_pad.py b/tools/binman/etype/u_boot_spl_bss_pad.py index 00b7ac5004..66a296a6f8 100644 --- a/tools/binman/etype/u_boot_spl_bss_pad.py +++ b/tools/binman/etype/u_boot_spl_bss_pad.py @@ -38,5 +38,5 @@ class Entry_u_boot_spl_bss_pad(Entry_blob): bss_size = elf.GetSymbolAddress(fname, '__bss_size') if not bss_size: self.Raise('Expected __bss_size symbol in spl/u-boot-spl') - self.SetContents(chr(0) * bss_size) + self.SetContents(tools.GetBytes(0, bss_size)) return True diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index e1fc9e8e9e..971fade343 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -551,8 +551,8 @@ class TestFunctional(unittest.TestCase): with open(fname, 'rb') as fd: data = fd.read() self.assertEqual(U_BOOT_DATA, data[3:7]) - self.assertEqual(chr(0) * 3, data[:3]) - self.assertEqual(chr(0) * 5, data[7:]) + self.assertEqual(tools.GetBytes(0, 3), data[:3]) + self.assertEqual(tools.GetBytes(0, 5), data[7:]) def testBadAlign(self): """Test that an invalid alignment value is detected""" @@ -731,7 +731,8 @@ class TestFunctional(unittest.TestCase): """Test that the image pad byte can be specified""" self._SetupSplElf() data = self._DoReadFile('021_image_pad.dts') - self.assertEqual(U_BOOT_SPL_DATA + (chr(0xff) * 1) + U_BOOT_DATA, data) + self.assertEqual(U_BOOT_SPL_DATA + tools.GetBytes(0xff, 1) + + U_BOOT_DATA, data) def testImageName(self): """Test that image files can be named""" @@ -754,8 +755,8 @@ class TestFunctional(unittest.TestCase): """Test that entries can be sorted""" self._SetupSplElf() data = self._DoReadFile('024_sorted.dts') - self.assertEqual(chr(0) * 1 + U_BOOT_SPL_DATA + chr(0) * 2 + - U_BOOT_DATA, data) + self.assertEqual(tools.GetBytes(0, 1) + U_BOOT_SPL_DATA + + tools.GetBytes(0, 2) + U_BOOT_DATA, data) def testPackZeroOffset(self): """Test that an entry at offset 0 is not given a new offset""" @@ -797,8 +798,8 @@ class TestFunctional(unittest.TestCase): """Test that a basic x86 ROM can be created""" self._SetupSplElf() data = self._DoReadFile('029_x86-rom.dts') - self.assertEqual(U_BOOT_DATA + chr(0) * 7 + U_BOOT_SPL_DATA + - chr(0) * 2, data) + self.assertEqual(U_BOOT_DATA + tools.GetBytes(0, 7) + U_BOOT_SPL_DATA + + tools.GetBytes(0, 2), data) def testPackX86RomMeNoDesc(self): """Test that an invalid Intel descriptor entry is detected""" @@ -1005,7 +1006,7 @@ class TestFunctional(unittest.TestCase): used_len = len(U_BOOT_NODTB_DATA) + fdt_len third = data[used_len:] - self.assertEqual(chr(0) * (0x200 - used_len), third) + self.assertEqual(tools.GetBytes(0, 0x200 - used_len), third) def testUnknownPosSize(self): """Test that microcode must be placed within the image""" @@ -1034,7 +1035,8 @@ class TestFunctional(unittest.TestCase): # ELF file with a '__bss_size' symbol self._SetupSplElf() data = self._DoReadFile('047_spl_bss_pad.dts') - self.assertEqual(U_BOOT_SPL_DATA + (chr(0) * 10) + U_BOOT_DATA, data) + self.assertEqual(U_BOOT_SPL_DATA + tools.GetBytes(0, 10) + U_BOOT_DATA, + data) def testSplBssPadMissing(self): """Test that a missing symbol is detected""" @@ -1108,9 +1110,9 @@ class TestFunctional(unittest.TestCase): self._SetupSplElf('u_boot_binman_syms') data = self._DoReadFile('053_symbols.dts') sym_values = struct.pack('= 3: + data = bytes([byte]) * size + else: + data = chr(byte) * size + return data From ade1e3864f1508ad0ccbce72a39d815c2d7d7c87 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:49 -0600 Subject: [PATCH 14/74] patman: Support use of stringIO in Python 3 With Python 3 this class has moved. Update the code to handle both cases. Signed-off-by: Simon Glass --- tools/patman/func_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index 31481157fc..2c4392dbc1 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -12,15 +12,20 @@ import sys import tempfile import unittest +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + import gitutil import patchstream import settings +import tools @contextlib.contextmanager def capture(): import sys - from cStringIO import StringIO oldout,olderr = sys.stdout, sys.stderr try: out=[StringIO(), StringIO()] From 513eace47d63161cabfd3cce82f3e075525b164a Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:50 -0600 Subject: [PATCH 15/74] patman: Move unicode helpers to tools Create helper functions in the tools module to deal with the differences between unicode in Python 2 (where we use the 'unicode' type) and Python 3 (where we use the 'str' type). Signed-off-by: Simon Glass --- tools/patman/gitutil.py | 4 ++-- tools/patman/series.py | 3 ++- tools/patman/settings.py | 16 +++------------- tools/patman/tools.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 11aeb73b74..fb9e67c0f0 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -12,6 +12,7 @@ import terminal import checkpatch import settings +import tools # True to use --no-decorate - we check this in Setup() use_no_decorate = True @@ -325,9 +326,8 @@ def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True): raw += LookupEmail(item, alias, raise_on_error=raise_on_error) result = [] for item in raw: + item = tools.FromUnicode(item) if not item in result: - if type(item) == unicode: - item = item.encode('utf-8') result.append(item) if tag: return ['%s %s%s%s' % (tag, quote, email, quote) for email in result] diff --git a/tools/patman/series.py b/tools/patman/series.py index 2735afaf88..0b71a896cb 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -11,6 +11,7 @@ import get_maintainer import gitutil import settings import terminal +import tools # Series-xxx tags that we understand valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes', 'name', @@ -249,7 +250,7 @@ class Series(dict): cover_cc = gitutil.BuildEmailList(self.get('cover_cc', '')) cover_cc = [m.encode('utf-8') if type(m) != str else m for m in cover_cc] - cc_list = ', '.join([x.decode('utf-8') + cc_list = ', '.join([tools.ToUnicode(x) for x in set(cover_cc + all_ccs)]) print(cover_fname, cc_list.encode('utf-8'), file=fd) diff --git a/tools/patman/settings.py b/tools/patman/settings.py index 07bf6a6ea4..b080136d88 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -14,6 +14,7 @@ import re import command import gitutil +import tools """Default settings per-project. @@ -99,17 +100,6 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser): for setting_name, setting_value in project_defaults.items(): self.set(project_settings, setting_name, setting_value) - def _to_unicode(self, val): - """Make sure a value is of type 'unicode' - - Args: - val: string or unicode object - - Returns: - unicode version of val - """ - return val if isinstance(val, unicode) else val.decode('utf-8') - def get(self, section, option, *args, **kwargs): """Extend SafeConfigParser to try project_section before section. @@ -127,7 +117,7 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser): val = ConfigParser.SafeConfigParser.get( self, section, option, *args, **kwargs ) - return self._to_unicode(val) + return tools.ToUnicode(val) def items(self, section, *args, **kwargs): """Extend SafeConfigParser to add project_section to section. @@ -162,7 +152,7 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser): item_dict = dict(top_items) item_dict.update(project_items) - return {(self._to_unicode(item), self._to_unicode(val)) + return {(tools.ToUnicode(item), tools.ToUnicode(val)) for item, val in item_dict.items()} def ReadGitAliases(fname): diff --git a/tools/patman/tools.py b/tools/patman/tools.py index 0ad0fb9705..7e6a45a3b0 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -258,3 +258,35 @@ def GetBytes(byte, size): else: data = chr(byte) * size return data + +def ToUnicode(val): + """Make sure a value is a unicode string + + This allows some amount of compatibility between Python 2 and Python3. For + the former, it returns a unicode object. + + Args: + val: string or unicode object + + Returns: + unicode version of val + """ + if sys.version_info[0] >= 3: + return val + return val if isinstance(val, unicode) else val.decode('utf-8') + +def FromUnicode(val): + """Make sure a value is a non-unicode string + + This allows some amount of compatibility between Python 2 and Python3. For + the former, it converts a unicode object to a string. + + Args: + val: string or unicode object + + Returns: + non-unicode version of val + """ + if sys.version_info[0] >= 3: + return val + return val if isinstance(val, str) else val.encode('utf-8') From b644c66f693c82750077b6f7530dde79f2ad7523 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:51 -0600 Subject: [PATCH 16/74] patman: Sort series output for repeatabily We use sets to produce the list of To and Cc lines for a series. This does not result in stable ordering of the recipients. Sort each list to ensure that the output is repeatable. This is necessary for tests. Signed-off-by: Simon Glass --- tools/patman/func_test.py | 12 ++++++------ tools/patman/series.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index 2c4392dbc1..f7d5ad68f6 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -183,10 +183,10 @@ class TestFunctional(unittest.TestCase): self.assertEqual('Prefix:\t RFC', lines[line + 3]) self.assertEqual('Cover: 4 lines', lines[line + 4]) line += 5 - self.assertEqual(' Cc: %s' % mel.encode('utf-8'), lines[line + 0]) - self.assertEqual(' Cc: %s' % rick, lines[line + 1]) - self.assertEqual(' Cc: %s' % fred, lines[line + 2]) - self.assertEqual(' Cc: %s' % ed.encode('utf-8'), lines[line + 3]) + self.assertEqual(' Cc: %s' % fred, lines[line + 0]) + self.assertEqual(' Cc: %s' % ed.encode('utf-8'), lines[line + 1]) + self.assertEqual(' Cc: %s' % mel.encode('utf-8'), lines[line + 2]) + self.assertEqual(' Cc: %s' % rick, lines[line + 3]) expected = ('Git command: git send-email --annotate ' '--in-reply-to="%s" --to "u-boot@lists.denx.de" ' '--cc "%s" --cc-cmd "%s --cc-cmd %s" %s %s' @@ -197,8 +197,8 @@ class TestFunctional(unittest.TestCase): self.assertEqual(('%s %s, %s' % (args[0], rick, stefan)) .encode('utf-8'), cc_lines[0]) - self.assertEqual(('%s %s, %s, %s, %s' % (args[1], fred, rick, stefan, - ed)).encode('utf-8'), cc_lines[1]) + self.assertEqual(('%s %s, %s, %s, %s' % (args[1], fred, ed, rick, + stefan)).encode('utf-8'), cc_lines[1]) expected = ''' This is a test of how the cover diff --git a/tools/patman/series.py b/tools/patman/series.py index 0b71a896cb..bbb30d849e 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -115,16 +115,16 @@ class Series(dict): commit = self.commits[upto] print(col.Color(col.GREEN, ' %s' % args[upto])) cc_list = list(self._generated_cc[commit.patch]) - for email in set(cc_list) - to_set - cc_set: + for email in sorted(set(cc_list) - to_set - cc_set): if email == None: email = col.Color(col.YELLOW, "" % tag) if email: print(' Cc: ', email) print - for item in to_set: + for item in sorted(to_set): print('To:\t ', item) - for item in cc_set - to_set: + for item in sorted(cc_set - to_set): print('Cc:\t ', item) print('Version: ', self.get('version')) print('Prefix:\t ', self.get('prefix')) @@ -132,7 +132,7 @@ class Series(dict): print('Cover: %d lines' % len(self.cover)) cover_cc = gitutil.BuildEmailList(self.get('cover_cc', '')) all_ccs = itertools.chain(cover_cc, *self._generated_cc.values()) - for email in set(all_ccs) - to_set - cc_set: + for email in sorted(set(all_ccs) - to_set - cc_set): print(' Cc: ', email) if cmd: print('Git command: %s' % cmd) @@ -243,7 +243,7 @@ class Series(dict): if limit is not None: cc = cc[:limit] all_ccs += cc - print(commit.patch, ', '.join(set(cc)), file=fd) + print(commit.patch, ', '.join(sorted(set(cc))), file=fd) self._generated_cc[commit.patch] = cc if cover_fname: @@ -251,7 +251,7 @@ class Series(dict): cover_cc = [m.encode('utf-8') if type(m) != str else m for m in cover_cc] cc_list = ', '.join([tools.ToUnicode(x) - for x in set(cover_cc + all_ccs)]) + for x in sorted(set(cover_cc + all_ccs))]) print(cover_fname, cc_list.encode('utf-8'), file=fd) fd.close() From 7ebb45c7e2e1b8355ec4802e0802ae7b383a6ea5 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:52 -0600 Subject: [PATCH 17/74] patman: Avoid unicode type in settings unit tests The unicode type does not exist in Python 3 and when displaying strings they do not have the 'u' prefix. Adjusts the settings unit tests to deal with this difference, by converting the comparison value to a string, thus dropping the 'u'. Signed-off-by: Simon Glass --- tools/patman/settings.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/patman/settings.py b/tools/patman/settings.py index b080136d88..c98911d522 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -58,26 +58,26 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser): # Check to make sure that bogus project gets general alias. >>> config = _ProjectConfigParser("zzz") >>> config.readfp(StringIO(sample_config)) - >>> config.get("alias", "enemies") - u'Evil ' + >>> str(config.get("alias", "enemies")) + 'Evil ' # Check to make sure that alias gets overridden by project. >>> config = _ProjectConfigParser("sm") >>> config.readfp(StringIO(sample_config)) - >>> config.get("alias", "enemies") - u'Green G. ' + >>> str(config.get("alias", "enemies")) + 'Green G. ' # Check to make sure that settings get merged with project. >>> config = _ProjectConfigParser("linux") >>> config.readfp(StringIO(sample_config)) - >>> sorted(config.items("settings")) - [(u'am_hero', u'True'), (u'process_tags', u'False')] + >>> sorted((str(a), str(b)) for (a, b) in config.items("settings")) + [('am_hero', 'True'), ('process_tags', 'False')] # Check to make sure that settings works with unknown project. >>> config = _ProjectConfigParser("unknown") >>> config.readfp(StringIO(sample_config)) - >>> sorted(config.items("settings")) - [(u'am_hero', u'True')] + >>> sorted((str(a), str(b)) for (a, b) in config.items("settings")) + [('am_hero', 'True')] """ def __init__(self, project_name): """Construct _ProjectConfigParser. From e6dca5e8f8d4160a6efed05f5275426e2d274542 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:53 -0600 Subject: [PATCH 18/74] patman: Adjust functional tests for Python 3 Change the code so that it works on both Python 2 and Python 3. This works by using unicode instead of latin1 for the test input, and ensuring that the output is converted to a string rather than a unicode object on Python 2. Signed-off-by: Simon Glass --- tools/patman/func_test.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index f7d5ad68f6..50a2741439 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -129,10 +129,10 @@ class TestFunctional(unittest.TestCase): """ process_tags = True ignore_bad_tags = True - stefan = u'Stefan Brüns ' + stefan = b'Stefan Br\xc3\xbcns '.decode('utf-8') rick = 'Richard III ' - mel = u'Lord Mëlchett ' - ed = u'Lond Edmund Blackaddër '.decode('utf-8') + ed = b'Lond Edmund Blackadd\xc3\xabr Date: Tue, 14 May 2019 15:53:54 -0600 Subject: [PATCH 19/74] patman: Tidy up a few more unicode conversions Use the new functions in the tools module to handle conversion. Signed-off-by: Simon Glass --- tools/patman/gitutil.py | 4 +--- tools/patman/series.py | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index fb9e67c0f0..dce7fa25b6 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -412,9 +412,7 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, if smtp_server: cmd.append('--smtp-server=%s' % smtp_server) if in_reply_to: - if type(in_reply_to) != str: - in_reply_to = in_reply_to.encode('utf-8') - cmd.append('--in-reply-to="%s"' % in_reply_to) + cmd.append('--in-reply-to="%s"' % tools.FromUnicode(in_reply_to)) if thread: cmd.append('--thread') diff --git a/tools/patman/series.py b/tools/patman/series.py index bbb30d849e..67103f03e6 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -239,7 +239,7 @@ class Series(dict): for x in set(cc) & set(settings.bounces): print(col.Color(col.YELLOW, 'Skipping "%s"' % x)) cc = set(cc) - set(settings.bounces) - cc = [m.encode('utf-8') if type(m) != str else m for m in cc] + cc = [tools.FromUnicode(m) for m in cc] if limit is not None: cc = cc[:limit] all_ccs += cc @@ -248,8 +248,7 @@ class Series(dict): if cover_fname: cover_cc = gitutil.BuildEmailList(self.get('cover_cc', '')) - cover_cc = [m.encode('utf-8') if type(m) != str else m - for m in cover_cc] + cover_cc = [tools.FromUnicode(m) for m in cover_cc] cc_list = ', '.join([tools.ToUnicode(x) for x in sorted(set(cover_cc + all_ccs))]) print(cover_fname, cc_list.encode('utf-8'), file=fd) From b2e73125b20a8f15d031d05dfec572b5014bc7fe Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 14 May 2019 15:53:55 -0600 Subject: [PATCH 20/74] patman: Don't require Python 2 Update the shebang to allow either Python 2 or Python 3. Signed-off-by: Simon Glass --- tools/patman/patman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/patman/patman.py b/tools/patman/patman.py index 27a2febf70..9605a36eff 100755 --- a/tools/patman/patman.py +++ b/tools/patman/patman.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # SPDX-License-Identifier: GPL-2.0+ # # Copyright (c) 2011 The Chromium OS Authors. From 90a8132f4de673dbb35ffee265df93b32d6582ca Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:31 -0600 Subject: [PATCH 21/74] dtoc: Adjust code for Python 3 Update a few things in this tool so that they support Python 3: - print statements - iteritems() - xrange() Signed-off-by: Simon Glass --- tools/dtoc/dtb_platdata.py | 4 ++-- tools/dtoc/dtoc.py | 8 +++++--- tools/dtoc/test_dtoc.py | 4 +++- tools/dtoc/test_fdt.py | 8 +++++--- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tools/dtoc/dtb_platdata.py b/tools/dtoc/dtb_platdata.py index 17a3dccb11..4aeeab6fba 100644 --- a/tools/dtoc/dtb_platdata.py +++ b/tools/dtoc/dtb_platdata.py @@ -449,7 +449,7 @@ class DtbPlatdata(object): self.out(';\n') self.out('};\n') - for alias, struct_name in self._aliases.iteritems(): + for alias, struct_name in self._aliases.items(): if alias not in sorted(structs): self.out('#define %s%s %s%s\n'% (STRUCT_PREFIX, alias, STRUCT_PREFIX, struct_name)) @@ -498,7 +498,7 @@ class DtbPlatdata(object): vals.append(get_value(prop.type, val)) # Put 8 values per line to avoid very long lines. - for i in xrange(0, len(vals), 8): + for i in range(0, len(vals), 8): if i: self.buf(',\n\t\t') self.buf(', '.join(vals[i:i + 8])) diff --git a/tools/dtoc/dtoc.py b/tools/dtoc/dtoc.py index 2277af9bf7..c1a1d3534d 100755 --- a/tools/dtoc/dtoc.py +++ b/tools/dtoc/dtoc.py @@ -25,6 +25,8 @@ options. For more information about the use of this options and tool please see doc/driver-model/of-plat.txt """ +from __future__ import print_function + from optparse import OptionParser import os import sys @@ -64,11 +66,11 @@ def run_tests(args): suite = unittest.TestLoader().loadTestsFromTestCase(module) suite.run(result) - print result + print(result) for _, err in result.errors: - print err + print(err) for _, err in result.failures: - print err + print(err) def RunTestCoverage(): """Run the tests and check that we get 100% coverage""" diff --git a/tools/dtoc/test_dtoc.py b/tools/dtoc/test_dtoc.py index cb6d6e7baf..ae59a0a52a 100644 --- a/tools/dtoc/test_dtoc.py +++ b/tools/dtoc/test_dtoc.py @@ -8,6 +8,8 @@ This includes unit tests for some functions and functional tests for the dtoc tool. """ +from __future__ import print_function + import collections import os import struct @@ -97,7 +99,7 @@ class TestDtoc(unittest.TestCase): if expected != actual: self._WritePythonString('/tmp/binman.expected', expected) self._WritePythonString('/tmp/binman.actual', actual) - print 'Failures written to /tmp/binman.{expected,actual}' + print('Failures written to /tmp/binman.{expected,actual}') self.assertEquals(expected, actual) def test_name(self): diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 8d70dd2a29..2d1d7dc452 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -4,6 +4,8 @@ # Written by Simon Glass # +from __future__ import print_function + from optparse import OptionParser import glob import os @@ -535,11 +537,11 @@ def RunTests(args): suite = unittest.TestLoader().loadTestsFromTestCase(module) suite.run(result) - print result + print(result) for _, err in result.errors: - print err + print(err) for _, err in result.failures: - print err + print(err) if __name__ != '__main__': sys.exit(1) From 1953ce75312ab50988f2c81f7250c99dfc2ed52b Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:32 -0600 Subject: [PATCH 22/74] dtoc: Sort platdata output from dtoc At present the order of struct field emitted by this tool depends on the internal workings of a Python dictionary. Sort the fields to remove this uncertainty, so that tests are deterministic. Signed-off-by: Simon Glass --- tools/dtoc/dtb_platdata.py | 3 ++- tools/dtoc/test_dtoc.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tools/dtoc/dtb_platdata.py b/tools/dtoc/dtb_platdata.py index 4aeeab6fba..3b66af8df7 100644 --- a/tools/dtoc/dtb_platdata.py +++ b/tools/dtoc/dtb_platdata.py @@ -464,7 +464,8 @@ class DtbPlatdata(object): var_name = conv_name_to_c(node.name) self.buf('static const struct %s%s %s%s = {\n' % (STRUCT_PREFIX, struct_name, VAL_PREFIX, var_name)) - for pname, prop in node.props.items(): + for pname in sorted(node.props): + prop = node.props[pname] if pname in PROP_IGNORE_LIST or pname[0] == '#': continue member_name = conv_name_to_c(prop.name) diff --git a/tools/dtoc/test_dtoc.py b/tools/dtoc/test_dtoc.py index ae59a0a52a..b915b27856 100644 --- a/tools/dtoc/test_dtoc.py +++ b/tools/dtoc/test_dtoc.py @@ -199,16 +199,16 @@ struct dtd_sandbox_spl_test_2 { data = infile.read() self._CheckStrings(C_HEADER + ''' static const struct dtd_sandbox_spl_test dtv_spl_test = { +\t.boolval\t\t= true, \t.bytearray\t\t= {0x6, 0x0, 0x0}, \t.byteval\t\t= 0x5, +\t.intarray\t\t= {0x2, 0x3, 0x4, 0x0}, \t.intval\t\t\t= 0x1, -\t.notstring\t\t= {0x20, 0x21, 0x22, 0x10, 0x0}, \t.longbytearray\t\t= {0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10, \t\t0x11}, -\t.stringval\t\t= "message", -\t.boolval\t\t= true, -\t.intarray\t\t= {0x2, 0x3, 0x4, 0x0}, +\t.notstring\t\t= {0x20, 0x21, 0x22, 0x10, 0x0}, \t.stringarray\t\t= {"multi-word", "message", ""}, +\t.stringval\t\t= "message", }; U_BOOT_DEVICE(spl_test) = { \t.name\t\t= "sandbox_spl_test", @@ -219,12 +219,12 @@ U_BOOT_DEVICE(spl_test) = { static const struct dtd_sandbox_spl_test dtv_spl_test2 = { \t.bytearray\t\t= {0x1, 0x23, 0x34}, \t.byteval\t\t= 0x8, +\t.intarray\t\t= {0x5, 0x0, 0x0, 0x0}, \t.intval\t\t\t= 0x3, \t.longbytearray\t\t= {0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, \t\t0x0}, -\t.stringval\t\t= "message2", -\t.intarray\t\t= {0x5, 0x0, 0x0, 0x0}, \t.stringarray\t\t= {"another", "multi-word", "message"}, +\t.stringval\t\t= "message2", }; U_BOOT_DEVICE(spl_test2) = { \t.name\t\t= "sandbox_spl_test", From 194b8d5e71a41dbb839fdaa49677b146e6a0fb5d Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:33 -0600 Subject: [PATCH 23/74] dtoc: Use GetBytes() to obtain repeating bytes Use this helper function which works on both Python 2 and Python 3. Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 6 ++++-- tools/dtoc/test_fdt.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 031b3a0084..9518a287a2 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -11,6 +11,7 @@ import sys import fdt_util import libfdt from libfdt import QUIET_NOTFOUND +import tools # This deals with a device tree, presenting it as an assortment of Node and # Prop objects, representing nodes and properties, respectively. This file @@ -334,7 +335,8 @@ class Node: Args: prop_name: Name of property """ - self.props[prop_name] = Prop(self, None, prop_name, '\0' * 4) + self.props[prop_name] = Prop(self, None, prop_name, + tools.GetBytes(0, 4)) def AddEmptyProp(self, prop_name, len): """Add a property with a fixed data size, for filling in later @@ -346,7 +348,7 @@ class Node: prop_name: Name of property len: Length of data in property """ - value = chr(0) * len + value = tools.GetBytes(0, len) self.props[prop_name] = Prop(self, None, prop_name, value) def SetInt(self, prop_name, val): diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 2d1d7dc452..79f97d8013 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -279,7 +279,7 @@ class TestProp(unittest.TestCase): """Tests the GetEmpty() function for the various supported types""" self.assertEqual(True, fdt.Prop.GetEmpty(fdt.TYPE_BOOL)) self.assertEqual(chr(0), fdt.Prop.GetEmpty(fdt.TYPE_BYTE)) - self.assertEqual(chr(0) * 4, fdt.Prop.GetEmpty(fdt.TYPE_INT)) + self.assertEqual(tools.GetBytes(0, 4), fdt.Prop.GetEmpty(fdt.TYPE_INT)) self.assertEqual('', fdt.Prop.GetEmpty(fdt.TYPE_STRING)) def testGetOffset(self): From 7e6952df36a28b570818ee1fd36b07c41ef14aea Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:34 -0600 Subject: [PATCH 24/74] dtoc: Move BytesToValue() out of the Prop class This method does not actually use any members of the Prop class. Move it out of the class so that it is easier to add unit tests. Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 104 +++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 51 deletions(-) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 9518a287a2..35453fbed9 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -29,6 +29,57 @@ def CheckErr(errnum, msg): raise ValueError('Error %d: %s: %s' % (errnum, libfdt.fdt_strerror(errnum), msg)) + +def BytesToValue(bytes): + """Converts a string of bytes into a type and value + + Args: + A string containing bytes + + Return: + A tuple: + Type of data + Data, either a single element or a list of elements. Each element + is one of: + TYPE_STRING: string value from the property + TYPE_INT: a byte-swapped integer stored as a 4-byte string + TYPE_BYTE: a byte stored as a single-byte string + """ + bytes = str(bytes) + size = len(bytes) + strings = bytes.split('\0') + is_string = True + count = len(strings) - 1 + if count > 0 and not strings[-1]: + for string in strings[:-1]: + if not string: + is_string = False + break + for ch in string: + if ch < ' ' or ch > '~': + is_string = False + break + else: + is_string = False + if is_string: + if count == 1: + return TYPE_STRING, strings[0] + else: + return TYPE_STRING, strings[:-1] + if size % 4: + if size == 1: + return TYPE_BYTE, bytes[0] + else: + return TYPE_BYTE, list(bytes) + val = [] + for i in range(0, size, 4): + val.append(bytes[i:i + 4]) + if size == 4: + return TYPE_INT, val[0] + else: + return TYPE_INT, val + + class Prop: """A device tree property @@ -49,7 +100,7 @@ class Prop: self.type = TYPE_BOOL self.value = True return - self.type, self.value = self.BytesToValue(bytes) + self.type, self.value = BytesToValue(bytes) def RefreshOffset(self, poffset): self._offset = poffset @@ -88,55 +139,6 @@ class Prop: while len(self.value) < len(newprop.value): self.value.append(val) - def BytesToValue(self, bytes): - """Converts a string of bytes into a type and value - - Args: - A string containing bytes - - Return: - A tuple: - Type of data - Data, either a single element or a list of elements. Each element - is one of: - TYPE_STRING: string value from the property - TYPE_INT: a byte-swapped integer stored as a 4-byte string - TYPE_BYTE: a byte stored as a single-byte string - """ - bytes = str(bytes) - size = len(bytes) - strings = bytes.split('\0') - is_string = True - count = len(strings) - 1 - if count > 0 and not strings[-1]: - for string in strings[:-1]: - if not string: - is_string = False - break - for ch in string: - if ch < ' ' or ch > '~': - is_string = False - break - else: - is_string = False - if is_string: - if count == 1: - return TYPE_STRING, strings[0] - else: - return TYPE_STRING, strings[:-1] - if size % 4: - if size == 1: - return TYPE_BYTE, bytes[0] - else: - return TYPE_BYTE, list(bytes) - val = [] - for i in range(0, size, 4): - val.append(bytes[i:i + 4]) - if size == 4: - return TYPE_INT, val[0] - else: - return TYPE_INT, val - @classmethod def GetEmpty(self, type): """Get an empty / zero value of the given type @@ -183,7 +185,7 @@ class Prop: bytes: New property value to set """ self.bytes = str(bytes) - self.type, self.value = self.BytesToValue(bytes) + self.type, self.value = BytesToValue(bytes) self.dirty = True def Sync(self, auto_resize=False): From 2b6ed5e92e3b29fbfa3a966433d4bbc2ec1e9c58 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:35 -0600 Subject: [PATCH 25/74] dtoc: Updates BytesToValue() for Python 3 The difference between the bytes and str types in Python 3 requires a number of minor changes to this function. Update it to handle the input data using the 'bytes' type. Create two useful helper functions which can be used by other modules too. Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 41 +++++++++++++++++++++++++---------------- tools/patman/tools.py | 27 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 35453fbed9..2e74bc15be 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -30,50 +30,59 @@ def CheckErr(errnum, msg): (errnum, libfdt.fdt_strerror(errnum), msg)) -def BytesToValue(bytes): +def BytesToValue(data): """Converts a string of bytes into a type and value Args: - A string containing bytes + A bytes value (which on Python 2 is an alias for str) Return: A tuple: Type of data Data, either a single element or a list of elements. Each element is one of: - TYPE_STRING: string value from the property - TYPE_INT: a byte-swapped integer stored as a 4-byte string - TYPE_BYTE: a byte stored as a single-byte string + TYPE_STRING: str/bytes value from the property + TYPE_INT: a byte-swapped integer stored as a 4-byte str/bytes + TYPE_BYTE: a byte stored as a single-byte str/bytes """ - bytes = str(bytes) - size = len(bytes) - strings = bytes.split('\0') + data = bytes(data) + size = len(data) + strings = data.split(b'\0') is_string = True count = len(strings) - 1 - if count > 0 and not strings[-1]: + if count > 0 and not len(strings[-1]): for string in strings[:-1]: if not string: is_string = False break for ch in string: - if ch < ' ' or ch > '~': + # Handle Python 2 treating bytes as str + if type(ch) == str: + ch = ord(ch) + if ch < 32 or ch > 127: is_string = False break else: is_string = False if is_string: - if count == 1: - return TYPE_STRING, strings[0] + if count == 1: + if sys.version_info[0] >= 3: # pragma: no cover + return TYPE_STRING, strings[0].decode() + else: + return TYPE_STRING, strings[0] else: - return TYPE_STRING, strings[:-1] + if sys.version_info[0] >= 3: # pragma: no cover + return TYPE_STRING, [s.decode() for s in strings[:-1]] + else: + return TYPE_STRING, strings[:-1] if size % 4: if size == 1: - return TYPE_BYTE, bytes[0] + return TYPE_BYTE, tools.ToChar(data[0]) else: - return TYPE_BYTE, list(bytes) + return TYPE_BYTE, [tools.ToChar(ch) for ch in list(data)] val = [] for i in range(0, size, 4): - val.append(bytes[i:i + 4]) + val.append(data[i:i + 4]) if size == 4: return TYPE_INT, val[0] else: diff --git a/tools/patman/tools.py b/tools/patman/tools.py index 7e6a45a3b0..976670ef00 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -290,3 +290,30 @@ def FromUnicode(val): if sys.version_info[0] >= 3: return val return val if isinstance(val, str) else val.encode('utf-8') + +def ToByte(ch): + """Convert a character to an ASCII value + + This is useful because in Python 2 bytes is an alias for str, but in + Python 3 they are separate types. This function converts the argument to + an ASCII value in either case. + + Args: + ch: A string (Python 2) or byte (Python 3) value + + Returns: + integer ASCII value for ch + """ + return ord(ch) if type(ch) == str else ch + +def ToChar(byte): + """Convert a byte to a character + + This is useful because in Python 2 bytes is an alias for str, but in + Python 3 they are separate types. This function converts an ASCII value to + a value with the appropriate type in either case. + + Args: + byte: A byte or str value + """ + return chr(byte) if type(byte) != str else byte From f6b64815dda947090f112bd4f7e0e430a16b48d7 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:36 -0600 Subject: [PATCH 26/74] dtoc: Use byte type instead of str in fdt In Python 3 bytes and str are separate types. Use bytes to ensure that the code functions correctly with Python 3. Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 14 +++++++++----- tools/dtoc/test_fdt.py | 18 +++++++++--------- tools/patman/tools.py | 25 +++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 2e74bc15be..956e0c9800 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -193,7 +193,7 @@ class Prop: Args: bytes: New property value to set """ - self.bytes = str(bytes) + self.bytes = bytes self.type, self.value = BytesToValue(bytes) self.dirty = True @@ -398,7 +398,9 @@ class Node: prop_name: Name of property to set val: String value to set (will be \0-terminated in DT) """ - self.props[prop_name].SetData(val + chr(0)) + if sys.version_info[0] >= 3: # pragma: no cover + val = bytes(val, 'utf-8') + self.props[prop_name].SetData(val + b'\0') def AddString(self, prop_name, val): """Add a new string property to a node @@ -410,7 +412,9 @@ class Node: prop_name: Name of property to add val: String value of property """ - self.props[prop_name] = Prop(self, None, prop_name, val + chr(0)) + if sys.version_info[0] >= 3: # pragma: no cover + val = bytes(val, 'utf-8') + self.props[prop_name] = Prop(self, None, prop_name, val + b'\0') def AddSubnode(self, name): """Add a new subnode to the node @@ -496,7 +500,7 @@ class Fdt: Fdt object containing the data """ fdt = Fdt(None) - fdt._fdt_obj = libfdt.Fdt(bytearray(data)) + fdt._fdt_obj = libfdt.Fdt(bytes(data)) return fdt def LookupPhandle(self, phandle): @@ -586,7 +590,7 @@ class Fdt: Returns: The FDT contents as a string of bytes """ - return self._fdt_obj.as_bytearray() + return bytes(self._fdt_obj.as_bytearray()) def GetFdtObj(self): """Get the contents of the FDT diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 79f97d8013..3cd34b745e 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -47,7 +47,7 @@ def _GetPropertyValue(dtb, node, prop_name): # Add 12, which is sizeof(struct fdt_property), to get to start of data offset = prop.GetOffset() + 12 data = dtb.GetContents()[offset:offset + len(prop.value)] - return prop, [chr(x) for x in data] + return prop, [tools.ToChar(x) for x in data] class TestFdt(unittest.TestCase): @@ -383,7 +383,7 @@ class TestProp(unittest.TestCase): self.node.AddString('string', val) self.dtb.Sync(auto_resize=True) data = self.fdt.getprop(self.node.Offset(), 'string') - self.assertEqual(val + '\0', data) + self.assertEqual(tools.ToBytes(val) + b'\0', data) self.fdt.pack() self.node.SetString('string', val + 'x') @@ -393,21 +393,21 @@ class TestProp(unittest.TestCase): self.node.SetString('string', val[:-1]) prop = self.node.props['string'] - prop.SetData(val) + prop.SetData(tools.ToBytes(val)) self.dtb.Sync(auto_resize=False) data = self.fdt.getprop(self.node.Offset(), 'string') - self.assertEqual(val, data) + self.assertEqual(tools.ToBytes(val), data) self.node.AddEmptyProp('empty', 5) self.dtb.Sync(auto_resize=True) prop = self.node.props['empty'] - prop.SetData(val) + prop.SetData(tools.ToBytes(val)) self.dtb.Sync(auto_resize=False) data = self.fdt.getprop(self.node.Offset(), 'empty') - self.assertEqual(val, data) + self.assertEqual(tools.ToBytes(val), data) - self.node.SetData('empty', '123') - self.assertEqual('123', prop.bytes) + self.node.SetData('empty', b'123') + self.assertEqual(b'123', prop.bytes) def testFromData(self): dtb2 = fdt.Fdt.FromData(self.dtb.GetContents()) @@ -508,7 +508,7 @@ class TestFdtUtil(unittest.TestCase): self.assertEqual(dtb, fdt_util.EnsureCompiled(dtb)) def testGetPlainBytes(self): - self.assertEqual('fred', fdt_util.get_plain_bytes('fred')) + self.assertEqual(b'fred', fdt_util.get_plain_bytes('fred')) def RunTestCoverage(): diff --git a/tools/patman/tools.py b/tools/patman/tools.py index 976670ef00..bdc1953936 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -317,3 +317,28 @@ def ToChar(byte): byte: A byte or str value """ return chr(byte) if type(byte) != str else byte + +def ToChars(byte_list): + """Convert a list of bytes to a str/bytes type + + Args: + byte_list: List of ASCII values representing the string + + Returns: + string made by concatenating all the ASCII values + """ + return ''.join([chr(byte) for byte in byte_list]) + +def ToBytes(string): + """Convert a str type into a bytes type + + Args: + string: string to convert value + + Returns: + Python 3: A bytes type + Python 2: A string type + """ + if sys.version_info[0] >= 3: + return string.encode('utf-8') + return string From 928527f6863a7176279f33e2e466b0d0f2d841d0 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:37 -0600 Subject: [PATCH 27/74] dtoc: Convert the Fdt.Prop class to Python 3 Update this class to work correctly on Python 3 and to pass its unit tests. Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 956e0c9800..2b2470c314 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -98,18 +98,18 @@ class Prop: bytes type: Value type """ - def __init__(self, node, offset, name, bytes): + def __init__(self, node, offset, name, data): self._node = node self._offset = offset self.name = name self.value = None - self.bytes = str(bytes) + self.bytes = bytes(data) self.dirty = False - if not bytes: + if not data: self.type = TYPE_BOOL self.value = True return - self.type, self.value = BytesToValue(bytes) + self.type, self.value = BytesToValue(bytes(data)) def RefreshOffset(self, poffset): self._offset = poffset From 635180538ecb7c0b109089830ff734d3d002376a Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:38 -0600 Subject: [PATCH 28/74] dtoc: Convert the Fdt.Node class to Python 3 Update this class to work correctly on Python 3 and to pass its unit tests. The only required change is to deal with a difference in the behaviour of sorting with a None value. Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 2b2470c314..d9471c4381 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -465,8 +465,11 @@ class Node: # Sync properties now, whose offsets should not have been disturbed. # We do this after subnodes, since this disturbs the offsets of these - # properties. - prop_list = sorted(self.props.values(), key=lambda prop: prop._offset, + # properties. Note that new properties will have an offset of None here, + # which Python 3 cannot sort against int. So use a large value instead + # to ensure that the new properties are added first. + prop_list = sorted(self.props.values(), + key=lambda prop: prop._offset or 1 << 31, reverse=True) for prop in prop_list: prop.Sync(auto_resize) From 2ab6e13e014c065a6168d31d6b8aeca135767816 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:39 -0600 Subject: [PATCH 29/74] dtoc: Use binary mode for reading files The .dtb files are binary so we should open them as binary files. This allows Python 3 to use the correct 'bytes' type. Signed-off-by: Simon Glass --- tools/dtoc/test_fdt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 3cd34b745e..4c39f9a3e2 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -85,13 +85,13 @@ class TestFdt(unittest.TestCase): def testFlush(self): """Check that we can flush the device tree out to its file""" fname = self.dtb._fname - with open(fname) as fd: + with open(fname, 'rb') as fd: data = fd.read() os.remove(fname) with self.assertRaises(IOError): - open(fname) + open(fname, 'rb') self.dtb.Flush() - with open(fname) as fd: + with open(fname, 'rb') as fd: data = fd.read() def testPack(self): From e66d318fef611241749d37b6b73ff80c84a8c866 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:40 -0600 Subject: [PATCH 30/74] dtoc: Test full 64-bit properties with FdtCellsToCpu() At present this test does not check the upper 32 bits of the returned value. Add some additional tests to cover this. Signed-off-by: Simon Glass --- tools/dtoc/test_fdt.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 4c39f9a3e2..a5234ce1e8 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -498,10 +498,17 @@ class TestFdtUtil(unittest.TestCase): self.assertEqual(2, fdt_util.fdt_cells_to_cpu(val, 1)) dtb2 = fdt.FdtScan('tools/dtoc/dtoc_test_addr64.dts') - node2 = dtb2.GetNode('/test1') - val = node2.props['reg'].value + node1 = dtb2.GetNode('/test1') + val = node1.props['reg'].value self.assertEqual(0x1234, fdt_util.fdt_cells_to_cpu(val, 2)) + node2 = dtb2.GetNode('/test2') + val = node2.props['reg'].value + self.assertEqual(0x1234567890123456, fdt_util.fdt_cells_to_cpu(val, 2)) + self.assertEqual(0x9876543210987654, fdt_util.fdt_cells_to_cpu(val[2:], + 2)) + self.assertEqual(0x12345678, fdt_util.fdt_cells_to_cpu(val, 1)) + def testEnsureCompiled(self): """Test a degenerate case of this function""" dtb = fdt_util.EnsureCompiled('tools/dtoc/dtoc_test_simple.dts') From b5f0dafd4b66f8fde3c968ec20186bc8465558b8 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:41 -0600 Subject: [PATCH 31/74] dtoc: Add a unit test for BytesToValue() Add a simple unit test for one of the cases of this function, so that any fault can be seen directly, rather than appearing through the failure of another test. Signed-off-by: Simon Glass --- tools/dtoc/test_fdt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index a5234ce1e8..32a020bad2 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -19,7 +19,7 @@ for dirname in ['../patman', '..']: import command import fdt -from fdt import TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL +from fdt import TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL, BytesToValue import fdt_util from fdt_util import fdt32_to_cpu import libfdt @@ -121,6 +121,10 @@ class TestFdt(unittest.TestCase): node = self.dtb.GetNode('/spl-test') self.assertEqual(self.dtb, node.GetFdt()) + def testBytesToValue(self): + self.assertEqual(BytesToValue(b'this\0is\0'), + (TYPE_STRING, ['this', 'is'])) + class TestNode(unittest.TestCase): """Test operation of the Node class""" From 209a55976bd2aa4ba7e70b263a70fa14dd0c7675 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:42 -0600 Subject: [PATCH 32/74] dtoc: Update fdt_util for Python 3 Since we are now using the bytes type in Python 3, the conversion in fdt32_to_cpu() is not necessary, so drop it. Also use 'int' instead of 'long' to convert the integer value, since 'long' is not present in Python 3. With this, test_fdt passes with both Python 2 and 3: PYTHONPATH=/tmp/b/sandbox_spl/scripts/dtc/pylibfdt python \ ./tools/dtoc/test_fdt -t PYTHONPATH=~/cosarm/dtc/pylibfdt:tools/patman python3 \ ./tools/dtoc/test_fdt -t Signed-off-by: Simon Glass --- tools/dtoc/fdt_util.py | 15 ++------------- tools/dtoc/test_fdt.py | 3 --- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 5fbfc8877b..f47879ac00 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -16,14 +16,6 @@ import tempfile import command import tools -VERSION3 = sys.version_info > (3, 0) - -def get_plain_bytes(val): - """Handle Python 3 strings""" - if isinstance(val, bytes): - val = val.decode('utf-8') - return val.encode('raw_unicode_escape') - def fdt32_to_cpu(val): """Convert a device tree cell to an integer @@ -33,9 +25,6 @@ def fdt32_to_cpu(val): Return: A native-endian integer value """ - if VERSION3: - # This code is not reached in Python 2 - val = get_plain_bytes(val) # pragma: no cover return struct.unpack('>I', val)[0] def fdt_cells_to_cpu(val, cells): @@ -45,11 +34,11 @@ def fdt_cells_to_cpu(val, cells): Value to convert (array of one or more 4-character strings) Return: - A native-endian long value + A native-endian integer value """ if not cells: return 0 - out = long(fdt32_to_cpu(val[0])) + out = int(fdt32_to_cpu(val[0])) if cells == 2: out = out << 32 | fdt32_to_cpu(val[1]) return out diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 32a020bad2..bf469dbd54 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -518,9 +518,6 @@ class TestFdtUtil(unittest.TestCase): dtb = fdt_util.EnsureCompiled('tools/dtoc/dtoc_test_simple.dts') self.assertEqual(dtb, fdt_util.EnsureCompiled(dtb)) - def testGetPlainBytes(self): - self.assertEqual(b'fred', fdt_util.get_plain_bytes('fred')) - def RunTestCoverage(): """Run the tests and check that we get 100% coverage""" From 9b044f7e3d1b085499877774557b2db47fb4339c Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:43 -0600 Subject: [PATCH 33/74] dtoc: Update dtb_platdata to support Python 3 The only change needed is to update get_value() to support the 'bytes' type correctly with Python 3. With this the dtoc unit tests pass with both Python 2 and 3: PYTHONPATH=/tmp/b/sandbox_spl/scripts/dtc/pylibfdt python \ ./tools/dtoc/dtoc -t PYTHONPATH=~/cosarm/dtc/pylibfdt:tools/patman python3 \ ./tools/dtoc/dtoc -t Signed-off-by: Simon Glass --- tools/dtoc/dtb_platdata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/dtoc/dtb_platdata.py b/tools/dtoc/dtb_platdata.py index 3b66af8df7..037e82c8bb 100644 --- a/tools/dtoc/dtb_platdata.py +++ b/tools/dtoc/dtb_platdata.py @@ -17,6 +17,7 @@ import sys import fdt import fdt_util +import tools # When we see these properties we ignore them - i.e. do not create a structure member PROP_IGNORE_LIST = [ @@ -99,7 +100,7 @@ def get_value(ftype, value): if ftype == fdt.TYPE_INT: return '%#x' % fdt_util.fdt32_to_cpu(value) elif ftype == fdt.TYPE_BYTE: - return '%#x' % ord(value[0]) + return '%#x' % tools.ToByte(value[0]) elif ftype == fdt.TYPE_STRING: return '"%s"' % value elif ftype == fdt.TYPE_BOOL: From 3c47e4105e0b467938660a986c261f2db10a2edd Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:44 -0600 Subject: [PATCH 34/74] patman: Allow reading files in text mode While reading files in binary mode is the norm, sometimes we want to use text mode. Add an optional parameter to handle this. Signed-off-by: Simon Glass --- tools/patman/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/patman/tools.py b/tools/patman/tools.py index bdc1953936..8e9f22afe8 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -214,7 +214,7 @@ def Filename(fname): # If not found, just return the standard, unchanged path return fname -def ReadFile(fname): +def ReadFile(fname, binary=True): """Read and return the contents of a file. Args: @@ -223,7 +223,7 @@ def ReadFile(fname): Returns: data read from file, as a string. """ - with open(Filename(fname), 'rb') as fd: + with open(Filename(fname), binary and 'rb' or 'r') as fd: data = fd.read() #self._out.Info("Read file '%s' size %d (%#0x)" % #(fname, len(data), len(data))) From 58632a7f44cb0aef2c5cd4ede966c89abc430968 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:45 -0600 Subject: [PATCH 35/74] binman: Avoid changing a dict during iteration This code works OK in Python 2 but Python 3 complains. Adjust it to avoid deleting elements from a dict while iterating through it. Signed-off-by: Simon Glass --- tools/binman/control.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/binman/control.py b/tools/binman/control.py index ce25eb5485..20186ee198 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -131,10 +131,13 @@ def Binman(options, args): if options.image: skip = [] + new_images = OrderedDict() for name, image in images.items(): - if name not in options.image: - del images[name] + if name in options.image: + new_images[name] = image + else: skip.append(name) + images = new_images if skip and options.verbosity >= 2: print('Skipping images: %s' % ', '.join(skip)) From c6c10e77fb48f3e55857e379be86384fbcdc19d5 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:46 -0600 Subject: [PATCH 36/74] binman: Convert to use bytes type With Python 3 we want to use the 'bytes' type instead of 'str'. Adjust the code accordingly so that it works on both Python 2 and Python 3. Signed-off-by: Simon Glass --- tools/binman/elf_test.py | 5 +- tools/binman/etype/_testing.py | 2 +- tools/binman/etype/u_boot_dtb_with_ucode.py | 4 +- tools/binman/etype/u_boot_ucode.py | 4 +- tools/binman/etype/vblock.py | 2 +- tools/binman/ftest.py | 132 ++++++++++---------- 6 files changed, 77 insertions(+), 72 deletions(-) diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index 3d55cb1946..42d94cbbbe 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -22,7 +22,7 @@ class FakeEntry: """ def __init__(self, contents_size): self.contents_size = contents_size - self.data = 'a' * contents_size + self.data = tools.GetBytes(ord('a'), contents_size) def GetPath(self): return 'entry_path' @@ -122,7 +122,8 @@ class TestElf(unittest.TestCase): section = FakeSection(sym_value=None) elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms') syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) - self.assertEqual(tools.GetBytes(255, 16) + 'a' * 4, entry.data) + self.assertEqual(tools.GetBytes(255, 16) + tools.GetBytes(ord('a'), 4), + entry.data) def testDebug(self): """Check that enabling debug in the elf module produced debug output""" diff --git a/tools/binman/etype/_testing.py b/tools/binman/etype/_testing.py index 3e345bd952..ac62d2e200 100644 --- a/tools/binman/etype/_testing.py +++ b/tools/binman/etype/_testing.py @@ -75,7 +75,7 @@ class Entry__testing(Entry): def ObtainContents(self): if self.return_unknown_contents or not self.return_contents: return False - self.data = 'a' + self.data = b'a' self.contents_size = len(self.data) if self.return_contents_once: self.return_contents = False diff --git a/tools/binman/etype/u_boot_dtb_with_ucode.py b/tools/binman/etype/u_boot_dtb_with_ucode.py index 444c51b8b7..188888e022 100644 --- a/tools/binman/etype/u_boot_dtb_with_ucode.py +++ b/tools/binman/etype/u_boot_dtb_with_ucode.py @@ -26,7 +26,7 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob_dtb): """ def __init__(self, section, etype, node): Entry_blob_dtb.__init__(self, section, etype, node) - self.ucode_data = '' + self.ucode_data = b'' self.collate = False self.ucode_offset = None self.ucode_size = None @@ -65,7 +65,7 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob_dtb): for node in self.ucode.subnodes: data_prop = node.props.get('data') if data_prop: - self.ucode_data += ''.join(data_prop.bytes) + self.ucode_data += data_prop.bytes if self.collate: node.DeleteProp('data') return True diff --git a/tools/binman/etype/u_boot_ucode.py b/tools/binman/etype/u_boot_ucode.py index a00e530295..dee8848db7 100644 --- a/tools/binman/etype/u_boot_ucode.py +++ b/tools/binman/etype/u_boot_ucode.py @@ -69,7 +69,7 @@ class Entry_u_boot_ucode(Entry_blob): if entry and entry.target_offset: found = True if not found: - self.data = '' + self.data = b'' return True # Get the microcode from the device tree entry. If it is not available # yet, return False so we will be called later. If the section simply @@ -87,7 +87,7 @@ class Entry_u_boot_ucode(Entry_blob): if not fdt_entry.collate: # This binary can be empty - self.data = '' + self.data = b'' return True # Write it out to a file diff --git a/tools/binman/etype/vblock.py b/tools/binman/etype/vblock.py index 334ff9f966..91fa2f7808 100644 --- a/tools/binman/etype/vblock.py +++ b/tools/binman/etype/vblock.py @@ -51,7 +51,7 @@ class Entry_vblock(Entry): def ObtainContents(self): # Join up the data files to be signed - input_data = '' + input_data = b'' for entry_phandle in self.content: data = self.section.GetContentsByPhandle(entry_phandle, self) if data is None: diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 971fade343..48fec50179 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -29,38 +29,38 @@ import tools import tout # Contents of test files, corresponding to different entry types -U_BOOT_DATA = '1234' -U_BOOT_IMG_DATA = 'img' -U_BOOT_SPL_DATA = '56780123456789abcde' -U_BOOT_TPL_DATA = 'tpl' -BLOB_DATA = '89' -ME_DATA = '0abcd' -VGA_DATA = 'vga' -U_BOOT_DTB_DATA = 'udtb' -U_BOOT_SPL_DTB_DATA = 'spldtb' -U_BOOT_TPL_DTB_DATA = 'tpldtb' -X86_START16_DATA = 'start16' -X86_START16_SPL_DATA = 'start16spl' -X86_START16_TPL_DATA = 'start16tpl' -PPC_MPC85XX_BR_DATA = 'ppcmpc85xxbr' -U_BOOT_NODTB_DATA = 'nodtb with microcode pointer somewhere in here' -U_BOOT_SPL_NODTB_DATA = 'splnodtb with microcode pointer somewhere in here' -U_BOOT_TPL_NODTB_DATA = 'tplnodtb with microcode pointer somewhere in here' -FSP_DATA = 'fsp' -CMC_DATA = 'cmc' -VBT_DATA = 'vbt' -MRC_DATA = 'mrc' +U_BOOT_DATA = b'1234' +U_BOOT_IMG_DATA = b'img' +U_BOOT_SPL_DATA = b'56780123456789abcde' +U_BOOT_TPL_DATA = b'tpl' +BLOB_DATA = b'89' +ME_DATA = b'0abcd' +VGA_DATA = b'vga' +U_BOOT_DTB_DATA = b'udtb' +U_BOOT_SPL_DTB_DATA = b'spldtb' +U_BOOT_TPL_DTB_DATA = b'tpldtb' +X86_START16_DATA = b'start16' +X86_START16_SPL_DATA = b'start16spl' +X86_START16_TPL_DATA = b'start16tpl' +PPC_MPC85XX_BR_DATA = b'ppcmpc85xxbr' +U_BOOT_NODTB_DATA = b'nodtb with microcode pointer somewhere in here' +U_BOOT_SPL_NODTB_DATA = b'splnodtb with microcode pointer somewhere in here' +U_BOOT_TPL_NODTB_DATA = b'tplnodtb with microcode pointer somewhere in here' +FSP_DATA = b'fsp' +CMC_DATA = b'cmc' +VBT_DATA = b'vbt' +MRC_DATA = b'mrc' TEXT_DATA = 'text' TEXT_DATA2 = 'text2' TEXT_DATA3 = 'text3' -CROS_EC_RW_DATA = 'ecrw' -GBB_DATA = 'gbbd' -BMPBLK_DATA = 'bmp' -VBLOCK_DATA = 'vblk' -FILES_DATA = ("sorry I'm late\nOh, don't bother apologising, I'm " + - "sorry you're alive\n") -COMPRESS_DATA = 'data to compress' -REFCODE_DATA = 'refcode' +CROS_EC_RW_DATA = b'ecrw' +GBB_DATA = b'gbbd' +BMPBLK_DATA = b'bmp' +VBLOCK_DATA = b'vblk' +FILES_DATA = (b"sorry I'm late\nOh, don't bother apologising, I'm " + + b"sorry you're alive\n") +COMPRESS_DATA = b'data to compress' +REFCODE_DATA = b'refcode' class TestFunctional(unittest.TestCase): @@ -803,7 +803,7 @@ class TestFunctional(unittest.TestCase): def testPackX86RomMeNoDesc(self): """Test that an invalid Intel descriptor entry is detected""" - TestFunctional._MakeInputFile('descriptor.bin', '') + TestFunctional._MakeInputFile('descriptor.bin', b'') with self.assertRaises(ValueError) as e: self._DoTestFile('031_x86-rom-me.dts') self.assertIn("Node '/binman/intel-descriptor': Cannot find FD " @@ -900,8 +900,8 @@ class TestFunctional(unittest.TestCase): """ first, pos_and_size = self._RunMicrocodeTest('034_x86_ucode.dts', U_BOOT_NODTB_DATA) - self.assertEqual('nodtb with microcode' + pos_and_size + - ' somewhere in here', first) + self.assertEqual(b'nodtb with microcode' + pos_and_size + + b' somewhere in here', first) def _RunPackUbootSingleMicrocode(self): """Test that x86 microcode can be handled correctly @@ -932,8 +932,8 @@ class TestFunctional(unittest.TestCase): pos_and_size = struct.pack('<2L', 0xfffffe00 + ucode_pos, len(ucode_data)) first = data[:len(U_BOOT_NODTB_DATA)] - self.assertEqual('nodtb with microcode' + pos_and_size + - ' somewhere in here', first) + self.assertEqual(b'nodtb with microcode' + pos_and_size + + b' somewhere in here', first) def testPackUbootSingleMicrocode(self): """Test that x86 microcode can be handled correctly with fdt_normal. @@ -1068,8 +1068,8 @@ class TestFunctional(unittest.TestCase): self._SetupSplElf('u_boot_ucode_ptr') first, pos_and_size = self._RunMicrocodeTest(dts, U_BOOT_SPL_NODTB_DATA, ucode_second=ucode_second) - self.assertEqual('splnodtb with microc' + pos_and_size + - 'ter somewhere in here', first) + self.assertEqual(b'splnodtb with microc' + pos_and_size + + b'ter somewhere in here', first) def testPackUbootSplMicrocode(self): """Test that x86 microcode can be handled correctly in SPL""" @@ -1123,8 +1123,9 @@ class TestFunctional(unittest.TestCase): def testSections(self): """Basic test of sections""" data = self._DoReadFile('055_sections.dts') - expected = (U_BOOT_DATA + '!' * 12 + U_BOOT_DATA + 'a' * 12 + - U_BOOT_DATA + '&' * 4) + expected = (U_BOOT_DATA + tools.GetBytes(ord('!'), 12) + + U_BOOT_DATA + tools.GetBytes(ord('a'), 12) + + U_BOOT_DATA + tools.GetBytes(ord('&'), 4)) self.assertEqual(expected, data) def testMap(self): @@ -1282,8 +1283,10 @@ class TestFunctional(unittest.TestCase): } data, _, _, _ = self._DoReadFileDtb('066_text.dts', entry_args=entry_args) - expected = (TEXT_DATA + tools.GetBytes(0, 8 - len(TEXT_DATA)) + - TEXT_DATA2 + TEXT_DATA3 + 'some text') + expected = (tools.ToBytes(TEXT_DATA) + + tools.GetBytes(0, 8 - len(TEXT_DATA)) + + tools.ToBytes(TEXT_DATA2) + tools.ToBytes(TEXT_DATA3) + + b'some text') self.assertEqual(expected, data) def testEntryDocs(self): @@ -1304,32 +1307,33 @@ class TestFunctional(unittest.TestCase): """Basic test of generation of a flashrom fmap""" data = self._DoReadFile('067_fmap.dts') fhdr, fentries = fmap_util.DecodeFmap(data[32:]) - expected = U_BOOT_DATA + '!' * 12 + U_BOOT_DATA + 'a' * 12 + expected = (U_BOOT_DATA + tools.GetBytes(ord('!'), 12) + + U_BOOT_DATA + tools.GetBytes(ord('a'), 12)) self.assertEqual(expected, data[:32]) - self.assertEqual('__FMAP__', fhdr.signature) + self.assertEqual(b'__FMAP__', fhdr.signature) self.assertEqual(1, fhdr.ver_major) self.assertEqual(0, fhdr.ver_minor) self.assertEqual(0, fhdr.base) self.assertEqual(16 + 16 + fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fhdr.image_size) - self.assertEqual('FMAP', fhdr.name) + self.assertEqual(b'FMAP', fhdr.name) self.assertEqual(3, fhdr.nareas) for fentry in fentries: self.assertEqual(0, fentry.flags) self.assertEqual(0, fentries[0].offset) self.assertEqual(4, fentries[0].size) - self.assertEqual('RO_U_BOOT', fentries[0].name) + self.assertEqual(b'RO_U_BOOT', fentries[0].name) self.assertEqual(16, fentries[1].offset) self.assertEqual(4, fentries[1].size) - self.assertEqual('RW_U_BOOT', fentries[1].name) + self.assertEqual(b'RW_U_BOOT', fentries[1].name) self.assertEqual(32, fentries[2].offset) self.assertEqual(fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fentries[2].size) - self.assertEqual('FMAP', fentries[2].name) + self.assertEqual(b'FMAP', fentries[2].name) def testBlobNamedByArg(self): """Test we can add a blob with the filename coming from an entry arg""" @@ -1597,7 +1601,7 @@ class TestFunctional(unittest.TestCase): files = entries['files'] entries = files._section._entries - orig = '' + orig = b'' for i in range(1, 3): key = '%d.dat' % i start = entries[key].image_pos @@ -1625,10 +1629,10 @@ class TestFunctional(unittest.TestCase): """Test an expanding entry""" data, _, map_data, _ = self._DoReadFileDtb('088_expand_size.dts', map=True) - expect = ('a' * 8 + U_BOOT_DATA + - MRC_DATA + 'b' * 1 + U_BOOT_DATA + - 'c' * 8 + U_BOOT_DATA + - 'd' * 8) + expect = (tools.GetBytes(ord('a'), 8) + U_BOOT_DATA + + MRC_DATA + tools.GetBytes(ord('b'), 1) + U_BOOT_DATA + + tools.GetBytes(ord('c'), 8) + U_BOOT_DATA + + tools.GetBytes(ord('d'), 8)) self.assertEqual(expect, data) self.assertEqual('''ImagePos Offset Size Name 00000000 00000000 00000028 main-section @@ -1660,7 +1664,7 @@ class TestFunctional(unittest.TestCase): hash_node = dtb.GetNode('/binman/u-boot/hash').props['value'] m = hashlib.sha256() m.update(U_BOOT_DATA) - self.assertEqual(m.digest(), ''.join(hash_node.value)) + self.assertEqual(m.digest(), b''.join(hash_node.value)) def testHashNoAlgo(self): with self.assertRaises(ValueError) as e: @@ -1683,8 +1687,8 @@ class TestFunctional(unittest.TestCase): hash_node = dtb.GetNode('/binman/section/hash').props['value'] m = hashlib.sha256() m.update(U_BOOT_DATA) - m.update(16 * 'a') - self.assertEqual(m.digest(), ''.join(hash_node.value)) + m.update(tools.GetBytes(ord('a'), 16)) + self.assertEqual(m.digest(), b''.join(hash_node.value)) def testPackUBootTplMicrocode(self): """Test that x86 microcode can be handled correctly in TPL @@ -1699,14 +1703,14 @@ class TestFunctional(unittest.TestCase): TestFunctional._MakeInputFile('tpl/u-boot-tpl', fd.read()) first, pos_and_size = self._RunMicrocodeTest('093_x86_tpl_ucode.dts', U_BOOT_TPL_NODTB_DATA) - self.assertEqual('tplnodtb with microc' + pos_and_size + - 'ter somewhere in here', first) + self.assertEqual(b'tplnodtb with microc' + pos_and_size + + b'ter somewhere in here', first) def testFmapX86(self): """Basic test of generation of a flashrom fmap""" data = self._DoReadFile('094_fmap_x86.dts') fhdr, fentries = fmap_util.DecodeFmap(data[32:]) - expected = U_BOOT_DATA + MRC_DATA + 'a' * (32 - 7) + expected = U_BOOT_DATA + MRC_DATA + tools.GetBytes(ord('a'), 32 - 7) self.assertEqual(expected, data[:32]) fhdr, fentries = fmap_util.DecodeFmap(data[32:]) @@ -1714,21 +1718,21 @@ class TestFunctional(unittest.TestCase): self.assertEqual(0, fentries[0].offset) self.assertEqual(4, fentries[0].size) - self.assertEqual('U_BOOT', fentries[0].name) + self.assertEqual(b'U_BOOT', fentries[0].name) self.assertEqual(4, fentries[1].offset) self.assertEqual(3, fentries[1].size) - self.assertEqual('INTEL_MRC', fentries[1].name) + self.assertEqual(b'INTEL_MRC', fentries[1].name) self.assertEqual(32, fentries[2].offset) self.assertEqual(fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fentries[2].size) - self.assertEqual('FMAP', fentries[2].name) + self.assertEqual(b'FMAP', fentries[2].name) def testFmapX86Section(self): """Basic test of generation of a flashrom fmap""" data = self._DoReadFile('095_fmap_x86_section.dts') - expected = U_BOOT_DATA + MRC_DATA + 'b' * (32 - 7) + expected = U_BOOT_DATA + MRC_DATA + tools.GetBytes(ord('b'), 32 - 7) self.assertEqual(expected, data[:32]) fhdr, fentries = fmap_util.DecodeFmap(data[36:]) @@ -1736,16 +1740,16 @@ class TestFunctional(unittest.TestCase): self.assertEqual(0, fentries[0].offset) self.assertEqual(4, fentries[0].size) - self.assertEqual('U_BOOT', fentries[0].name) + self.assertEqual(b'U_BOOT', fentries[0].name) self.assertEqual(4, fentries[1].offset) self.assertEqual(3, fentries[1].size) - self.assertEqual('INTEL_MRC', fentries[1].name) + self.assertEqual(b'INTEL_MRC', fentries[1].name) self.assertEqual(36, fentries[2].offset) self.assertEqual(fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fentries[2].size) - self.assertEqual('FMAP', fentries[2].name) + self.assertEqual(b'FMAP', fentries[2].name) def testElf(self): """Basic test of ELF entries""" From 0199d19349ee0e5cc65dae4a453eb9c4074fb685 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:47 -0600 Subject: [PATCH 37/74] binman: Update entry_test to support Python 3 The reload() function is in a different place in Python 3. Update the code to handle this. Signed-off-by: Simon Glass --- tools/binman/entry_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/binman/entry_test.py b/tools/binman/entry_test.py index 1f7ff5b4e4..b30a7beecc 100644 --- a/tools/binman/entry_test.py +++ b/tools/binman/entry_test.py @@ -41,7 +41,11 @@ class TestEntry(unittest.TestCase): del sys.modules['importlib'] global entry if entry: - reload(entry) + if sys.version_info[0] >= 3: + import importlib + importlib.reload(entry) + else: + reload(entry) else: import entry entry.Entry.Create(None, self.GetNode(), 'u-boot-spl') From f3a58c8a7858bfb38456aae7cadecdfbd0f3da98 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:48 -0600 Subject: [PATCH 38/74] patman: Update fmap code for Python 3 This needs special care to ensure that the bytes type is used for binary data. Add conversion code to deal with strings and bytes correctly. Signed-off-by: Simon Glass --- tools/binman/etype/fmap.py | 3 ++- tools/binman/fmap_util.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/binman/etype/fmap.py b/tools/binman/etype/fmap.py index bf35a5bbf4..e6b5c5c74c 100644 --- a/tools/binman/etype/fmap.py +++ b/tools/binman/etype/fmap.py @@ -7,6 +7,7 @@ from entry import Entry import fmap_util +import tools class Entry_fmap(Entry): @@ -46,7 +47,7 @@ class Entry_fmap(Entry): if pos is not None: pos -= entry.section.GetRootSkipAtStart() areas.append(fmap_util.FmapArea(pos or 0, entry.size or 0, - entry.name, 0)) + tools.FromUnicode(entry.name), 0)) entries = self.section._image.GetEntries() areas = [] diff --git a/tools/binman/fmap_util.py b/tools/binman/fmap_util.py index be3cbee87b..d0f956b622 100644 --- a/tools/binman/fmap_util.py +++ b/tools/binman/fmap_util.py @@ -8,9 +8,12 @@ import collections import struct +import sys + +import tools # constants imported from lib/fmap.h -FMAP_SIGNATURE = '__FMAP__' +FMAP_SIGNATURE = b'__FMAP__' FMAP_VER_MAJOR = 1 FMAP_VER_MINOR = 0 FMAP_STRLEN = 32 @@ -50,6 +53,8 @@ FmapArea = collections.namedtuple('FmapArea', FMAP_AREA_NAMES) def NameToFmap(name): + if type(name) == bytes and sys.version_info[0] >= 3: + name = name.decode('utf-8') # pragma: no cover (for Python 2) return name.replace('\0', '').replace('-', '_').upper() def ConvertName(field_names, fields): @@ -65,7 +70,7 @@ def ConvertName(field_names, fields): value: value of that field (string for the ones we support) """ name_index = field_names.index('name') - fields[name_index] = NameToFmap(fields[name_index]) + fields[name_index] = tools.ToBytes(NameToFmap(fields[name_index])) def DecodeFmap(data): """Decode a flashmap into a header and list of areas @@ -106,7 +111,8 @@ def EncodeFmap(image_size, name, areas): ConvertName(names, params) return struct.pack(fmt, *params) - values = FmapHeader(FMAP_SIGNATURE, 1, 0, 0, image_size, name, len(areas)) + values = FmapHeader(FMAP_SIGNATURE, 1, 0, 0, image_size, + tools.FromUnicode(name), len(areas)) blob = _FormatBlob(FMAP_HEADER_FORMAT, FMAP_HEADER_NAMES, values) for area in areas: blob += _FormatBlob(FMAP_AREA_FORMAT, FMAP_AREA_NAMES, area) From e16b7b6c49ff80c8cc2eb53b1f210dc4b743dc90 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:49 -0600 Subject: [PATCH 39/74] binman: Update 'text' entry for Python 3 This code reads a binary value and then uses it as a string to look up another value. Add conversions to make this work as expected on Python 3. Signed-off-by: Simon Glass --- tools/binman/etype/text.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/binman/etype/text.py b/tools/binman/etype/text.py index c4aa510a87..9ee04d7c9d 100644 --- a/tools/binman/etype/text.py +++ b/tools/binman/etype/text.py @@ -7,6 +7,7 @@ from collections import OrderedDict from entry import Entry, EntryArg import fdt_util +import tools class Entry_text(Entry): @@ -48,9 +49,11 @@ class Entry_text(Entry): """ def __init__(self, section, etype, node): Entry.__init__(self, section, etype, node) - self.text_label, = self.GetEntryArgsOrProps( - [EntryArg('text-label', str)]) - self.value, = self.GetEntryArgsOrProps([EntryArg(self.text_label, str)]) + label, = self.GetEntryArgsOrProps([EntryArg('text-label', str)]) + self.text_label = tools.ToStr(label) if type(label) != str else label + value, = self.GetEntryArgsOrProps([EntryArg(self.text_label, str)]) + value = tools.ToBytes(value) if value is not None else value + self.value = value def ObtainContents(self): if not self.value: From 1fc62de19fff9ab5228af72038753ec3a529eef1 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:50 -0600 Subject: [PATCH 40/74] binman: Fix up a format string in AssertInList() Add the missing 's' to the required '%s' here. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 48fec50179..d0a8b751a2 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -395,7 +395,7 @@ class TestFunctional(unittest.TestCase): for grep in grep_list: if grep in target: return - self.fail("Error: '%' not found in '%s'" % (grep_list, target)) + self.fail("Error: '%s' not found in '%s'" % (grep_list, target)) def CheckNoGaps(self, entries): """Check that all entries fit together without gaps From eb546ac181ba93a16949167a4a41a38fe539b378 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:51 -0600 Subject: [PATCH 41/74] binman: Read map files as text These files are text files so should be read as such, so that string-equality assertions work as expected. With this binman tests work correctly on Python 2 and Python 3: PYTHONPATH=/tmp/b/sandbox_spl/scripts/dtc/pylibfdt \ python ./tools/binman/binman -t Change first line of binman.py to end "python3": PYTHONPATH=~/cosarm/dtc/pylibfdt:tools/patman \ python3 ./tools/binman/binman -t Signed-off-by: Simon Glass --- tools/binman/ftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index d0a8b751a2..cc57ef3e04 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -1777,7 +1777,7 @@ class TestFunctional(unittest.TestCase): # We should not get an inmage, but there should be a map file self.assertFalse(os.path.exists(tools.GetOutputFilename('image.bin'))) self.assertTrue(os.path.exists(map_fname)) - map_data = tools.ReadFile(map_fname) + map_data = tools.ReadFile(map_fname, binary=False) self.assertEqual('''ImagePos Offset Size Name 00000000 00000007 main-section 00000000 00000004 u-boot From 55660d08a095158069ec37915ec4f0843588cd58 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:52 -0600 Subject: [PATCH 42/74] binman: Document parallel tests Since binman can run tests in parallel, document this. Signed-off-by: Simon Glass --- tools/binman/README | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/binman/README b/tools/binman/README index 927fa856ac..ac193f16cf 100644 --- a/tools/binman/README +++ b/tools/binman/README @@ -702,6 +702,20 @@ To enable Python test coverage on Debian-type distributions (e.g. Ubuntu): $ sudo apt-get install python-coverage python-pytest +Concurrent tests +---------------- + +Binman tries to run tests concurrently. This means that the tests make use of +all available CPUs to run. + + To enable this: + + $ sudo apt-get install python-subunit python3-subunit + +Use '-P 1' to disable this. It is automatically disabled when code coverage is +being used (-T) since they are incompatible. + + Advanced Features / Technical docs ---------------------------------- From 5385f5a0183cfb0b6f86ec2b55d40b67a4acfe57 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:53 -0600 Subject: [PATCH 43/74] binman: Update the README.entries file A few minor changes have been made including one new entry. Update the documentation with: $ binman -E >tools/binman/README.entries Signed-off-by: Simon Glass --- tools/binman/README.entries | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/binman/README.entries b/tools/binman/README.entries index 9fc2f83280..357946d630 100644 --- a/tools/binman/README.entries +++ b/tools/binman/README.entries @@ -224,6 +224,20 @@ See README.x86 for information about x86 binary blobs. +Entry: intel-refcode: Entry containing an Intel Reference Code file +------------------------------------------------------------------- + +Properties / Entry arguments: + - filename: Filename of file to read into entry + +This file contains code for setting up the platform on some Intel systems. +This is executed by U-Boot when needed early during startup. A typical +filename is 'refcode.bin'. + +See README.x86 for information about x86 binary blobs. + + + Entry: intel-vbt: Entry containing an Intel Video BIOS Table (VBT) file ----------------------------------------------------------------------- @@ -627,6 +641,7 @@ Entry: vblock: An entry which contains a Chromium OS verified boot block ------------------------------------------------------------------------ Properties / Entry arguments: + - content: List of phandles to entries to sign - keydir: Directory containing the public keys to use - keyblock: Name of the key file to use (inside keydir) - signprivate: Name of provide key file to use (inside keydir) From 9550f9acd64449a739b2db90c64e63a6269d81d8 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 17 May 2019 22:00:54 -0600 Subject: [PATCH 44/74] patman: Update cover-coverage tests for Python 3 We need slightly different commands to run code coverage with Python 3. Update the RunTestCoverage() function to handle this. Signed-off-by: Simon Glass --- tools/patman/test_util.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/patman/test_util.py b/tools/patman/test_util.py index e462ec8f72..ea36cd1633 100644 --- a/tools/patman/test_util.py +++ b/tools/patman/test_util.py @@ -17,6 +17,8 @@ try: except ImportError: from io import StringIO +PYTHON = 'python%d' % sys.version_info[0] + def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): """Run tests and check that we get 100% coverage @@ -43,11 +45,12 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): else: glob_list = [] glob_list += exclude_list - glob_list += ['*libfdt.py', '*site-packages*'] - cmd = ('PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools python-coverage run ' - '--omit "%s" %s -P1 -t' % (build_dir, ','.join(glob_list), prog)) + glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*'] + cmd = ('PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools %s-coverage run ' + '--omit "%s" %s -P1 -t' % (build_dir, PYTHON, ','.join(glob_list), + prog)) os.system(cmd) - stdout = command.Output('python-coverage', 'report') + stdout = command.Output('%s-coverage' % PYTHON, 'report') lines = stdout.splitlines() if required: # Convert '/path/to/name.py' just the module name 'name' @@ -65,8 +68,8 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): print(coverage) if coverage != '100%': print(stdout) - print("Type 'python-coverage html' to get a report in " - 'htmlcov/index.html') + print("Type '%s-coverage html' to get a report in " + 'htmlcov/index.html' % PYTHON) print('Coverage error: %s, but should be 100%%' % coverage) ok = False if not ok: From 00ba3b1edc74ed5515eae2d7e1f733b8516f908a Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 10:07:17 -0600 Subject: [PATCH 45/74] Add a simple script to remove boards This script attempts to create a git commit which removes a single board. It is quite fallible and everything it does needs checking. But it can help speed up the process. Signed-off-by: Simon Glass Reviewed-by: Chris Packham --- tools/rmboard.py | 150 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100755 tools/rmboard.py diff --git a/tools/rmboard.py b/tools/rmboard.py new file mode 100755 index 0000000000..df4f04b01c --- /dev/null +++ b/tools/rmboard.py @@ -0,0 +1,150 @@ +#! /usr/bin/python +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2019 Google LLC +# + +""" +Script to remove boards + +Usage: + rmboard.py ... + +A single commit is created for each board removed. + +Some boards may depend on files provided by another and this will cause +problems, generally the removal of files which should not be removed. + +This script works by: + - Looking through the MAINTAINERS files which mention a board to find out + what files the board uses + - Looking through the Kconfig files which mention a board to find one that + needs to have material removed + +Search for ## to update the commit message manually. +""" + +from __future__ import print_function + +import glob +import os +import re +import sys + +# Bring in the patman libraries +our_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(our_path, '../tools/patman')) + +import command + +def rm_kconfig_include(path): + """Remove a path from Kconfig files + + This function finds the given path in a 'source' statement in a Kconfig + file and removes that line from the file. This is needed because the path + is going to be removed, so any reference to it will cause a problem with + Kconfig parsing. + + The changes are made locally and then added to the git staging area. + + Args: + path: Path to search for and remove + """ + cmd = ['git', 'grep', path] + stdout = command.RunPipe([cmd], capture=True, raise_on_error=False).stdout + if not stdout: + return + fname = stdout.split(':')[0] + + print("Fixing up '%s' to remove reference to '%s'" % (fname, path)) + cmd = ['sed', '-i', '\|%s|d' % path, fname] + stdout = command.RunPipe([cmd], capture=True).stdout + + cmd = ['git', 'add', fname] + stdout = command.RunPipe([cmd], capture=True).stdout + +def rm_board(board): + """Create a commit which removes a single board + + This looks up the MAINTAINERS file to file files that need to be removed, + then removes pieces from the Kconfig files that mention the board. + + + Args: + board: Board name to remove + """ + + # Find all MAINTAINERS and Kconfig files which mention the board + cmd = ['git', 'grep', '-l', board] + stdout = command.RunPipe([cmd], capture=True).stdout + maintain = [] + kconfig = [] + for line in stdout.splitlines(): + line = line.strip() + if 'MAINTAINERS' in line: + if line not in maintain: + maintain.append(line) + elif 'Kconfig' in line: + kconfig.append(line) + paths = [] + cc = [] + + # Look through the MAINTAINERS file to find things to remove + for fname in maintain: + with open(fname) as fd: + for line in fd: + line = line.strip() + fields = re.split('[ \t]', line, 1) + if len(fields) == 2: + if fields[0] == 'M:': + cc.append(fields[1]) + elif fields[0] == 'F:': + paths.append(fields[1].strip()) + + # Expand any wildcards in the MAINTAINERS file + real = [] + for path in paths: + if path[-1] == '/': + path = path[:-1] + if '*' in path: + globbed = glob.glob(path) + print("Expanded '%s' to '%s'" % (path, globbed)) + real += globbed + else: + real.append(path) + + # Search for Kconfig files in the resulting list. Remove any 'source' lines + # which reference Kconfig files we want to remove + for path in real: + cmd = ['find', path] + stdout = (command.RunPipe([cmd], capture=True, raise_on_error=False). + stdout) + for fname in stdout.splitlines(): + if fname.endswith('Kconfig'): + rm_kconfig_include(fname) + + # Remove unwanted files + cmd = ['git', 'rm', '-r'] + real + stdout = command.RunPipe([cmd], capture=True).stdout + + ## Change the messages as needed + msg = '''arm: Remove %s board + +This board has not been converted to CONFIG_DM_MMC by the deadline. +Remove it. + +''' % board + for name in cc: + msg += 'Patch-cc: %s\n' % name + + # Create the commit + cmd = ['git', 'commit', '-s', '-m', msg] + stdout = command.RunPipe([cmd], capture=True).stdout + + # Check if the board is mentioned anywhere else. The user will need to deal + # with this + cmd = ['git', 'grep', '-il', board] + print(command.RunPipe([cmd], capture=True, raise_on_error=False).stdout) + print(' '.join(cmd)) + +for board in sys.argv[1:]: + rm_board(board) From e8726da2269cd80f60db92e2ba03e3e699c4fee6 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:40 -0600 Subject: [PATCH 46/74] sandbox: Sync up sandbox64.dts with main DT Various nodes have been added and adjusted with sandbox. Move these changes over to sandbox64.dts to keep these in sync. Signed-off-by: Simon Glass --- arch/sandbox/dts/sandbox64.dts | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/arch/sandbox/dts/sandbox64.dts b/arch/sandbox/dts/sandbox64.dts index a3c95f2cdb..e5b4426426 100644 --- a/arch/sandbox/dts/sandbox64.dts +++ b/arch/sandbox/dts/sandbox64.dts @@ -11,14 +11,21 @@ i2c0 = &i2c_0; pci0 = &pci; rtc0 = &rtc_0; + axi0 = &axi; }; chosen { stdout-path = "/serial"; }; + audio: audio-codec { + compatible = "sandbox,audio-codec"; + #sound-dai-cells = <1>; + }; + cros_ec: cros-ec { reg = <0 0 0 0>; + u-boot,dm-pre-reloc; compatible = "google,cros-ec-sandbox"; /* @@ -26,6 +33,7 @@ * that the STM32L flash erases to 0, not 0xff. */ flash { + u-boot,dm-pre-reloc; image-pos = <0x08000000>; size = <0x20000>; erase-value = <0>; @@ -58,6 +66,7 @@ }; gpio_a: gpios@0 { + u-boot,dm-pre-reloc; gpio-controller; compatible = "sandbox,gpio"; #gpio-cells = <1>; @@ -66,6 +75,7 @@ }; gpio_b: gpios@1 { + u-boot,dm-pre-reloc; gpio-controller; compatible = "sandbox,gpio"; #gpio-cells = <2>; @@ -120,6 +130,11 @@ }; }; + i2s: i2s { + compatible = "sandbox,i2s"; + #sound-dai-cells = <1>; + }; + lcd { u-boot,dm-pre-reloc; compatible = "sandbox,lcd-sdl"; @@ -159,6 +174,7 @@ pinctrl { compatible = "sandbox,pinctrl"; + status = "okay"; pinctrl_i2c0: i2c0 { groups = "i2c"; @@ -170,19 +186,38 @@ groups = "serial_a"; function = "serial"; }; + + pinctrl_onewire0: onewire0 { + groups = "w1"; + function = "w1"; + bias-pull-up; + }; }; reset@1 { compatible = "sandbox,reset"; }; + sound { + compatible = "sandbox,sound"; + cpu { + sound-dai = <&i2s 0>; + }; + + codec { + sound-dai = <&audio 0>; + }; + }; + spi@0 { + u-boot,dm-pre-reloc; #address-cells = <1>; #size-cells = <0>; reg = <0 0 0 0>; compatible = "sandbox,spi"; cs-gpios = <0>, <&gpio_a 0>; firmware_storage_spi: flash@0 { + u-boot,dm-pre-reloc; reg = <0>; compatible = "spansion,m25p16", "sandbox,spi-flash"; spi-max-frequency = <40000000>; @@ -238,6 +273,7 @@ }; tpm { + u-boot,dm-pre-reloc; compatible = "google,sandbox-tpm"; }; @@ -255,6 +291,7 @@ /* Needs to be available prior to relocation */ uart0: serial { + u-boot,dm-spl; compatible = "sandbox,serial"; sandbox,text-colour = "cyan"; pinctrl-names = "default"; @@ -319,6 +356,29 @@ }; }; + axi: axi@0 { + compatible = "sandbox,axi"; + #address-cells = <0x1>; + #size-cells = <0x1>; + store@0 { + compatible = "sandbox,sandbox_store"; + reg = <0x0 0x400>; + }; + }; + + onewire0: onewire { + compatible = "w1-gpio"; + gpios = <&gpio_a 8>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_onewire0>; + status = "okay"; + + sandbox_eeprom0: sandbox_eeprom@0 { + compatible = "sandbox,w1-eeprom"; + status = "okay"; + }; + }; + sandbox_tee { compatible = "sandbox,tee"; }; @@ -326,3 +386,10 @@ #include "cros-ec-keyboard.dtsi" #include "sandbox_pmic.dtsi" + +&cros_ec { + u-boot,dm-pre-reloc; + keyboard-controller { + u-boot,dm-pre-reloc; + }; +}; From dc8c91abb7f80d4c57775567402ad63ff7a0f844 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:41 -0600 Subject: [PATCH 47/74] sandbox: Create a common sandbox DT At present sandbox and sandbox64 have duplicated nodes. This is hard to maintain since changes in one need to be manually added to the other. Create a common file to solve this problem. Signed-off-by: Simon Glass --- arch/sandbox/dts/sandbox.dts | 333 +------------------------------ arch/sandbox/dts/sandbox.dtsi | 346 +++++++++++++++++++++++++++++++++ arch/sandbox/dts/sandbox64.dts | 333 +------------------------------ 3 files changed, 348 insertions(+), 664 deletions(-) create mode 100644 arch/sandbox/dts/sandbox.dtsi diff --git a/arch/sandbox/dts/sandbox.dts b/arch/sandbox/dts/sandbox.dts index a41b5f052d..c0bdba94a7 100644 --- a/arch/sandbox/dts/sandbox.dts +++ b/arch/sandbox/dts/sandbox.dts @@ -1,7 +1,5 @@ /dts-v1/; -#define USB_CLASS_HUB 9 - / { #address-cells = <1>; #size-cells = <1>; @@ -14,44 +12,10 @@ axi0 = &axi; }; - chosen { - stdout-path = "/serial"; - }; - - audio: audio-codec { - compatible = "sandbox,audio-codec"; - #sound-dai-cells = <1>; - }; - cros_ec: cros-ec { reg = <0 0>; u-boot,dm-pre-reloc; compatible = "google,cros-ec-sandbox"; - - /* - * This describes the flash memory within the EC. Note - * that the STM32L flash erases to 0, not 0xff. - */ - flash { - u-boot,dm-pre-reloc; - image-pos = <0x08000000>; - size = <0x20000>; - erase-value = <0>; - - /* Information for sandbox */ - ro { - image-pos = <0>; - size = <0xf000>; - }; - wp-ro { - image-pos = <0xf000>; - size = <0x1000>; - }; - rw { - image-pos = <0x10000>; - size = <0x10000>; - }; - }; }; ethrawbus { @@ -65,30 +29,6 @@ fake-host-hwaddr = [00 00 66 44 22 00]; }; - gpio_a: gpios@0 { - u-boot,dm-pre-reloc; - gpio-controller; - compatible = "sandbox,gpio"; - #gpio-cells = <1>; - gpio-bank-name = "a"; - sandbox,gpio-count = <20>; - }; - - gpio_b: gpios@1 { - u-boot,dm-pre-reloc; - gpio-controller; - compatible = "sandbox,gpio"; - #gpio-cells = <2>; - gpio-bank-name = "b"; - sandbox,gpio-count = <10>; - }; - - hexagon { - compatible = "demo-simple"; - colour = "white"; - sides = <6>; - }; - i2c_0: i2c@0 { #address-cells = <1>; #size-cells = <0>; @@ -97,63 +37,6 @@ clock-frequency = <400000>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; - eeprom@2c { - reg = <0x2c>; - compatible = "i2c-eeprom"; - sandbox,emul = <&emul_eeprom>; - }; - - rtc_0: rtc@43 { - reg = <0x43>; - compatible = "sandbox-rtc"; - sandbox,emul = <&emul0>; - }; - sandbox_pmic: sandbox_pmic { - reg = <0x40>; - }; - - mc34708: pmic@41 { - reg = <0x41>; - }; - - i2c_emul: emul { - reg = <0xff>; - compatible = "sandbox,i2c-emul-parent"; - emul_eeprom: emul-eeprom { - compatible = "sandbox,i2c-eeprom"; - sandbox,filename = "i2c.bin"; - sandbox,size = <256>; - }; - emul0: emul0 { - compatible = "sandbox,i2c-rtc"; - }; - }; - }; - - i2s: i2s { - compatible = "sandbox,i2s"; - #sound-dai-cells = <1>; - }; - - lcd { - u-boot,dm-pre-reloc; - compatible = "sandbox,lcd-sdl"; - xres = <1366>; - yres = <768>; - }; - - leds { - compatible = "gpio-leds"; - - iracibble { - gpios = <&gpio_a 1 0>; - label = "sandbox:red"; - }; - - martinet { - gpios = <&gpio_a 2 0>; - label = "sandbox:green"; - }; }; pci: pci-controller { @@ -163,50 +46,6 @@ #size-cells = <2>; ranges = <0x02000000 0 0x10000000 0x10000000 0 0x2000 0x01000000 0 0x20000000 0x20000000 0 0x2000>; - pci@1f,0 { - compatible = "pci-generic"; - reg = <0xf800 0 0 0 0>; - emul@1f,0 { - compatible = "sandbox,swap-case"; - }; - }; - }; - - pinctrl { - compatible = "sandbox,pinctrl"; - status = "okay"; - - pinctrl_i2c0: i2c0 { - groups = "i2c"; - function = "i2c"; - bias-pull-up; - }; - - pinctrl_serial0: uart0 { - groups = "serial_a"; - function = "serial"; - }; - - pinctrl_onewire0: onewire0 { - groups = "w1"; - function = "w1"; - bias-pull-up; - }; - }; - - reset@1 { - compatible = "sandbox,reset"; - }; - - sound { - compatible = "sandbox,sound"; - cpu { - sound-dai = <&i2s 0>; - }; - - codec { - sound-dai = <&audio 0>; - }; }; spi@0 { @@ -216,180 +55,10 @@ reg = <0 0>; compatible = "sandbox,spi"; cs-gpios = <0>, <&gpio_a 0>; - firmware_storage_spi: flash@0 { - u-boot,dm-pre-reloc; - reg = <0>; - compatible = "spansion,m25p16", "sandbox,spi-flash"; - spi-max-frequency = <40000000>; - sandbox,filename = "spi.bin"; - }; }; - spl-test { - u-boot,dm-pre-reloc; - compatible = "sandbox,spl-test"; - boolval; - intval = <1>; - intarray = <2 3 4>; - byteval = [05]; - bytearray = [06]; - longbytearray = [09 0a 0b 0c 0d 0e 0f 10 11]; - stringval = "message"; - stringarray = "multi-word", "message"; - }; - - spl-test2 { - u-boot,dm-pre-reloc; - compatible = "sandbox,spl-test"; - intval = <3>; - intarray = <5>; - byteval = [08]; - bytearray = [01 23 34]; - longbytearray = [09 0a 0b 0c]; - stringval = "message2"; - stringarray = "another", "multi-word", "message"; - }; - - spl-test3 { - u-boot,dm-pre-reloc; - compatible = "sandbox,spl-test"; - stringarray = "one"; - }; - - spl-test4 { - u-boot,dm-pre-reloc; - compatible = "sandbox,spl-test.2"; - }; - - square { - compatible = "demo-shape"; - colour = "blue"; - sides = <4>; - }; - - timer { - compatible = "sandbox,timer"; - clock-frequency = <1000000>; - }; - - tpm { - u-boot,dm-pre-reloc; - compatible = "google,sandbox-tpm"; - }; - - tpm2 { - compatible = "sandbox,tpm2"; - }; - - triangle { - compatible = "demo-shape"; - colour = "cyan"; - sides = <3>; - character = <83>; - light-gpios = <&gpio_a 2>, <&gpio_b 6 0>; - }; - - /* Needs to be available prior to relocation */ - uart0: serial { - u-boot,dm-spl; - compatible = "sandbox,serial"; - sandbox,text-colour = "cyan"; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_serial0>; - }; - - usb@0 { - compatible = "sandbox,usb"; - status = "disabled"; - hub { - compatible = "sandbox,usb-hub"; - #address-cells = <1>; - #size-cells = <0>; - flash-stick { - reg = <0>; - compatible = "sandbox,usb-flash"; - }; - }; - }; - - usb@1 { - compatible = "sandbox,usb"; - hub { - compatible = "usb-hub"; - usb,device-class = ; - hub-emul { - compatible = "sandbox,usb-hub"; - #address-cells = <1>; - #size-cells = <0>; - flash-stick { - reg = <0>; - compatible = "sandbox,usb-flash"; - sandbox,filepath = "flash.bin"; - }; - }; - }; - }; - - usb@2 { - compatible = "sandbox,usb"; - status = "disabled"; - }; - - spmi: spmi@0 { - compatible = "sandbox,spmi"; - #address-cells = <0x1>; - #size-cells = <0x1>; - pm8916@0 { - compatible = "qcom,spmi-pmic"; - reg = <0x0 0x1>; - #address-cells = <0x1>; - #size-cells = <0x1>; - - spmi_gpios: gpios@c000 { - compatible = "qcom,pm8916-gpio"; - reg = <0xc000 0x400>; - gpio-controller; - gpio-count = <4>; - #gpio-cells = <2>; - gpio-bank-name="spmi"; - }; - }; - }; - - axi: axi@0 { - compatible = "sandbox,axi"; - #address-cells = <0x1>; - #size-cells = <0x1>; - store@0 { - compatible = "sandbox,sandbox_store"; - reg = <0x0 0x400>; - }; - }; - - onewire0: onewire { - compatible = "w1-gpio"; - gpios = <&gpio_a 8>; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_onewire0>; - status = "okay"; - - sandbox_eeprom0: sandbox_eeprom@0 { - compatible = "sandbox,w1-eeprom"; - status = "okay"; - }; - }; - - sandbox_tee { - compatible = "sandbox,tee"; - }; }; +#include "sandbox.dtsi" #include "cros-ec-keyboard.dtsi" #include "sandbox_pmic.dtsi" - -&cros_ec { - u-boot,dm-pre-reloc; - keyboard-controller { - u-boot,dm-pre-reloc; - }; -}; diff --git a/arch/sandbox/dts/sandbox.dtsi b/arch/sandbox/dts/sandbox.dtsi new file mode 100644 index 0000000000..ebc4ece824 --- /dev/null +++ b/arch/sandbox/dts/sandbox.dtsi @@ -0,0 +1,346 @@ +/* + * This is the common sandbox device-tree nodes. This is shared between sandbox + * and sandbox64 builds. + */ + +#define USB_CLASS_HUB 9 + +/ { + chosen { + stdout-path = "/serial"; + }; + + audio: audio-codec { + compatible = "sandbox,audio-codec"; + #sound-dai-cells = <1>; + }; + + gpio_a: gpios@0 { + u-boot,dm-pre-reloc; + gpio-controller; + compatible = "sandbox,gpio"; + #gpio-cells = <1>; + gpio-bank-name = "a"; + sandbox,gpio-count = <20>; + }; + + gpio_b: gpios@1 { + u-boot,dm-pre-reloc; + gpio-controller; + compatible = "sandbox,gpio"; + #gpio-cells = <2>; + gpio-bank-name = "b"; + sandbox,gpio-count = <10>; + }; + + hexagon { + compatible = "demo-simple"; + colour = "white"; + sides = <6>; + }; + + i2c_0: i2c@0 { + eeprom@2c { + reg = <0x2c>; + compatible = "i2c-eeprom"; + sandbox,emul = <&emul_eeprom>; + }; + + rtc_0: rtc@43 { + reg = <0x43>; + compatible = "sandbox-rtc"; + sandbox,emul = <&emul0>; + }; + sandbox_pmic: sandbox_pmic { + reg = <0x40>; + }; + + mc34708: pmic@41 { + reg = <0x41>; + }; + + i2c_emul: emul { + reg = <0xff>; + compatible = "sandbox,i2c-emul-parent"; + emul_eeprom: emul-eeprom { + compatible = "sandbox,i2c-eeprom"; + sandbox,filename = "i2c.bin"; + sandbox,size = <256>; + }; + emul0: emul0 { + compatible = "sandbox,i2c-rtc"; + }; + }; + }; + + i2s: i2s { + compatible = "sandbox,i2s"; + #sound-dai-cells = <1>; + }; + + lcd { + u-boot,dm-pre-reloc; + compatible = "sandbox,lcd-sdl"; + xres = <1366>; + yres = <768>; + }; + + leds { + compatible = "gpio-leds"; + + iracibble { + gpios = <&gpio_a 1 0>; + label = "sandbox:red"; + }; + + martinet { + gpios = <&gpio_a 2 0>; + label = "sandbox:green"; + }; + }; + + pci-controller { + pci@1f,0 { + compatible = "pci-generic"; + reg = <0xf800 0 0 0 0>; + emul@1f,0 { + compatible = "sandbox,swap-case"; + }; + }; + }; + + pinctrl { + compatible = "sandbox,pinctrl"; + status = "okay"; + + pinctrl_i2c0: i2c0 { + groups = "i2c"; + function = "i2c"; + bias-pull-up; + }; + + pinctrl_serial0: uart0 { + groups = "serial_a"; + function = "serial"; + }; + + pinctrl_onewire0: onewire0 { + groups = "w1"; + function = "w1"; + bias-pull-up; + }; + }; + + reset@1 { + compatible = "sandbox,reset"; + }; + + sound { + compatible = "sandbox,sound"; + cpu { + sound-dai = <&i2s 0>; + }; + + codec { + sound-dai = <&audio 0>; + }; + }; + + spi@0 { + firmware_storage_spi: flash@0 { + u-boot,dm-pre-reloc; + reg = <0>; + compatible = "spansion,m25p16", "sandbox,spi-flash"; + spi-max-frequency = <40000000>; + sandbox,filename = "spi.bin"; + }; + }; + + spl-test { + u-boot,dm-pre-reloc; + compatible = "sandbox,spl-test"; + boolval; + intval = <1>; + intarray = <2 3 4>; + byteval = [05]; + bytearray = [06]; + longbytearray = [09 0a 0b 0c 0d 0e 0f 10 11]; + stringval = "message"; + stringarray = "multi-word", "message"; + }; + + spl-test2 { + u-boot,dm-pre-reloc; + compatible = "sandbox,spl-test"; + intval = <3>; + intarray = <5>; + byteval = [08]; + bytearray = [01 23 34]; + longbytearray = [09 0a 0b 0c]; + stringval = "message2"; + stringarray = "another", "multi-word", "message"; + }; + + spl-test3 { + u-boot,dm-pre-reloc; + compatible = "sandbox,spl-test"; + stringarray = "one"; + }; + + spl-test4 { + u-boot,dm-pre-reloc; + compatible = "sandbox,spl-test.2"; + }; + + square { + compatible = "demo-shape"; + colour = "blue"; + sides = <4>; + }; + + timer { + compatible = "sandbox,timer"; + clock-frequency = <1000000>; + }; + + tpm { + u-boot,dm-pre-reloc; + compatible = "google,sandbox-tpm"; + }; + + tpm2 { + compatible = "sandbox,tpm2"; + }; + + triangle { + compatible = "demo-shape"; + colour = "cyan"; + sides = <3>; + character = <83>; + light-gpios = <&gpio_a 2>, <&gpio_b 6 0>; + }; + + /* Needs to be available prior to relocation */ + uart0: serial { + u-boot,dm-spl; + compatible = "sandbox,serial"; + sandbox,text-colour = "cyan"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_serial0>; + }; + + usb@0 { + compatible = "sandbox,usb"; + status = "disabled"; + hub { + compatible = "sandbox,usb-hub"; + #address-cells = <1>; + #size-cells = <0>; + flash-stick { + reg = <0>; + compatible = "sandbox,usb-flash"; + }; + }; + }; + + usb@1 { + compatible = "sandbox,usb"; + hub { + compatible = "usb-hub"; + usb,device-class = ; + hub-emul { + compatible = "sandbox,usb-hub"; + #address-cells = <1>; + #size-cells = <0>; + flash-stick { + reg = <0>; + compatible = "sandbox,usb-flash"; + sandbox,filepath = "flash.bin"; + }; + }; + }; + }; + + usb@2 { + compatible = "sandbox,usb"; + status = "disabled"; + }; + + spmi: spmi@0 { + compatible = "sandbox,spmi"; + #address-cells = <0x1>; + #size-cells = <0x1>; + pm8916@0 { + compatible = "qcom,spmi-pmic"; + reg = <0x0 0x1>; + #address-cells = <0x1>; + #size-cells = <0x1>; + + spmi_gpios: gpios@c000 { + compatible = "qcom,pm8916-gpio"; + reg = <0xc000 0x400>; + gpio-controller; + gpio-count = <4>; + #gpio-cells = <2>; + gpio-bank-name="spmi"; + }; + }; + }; + + axi: axi@0 { + compatible = "sandbox,axi"; + #address-cells = <0x1>; + #size-cells = <0x1>; + store@0 { + compatible = "sandbox,sandbox_store"; + reg = <0x0 0x400>; + }; + }; + + onewire0: onewire { + compatible = "w1-gpio"; + gpios = <&gpio_a 8>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_onewire0>; + status = "okay"; + + sandbox_eeprom0: sandbox_eeprom@0 { + compatible = "sandbox,w1-eeprom"; + status = "okay"; + }; + }; + + sandbox_tee { + compatible = "sandbox,tee"; + }; +}; + +&cros_ec { + /* + * This describes the flash memory within the EC. Note + * that the STM32L flash erases to 0, not 0xff. + */ + flash { + image-pos = <0x08000000>; + size = <0x20000>; + erase-value = <0>; + + /* Information for sandbox */ + ro { + image-pos = <0>; + size = <0xf000>; + }; + wp-ro { + image-pos = <0xf000>; + size = <0x1000>; + }; + rw { + image-pos = <0x10000>; + size = <0x10000>; + }; + }; + + keyboard-controller { + u-boot,dm-pre-reloc; + }; +}; diff --git a/arch/sandbox/dts/sandbox64.dts b/arch/sandbox/dts/sandbox64.dts index e5b4426426..a263f78c95 100644 --- a/arch/sandbox/dts/sandbox64.dts +++ b/arch/sandbox/dts/sandbox64.dts @@ -1,7 +1,5 @@ /dts-v1/; -#define USB_CLASS_HUB 9 - / { #address-cells = <2>; #size-cells = <2>; @@ -14,44 +12,10 @@ axi0 = &axi; }; - chosen { - stdout-path = "/serial"; - }; - - audio: audio-codec { - compatible = "sandbox,audio-codec"; - #sound-dai-cells = <1>; - }; - cros_ec: cros-ec { reg = <0 0 0 0>; u-boot,dm-pre-reloc; compatible = "google,cros-ec-sandbox"; - - /* - * This describes the flash memory within the EC. Note - * that the STM32L flash erases to 0, not 0xff. - */ - flash { - u-boot,dm-pre-reloc; - image-pos = <0x08000000>; - size = <0x20000>; - erase-value = <0>; - - /* Information for sandbox */ - ro { - image-pos = <0>; - size = <0xf000>; - }; - wp-ro { - image-pos = <0xf000>; - size = <0x1000>; - }; - rw { - image-pos = <0x10000>; - size = <0x10000>; - }; - }; }; ethrawbus { @@ -65,30 +29,6 @@ fake-host-hwaddr = [00 00 66 44 22 00]; }; - gpio_a: gpios@0 { - u-boot,dm-pre-reloc; - gpio-controller; - compatible = "sandbox,gpio"; - #gpio-cells = <1>; - gpio-bank-name = "a"; - sandbox,gpio-count = <20>; - }; - - gpio_b: gpios@1 { - u-boot,dm-pre-reloc; - gpio-controller; - compatible = "sandbox,gpio"; - #gpio-cells = <2>; - gpio-bank-name = "b"; - sandbox,gpio-count = <10>; - }; - - hexagon { - compatible = "demo-simple"; - colour = "white"; - sides = <6>; - }; - i2c_0: i2c@0 { #address-cells = <1>; #size-cells = <0>; @@ -97,63 +37,6 @@ clock-frequency = <400000>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; - eeprom@2c { - reg = <0x2c>; - compatible = "i2c-eeprom"; - sandbox,emul = <&emul_eeprom>; - }; - - rtc_0: rtc@43 { - reg = <0x43>; - compatible = "sandbox-rtc"; - sandbox,emul = <&emul0>; - }; - sandbox_pmic: sandbox_pmic { - reg = <0x40>; - }; - - mc34708: pmic@41 { - reg = <0x41>; - }; - - i2c_emul: emul { - reg = <0xff>; - compatible = "sandbox,i2c-emul-parent"; - emul_eeprom: emul-eeprom { - compatible = "sandbox,i2c-eeprom"; - sandbox,filename = "i2c.bin"; - sandbox,size = <256>; - }; - emul0: emul0 { - compatible = "sandbox,i2c-rtc"; - }; - }; - }; - - i2s: i2s { - compatible = "sandbox,i2s"; - #sound-dai-cells = <1>; - }; - - lcd { - u-boot,dm-pre-reloc; - compatible = "sandbox,lcd-sdl"; - xres = <1366>; - yres = <768>; - }; - - leds { - compatible = "gpio-leds"; - - iracibble { - gpios = <&gpio_a 1 0>; - label = "sandbox:red"; - }; - - martinet { - gpios = <&gpio_a 2 0>; - label = "sandbox:green"; - }; }; pci: pci-controller { @@ -163,50 +46,6 @@ #size-cells = <2>; ranges = <0x02000000 0 0x10000000 0 0x10000000 0 0x2000 0x01000000 0 0x20000000 0 0x20000000 0 0x2000>; - pci@1f,0 { - compatible = "pci-generic"; - reg = <0xf800 0 0 0 0>; - emul@1f,0 { - compatible = "sandbox,swap-case"; - }; - }; - }; - - pinctrl { - compatible = "sandbox,pinctrl"; - status = "okay"; - - pinctrl_i2c0: i2c0 { - groups = "i2c"; - function = "i2c"; - bias-pull-up; - }; - - pinctrl_serial0: uart0 { - groups = "serial_a"; - function = "serial"; - }; - - pinctrl_onewire0: onewire0 { - groups = "w1"; - function = "w1"; - bias-pull-up; - }; - }; - - reset@1 { - compatible = "sandbox,reset"; - }; - - sound { - compatible = "sandbox,sound"; - cpu { - sound-dai = <&i2s 0>; - }; - - codec { - sound-dai = <&audio 0>; - }; }; spi@0 { @@ -216,180 +55,10 @@ reg = <0 0 0 0>; compatible = "sandbox,spi"; cs-gpios = <0>, <&gpio_a 0>; - firmware_storage_spi: flash@0 { - u-boot,dm-pre-reloc; - reg = <0>; - compatible = "spansion,m25p16", "sandbox,spi-flash"; - spi-max-frequency = <40000000>; - sandbox,filename = "spi.bin"; - }; }; - spl-test { - u-boot,dm-pre-reloc; - compatible = "sandbox,spl-test"; - boolval; - intval = <1>; - intarray = <2 3 4>; - byteval = [05]; - bytearray = [06]; - longbytearray = [09 0a 0b 0c 0d 0e 0f 10 11]; - stringval = "message"; - stringarray = "multi-word", "message"; - }; - - spl-test2 { - u-boot,dm-pre-reloc; - compatible = "sandbox,spl-test"; - intval = <3>; - intarray = <5>; - byteval = [08]; - bytearray = [01 23 34]; - longbytearray = [09 0a 0b 0c]; - stringval = "message2"; - stringarray = "another", "multi-word", "message"; - }; - - spl-test3 { - u-boot,dm-pre-reloc; - compatible = "sandbox,spl-test"; - stringarray = "one"; - }; - - spl-test4 { - u-boot,dm-pre-reloc; - compatible = "sandbox,spl-test.2"; - }; - - square { - compatible = "demo-shape"; - colour = "blue"; - sides = <4>; - }; - - timer { - compatible = "sandbox,timer"; - clock-frequency = <1000000>; - }; - - tpm { - u-boot,dm-pre-reloc; - compatible = "google,sandbox-tpm"; - }; - - tpm2 { - compatible = "sandbox,tpm2"; - }; - - triangle { - compatible = "demo-shape"; - colour = "cyan"; - sides = <3>; - character = <83>; - light-gpios = <&gpio_a 2>, <&gpio_b 6 0>; - }; - - /* Needs to be available prior to relocation */ - uart0: serial { - u-boot,dm-spl; - compatible = "sandbox,serial"; - sandbox,text-colour = "cyan"; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_serial0>; - }; - - usb@0 { - compatible = "sandbox,usb"; - status = "disabled"; - hub { - compatible = "sandbox,usb-hub"; - #address-cells = <1>; - #size-cells = <0>; - flash-stick { - reg = <0>; - compatible = "sandbox,usb-flash"; - }; - }; - }; - - usb@1 { - compatible = "sandbox,usb"; - hub { - compatible = "usb-hub"; - usb,device-class = ; - hub-emul { - compatible = "sandbox,usb-hub"; - #address-cells = <1>; - #size-cells = <0>; - flash-stick { - reg = <0>; - compatible = "sandbox,usb-flash"; - sandbox,filepath = "flash.bin"; - }; - }; - }; - }; - - usb@2 { - compatible = "sandbox,usb"; - status = "disabled"; - }; - - spmi: spmi@0 { - compatible = "sandbox,spmi"; - #address-cells = <0x1>; - #size-cells = <0x1>; - pm8916@0 { - compatible = "qcom,spmi-pmic"; - reg = <0x0 0x1>; - #address-cells = <0x1>; - #size-cells = <0x1>; - - spmi_gpios: gpios@c000 { - compatible = "qcom,pm8916-gpio"; - reg = <0xc000 0x400>; - gpio-controller; - gpio-count = <4>; - #gpio-cells = <2>; - gpio-bank-name="spmi"; - }; - }; - }; - - axi: axi@0 { - compatible = "sandbox,axi"; - #address-cells = <0x1>; - #size-cells = <0x1>; - store@0 { - compatible = "sandbox,sandbox_store"; - reg = <0x0 0x400>; - }; - }; - - onewire0: onewire { - compatible = "w1-gpio"; - gpios = <&gpio_a 8>; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_onewire0>; - status = "okay"; - - sandbox_eeprom0: sandbox_eeprom@0 { - compatible = "sandbox,w1-eeprom"; - status = "okay"; - }; - }; - - sandbox_tee { - compatible = "sandbox,tee"; - }; }; +#include "sandbox.dtsi" #include "cros-ec-keyboard.dtsi" #include "sandbox_pmic.dtsi" - -&cros_ec { - u-boot,dm-pre-reloc; - keyboard-controller { - u-boot,dm-pre-reloc; - }; -}; From 87dac6b0749961032185833b760e1c31fa73c64c Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:42 -0600 Subject: [PATCH 48/74] sandbox: Add an alias for SPI At present 'sf probe' does not work since it cannot find SPI bus 0. Add an alias to correct this, now that we no-longer have the --spi_sf option. Signed-off-by: Simon Glass --- arch/sandbox/dts/sandbox.dts | 3 ++- arch/sandbox/dts/sandbox64.dts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/sandbox/dts/sandbox.dts b/arch/sandbox/dts/sandbox.dts index c0bdba94a7..9a949791a0 100644 --- a/arch/sandbox/dts/sandbox.dts +++ b/arch/sandbox/dts/sandbox.dts @@ -10,6 +10,7 @@ pci0 = &pci; rtc0 = &rtc_0; axi0 = &axi; + spi0 = &spi; }; cros_ec: cros-ec { @@ -48,7 +49,7 @@ 0x01000000 0 0x20000000 0x20000000 0 0x2000>; }; - spi@0 { + spi: spi@0 { u-boot,dm-pre-reloc; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/sandbox/dts/sandbox64.dts b/arch/sandbox/dts/sandbox64.dts index a263f78c95..da2b4423fd 100644 --- a/arch/sandbox/dts/sandbox64.dts +++ b/arch/sandbox/dts/sandbox64.dts @@ -10,6 +10,7 @@ pci0 = &pci; rtc0 = &rtc_0; axi0 = &axi; + spi0 = &spi; }; cros_ec: cros-ec { @@ -48,7 +49,7 @@ 0x01000000 0 0x20000000 0 0x20000000 0 0x2000>; }; - spi@0 { + spi: spi@0 { u-boot,dm-pre-reloc; #address-cells = <1>; #size-cells = <0>; From 9072326a6ab47ad87a0be46f4d8619b60c2d8515 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:43 -0600 Subject: [PATCH 49/74] sandbox: Exit when SYSRESET_POWER_OFF is requested At present this returns but it seems better to just exit sandbox immediately. Signed-off-by: Simon Glass reset --- arch/sandbox/cpu/state.c | 1 + drivers/sysreset/sysreset_sandbox.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/sandbox/cpu/state.c b/arch/sandbox/cpu/state.c index d3b9c05985..2333240fda 100644 --- a/arch/sandbox/cpu/state.c +++ b/arch/sandbox/cpu/state.c @@ -356,6 +356,7 @@ void state_reset_for_test(struct sandbox_state *state) /* No reset yet, so mark it as such. Always allow power reset */ state->last_sysreset = SYSRESET_COUNT; state->sysreset_allowed[SYSRESET_POWER] = true; + state->sysreset_allowed[SYSRESET_POWER_OFF] = true; memset(&state->wdt, '\0', sizeof(state->wdt)); memset(state->spi, '\0', sizeof(state->spi)); diff --git a/drivers/sysreset/sysreset_sandbox.c b/drivers/sysreset/sysreset_sandbox.c index 38e2a7e241..7dfd89460f 100644 --- a/drivers/sysreset/sysreset_sandbox.c +++ b/drivers/sysreset/sysreset_sandbox.c @@ -66,6 +66,7 @@ static int sandbox_sysreset_request(struct udevice *dev, enum sysreset_t type) case SYSRESET_POWER_OFF: if (!state->sysreset_allowed[type]) return -EACCES; + sandbox_exit(); default: return -ENOSYS; } From f2980ece06363e584541829bf40d46ad9760d535 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:44 -0600 Subject: [PATCH 50/74] sandbox: Quit when hang() is called It doesn't make a lot of sense to hang on sandbox when hang() is called, since the only way out is Ctrl-C. In fact, Ctrl-C does not work if the terminal is in raw mode, which it will be if the command-line has not been reached yet. In that case, Ctrl-Z / kill -9 must be used, which is not very friendly. Avoid all of this by quiting when hang() is called. Signed-off-by: Simon Glass --- lib/hang.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/hang.c b/lib/hang.c index c5a78694be..4d026a3e64 100644 --- a/lib/hang.c +++ b/lib/hang.c @@ -9,6 +9,7 @@ #include #include +#include /** * hang - stop processing by staying in an endless loop @@ -26,6 +27,8 @@ void hang(void) puts("### ERROR ### Please RESET the board ###\n"); #endif bootstage_error(BOOTSTAGE_ID_NEED_RESET); + if (IS_ENABLED(CONFIG_SANDBOX)) + os_exit(1); for (;;) ; } From 3f2f5cf62a45656c78b5317a5cbfbb077a95fc19 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:45 -0600 Subject: [PATCH 51/74] sandbox: spl: Lower priority of standard loader We normally want to load U-Boot from SPL, but if a board wants to do something else, it is currently not possible since the standard loader has the top priority. Lower it to allow other SPL_LOAD_IMAGE_METHOD() declarations to override it. Signed-off-by: Simon Glass --- arch/sandbox/cpu/spl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sandbox/cpu/spl.c b/arch/sandbox/cpu/spl.c index 2ca4cd6e35..106a78ac1c 100644 --- a/arch/sandbox/cpu/spl.c +++ b/arch/sandbox/cpu/spl.c @@ -44,7 +44,7 @@ static int spl_board_load_image(struct spl_image_info *spl_image, return 0; } -SPL_LOAD_IMAGE_METHOD("sandbox", 0, BOOT_DEVICE_BOARD, spl_board_load_image); +SPL_LOAD_IMAGE_METHOD("sandbox", 9, BOOT_DEVICE_BOARD, spl_board_load_image); void spl_board_init(void) { From 912dcb1b178e165a54d7c0a8d9e36afc9d46d3af Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:46 -0600 Subject: [PATCH 52/74] sandbox: Add a comment to board_init_f() This function is used for both SPL and TPL. Add a comment to that effect. Signed-off-by: Simon Glass --- arch/sandbox/cpu/spl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sandbox/cpu/spl.c b/arch/sandbox/cpu/spl.c index 106a78ac1c..4f415c71d6 100644 --- a/arch/sandbox/cpu/spl.c +++ b/arch/sandbox/cpu/spl.c @@ -12,6 +12,7 @@ DECLARE_GLOBAL_DATA_PTR; +/* SPL / TPL init function */ void board_init_f(ulong flag) { struct sandbox_state *state = state_get_current(); From e8a7b3051b50e0ea1422b5bc5983539ee17a6b55 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:47 -0600 Subject: [PATCH 53/74] sandbox: Allo sdl-config to be overridden When cross-compiling, sometimes sdl-config must come from a different path from the default. Add a way to override it, by adding SDL_CONFIG to the environment before building U-Boot. Signed-off-by: Simon Glass --- arch/sandbox/config.mk | 5 +++-- board/sandbox/README.sandbox | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/sandbox/config.mk b/arch/sandbox/config.mk index 31a12db103..05fbbd7bcc 100644 --- a/arch/sandbox/config.mk +++ b/arch/sandbox/config.mk @@ -5,14 +5,15 @@ PLATFORM_CPPFLAGS += -D__SANDBOX__ -U_FORTIFY_SOURCE PLATFORM_CPPFLAGS += -DCONFIG_ARCH_MAP_SYSMEM PLATFORM_CPPFLAGS += -fPIC PLATFORM_LIBS += -lrt +SDL_CONFIG ?= sdl-config # Define this to avoid linking with SDL, which requires SDL libraries # This can solve 'sdl-config: Command not found' errors ifneq ($(NO_SDL),) PLATFORM_CPPFLAGS += -DSANDBOX_NO_SDL else -PLATFORM_LIBS += $(shell sdl-config --libs) -PLATFORM_CPPFLAGS += $(shell sdl-config --cflags) +PLATFORM_LIBS += $(shell $(SDL_CONFIG) --libs) +PLATFORM_CPPFLAGS += $(shell $(SDL_CONFIG) --cflags) endif cmd_u-boot__ = $(CC) -o $@ -Wl,-T u-boot.lds $(u-boot-init) \ diff --git a/board/sandbox/README.sandbox b/board/sandbox/README.sandbox index 48c1e2b9e7..5e8fa58ba2 100644 --- a/board/sandbox/README.sandbox +++ b/board/sandbox/README.sandbox @@ -435,6 +435,13 @@ board_init_f() and board_init_r(). This approach can be used on normal boards as well as sandbox. +SDL_CONFIG +---------- + +If sdl-config is on a different path from the default, set the SDL_CONFIG +environment variable to the correct pathname before building U-Boot. + + Testing ------- From e676f4397c019840e2b0d7330506a857c6dc5a81 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:48 -0600 Subject: [PATCH 54/74] sandbox: Add a memory {} node Add a memory node which indicates the size of sandbox memory. Signed-off-by: Simon Glass --- arch/sandbox/dts/sandbox.dts | 6 ++++++ arch/sandbox/dts/sandbox64.dts | 6 ++++++ include/configs/sandbox.h | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/arch/sandbox/dts/sandbox.dts b/arch/sandbox/dts/sandbox.dts index 9a949791a0..16a33db87d 100644 --- a/arch/sandbox/dts/sandbox.dts +++ b/arch/sandbox/dts/sandbox.dts @@ -1,5 +1,7 @@ /dts-v1/; +#include + / { #address-cells = <1>; #size-cells = <1>; @@ -13,6 +15,10 @@ spi0 = &spi; }; + memory { + reg = <0 CONFIG_SYS_SDRAM_SIZE>; + }; + cros_ec: cros-ec { reg = <0 0>; u-boot,dm-pre-reloc; diff --git a/arch/sandbox/dts/sandbox64.dts b/arch/sandbox/dts/sandbox64.dts index da2b4423fd..37a5539ff4 100644 --- a/arch/sandbox/dts/sandbox64.dts +++ b/arch/sandbox/dts/sandbox64.dts @@ -1,5 +1,7 @@ /dts-v1/; +#include + / { #address-cells = <2>; #size-cells = <2>; @@ -13,6 +15,10 @@ spi0 = &spi; }; + memory { + reg = /bits/ 64 <0 CONFIG_SYS_SDRAM_SIZE>; + }; + cros_ec: cros-ec { reg = <0 0 0 0>; u-boot,dm-pre-reloc; diff --git a/include/configs/sandbox.h b/include/configs/sandbox.h index 50affaf1a8..5d75021ed6 100644 --- a/include/configs/sandbox.h +++ b/include/configs/sandbox.h @@ -63,7 +63,11 @@ func(HOST, host, 1) \ func(HOST, host, 0) +#ifdef __ASSEMBLY__ +#define BOOTENV +#else #include +#endif #define CONFIG_KEEP_SERVERADDR #define CONFIG_UDP_CHECKSUM From 24c2776ba9002fadbf7cf86a0f5bc6a17cc08290 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:49 -0600 Subject: [PATCH 55/74] sandbox: Correct spi flash operation Since the SPI nor conversion, 'sf probe' does not work on sandbox. Fix this by using the expected compatible string in the flash node. Signed-off-by: Simon Glass Fixes: cd35365762 (mtd: sf_probe: remove spi-flash compatible) --- arch/sandbox/dts/sandbox.dtsi | 2 +- drivers/spi/Kconfig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sandbox/dts/sandbox.dtsi b/arch/sandbox/dts/sandbox.dtsi index ebc4ece824..2fb365d86e 100644 --- a/arch/sandbox/dts/sandbox.dtsi +++ b/arch/sandbox/dts/sandbox.dtsi @@ -150,7 +150,7 @@ firmware_storage_spi: flash@0 { u-boot,dm-pre-reloc; reg = <0>; - compatible = "spansion,m25p16", "sandbox,spi-flash"; + compatible = "spansion,m25p16", "jedec,spi-nor"; spi-max-frequency = <40000000>; sandbox,filename = "spi.bin"; }; diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index f9b282313a..8e1424ba42 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -226,7 +226,7 @@ config SANDBOX_SPI cs-gpios = <0>, <&gpio_a 0>; flash@0 { reg = <0>; - compatible = "spansion,m25p16", "sandbox,spi-flash"; + compatible = "spansion,m25p16", "jedec,spi-nor"; spi-max-frequency = <40000000>; sandbox,filename = "spi.bin"; }; From 80b7cb8c3230bb29b065633c803694d8720ecadf Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:50 -0600 Subject: [PATCH 56/74] sandbox: Add documentation on how to run valgrind U-Boot sandbox can run with valgrind to check memory allocations. Add documentation on how to do this. Signed-off-by: Simon Glass --- board/sandbox/README.sandbox | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/board/sandbox/README.sandbox b/board/sandbox/README.sandbox index 5e8fa58ba2..c10dd444df 100644 --- a/board/sandbox/README.sandbox +++ b/board/sandbox/README.sandbox @@ -442,6 +442,20 @@ If sdl-config is on a different path from the default, set the SDL_CONFIG environment variable to the correct pathname before building U-Boot. +Using valgrind / memcheck +------------------------- + +It is possible to run U-Boot under valgrind to check memory allocations: + + valgrind u-boot + +If you are running sandbox SPL or TPL, then valgrind will not by default +notice when U-Boot jumps from TPL to SPL, or from SPL to U-Boot proper. To +fix this, use: + + valgrind --trace-children=yes u-boot + + Testing ------- From 44c6a0ed6e5a37b1d4c63d2351213cf736979858 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:51 -0600 Subject: [PATCH 57/74] x86: Add a forward struct declaration in coreboot_tables.h This struct is not defined in this header file. Add a forward declaration so that it can be included in any context. Signed-off-by: Simon Glass Reviewed-by: Bin Meng --- arch/x86/include/asm/coreboot_tables.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/include/asm/coreboot_tables.h b/arch/x86/include/asm/coreboot_tables.h index c42175b94d..2c54e24e02 100644 --- a/arch/x86/include/asm/coreboot_tables.h +++ b/arch/x86/include/asm/coreboot_tables.h @@ -8,6 +8,8 @@ #ifndef _COREBOOT_TABLES_H #define _COREBOOT_TABLES_H +struct memory_area; + struct cbuint64 { u32 lo; u32 hi; From 59d0452df36c92015eb68163519f1581e68d6392 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:52 -0600 Subject: [PATCH 58/74] bootstage: Add support for TPL record count If bootstage is enabled in TPL it lacks a record count and so does not build. Fix this by adding a new Kconfig option. Signed-off-by: Simon Glass --- common/Kconfig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/Kconfig b/common/Kconfig index af66496e75..4865a4dfc8 100644 --- a/common/Kconfig +++ b/common/Kconfig @@ -69,6 +69,13 @@ config SPL_BOOTSTAGE_RECORD_COUNT This is the size of the bootstage record list and is the maximum number of bootstage records that can be recorded. +config TPL_BOOTSTAGE_RECORD_COUNT + int "Number of boot stage records to store for TPL" + default 5 + help + This is the size of the bootstage record list and is the maximum + number of bootstage records that can be recorded. + config BOOTSTAGE_FDT bool "Store boot timing information in the OS device tree" depends on BOOTSTAGE From 731ba3c0beaf4318cc97ece1e422a61fc5721f5b Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:53 -0600 Subject: [PATCH 59/74] blk: Allow control of the block cache in TPL Generally block devices are not enabled in TPL, but in case they are, add a Kconfig option for the block cache. This allows the setting (default off) to be found with CONFIG_IS_ENABLED(). Signed-off-by: Simon Glass --- drivers/block/Kconfig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index 33f4aa2418..1e6dad8692 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -56,6 +56,13 @@ config SPL_BLOCK_CACHE help This option enables the disk-block cache in SPL +config TPL_BLOCK_CACHE + bool "Use block device cache in TPL" + depends on TPL_BLK + default n + help + This option enables the disk-block cache in TPL + config IDE bool "Support IDE controllers" select HAVE_BLOCK_DEVICE From ca2abb75a086d8b0bdb51689f331ba8f1a146379 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 18 May 2019 11:59:54 -0600 Subject: [PATCH 60/74] spi: Avoid using malloc() in a critical function In general we should avoid calling malloc() and free() repeatedly in U-Boot lest we turn it into tianocore. In SPL this can make SPI flash unusable since free() is often a nop and allocation space is limited. In any case, these seems no need for malloc() since the number of bytes is very small, perhaps less than 8. Signed-off-by: Simon Glass Fixes: d13f5b254a (spi: Extend the core to ease integration of SPI memory controllers) --- drivers/spi/spi-mem.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c index b86eee75bc..7aabebeff5 100644 --- a/drivers/spi/spi-mem.c +++ b/drivers/spi/spi-mem.c @@ -201,7 +201,6 @@ int spi_mem_exec_op(struct spi_slave *slave, const struct spi_mem_op *op) unsigned int pos = 0; const u8 *tx_buf = NULL; u8 *rx_buf = NULL; - u8 *op_buf; int op_len; u32 flag; int ret; @@ -338,7 +337,17 @@ int spi_mem_exec_op(struct spi_slave *slave, const struct spi_mem_op *op) } op_len = sizeof(op->cmd.opcode) + op->addr.nbytes + op->dummy.nbytes; - op_buf = calloc(1, op_len); + + /* + * Avoid using malloc() here so that we can use this code in SPL where + * simple malloc may be used. That implementation does not allow free() + * so repeated calls to this code can exhaust the space. + * + * The value of op_len is small, since it does not include the actual + * data being sent, only the op-code and address. In fact, it should be + * possible to just use a small fixed value here instead of op_len. + */ + u8 op_buf[op_len]; op_buf[pos++] = op->cmd.opcode; @@ -382,8 +391,6 @@ int spi_mem_exec_op(struct spi_slave *slave, const struct spi_mem_op *op) debug("%02x ", tx_buf ? tx_buf[i] : rx_buf[i]); debug("[ret %d]\n", ret); - free(op_buf); - if (ret < 0) return ret; #endif /* __UBOOT__ */ From 1edc5688df70ee3ddb692ef3429ac28ed8666a4b Mon Sep 17 00:00:00 2001 From: Vabhav Sharma Date: Thu, 31 Jan 2019 12:08:10 +0000 Subject: [PATCH 61/74] drivers: serial: lpuart: Enable Little Endian Support By default LPUART driver with compatible string "fsl,ls1021a-lpuart" support big-endian mode.On NXP SoC like LS1028A LPUART IP is little-endian,Added support to Fetch LPUART IP Endianness from lpuart device-tree node. Signed-off-by: Vabhav Sharma Reviewed-by: Simon Glass --- drivers/serial/serial_lpuart.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/serial/serial_lpuart.c b/drivers/serial/serial_lpuart.c index a357b00d28..57dd4a72c6 100644 --- a/drivers/serial/serial_lpuart.c +++ b/drivers/serial/serial_lpuart.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ /* + * Copyright 2019 NXP * Copyright 2013 Freescale Semiconductor, Inc. */ @@ -502,6 +503,9 @@ static int lpuart_serial_ofdata_to_platdata(struct udevice *dev) plat->reg = (void *)addr; plat->flags = dev_get_driver_data(dev); + if (fdtdec_get_bool(blob, node, "little-endian")) + plat->flags &= ~LPUART_FLAG_REGMAP_ENDIAN_BIG; + if (!fdt_node_check_compatible(blob, node, "fsl,ls1021a-lpuart")) plat->devtype = DEV_LS1021A; else if (!fdt_node_check_compatible(blob, node, "fsl,imx7ulp-lpuart")) From 848e94d0ba7296d81e3c00b01bad1c13741497fb Mon Sep 17 00:00:00 2001 From: Thierry Reding Date: Mon, 20 May 2019 18:05:04 +0200 Subject: [PATCH 62/74] fdtdec: test: Fix memory leak Free the memory allocated to store the test FDT upon test completion to avoid leaking the memory. We don't bother cleaning up on test failure since the code is broken in that case and should be fixed, in which case the leak would also go away. Reported-by: Tom Rini Suggested-by: Heinrich Schuchardt Signed-off-by: Thierry Reding Reviewed-by: Simon Glass --- lib/fdtdec_test.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/fdtdec_test.c b/lib/fdtdec_test.c index 1f4f270540..e8bfd1fb1e 100644 --- a/lib/fdtdec_test.c +++ b/lib/fdtdec_test.c @@ -138,6 +138,7 @@ static int run_test(const char *aliases, const char *nodes, const char *expect) } printf("pass\n"); + free(blob); return 0; } @@ -292,6 +293,7 @@ static int check_carveout(void) CHECKVAL(make_fdt_carveout(fdt, FDT_SIZE, 2, 2), 0); CHECKOK(check_fdt_carveout(fdt, 2, 2)); + free(fdt); return 0; } From 099ed45c6c6e88eb2783589027b610f2b9a2f71f Mon Sep 17 00:00:00 2001 From: Patrick Delaunay Date: Tue, 21 May 2019 19:19:11 +0200 Subject: [PATCH 63/74] test: check u-boot properties in SPL device tree Add a test to check the management of the U-boot relocation properties for device tree SPL generation (fdtgrep result) and platdata: - 'dm-pre-proper' and 'dm-tpl' not included in SPL - 'dm-pre-reloc' and 'dm-spl' included in SPL Signed-off-by: Patrick Delaunay Reviewed-by: Simon Glass --- arch/sandbox/dts/sandbox.dtsi | 18 ++++++++++++++++++ test/py/tests/test_ofplatdata.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/arch/sandbox/dts/sandbox.dtsi b/arch/sandbox/dts/sandbox.dtsi index 2fb365d86e..c6d5650c20 100644 --- a/arch/sandbox/dts/sandbox.dtsi +++ b/arch/sandbox/dts/sandbox.dtsi @@ -192,6 +192,24 @@ compatible = "sandbox,spl-test.2"; }; + spl-test5 { + u-boot,dm-tpl; + compatible = "sandbox,spl-test"; + stringarray = "tpl"; + }; + + spl-test6 { + u-boot,dm-pre-proper; + compatible = "sandbox,spl-test"; + stringarray = "pre-proper"; + }; + + spl-test7 { + u-boot,dm-spl; + compatible = "sandbox,spl-test"; + stringarray = "spl"; + }; + square { compatible = "demo-shape"; colour = "blue"; diff --git a/test/py/tests/test_ofplatdata.py b/test/py/tests/test_ofplatdata.py index 98103ee71a..263334b074 100644 --- a/test/py/tests/test_ofplatdata.py +++ b/test/py/tests/test_ofplatdata.py @@ -2,6 +2,7 @@ # Copyright (c) 2016 Google, Inc import pytest +import u_boot_utils as util OF_PLATDATA_OUTPUT = ''' of-platdata probe: @@ -31,6 +32,15 @@ intarray 0 0 0 0 longbytearray 00 00 00 00 00 00 00 00 00 string stringarray "one" "" "" +of-platdata probe: +bool 0 +byte 00 +bytearray 00 00 00 +int 0 +intarray 0 0 0 0 +longbytearray 00 00 00 00 00 00 00 00 00 +string +stringarray "spl" "" "" ''' @pytest.mark.buildconfigspec('spl_of_platdata') @@ -40,3 +50,21 @@ def test_ofplatdata(u_boot_console): cons.restart_uboot_with_flags(['--show_of_platdata']) output = cons.get_spawn_output().replace('\r', '') assert OF_PLATDATA_OUTPUT in output + +@pytest.mark.buildconfigspec('spl_of_platdata') +def test_spl_devicetree(u_boot_console): + """Test content of spl device-tree""" + cons = u_boot_console + dtb = cons.config.build_dir + '/spl/u-boot-spl.dtb' + fdtgrep = cons.config.build_dir + '/tools/fdtgrep' + output = util.run_and_log(cons, [fdtgrep, '-l', dtb]) + + assert "u-boot,dm-pre-reloc" not in output + assert "u-boot,dm-pre-proper" not in output + assert "u-boot,dm-spl" not in output + assert "u-boot,dm-tpl" not in output + + assert "spl-test4" in output + assert "spl-test5" not in output + assert "spl-test6" not in output + assert "spl-test7" in output From 6998974926df9b41b299cd9f1aaff37649ca9971 Mon Sep 17 00:00:00 2001 From: Patrick Delaunay Date: Tue, 21 May 2019 19:19:12 +0200 Subject: [PATCH 64/74] fdt: Allow indicating a node is for U-Boot proper only This add missing parts for previous commit 06f94461a9f4 ("fdt: Allow indicating a node is for U-Boot proper only") At present it is not possible to specify that a node should be used before relocation (in U-Boot proper) without it also ending up in SPL and TPL device trees. Add a new "u-boot,dm-pre-proper" boolean property for this. Signed-off-by: Patrick Delaunay Reviewed-by: Simon Glass --- drivers/core/util.c | 2 ++ drivers/video/video-uclass.c | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/core/util.c b/drivers/core/util.c index 96e47dc707..60b939a924 100644 --- a/drivers/core/util.c +++ b/drivers/core/util.c @@ -42,6 +42,8 @@ bool dm_ofnode_pre_reloc(ofnode node) #else if (ofnode_read_bool(node, "u-boot,dm-pre-reloc")) return true; + if (ofnode_read_bool(node, "u-boot,dm-pre-proper")) + return true; /* * In regular builds individual spl and tpl handling both diff --git a/drivers/video/video-uclass.c b/drivers/video/video-uclass.c index b19bfb4f2f..d4071c0661 100644 --- a/drivers/video/video-uclass.c +++ b/drivers/video/video-uclass.c @@ -291,7 +291,9 @@ static int video_post_bind(struct udevice *dev) return 0; size = alloc_fb(dev, &addr); if (addr < gd->video_bottom) { - /* Device tree node may need the 'u-boot,dm-pre-reloc' tag */ + /* Device tree node may need the 'u-boot,dm-pre-reloc' or + * 'u-boot,dm-pre-proper' tag + */ printf("Video device '%s' cannot allocate frame buffer memory -ensure the device is set up before relocation\n", dev->name); return -ENOSPC; From 54e1223aa4ca5d65fe3c6497e92bee4453aeddac Mon Sep 17 00:00:00 2001 From: Patrick Delaunay Date: Tue, 21 May 2019 19:19:13 +0200 Subject: [PATCH 65/74] dm: doc: add documentation for pre-reloc properties in SPL and TPL Add documentation for the pre-reloc property in SPL and TPL device-tree: - u-boot,dm-pre-proper - u-boot,dm-pre-reloc - u-boot,dm-spl - u-boot,dm-tpl Signed-off-by: Patrick Delaunay Reviewed-by: Simon Glass --- doc/README.SPL | 16 ++++++++++++++++ doc/README.TPL | 4 ++++ doc/driver-model/README.txt | 4 ++++ include/dm/ofnode.h | 6 ++++-- include/dm/util.h | 6 ++++-- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/doc/README.SPL b/doc/README.SPL index 7a30fefa96..6eed83f8c5 100644 --- a/doc/README.SPL +++ b/doc/README.SPL @@ -66,6 +66,22 @@ CONFIG_SPL_SPI_LOAD (drivers/mtd/spi/spi_spl_load.o) CONFIG_SPL_RAM_DEVICE (common/spl/spl.c) CONFIG_SPL_WATCHDOG_SUPPORT (drivers/watchdog/libwatchdog.o) +Device tree +----------- +The U-Boot device tree is filtered by the fdtgrep tools during the build +process to generate a much smaller device tree used in SPL (spl/u-boot-spl.dtb) +with: +- the mandatory nodes (/alias, /chosen, /config) +- the nodes with one pre-relocation property: + 'u-boot,dm-pre-reloc' or 'u-boot,dm-spl' + +ftgrep is also used to remove: +- the properties defined in CONFIG_OF_SPL_REMOVE_PROPS +- all the pre-relocation properties + ('u-boot,dm-pre-reloc', 'u-boot,dm-spl' and 'u-boot,dm-tpl') + +All the nodes remaining in the SPL devicetree are bound +(see driver-model/README.txt). Debugging --------- diff --git a/doc/README.TPL b/doc/README.TPL index 980debe777..c94129fdc9 100644 --- a/doc/README.TPL +++ b/doc/README.TPL @@ -34,6 +34,10 @@ determine which SPL options to choose based on whether CONFIG_TPL_BUILD is set. Source files can be compiled for TPL with options choosed in the board config file. +TPL use a small device tree (u-boot-tpl.dtb), containing only the nodes with +the pre-relocation properties: 'u-boot,dm-pre-reloc' and 'u-boot,dm-tpl' +(see README.SPL for details). + For example: spl/Makefile: diff --git a/doc/driver-model/README.txt b/doc/driver-model/README.txt index 07b120d512..532a771f68 100644 --- a/doc/driver-model/README.txt +++ b/doc/driver-model/README.txt @@ -849,6 +849,10 @@ in the device tree node. For U-Boot proper you can use 'u-boot,dm-pre-proper' which means that it will be processed (and a driver bound) in U-Boot proper prior to relocation, but will not be available in SPL or TPL. +To reduce the size of SPL and TPL, only the nodes with pre-relocation properties +('u-boot,dm-pre-reloc', 'u-boot,dm-spl' or 'u-boot,dm-tpl') are keept in their +device trees (see README.SPL for details); the remaining nodes are always bound. + Then post relocation we throw that away and re-init driver model again. For drivers which require some sort of continuity between pre- and post-relocation devices, we can provide access to the pre-relocation diff --git a/include/dm/ofnode.h b/include/dm/ofnode.h index 4ab2ae1ba5..704f91589a 100644 --- a/include/dm/ofnode.h +++ b/include/dm/ofnode.h @@ -676,12 +676,14 @@ int ofnode_read_simple_size_cells(ofnode node); * After relocation and jumping into the real U-Boot binary it is possible to * determine if a node was bound in one of SPL/TPL stages. * - * There are 3 settings currently in use - * - + * There are 4 settings currently in use + * - u-boot,dm-pre-proper: U-Boot proper pre-relocation only * - u-boot,dm-pre-reloc: legacy and indicates any of TPL or SPL * Existing platforms only use it to indicate nodes needed in * SPL. Should probably be replaced by u-boot,dm-spl for * new platforms. + * - u-boot,dm-spl: SPL and U-Boot pre-relocation + * - u-boot,dm-tpl: TPL and U-Boot pre-relocation * * @node: node to check * @return true if node is needed in SPL/TL, false otherwise diff --git a/include/dm/util.h b/include/dm/util.h index 60d3b93dec..348c2ace3c 100644 --- a/include/dm/util.h +++ b/include/dm/util.h @@ -52,12 +52,14 @@ static inline void dm_dump_devres(void) * it is possible to determine if a node was bound in one of * SPL/TPL stages. * - * There are 3 settings currently in use - * - + * There are 4 settings currently in use + * - u-boot,dm-pre-proper: U-Boot proper pre-relocation only * - u-boot,dm-pre-reloc: legacy and indicates any of TPL or SPL * Existing platforms only use it to indicate nodes needed in * SPL. Should probably be replaced by u-boot,dm-spl for * existing platforms. + * - u-boot,dm-spl: SPL and U-Boot pre-relocation + * - u-boot,dm-tpl: TPL and U-Boot pre-relocation * @node: of node * * Returns true if node is needed in SPL/TL, false otherwise. From 7ef8e9b09a74567c529f168a4da5c1d5cffd58c2 Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Sun, 2 Jun 2019 00:53:24 +0200 Subject: [PATCH 66/74] trace: undefined reference to `trace_early_init' Compiling with TRACE but without TRACE_EARLY results in an error aarch64-linux-gnu-ld.bfd: common/built-in.o:(.rodata.init_sequence_f+0x10): undefined reference to `trace_early_init' trace_early_init() should not be called if CONFIG_TRACE_EARLY is not defined. Signed-off-by: Heinrich Schuchardt Reviewed-by: Simon Glass --- common/board_f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/board_f.c b/common/board_f.c index c25eb188fb..0fc938dc87 100644 --- a/common/board_f.c +++ b/common/board_f.c @@ -839,7 +839,7 @@ static const init_fnc_t init_sequence_f[] = { #ifdef CONFIG_OF_CONTROL fdtdec_setup, #endif -#ifdef CONFIG_TRACE +#ifdef CONFIG_TRACE_EARLY trace_early_init, #endif initf_malloc, From a2fa38da200cbd454bcb8dc7659bde00044302f7 Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Sun, 2 Jun 2019 13:05:08 +0200 Subject: [PATCH 67/74] trace: conserve gd register An UEFI application may change the value of the register that gd lives in. But some of our functions like get_ticks() access this register. So we have to set the gd register to the U-Boot value when entering a trace point and set it back to the application value when exiting the trace point. Signed-off-by: Heinrich Schuchardt Reviewed-by: Simon Glass --- lib/trace.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/lib/trace.c b/lib/trace.c index 9956442fef..a13e5bf228 100644 --- a/lib/trace.c +++ b/lib/trace.c @@ -56,6 +56,49 @@ static inline uintptr_t __attribute__((no_instrument_function)) return offset / FUNC_SITE_SIZE; } +#ifdef CONFIG_EFI_LOADER + +/** + * trace_gd - the value of the gd register + */ +static volatile void *trace_gd; + +/** + * trace_save_gd() - save the value of the gd register + */ +static void __attribute__((no_instrument_function)) trace_save_gd(void) +{ + trace_gd = gd; +} + +/** + * trace_swap_gd() - swap between U-Boot and application gd register value + * + * An UEFI application may change the value of the register that gd lives in. + * But some of our functions like get_ticks() access this register. So we + * have to set the gd register to the U-Boot value when entering a trace + * point and set it back to the application value when exiting the trace point. + */ +static void __attribute__((no_instrument_function)) trace_swap_gd(void) +{ + volatile void *temp_gd = trace_gd; + + trace_gd = gd; + gd = temp_gd; +} + +#else + +static void __attribute__((no_instrument_function)) trace_save_gd(void) +{ +} + +static void __attribute__((no_instrument_function)) trace_swap_gd(void) +{ +} + +#endif + static void __attribute__((no_instrument_function)) add_ftrace(void *func_ptr, void *caller, ulong flags) { @@ -100,6 +143,7 @@ void __attribute__((no_instrument_function)) __cyg_profile_func_enter( if (trace_enabled) { int func; + trace_swap_gd(); add_ftrace(func_ptr, caller, FUNCF_ENTRY); func = func_ptr_to_num(func_ptr); if (func < hdr->func_count) { @@ -111,6 +155,7 @@ void __attribute__((no_instrument_function)) __cyg_profile_func_enter( hdr->depth++; if (hdr->depth > hdr->depth_limit) hdr->max_depth = hdr->depth; + trace_swap_gd(); } } @@ -126,8 +171,10 @@ void __attribute__((no_instrument_function)) __cyg_profile_func_exit( void *func_ptr, void *caller) { if (trace_enabled) { + trace_swap_gd(); add_ftrace(func_ptr, caller, FUNCF_EXIT); hdr->depth--; + trace_swap_gd(); } } @@ -284,6 +331,8 @@ int __attribute__((no_instrument_function)) trace_init(void *buff, size_t needed; int was_disabled = !trace_enabled; + trace_save_gd(); + if (!was_disabled) { #ifdef CONFIG_TRACE_EARLY char *end; From da0fb5fdd430abef136bc5e1636f8d1855cf0008 Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Sun, 2 Jun 2019 13:30:09 +0200 Subject: [PATCH 68/74] trace: make call depth limit customizable Up to now we had hard coded values for the call depth up to which trace records are created: 200 for early tracing, 15 thereafter. UEFI applications reach a call depth of 80 or above. Provide customizing settings for the call trace depth limit and the early call trace depth limit. Use the old values as defaults. Signed-off-by: Heinrich Schuchardt Reviewed-by: Simon Glass --- lib/Kconfig | 15 +++++++++++++++ lib/trace.c | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/Kconfig b/lib/Kconfig index 416e63c1c7..e717eb3de5 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -192,6 +192,13 @@ config TRACE_BUFFER_SIZE the size is too small then 'trace stats' will show a message saying how many records were dropped due to buffer overflow. +config TRACE_CALL_DEPTH_LIMIT + int "Trace call depth limit" + depends on TRACE + default 15 + help + Sets the maximum call depth up to which function calls are recorded. + config TRACE_EARLY bool "Enable tracing before relocation" depends on TRACE @@ -209,6 +216,14 @@ config TRACE_EARLY_SIZE Sets the size of the early trace buffer in bytes. This is used to hold tracing information before relocation. +config TRACE_EARLY_CALL_DEPTH_LIMIT + int "Early trace call depth limit" + depends on TRACE_EARLY + default 200 + help + Sets the maximum call depth up to which function calls are recorded + during early tracing. + config TRACE_EARLY_ADDR hex "Address of early trace buffer in U-Boot" depends on TRACE_EARLY diff --git a/lib/trace.c b/lib/trace.c index a13e5bf228..04780f59d3 100644 --- a/lib/trace.c +++ b/lib/trace.c @@ -376,7 +376,7 @@ int __attribute__((no_instrument_function)) trace_init(void *buff, add_textbase(); puts("trace: enabled\n"); - hdr->depth_limit = 15; + hdr->depth_limit = CONFIG_TRACE_CALL_DEPTH_LIMIT; trace_enabled = 1; trace_inited = 1; @@ -410,7 +410,7 @@ int __attribute__((no_instrument_function)) trace_early_init(void) hdr->ftrace = (struct trace_call *)((char *)hdr + needed); hdr->ftrace_size = (buff_size - needed) / sizeof(*hdr->ftrace); add_textbase(); - hdr->depth_limit = 200; + hdr->depth_limit = CONFIG_TRACE_EARLY_CALL_DEPTH_LIMIT; printf("trace: early enable at %08x\n", CONFIG_TRACE_EARLY_ADDR); trace_enabled = 1; From 2204bc1bd3abedc37a72282b4ddcfb3eca8cfccb Mon Sep 17 00:00:00 2001 From: Alex Marginean Date: Fri, 7 Jun 2019 11:24:22 +0300 Subject: [PATCH 69/74] pci: fixed dm_pci_map_bar comment The comment now indicates that the input argument bar is a register offset, not a BAR index. It also mentions which BARs are supported for type 0/1 and that the function can return 0 on error. Signed-off-by: Alex Marginean Reviewed-by: Bin Meng --- include/pci.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/pci.h b/include/pci.h index 508f7bca81..40c7751acf 100644 --- a/include/pci.h +++ b/include/pci.h @@ -1309,12 +1309,14 @@ pci_addr_t dm_pci_phys_to_bus(struct udevice *dev, phys_addr_t addr, * dm_pci_map_bar() - get a virtual address associated with a BAR region * * Looks up a base address register and finds the physical memory address - * that corresponds to it + * that corresponds to it. + * Can be used for 32b BARs 0-5 on type 0 functions and for 32b BARs 0-1 on + * type 1 functions. * * @dev: Device to check - * @bar: Bar number to read (numbered from 0) + * @bar: Bar register offset (PCI_BASE_ADDRESS_...) * @flags: Flags for the region type (PCI_REGION_...) - * @return: pointer to the virtual address to use + * @return: pointer to the virtual address to use or 0 on error */ void *dm_pci_map_bar(struct udevice *dev, int bar, int flags); From 0b143d8ab2b84219552d652e46619360a38888d1 Mon Sep 17 00:00:00 2001 From: Alex Marginean Date: Fri, 7 Jun 2019 11:24:23 +0300 Subject: [PATCH 70/74] drivers: pci: add map_bar support for Enhanced Allocation Makes dm_pci_map_bar API available for integrated PCI devices that support Enhanced Allocation instead of the original PCI BAR mechanism. Signed-off-by: Alex Marginean Reviewed-by: Bin Meng --- drivers/pci/pci-uclass.c | 46 ++++++++++++++++++++++++++++++++++++++++ include/pci.h | 13 ++++++++++++ 2 files changed, 59 insertions(+) diff --git a/drivers/pci/pci-uclass.c b/drivers/pci/pci-uclass.c index cf1e7617ae..389aec15ce 100644 --- a/drivers/pci/pci-uclass.c +++ b/drivers/pci/pci-uclass.c @@ -1341,10 +1341,56 @@ pci_addr_t dm_pci_phys_to_bus(struct udevice *dev, phys_addr_t phys_addr, return bus_addr; } +static void *dm_pci_map_ea_bar(struct udevice *dev, int bar, int flags, + int ea_off) +{ + int ea_cnt, i, entry_size; + int bar_id = (bar - PCI_BASE_ADDRESS_0) >> 2; + u32 ea_entry; + phys_addr_t addr; + + /* EA capability structure header */ + dm_pci_read_config32(dev, ea_off, &ea_entry); + ea_cnt = (ea_entry >> 16) & PCI_EA_NUM_ENT_MASK; + ea_off += PCI_EA_FIRST_ENT; + + for (i = 0; i < ea_cnt; i++, ea_off += entry_size) { + /* Entry header */ + dm_pci_read_config32(dev, ea_off, &ea_entry); + entry_size = ((ea_entry & PCI_EA_ES) + 1) << 2; + + if (((ea_entry & PCI_EA_BEI) >> 4) != bar_id) + continue; + + /* Base address, 1st DW */ + dm_pci_read_config32(dev, ea_off + 4, &ea_entry); + addr = ea_entry & PCI_EA_FIELD_MASK; + if (ea_entry & PCI_EA_IS_64) { + /* Base address, 2nd DW, skip over 4B MaxOffset */ + dm_pci_read_config32(dev, ea_off + 12, &ea_entry); + addr |= ((u64)ea_entry) << 32; + } + + /* size ignored for now */ + return map_physmem(addr, flags, 0); + } + + return 0; +} + void *dm_pci_map_bar(struct udevice *dev, int bar, int flags) { pci_addr_t pci_bus_addr; u32 bar_response; + int ea_off; + + /* + * if the function supports Enhanced Allocation use that instead of + * BARs + */ + ea_off = dm_pci_find_capability(dev, PCI_CAP_ID_EA); + if (ea_off) + return dm_pci_map_ea_bar(dev, bar, flags, ea_off); /* read BAR address */ dm_pci_read_config32(dev, bar, &bar_response); diff --git a/include/pci.h b/include/pci.h index 40c7751acf..0aab438159 100644 --- a/include/pci.h +++ b/include/pci.h @@ -455,6 +455,17 @@ #define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */ #define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_PTM +/* Enhanced Allocation Registers */ +#define PCI_EA_NUM_ENT 2 /* Number of Capability Entries */ +#define PCI_EA_NUM_ENT_MASK 0x3f /* Num Entries Mask */ +#define PCI_EA_FIRST_ENT 4 /* First EA Entry in List */ +#define PCI_EA_ES 0x00000007 /* Entry Size */ +#define PCI_EA_BEI 0x000000f0 /* BAR Equivalent Indicator */ +/* Base, MaxOffset registers */ +/* bit 0 is reserved */ +#define PCI_EA_IS_64 0x00000002 /* 64-bit field flag */ +#define PCI_EA_FIELD_MASK 0xfffffffc /* For Base & Max Offset */ + /* Include the ID list */ #include @@ -1312,6 +1323,8 @@ pci_addr_t dm_pci_phys_to_bus(struct udevice *dev, phys_addr_t addr, * that corresponds to it. * Can be used for 32b BARs 0-5 on type 0 functions and for 32b BARs 0-1 on * type 1 functions. + * Can also be used on type 0 functions that support Enhanced Allocation for + * 32b/64b BARs. Note that duplicate BEI entries are not supported. * * @dev: Device to check * @bar: Bar register offset (PCI_BASE_ADDRESS_...) From 21ebbafde8dbbc5903a302ba0c126ff2ac1423c8 Mon Sep 17 00:00:00 2001 From: Alex Marginean Date: Fri, 7 Jun 2019 11:24:24 +0300 Subject: [PATCH 71/74] test: dm: Add a test for PCI Enhanced Allocation This test is built on top of the existing swap_case driver. It adds EA capability structure support to swap_case and uses that to map BARs. BAR1 works as it used to, swapping upper/lower case. BARs 2,4 map to a couple of magic values. Signed-off-by: Alex Marginean Reviewed-by: Bin Meng Tested-by: Bin Meng --- arch/sandbox/dts/test.dts | 8 +++ arch/sandbox/include/asm/test.h | 13 ++++ drivers/misc/swap_case.c | 102 +++++++++++++++++++++++++++++++- test/dm/pci.c | 49 +++++++++++++++ 4 files changed, 171 insertions(+), 1 deletion(-) diff --git a/arch/sandbox/dts/test.dts b/arch/sandbox/dts/test.dts index c328258901..ee0f631b29 100644 --- a/arch/sandbox/dts/test.dts +++ b/arch/sandbox/dts/test.dts @@ -446,6 +446,14 @@ compatible = "sandbox,swap-case"; }; }; + pci@1,0 { + compatible = "pci-generic"; + reg = <0x0800 0 0 0 0>; + emul@0,0 { + compatible = "sandbox,swap-case"; + use-ea; + }; + }; pci@1f,0 { compatible = "pci-generic"; reg = <0xf800 0 0 0 0>; diff --git a/arch/sandbox/include/asm/test.h b/arch/sandbox/include/asm/test.h index e956a05262..32125f3037 100644 --- a/arch/sandbox/include/asm/test.h +++ b/arch/sandbox/include/asm/test.h @@ -19,6 +19,7 @@ #define PCI_CAP_ID_PM_OFFSET 0x50 #define PCI_CAP_ID_EXP_OFFSET 0x60 #define PCI_CAP_ID_MSIX_OFFSET 0x70 +#define PCI_CAP_ID_EA_OFFSET 0x80 #define PCI_EXT_CAP_ID_ERR_OFFSET 0x100 #define PCI_EXT_CAP_ID_VC_OFFSET 0x200 @@ -30,6 +31,18 @@ #define SANDBOX_CLK_RATE 32768 +/* Macros used to test PCI EA capability structure */ +#define PCI_CAP_EA_BASE_LO0 0x00100000 +#define PCI_CAP_EA_BASE_LO1 0x00110000 +#define PCI_CAP_EA_BASE_LO2 0x00120000 +#define PCI_CAP_EA_BASE_LO4 0x00140000 +#define PCI_CAP_EA_BASE_HI2 0x00020000ULL +#define PCI_CAP_EA_BASE_HI4 0x00040000ULL +#define PCI_CAP_EA_SIZE_LO 0x0000ffff +#define PCI_CAP_EA_SIZE_HI 0x00000010ULL +#define PCI_EA_BAR2_MAGIC 0x72727272 +#define PCI_EA_BAR4_MAGIC 0x74747474 + /* System controller driver data */ enum { SYSCON0 = 32, diff --git a/drivers/misc/swap_case.c b/drivers/misc/swap_case.c index fa608cec1b..6afc6d9466 100644 --- a/drivers/misc/swap_case.c +++ b/drivers/misc/swap_case.c @@ -61,11 +61,63 @@ static int sandbox_swap_case_get_devfn(struct udevice *dev) return plat->devfn; } +static int sandbox_swap_case_use_ea(struct udevice *dev) +{ + return !!ofnode_get_property(dev->node, "use-ea", NULL); +} + +/* Please keep these macros in sync with ea_regs below */ +#define PCI_CAP_ID_EA_SIZE (sizeof(ea_regs) + 4) +#define PCI_CAP_ID_EA_ENTRY_CNT 4 +/* Hardcoded EA structure, excluding 1st DW. */ +static const u32 ea_regs[] = { + /* BEI=0, ES=2, BAR0 32b Base + 32b MaxOffset, I/O space */ + (2 << 8) | 2, + PCI_CAP_EA_BASE_LO0, + 0, + /* BEI=1, ES=2, BAR1 32b Base + 32b MaxOffset */ + (1 << 4) | 2, + PCI_CAP_EA_BASE_LO1, + MEM_TEXT_SIZE - 1, + /* BEI=2, ES=3, BAR2 64b Base + 32b MaxOffset */ + (2 << 4) | 3, + PCI_CAP_EA_BASE_LO2 | PCI_EA_IS_64, + PCI_CAP_EA_SIZE_LO, + PCI_CAP_EA_BASE_HI2, + /* BEI=4, ES=4, BAR4 64b Base + 64b MaxOffset */ + (4 << 4) | 4, + PCI_CAP_EA_BASE_LO4 | PCI_EA_IS_64, + PCI_CAP_EA_SIZE_LO | PCI_EA_IS_64, + PCI_CAP_EA_BASE_HI4, + PCI_CAP_EA_SIZE_HI, +}; + +static int sandbox_swap_case_read_ea(struct udevice *emul, uint offset, + ulong *valuep, enum pci_size_t size) +{ + u32 reg; + + offset = offset - PCI_CAP_ID_EA_OFFSET - 4; + reg = ea_regs[offset >> 2]; + reg >>= (offset % 4) * 8; + + *valuep = reg; + return 0; +} + static int sandbox_swap_case_read_config(struct udevice *emul, uint offset, ulong *valuep, enum pci_size_t size) { struct swap_case_platdata *plat = dev_get_platdata(emul); + /* + * The content of the EA capability structure is handled elsewhere to + * keep the switch/case below sane + */ + if (offset > PCI_CAP_ID_EA_OFFSET + PCI_CAP_LIST_NEXT && + offset < PCI_CAP_ID_EA_OFFSET + PCI_CAP_ID_EA_SIZE) + return sandbox_swap_case_read_ea(emul, offset, valuep, size); + switch (offset) { case PCI_COMMAND: *valuep = plat->command; @@ -134,9 +186,21 @@ static int sandbox_swap_case_read_config(struct udevice *emul, uint offset, *valuep = PCI_CAP_ID_MSIX_OFFSET; break; case PCI_CAP_ID_MSIX_OFFSET: - *valuep = PCI_CAP_ID_MSIX; + if (sandbox_swap_case_use_ea(emul)) + *valuep = (PCI_CAP_ID_EA_OFFSET << 8) | PCI_CAP_ID_MSIX; + else + *valuep = PCI_CAP_ID_MSIX; break; case PCI_CAP_ID_MSIX_OFFSET + PCI_CAP_LIST_NEXT: + if (sandbox_swap_case_use_ea(emul)) + *valuep = PCI_CAP_ID_EA_OFFSET; + else + *valuep = 0; + break; + case PCI_CAP_ID_EA_OFFSET: + *valuep = (PCI_CAP_ID_EA_ENTRY_CNT << 16) | PCI_CAP_ID_EA; + break; + case PCI_CAP_ID_EA_OFFSET + PCI_CAP_LIST_NEXT: *valuep = 0; break; case PCI_EXT_CAP_ID_ERR_OFFSET: @@ -257,6 +321,9 @@ int sandbox_swap_case_write_io(struct udevice *dev, unsigned int addr, return 0; } +static int pci_ea_bar2_magic = PCI_EA_BAR2_MAGIC; +static int pci_ea_bar4_magic = PCI_EA_BAR4_MAGIC; + static int sandbox_swap_case_map_physmem(struct udevice *dev, phys_addr_t addr, unsigned long *lenp, void **ptrp) { @@ -265,9 +332,42 @@ static int sandbox_swap_case_map_physmem(struct udevice *dev, int barnum; int ret; + if (sandbox_swap_case_use_ea(dev)) { + /* + * only support mapping base address in EA test for now, we + * don't handle mapping an offset inside a BAR. Seems good + * enough for the current test. + */ + switch (addr) { + case (phys_addr_t)PCI_CAP_EA_BASE_LO0: + *ptrp = &priv->op; + *lenp = 4; + break; + case (phys_addr_t)PCI_CAP_EA_BASE_LO1: + *ptrp = priv->mem_text; + *lenp = barinfo[1].size - 1; + break; + case (phys_addr_t)((PCI_CAP_EA_BASE_HI2 << 32) | + PCI_CAP_EA_BASE_LO2): + *ptrp = &pci_ea_bar2_magic; + *lenp = PCI_CAP_EA_SIZE_LO; + break; + case (phys_addr_t)((PCI_CAP_EA_BASE_HI4 << 32) | + PCI_CAP_EA_BASE_LO4): + *ptrp = &pci_ea_bar4_magic; + *lenp = (PCI_CAP_EA_SIZE_HI << 32) | + PCI_CAP_EA_SIZE_LO; + break; + default: + return -ENOENT; + } + return 0; + } + ret = sandbox_swap_case_find_bar(dev, addr, &barnum, &offset); if (ret) return ret; + if (barnum == 1) { *ptrp = priv->mem_text + offset; avail = barinfo[1].size - offset; diff --git a/test/dm/pci.c b/test/dm/pci.c index a1febd54b7..c325f6600e 100644 --- a/test/dm/pci.c +++ b/test/dm/pci.c @@ -245,3 +245,52 @@ static int dm_test_pci_cap(struct unit_test_state *uts) return 0; } DM_TEST(dm_test_pci_cap, DM_TESTF_SCAN_PDATA | DM_TESTF_SCAN_FDT); + +/* Test looking up BARs in EA capability structure */ +static int dm_test_pci_ea(struct unit_test_state *uts) +{ + struct udevice *bus, *swap; + void *bar; + int cap; + + /* + * use emulated device mapping function, we're not using real physical + * addresses in this test + */ + sandbox_set_enable_pci_map(true); + + ut_assertok(uclass_get_device_by_seq(UCLASS_PCI, 0, &bus)); + ut_assertok(dm_pci_bus_find_bdf(PCI_BDF(0, 0x01, 0), &swap)); + + /* look up PCI_CAP_ID_EA */ + cap = dm_pci_find_capability(swap, PCI_CAP_ID_EA); + ut_asserteq(PCI_CAP_ID_EA_OFFSET, cap); + + /* test swap case in BAR 1 */ + bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_0, 0); + ut_assertnonnull(bar); + *(int *)bar = 2; /* swap upper/lower */ + + bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_1, 0); + ut_assertnonnull(bar); + strcpy(bar, "ea TEST"); + unmap_sysmem(bar); + bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_1, 0); + ut_assertnonnull(bar); + ut_asserteq_str("EA test", bar); + + /* test magic values in BARs2, 4; BAR 3 is n/a */ + bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_2, 0); + ut_assertnonnull(bar); + ut_asserteq(PCI_EA_BAR2_MAGIC, *(u32 *)bar); + + bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_3, 0); + ut_assertnull(bar); + + bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_4, 0); + ut_assertnonnull(bar); + ut_asserteq(PCI_EA_BAR4_MAGIC, *(u32 *)bar); + + return 0; +} +DM_TEST(dm_test_pci_ea, DM_TESTF_SCAN_PDATA | DM_TESTF_SCAN_FDT); From b8e1f8270ca9667e84eaa4049e006c27e37f3cca Mon Sep 17 00:00:00 2001 From: Alex Marginean Date: Fri, 7 Jun 2019 11:24:25 +0300 Subject: [PATCH 72/74] drivers: pci: add API to issue FLR on a PCI function if supported Adds dm_pci_flr API that issues a Function Level reset on a PCI-e function, if FLR is supported. Signed-off-by: Alex Marginean Reviewed-by: Bin Meng --- drivers/pci/pci-uclass.c | 24 ++++++++++++++++++++++++ include/pci.h | 14 ++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/drivers/pci/pci-uclass.c b/drivers/pci/pci-uclass.c index 389aec15ce..c74ebf6a76 100644 --- a/drivers/pci/pci-uclass.c +++ b/drivers/pci/pci-uclass.c @@ -1494,6 +1494,30 @@ int dm_pci_find_ext_capability(struct udevice *dev, int cap) return dm_pci_find_next_ext_capability(dev, 0, cap); } +int dm_pci_flr(struct udevice *dev) +{ + int pcie_off; + u32 cap; + + /* look for PCI Express Capability */ + pcie_off = dm_pci_find_capability(dev, PCI_CAP_ID_EXP); + if (!pcie_off) + return -ENOENT; + + /* check FLR capability */ + dm_pci_read_config32(dev, pcie_off + PCI_EXP_DEVCAP, &cap); + if (!(cap & PCI_EXP_DEVCAP_FLR)) + return -ENOENT; + + dm_pci_clrset_config16(dev, pcie_off + PCI_EXP_DEVCTL, 0, + PCI_EXP_DEVCTL_BCR_FLR); + + /* wait 100ms, per PCI spec */ + mdelay(100); + + return 0; +} + UCLASS_DRIVER(pci) = { .id = UCLASS_PCI, .name = "pci", diff --git a/include/pci.h b/include/pci.h index 0aab438159..298d0d4355 100644 --- a/include/pci.h +++ b/include/pci.h @@ -466,6 +466,12 @@ #define PCI_EA_IS_64 0x00000002 /* 64-bit field flag */ #define PCI_EA_FIELD_MASK 0xfffffffc /* For Base & Max Offset */ +/* PCI Express capabilities */ +#define PCI_EXP_DEVCAP 4 /* Device capabilities */ +#define PCI_EXP_DEVCAP_FLR 0x10000000 /* Function Level Reset */ +#define PCI_EXP_DEVCTL 8 /* Device Control */ +#define PCI_EXP_DEVCTL_BCR_FLR 0x8000 /* Bridge Configuration Retry / FLR */ + /* Include the ID list */ #include @@ -1426,6 +1432,14 @@ int dm_pci_find_next_ext_capability(struct udevice *dev, int start, int cap); */ int dm_pci_find_ext_capability(struct udevice *dev, int cap); +/** + * dm_pci_flr() - Perform FLR if the device suppoorts it + * + * @dev: PCI device to reset + * @return: 0 if OK, -ENOENT if FLR is not supported by dev + */ +int dm_pci_flr(struct udevice *dev); + #define dm_pci_virt_to_bus(dev, addr, flags) \ dm_pci_phys_to_bus(dev, (virt_to_phys(addr)), (flags)) #define dm_pci_bus_to_virt(dev, addr, flags, len, map_flags) \ From 2b7a3882e0f70d253fb6a2da5682c94aea4c46f1 Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Fri, 14 Jun 2019 21:50:55 +0200 Subject: [PATCH 73/74] trace: do not limit trace buffer to 2GiB There is no good reason to limit the trace buffer to 2GiB on a 64bit system. Adjust the types of the relevant parameters. Signed-off-by: Heinrich Schuchardt Reviewed-by: Simon Glass --- cmd/trace.c | 10 ++++------ include/trace.h | 6 +++--- lib/trace.c | 14 +++++++------- tools/proftool.c | 4 ++-- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/cmd/trace.c b/cmd/trace.c index 26bf0960d4..7d328f88be 100644 --- a/cmd/trace.c +++ b/cmd/trace.c @@ -30,8 +30,7 @@ static int get_args(int argc, char * const argv[], char **buff, static int create_func_list(int argc, char * const argv[]) { - size_t buff_size, avail, buff_ptr, used; - unsigned int needed; + size_t buff_size, avail, buff_ptr, needed, used; char *buff; int err; @@ -41,7 +40,7 @@ static int create_func_list(int argc, char * const argv[]) avail = buff_size - buff_ptr; err = trace_list_functions(buff + buff_ptr, avail, &needed); if (err) - printf("Error: truncated (%#x bytes needed)\n", needed); + printf("Error: truncated (%#zx bytes needed)\n", needed); used = min(avail, (size_t)needed); printf("Function trace dumped to %08lx, size %#zx\n", (ulong)map_to_sysmem(buff + buff_ptr), used); @@ -54,8 +53,7 @@ static int create_func_list(int argc, char * const argv[]) static int create_call_list(int argc, char * const argv[]) { - size_t buff_size, avail, buff_ptr, used; - unsigned int needed; + size_t buff_size, avail, buff_ptr, needed, used; char *buff; int err; @@ -65,7 +63,7 @@ static int create_call_list(int argc, char * const argv[]) avail = buff_size - buff_ptr; err = trace_list_calls(buff + buff_ptr, avail, &needed); if (err) - printf("Error: truncated (%#x bytes needed)\n", needed); + printf("Error: truncated (%#zx bytes needed)\n", needed); used = min(avail, (size_t)needed); printf("Call list dumped to %08lx, size %#zx\n", (ulong)map_to_sysmem(buff + buff_ptr), used); diff --git a/include/trace.h b/include/trace.h index 99f34f72bc..606dba9768 100644 --- a/include/trace.h +++ b/include/trace.h @@ -39,7 +39,7 @@ struct trace_output_func { /* A header at the start of the trace output buffer */ struct trace_output_hdr { enum trace_chunk_type type; /* Record type */ - uint32_t rec_count; /* Number of records */ + size_t rec_count; /* Number of records */ }; /* Print statistics about traced function calls */ @@ -57,7 +57,7 @@ void trace_print_stats(void); * @param needed Returns number of bytes used / needed * @return 0 if ok, -1 on error (buffer exhausted) */ -int trace_list_functions(void *buff, int buff_size, unsigned *needed); +int trace_list_functions(void *buff, size_t buff_size, size_t *needed); /* Flags for ftrace_record */ enum ftrace_flags { @@ -77,7 +77,7 @@ struct trace_call { uint32_t flags; /* Flags and timestamp */ }; -int trace_list_calls(void *buff, int buff_size, unsigned int *needed); +int trace_list_calls(void *buff, size_t buff_size, size_t *needed); /** * Turn function tracing on and off diff --git a/lib/trace.c b/lib/trace.c index 04780f59d3..f2402b9359 100644 --- a/lib/trace.c +++ b/lib/trace.c @@ -190,12 +190,12 @@ void __attribute__((no_instrument_function)) __cyg_profile_func_exit( * greater than buff_size if we ran out of space. * @return 0 if ok, -1 if space was exhausted */ -int trace_list_functions(void *buff, int buff_size, unsigned int *needed) +int trace_list_functions(void *buff, size_t buff_size, size_t *needed) { struct trace_output_hdr *output_hdr = NULL; void *end, *ptr = buff; - int func; - int upto; + size_t func; + size_t upto; end = buff ? buff + buff_size : NULL; @@ -206,7 +206,7 @@ int trace_list_functions(void *buff, int buff_size, unsigned int *needed) /* Add information about each function */ for (func = upto = 0; func < hdr->func_count; func++) { - int calls = hdr->call_accum[func]; + size_t calls = hdr->call_accum[func]; if (!calls) continue; @@ -235,12 +235,12 @@ int trace_list_functions(void *buff, int buff_size, unsigned int *needed) return 0; } -int trace_list_calls(void *buff, int buff_size, unsigned *needed) +int trace_list_calls(void *buff, size_t buff_size, size_t *needed) { struct trace_output_hdr *output_hdr = NULL; void *end, *ptr = buff; - int rec, upto; - int count; + size_t rec, upto; + size_t count; end = buff ? buff + buff_size : NULL; diff --git a/tools/proftool.c b/tools/proftool.c index c1803fa78a..fecb9d6e99 100644 --- a/tools/proftool.c +++ b/tools/proftool.c @@ -205,12 +205,12 @@ static struct func_info *find_caller_by_offset(uint32_t offset) return low >= 0 ? &func_list[low] : NULL; } -static int read_calls(FILE *fin, int count) +static int read_calls(FILE *fin, size_t count) { struct trace_call *call_data; int i; - notice("call count: %d\n", count); + notice("call count: %zu\n", count); call_list = (struct trace_call *)calloc(count, sizeof(*call_data)); if (!call_list) { error("Cannot allocate call_list\n"); From 7ea33579576d2bcd19df76bd8769e7ab3b4a169b Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Fri, 14 Jun 2019 21:52:22 +0200 Subject: [PATCH 74/74] trace: trace buffer may exceed 2GiB Correct the debug output for the trace buffer size to accommodate trace buffers exceeding 2GiB. Signed-off-by: Heinrich Schuchardt Reviewed-by: Simon Glass --- common/board_f.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/board_f.c b/common/board_f.c index 0fc938dc87..4760d728f3 100644 --- a/common/board_f.c +++ b/common/board_f.c @@ -442,8 +442,8 @@ static int reserve_trace(void) #ifdef CONFIG_TRACE gd->relocaddr -= CONFIG_TRACE_BUFFER_SIZE; gd->trace_buff = map_sysmem(gd->relocaddr, CONFIG_TRACE_BUFFER_SIZE); - debug("Reserving %dk for trace data at: %08lx\n", - CONFIG_TRACE_BUFFER_SIZE >> 10, gd->relocaddr); + debug("Reserving %luk for trace data at: %08lx\n", + (unsigned long)CONFIG_TRACE_BUFFER_SIZE >> 10, gd->relocaddr); #endif return 0;