From a8b309ce9760943486e0585285e0125588a31650 Mon Sep 17 00:00:00 2001 From: Thomas Bracht Laumann Jespersen Date: Mon, 31 Jan 2022 12:20:28 +0100 Subject: scripts/dtc: Call pkg-config POSIXly correct Running with POSIXLY_CORRECT=1 in the environment the scripts/dtc build fails, because pkg-config doesn't output anything when the flags come after the arguments. Fixes: 067c650c456e ("dtc: Use pkg-config to locate libyaml") Signed-off-by: Thomas Bracht Laumann Jespersen Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20220131112028.7907-1-t@laumann.xyz --- scripts/dtc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/dtc/Makefile b/scripts/dtc/Makefile index 95aaf7431bff..1cba78e1dce6 100644 --- a/scripts/dtc/Makefile +++ b/scripts/dtc/Makefile @@ -29,7 +29,7 @@ dtc-objs += yamltree.o # To include installed in a non-default path HOSTCFLAGS_yamltree.o := $(shell pkg-config --cflags yaml-0.1) # To link libyaml installed in a non-default path -HOSTLDLIBS_dtc := $(shell pkg-config yaml-0.1 --libs) +HOSTLDLIBS_dtc := $(shell pkg-config --libs yaml-0.1) endif # Generated files need one more search path to include headers in source tree -- cgit v1.2.3 From 8e4296c286ed7a3b69e619d5069493bdcc9a2b25 Mon Sep 17 00:00:00 2001 From: Frank Rowand Date: Tue, 1 Feb 2022 12:14:13 -0600 Subject: of: unittest: add program to process EXPECT messages If unittest detects a problem it will print a warning or error message to the console. Unittest also triggers warning and error messages from other kernel code as a result of intentionally bad unittest data. This has led to confusion as to whether the triggered messages are an expected result of a test or whether there is a real problem that is independent of unittest. EXPECT messages were added to unittest to report each triggered message that is expected, resulting in verbose console output. scripts/dtc/of_unittest is a new program that processes the EXPECT messages to determine whether the triggered messages occurred and also removes the excess verbosity of the EXPECT messages. More information is available from 'scripts/dtc/of_unittest_expect --help'. Signed-off-by: Frank Rowand Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20220201181413.2719955-1-frowand.list@gmail.com --- scripts/dtc/of_unittest_expect | 408 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100755 scripts/dtc/of_unittest_expect (limited to 'scripts') diff --git a/scripts/dtc/of_unittest_expect b/scripts/dtc/of_unittest_expect new file mode 100755 index 000000000000..96b12d9ea606 --- /dev/null +++ b/scripts/dtc/of_unittest_expect @@ -0,0 +1,408 @@ +#!/usr/bin/perl +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright 2020, 2022 Sony Corporation +# +# Author: Frank Rowand + +# This program is meant to be an aid to reading the verbose output of +# on the console log that results from executing the Linux kernel +# devicetree unittest (drivers/of/unitest.c). + +$VUFX = "220201a"; + +use strict 'refs'; +use strict subs; + +use Getopt::Long; +use Text::Wrap; + +# strip off everything before final "/" +(undef, $script_name) = split(/^.*\//, $0); + +# following /usr/include/sysexits.h +$EX_OK=0; +$EX_USAGE=64; + + +#______________________________________________________________________________ +sub compare { + my ($expect, $got) = @_; + my $expect_next; + my $expect_next_lit; + my $got_next; + my $type; + + while ($expect) { + + ($expect_next, $type) = split(/<>/, $type); + $expect =~ s/^.*?>>//; # '?' is non-greedy, minimal match + + # literal, ignore all metacharacters when used in a regex + $expect_next_lit = quotemeta($expect_next); + + $got_next = $got; + $got_next =~ s/^($expect_next_lit).*/\1/; + $got =~ s/^$expect_next_lit//; + + if ($expect_next ne $got_next) { + return 0; + } + + if ($type eq "int") { + if ($got =~ /^[+-]*[0-9]+/) { + $got =~ s/^[+-]*[0-9]+//; + } else { + return 0; + } + } elsif ($type eq "hex") { + if ($got =~ /^(0x)*[0-9a-f]+/) { + $got =~ s/^(0x)*[0-9a-f]+//; + } else { + return 0; + } + } elsif ($type eq "") { + if ($expect_next ne $got_next) { + return 0; + } else { + return 1; + } + } else { + $internal_err++; + print "** ERROR: special pattern not recognized: <<$type>>, CONSOLE_LOG line: $.\n"; + return 0; + } + + } + + # should not get here + $internal_err++; + print "** ERROR: $script_name internal error, at end of compare(), CONSOLE_LOG line: $.\n"; + + return 0; +} + + +#______________________________________________________________________________ +sub usage { + +# ***** when editing, be careful to not put tabs in the string printed: + + print STDERR +" +usage: + + $script_name CONSOLE_LOG + + -h print program usage + --help print program usage + --hide-expect suppress output of EXPECTed lines + --line-num report line number of CONSOLE_LOG + --no-expect-stats do not report EXPECT statistics + --no-strip-ts do not strip leading console timestamps + --verbose do not suppress EXPECT begin and end lines + --version print program version and exit + + + Process a console log for EXPECTed test related messages to either + highlight expected devicetree unittest related messages or suppress + the messages. Leading console timestamps will be stripped. + + Various unittests may trigger kernel messages from outside the + unittest code. The unittest annotates that it expects the message + to occur with an 'EXPECT \\ : text' (begin) before triggering the + message, and an 'EXPECT / : text' (end) after triggering the message. + + If an expected message does not occur, that will be reported. + + For each expected message, the 'EXPECT \\ : text' (begin) and + 'EXPECT / : text' (end), 'text' will contain the message text. + + If 'EXPECT \\' (begin) and 'EXPECT /' (end) lines do not contain + matching 'text', that will be reported. + + If EXPECT lines are nested, 'EXPECT /' (end) lines must be in the + reverse order of the corresponding 'EXPECT \\' (begin) lines. + + 'EXPECT \\ : text' (begin) and 'EXPECT / : text' (end) lines can + contain special patterns in 'text': + + <> matches: [+-]*[0-9]+ + <> matches: (0x)*[0-9a-f]+ + + 'EXPECT \\' (begin) and 'EXPECT /' (end) lines are suppressed. + + A prefix is added to every line of output: + + 'ok ' Line matches an enclosing EXPECT begin/end pair + + '** ' Line reports $script_name warning or error + + '-> ' Line reports start or end of the unittests + + '>> ' Line reports a unittest test FAIL + + ' ' Lines that are not otherwise prefixed + + Issues detected in CONSOLE_LOG are reported to STDOUT, not to STDERR. + + Known Issues: + + --line-num causes the CONSOLE_LOG line number to be printed in 4 columns. + If CONSOLE_LOG contains more than 9999 lines then more columns will be + used to report the line number for lines greater than 9999 (eg for + lines 10000 - 99999, 5 columns will be used). +"; + + return {}; +} + +#______________________________________________________________________________ +#______________________________________________________________________________ + +if (!GetOptions( + "h" => \$help, + "help" => \$help, + "hide-expect" => \$hide_expect, + "line-num" => \$print_line_num, + "no-expect-stats" => \$no_expect_stats, + "no-strip-ts" => \$no_strip_ts, + "verbose" => \$verbose, + "version" => \$version, + )) { + print STDERR "\n"; + print STDERR "ERROR processing command line options\n"; + print STDERR "\n"; + print STDERR "For help, type '$script_name --help'\n"; + print STDERR "\n"; + + exit $EX_OK; +} + + +if ($no_strip_ts) { + $strip_ts = 1; + $no_strip_ts = 0; +} else { + $strip_ts = 0; + $no_strip_ts = 1; +} + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +if ($help){ + + &usage; + + exit $EX_OK; +} + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +if ($version) { + print STDERR "\n$script_name $VUFX\n\n"; + print STDERR "\n"; + + exit $EX_OK; +} + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +if ($#ARGV != 0) { + + # Limit input files to exactly one. + # + # 'while ($line = ) {' in the code below supports multiple file + # names on the command line, but the EXPECT statistics are reported + # once for all input - it is not an expected use case to generate one + # set of statistics for multiple input files. + + print STDERR "\n"; + print STDERR "Required arguments: CONSOLE_LOG\n"; + print STDERR "\n"; + + exit $EX_USAGE; +} + + +#______________________________________________________________________________ + +# Patterns to match 'EXPECT \ : ' (begin) and 'EXPECT / : ' (end) +# +# $exp_* are used as regex match patterns, +# so '\\\\' in $exp_begin matches a single '\' +# quotemeta() does not do the right thing in this case +# +# $pr_fmt is the prefix that unittest prints for every message + +$pr_fmt = "### dt-test ### "; +$exp_begin = "${pr_fmt}EXPECT \\\\ : "; +$exp_end = "${pr_fmt}EXPECT / : "; + + +$line_num = ""; +$timestamp = ""; + +LINE: +while ($line = ) { + + chomp $line; + + $prefix = " "; ## 2 characters + + + if ($strip_ts) { + + $timestamp = $line; + + if ($timestamp =~ /^\[\s*[0-9]+\.[0-9]*\] /) { + ($timestamp, $null) = split(/]/, $line); + $timestamp = $timestamp . "] "; + + } else { + $timestamp = ""; + } + } + + $line =~ s/^\[\s*[0-9]+\.[0-9]*\] //; + + + # ----- find EXPECT begin + + if ($line =~ /^\s*$exp_begin/) { + $data = $line; + $data =~ s/^\s*$exp_begin//; + push @begin, $data; + + if ($verbose) { + if ($print_line_num) { + $line_num = sprintf("%4s ", $.); + } + printf "%s %s%s%s\n", $prefix, $line_num, $timestamp, $line; + } + + next LINE; + } + + + # ----- find EXPECT end + + if ($line =~ /^\s*$exp_end/) { + $data = $line; + $data =~ s/^\s*$exp_end//; + + if ($verbose) { + if ($print_line_num) { + $line_num = sprintf("%4s ", $.); + } + printf "%s %s%s%s\n", $prefix, $line_num, $timestamp, $line; + } + + $found = 0; + $no_begin = 0; + if (@found_or_begin > 0) { + $begin = pop @found_or_begin; + if (compare($data, $begin)) { + $found = 1; + } + } elsif (@begin > 0) { + $begin = pop @begin; + } else { + $no_begin = 1; + } + + if ($no_begin) { + + $expect_missing_begin++; + print "** ERROR: EXPECT end without any EXPECT begin:\n"; + print " end ---> $line\n"; + + } elsif (! $found) { + + if ($print_line_num) { + $line_num = sprintf("%4s ", $.); + } + + $expect_not_found++; + printf "** %s%s$script_name WARNING - not found ---> %s\n", + $line_num, $timestamp, $data; + + } elsif (! compare($data, $begin)) { + + $expect_missing_end++; + print "** ERROR: EXPECT end does not match EXPECT begin:\n"; + print " begin -> $begin\n"; + print " end ---> $line\n"; + + } else { + + $expect_found++; + + } + + next LINE; + } + + + # ----- not an EXPECT line + + if (($line =~ /^${pr_fmt}start of unittest - you will see error messages$/) || + ($line =~ /^${pr_fmt}end of unittest - [0-9]+ passed, [0-9]+ failed$/ ) ) { + $prefix = "->"; # 2 characters + } elsif ($line =~ /^${pr_fmt}FAIL /) { + $unittest_fail++; + $prefix = ">>"; # 2 characters + } + + $found = 0; + foreach $begin (@begin) { + if (compare($begin, $line)) { + $found = 1; + last; + } + } + + if ($found) { + $begin = shift @begin; + while (! compare($begin, $line)) { + push @found_or_begin, $begin; + $begin = shift @begin; + } + push @found_or_begin, $line; + + if ($hide_expect) { + $suppress_line = 1; + next LINE; + } + $prefix = "ok"; # 2 characters + } + + + if ($print_line_num) { + $line_num = sprintf("%4s ", $.); + } + + printf "%s %s%s%s\n", $prefix, $line_num, $timestamp, $line; +} + +if (! $no_expect_stats) { + print "\n"; + print "** EXPECT statistics:\n"; + print "**\n"; + printf "** EXPECT found : %4i\n", $expect_found; + printf "** EXPECT not found : %4i\n", $expect_not_found; + printf "** missing EXPECT begin : %4i\n", $expect_missing_begin; + printf "** missing EXPECT end : %4i\n", $expect_missing_end; + printf "** unittest FAIL : %4i\n", $unittest_fail; + printf "** internal error : %4i\n", $internal_err; +} + +if (@begin) { + print "** ERROR: EXPECT begin without any EXPECT end:\n"; + print " This list may be misleading.\n"; + foreach $begin (@begin) { + print " begin ---> $begin\n"; + } +} -- cgit v1.2.3 From 2783a7f56f9980f61ca809b826bcd14dc77eb7b9 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Thu, 10 Mar 2022 10:05:12 -0600 Subject: dt-bindings: kbuild: Pass DT_SCHEMA_FILES to dt-validate In preparation for supporting validation of DTB files, the full processed schema will always be needed in order to extract type information from it. Therefore, the processed schema containing only what DT_SCHEMA_FILES specifies won't work. Instead, dt-validate has gained an option, -l or --limit, to specify which schema(s) to use for validation. As the command line option is new, we the minimum dtschema version must be updated. Cc: Masahiro Yamada Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20220310160513.1708182-2-robh@kernel.org --- scripts/Makefile.lib | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 79be57fdd32a..c8c6f1745d03 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -361,10 +361,9 @@ $(multi-dtb-y): FORCE $(call multi_depend, $(multi-dtb-y), .dtb, -dtbs) DT_CHECKER ?= dt-validate -DT_CHECKER_FLAGS ?= $(if $(DT_SCHEMA_FILES),,-m) +DT_CHECKER_FLAGS ?= $(if $(DT_SCHEMA_FILES),-l $(DT_SCHEMA_FILES),-m) DT_BINDING_DIR := Documentation/devicetree/bindings -# DT_TMP_SCHEMA may be overridden from Documentation/devicetree/bindings/Makefile -DT_TMP_SCHEMA ?= $(objtree)/$(DT_BINDING_DIR)/processed-schema.json +DT_TMP_SCHEMA := $(objtree)/$(DT_BINDING_DIR)/processed-schema.json quiet_cmd_dtb_check = CHECK $@ cmd_dtb_check = $(DT_CHECKER) $(DT_CHECKER_FLAGS) -u $(srctree)/$(DT_BINDING_DIR) -p $(DT_TMP_SCHEMA) $@ -- cgit v1.2.3 From ef8795f3f1cef2b2d2cd5dfab3758a7601898bc9 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Thu, 10 Mar 2022 10:05:13 -0600 Subject: dt-bindings: kbuild: Use DTB files for validation Switch the DT validation to use DTB files directly instead of a DTS to YAML conversion. The original motivation for supporting validation on DTB files was to enable running validation on a running system (e.g. 'dt-validate /sys/firmware/fdt') or other cases where the original source DTS is not available. The YAML format was not without issues. Using DTBs with the schema type information solves some of those problems. The YAML format relies on the DTS source level information including bracketing of properties, size directives, and phandle tags all of which are lost in a DTB file. While standardizing the bracketing is a good thing, it does cause a lot of extra warnings and churn to fix them. Another issue has been signed types are not validated correctly as sign information is not propagated to YAML. Using the schema type information allows for proper handling of signed types. YAML also can't represent the full range of 64-bit integers as numbers are stored as floats by most/all parsers. The DTB validation works by decoding property values using the type information in the schemas themselves. The main corner case this does not work for is matrix types where neither dimension is fixed. For now, checking the dimensions in these cases are skipped. Signed-off-by: Rob Herring Tested-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20220310160513.1708182-3-robh@kernel.org --- scripts/Makefile.lib | 20 +++++++------------- scripts/dtc/Makefile | 13 ------------- scripts/dtc/update-dtc-source.sh | 2 +- 3 files changed, 8 insertions(+), 27 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index c8c6f1745d03..9d5320a47ef8 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -87,11 +87,6 @@ base-dtb-y := $(foreach m, $(multi-dtb-y), $(firstword $(call suffix-search, $m, always-y += $(dtb-y) -ifneq ($(CHECK_DTBS),) -always-y += $(patsubst %.dtb,%.dt.yaml, $(real-dtb-y)) -always-y += $(patsubst %.dtbo,%.dt.yaml, $(real-dtb-y)) -endif - # Add subdir path extra-y := $(addprefix $(obj)/,$(extra-y)) @@ -347,12 +342,6 @@ cmd_dtc = $(HOSTCC) -E $(dtc_cpp_flags) -x assembler-with-cpp -o $(dtc-tmp) $< ; -d $(depfile).dtc.tmp $(dtc-tmp) ; \ cat $(depfile).pre.tmp $(depfile).dtc.tmp > $(depfile) -$(obj)/%.dtb: $(src)/%.dts $(DTC) FORCE - $(call if_changed_dep,dtc) - -$(obj)/%.dtbo: $(src)/%.dts $(DTC) FORCE - $(call if_changed_dep,dtc) - quiet_cmd_fdtoverlay = DTOVL $@ cmd_fdtoverlay = $(objtree)/scripts/dtc/fdtoverlay -o $@ -i $(real-prereqs) @@ -360,22 +349,27 @@ $(multi-dtb-y): FORCE $(call if_changed,fdtoverlay) $(call multi_depend, $(multi-dtb-y), .dtb, -dtbs) +ifneq ($(CHECK_DTBS)$(CHECK_DT_BINDING),) DT_CHECKER ?= dt-validate DT_CHECKER_FLAGS ?= $(if $(DT_SCHEMA_FILES),-l $(DT_SCHEMA_FILES),-m) DT_BINDING_DIR := Documentation/devicetree/bindings DT_TMP_SCHEMA := $(objtree)/$(DT_BINDING_DIR)/processed-schema.json quiet_cmd_dtb_check = CHECK $@ - cmd_dtb_check = $(DT_CHECKER) $(DT_CHECKER_FLAGS) -u $(srctree)/$(DT_BINDING_DIR) -p $(DT_TMP_SCHEMA) $@ + cmd_dtb_check = $(DT_CHECKER) $(DT_CHECKER_FLAGS) -u $(srctree)/$(DT_BINDING_DIR) -p $(DT_TMP_SCHEMA) $@ || true +endif define rule_dtc $(call cmd_and_fixdep,dtc) $(call cmd,dtb_check) endef -$(obj)/%.dt.yaml: $(src)/%.dts $(DTC) $(DT_TMP_SCHEMA) FORCE +$(obj)/%.dtb: $(src)/%.dts $(DTC) $(DT_TMP_SCHEMA) FORCE $(call if_changed_rule,dtc) +$(obj)/%.dtbo: $(src)/%.dts $(DTC) FORCE + $(call if_changed_dep,dtc) + dtc-tmp = $(subst $(comma),_,$(dot-target).dts.tmp) # Bzip2 diff --git a/scripts/dtc/Makefile b/scripts/dtc/Makefile index 1cba78e1dce6..4d32b9497da9 100644 --- a/scripts/dtc/Makefile +++ b/scripts/dtc/Makefile @@ -17,20 +17,7 @@ fdtoverlay-objs := $(libfdt) fdtoverlay.o util.o # Source files need to get at the userspace version of libfdt_env.h to compile HOST_EXTRACFLAGS += -I $(srctree)/$(src)/libfdt - -ifeq ($(shell pkg-config --exists yaml-0.1 2>/dev/null && echo yes),) -ifneq ($(CHECK_DT_BINDING)$(CHECK_DTBS),) -$(error dtc needs libyaml for DT schema validation support. \ - Install the necessary libyaml development package.) -endif HOST_EXTRACFLAGS += -DNO_YAML -else -dtc-objs += yamltree.o -# To include installed in a non-default path -HOSTCFLAGS_yamltree.o := $(shell pkg-config --cflags yaml-0.1) -# To link libyaml installed in a non-default path -HOSTLDLIBS_dtc := $(shell pkg-config --libs yaml-0.1) -endif # Generated files need one more search path to include headers in source tree HOSTCFLAGS_dtc-lexer.lex.o := -I $(srctree)/$(src) diff --git a/scripts/dtc/update-dtc-source.sh b/scripts/dtc/update-dtc-source.sh index 32ff17ffd089..94627541533e 100755 --- a/scripts/dtc/update-dtc-source.sh +++ b/scripts/dtc/update-dtc-source.sh @@ -32,7 +32,7 @@ DTC_UPSTREAM_PATH=`pwd`/../dtc DTC_LINUX_PATH=`pwd`/scripts/dtc DTC_SOURCE="checks.c data.c dtc.c dtc.h flattree.c fstree.c livetree.c srcpos.c \ - srcpos.h treesource.c util.c util.h version_gen.h yamltree.c \ + srcpos.h treesource.c util.c util.h version_gen.h \ dtc-lexer.l dtc-parser.y" LIBFDT_SOURCE="fdt.c fdt.h fdt_addresses.c fdt_empty_tree.c \ fdt_overlay.c fdt_ro.c fdt_rw.c fdt_strerror.c fdt_sw.c \ -- cgit v1.2.3