Blame view
scripts/checkkconfigsymbols.py
15.5 KB
7c5227af2
|
1 |
#!/usr/bin/env python3 |
24fe1f03e
|
2 |
|
b1a3f2434
|
3 |
"""Find Kconfig symbols that are referenced but not defined.""" |
24fe1f03e
|
4 |
|
f175ba174
|
5 |
# (c) 2014-2016 Valentin Rothberg <valentinrothberg@gmail.com> |
cc641d552
|
6 |
# (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de> |
24fe1f03e
|
7 |
# |
cc641d552
|
8 |
# Licensed under the terms of the GNU GPL License version 2 |
24fe1f03e
|
9 |
|
14390e316
|
10 |
import argparse |
1b2c84146
|
11 |
import difflib |
24fe1f03e
|
12 13 |
import os import re |
e2042a8a8
|
14 |
import signal |
f175ba174
|
15 |
import subprocess |
b1a3f2434
|
16 |
import sys |
e2042a8a8
|
17 |
from multiprocessing import Pool, cpu_count |
24fe1f03e
|
18 |
|
cc641d552
|
19 20 |
# regex expressions |
24fe1f03e
|
21 |
OPERATORS = r"&|\(|\)|\||\!" |
ef3f55438
|
22 23 24 |
SYMBOL = r"(?:\w*[A-Z0-9]\w*){2,}" DEF = r"^\s*(?:menu){,1}config\s+(" + SYMBOL + r")\s*" EXPR = r"(?:" + OPERATORS + r"|\s|" + SYMBOL + r")+" |
0bd38ae35
|
25 26 |
DEFAULT = r"default\s+.*?(?:if\s.+){,1}" STMT = r"^\s*(?:if|select|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR |
ef3f55438
|
27 |
SOURCE_SYMBOL = r"(?:\W|\b)+[D]{,1}CONFIG_(" + SYMBOL + r")" |
24fe1f03e
|
28 |
|
cc641d552
|
29 |
# regex objects |
24fe1f03e
|
30 |
REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$") |
ef3f55438
|
31 32 |
REGEX_SYMBOL = re.compile(r'(?!\B)' + SYMBOL + r'(?!\B)') REGEX_SOURCE_SYMBOL = re.compile(SOURCE_SYMBOL) |
cc641d552
|
33 |
REGEX_KCONFIG_DEF = re.compile(DEF) |
24fe1f03e
|
34 35 36 |
REGEX_KCONFIG_EXPR = re.compile(EXPR) REGEX_KCONFIG_STMT = re.compile(STMT) REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$") |
ef3f55438
|
37 |
REGEX_FILTER_SYMBOLS = re.compile(r"[A-Za-z0-9]$") |
0bd38ae35
|
38 |
REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+") |
e2042a8a8
|
39 |
REGEX_QUOTES = re.compile("(\"(.*?)\")") |
24fe1f03e
|
40 |
|
b1a3f2434
|
41 42 |
def parse_options(): """The user interface of this module.""" |
14390e316
|
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
usage = "Run this tool to detect Kconfig symbols that are referenced but " \ "not defined in Kconfig. If no option is specified, " \ "checkkconfigsymbols defaults to check your current tree. " \ "Please note that specifying commits will 'git reset --hard\' " \ "your current tree! You may save uncommitted changes to avoid " \ "losing data." parser = argparse.ArgumentParser(description=usage) parser.add_argument('-c', '--commit', dest='commit', action='store', default="", help="check if the specified commit (hash) introduces " "undefined Kconfig symbols") parser.add_argument('-d', '--diff', dest='diff', action='store', default="", help="diff undefined symbols between two commits " "(e.g., -d commmit1..commit2)") parser.add_argument('-f', '--find', dest='find', action='store_true', default=False, help="find and show commits that may cause symbols to be " "missing (required to run with --diff)") parser.add_argument('-i', '--ignore', dest='ignore', action='store', default="", help="ignore files matching this Python regex " "(e.g., -i '.*defconfig')") parser.add_argument('-s', '--sim', dest='sim', action='store', default="", help="print a list of max. 10 string-similar symbols") parser.add_argument('--force', dest='force', action='store_true', default=False, help="reset current Git tree even when it's dirty") parser.add_argument('--no-color', dest='color', action='store_false', default=True, help="don't print colored output (default when not " "outputting to a terminal)") args = parser.parse_args() if args.commit and args.diff: |
b1a3f2434
|
87 |
sys.exit("Please specify only one option at once.") |
14390e316
|
88 |
if args.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", args.diff): |
b1a3f2434
|
89 |
sys.exit("Please specify valid input in the following format: " |
38cbfe4fe
|
90 |
"\'commit1..commit2\'") |
b1a3f2434
|
91 |
|
14390e316
|
92 93 |
if args.commit or args.diff: if not args.force and tree_is_dirty(): |
b1a3f2434
|
94 95 96 97 98 99 100 101 102 |
sys.exit("The current Git tree is dirty (see 'git status'). " "Running this script may delete important data since it " "calls 'git reset --hard' for some performance reasons. " " Please run this script in a clean Git tree or pass " "'--force' if you want to ignore this warning and " "continue.") |
14390e316
|
103 104 |
if args.commit: args.find = False |
a42fa92ce
|
105 |
|
14390e316
|
106 |
if args.ignore: |
cf132e4a8
|
107 |
try: |
14390e316
|
108 |
re.match(args.ignore, "this/is/just/a/test.c") |
cf132e4a8
|
109 110 |
except: sys.exit("Please specify a valid Python regex.") |
14390e316
|
111 |
return args |
b1a3f2434
|
112 |
|
24fe1f03e
|
113 114 |
def main(): """Main function of this module.""" |
14390e316
|
115 |
args = parse_options() |
b1a3f2434
|
116 |
|
36c79c7fa
|
117 118 |
global COLOR COLOR = args.color and sys.stdout.isatty() |
4c73c0882
|
119 |
|
14390e316
|
120 121 |
if args.sim and not args.commit and not args.diff: sims = find_sims(args.sim, args.ignore) |
1b2c84146
|
122 |
if sims: |
7c5227af2
|
123 |
print("%s: %s" % (yel("Similar symbols"), ', '.join(sims))) |
1b2c84146
|
124 |
else: |
7c5227af2
|
125 |
print("%s: no similar symbols found" % yel("Similar symbols")) |
1b2c84146
|
126 127 128 129 130 |
sys.exit(0) # dictionary of (un)defined symbols defined = {} undefined = {} |
14390e316
|
131 |
if args.commit or args.diff: |
b1a3f2434
|
132 133 134 135 136 |
head = get_head() # get commit range commit_a = None commit_b = None |
14390e316
|
137 138 139 140 141 |
if args.commit: commit_a = args.commit + "~" commit_b = args.commit elif args.diff: split = args.diff.split("..") |
b1a3f2434
|
142 143 144 145 146 147 |
commit_a = split[0] commit_b = split[1] undefined_a = {} undefined_b = {} # get undefined items before the commit |
2f9cc12bb
|
148 |
reset(commit_a) |
14390e316
|
149 |
undefined_a, _ = check_symbols(args.ignore) |
b1a3f2434
|
150 151 |
# get undefined items for the commit |
2f9cc12bb
|
152 |
reset(commit_b) |
14390e316
|
153 |
undefined_b, defined = check_symbols(args.ignore) |
b1a3f2434
|
154 155 |
# report cases that are present for the commit but not before |
ef3f55438
|
156 157 158 159 160 161 |
for symbol in sorted(undefined_b): # symbol has not been undefined before if symbol not in undefined_a: files = sorted(undefined_b.get(symbol)) undefined[symbol] = files # check if there are new files that reference the undefined symbol |
b1a3f2434
|
162 |
else: |
ef3f55438
|
163 164 |
files = sorted(undefined_b.get(symbol) - undefined_a.get(symbol)) |
b1a3f2434
|
165 |
if files: |
ef3f55438
|
166 |
undefined[symbol] = files |
b1a3f2434
|
167 168 |
# reset to head |
2f9cc12bb
|
169 |
reset(head) |
b1a3f2434
|
170 171 172 |
# default to check the entire tree else: |
14390e316
|
173 |
undefined, defined = check_symbols(args.ignore) |
1b2c84146
|
174 175 |
# now print the output |
ef3f55438
|
176 177 |
for symbol in sorted(undefined): print(red(symbol)) |
1b2c84146
|
178 |
|
ef3f55438
|
179 |
files = sorted(undefined.get(symbol)) |
7c5227af2
|
180 |
print("%s: %s" % (yel("Referencing files"), ", ".join(files))) |
1b2c84146
|
181 |
|
ef3f55438
|
182 |
sims = find_sims(symbol, args.ignore, defined) |
1b2c84146
|
183 184 |
sims_out = yel("Similar symbols") if sims: |
7c5227af2
|
185 |
print("%s: %s" % (sims_out, ', '.join(sims))) |
1b2c84146
|
186 |
else: |
7c5227af2
|
187 |
print("%s: %s" % (sims_out, "no similar symbols found")) |
1b2c84146
|
188 |
|
14390e316
|
189 |
if args.find: |
7c5227af2
|
190 |
print("%s:" % yel("Commits changing symbol")) |
ef3f55438
|
191 |
commits = find_commits(symbol, args.diff) |
1b2c84146
|
192 193 194 |
if commits: for commit in commits: commit = commit.split(" ", 1) |
7c5227af2
|
195 |
print("\t- %s (\"%s\")" % (yel(commit[0]), commit[1])) |
1b2c84146
|
196 |
else: |
7c5227af2
|
197 |
print("\t- no commit found") |
36c79c7fa
|
198 |
print() # new line |
c74556630
|
199 |
|
2f9cc12bb
|
200 201 202 |
def reset(commit): """Reset current git tree to %commit.""" execute(["git", "reset", "--hard", commit]) |
c74556630
|
203 204 205 206 |
def yel(string): """ Color %string yellow. """ |
36c79c7fa
|
207 |
return "\033[33m%s\033[0m" % string if COLOR else string |
c74556630
|
208 209 210 211 212 213 |
def red(string): """ Color %string red. """ |
36c79c7fa
|
214 |
return "\033[31m%s\033[0m" % string if COLOR else string |
b1a3f2434
|
215 216 217 218 |
def execute(cmd): """Execute %cmd and return stdout. Exit in case of error.""" |
f175ba174
|
219 |
try: |
2f9cc12bb
|
220 |
stdout = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False) |
7c5227af2
|
221 |
stdout = stdout.decode(errors='replace') |
f175ba174
|
222 |
except subprocess.CalledProcessError as fail: |
2f9cc12bb
|
223 |
exit(fail) |
b1a3f2434
|
224 |
return stdout |
a42fa92ce
|
225 226 |
def find_commits(symbol, diff): """Find commits changing %symbol in the given range of %diff.""" |
2f9cc12bb
|
227 228 229 |
commits = execute(["git", "log", "--pretty=oneline", "--abbrev-commit", "-G", symbol, diff]) |
1b2c84146
|
230 231 |
return [x for x in commits.split(" ") if x] |
a42fa92ce
|
232 |
|
b1a3f2434
|
233 234 235 |
def tree_is_dirty(): """Return true if the current working tree is dirty (i.e., if any file has been added, deleted, modified, renamed or copied but not committed).""" |
2f9cc12bb
|
236 |
stdout = execute(["git", "status", "--porcelain"]) |
b1a3f2434
|
237 238 239 240 241 242 243 244 |
for line in stdout: if re.findall(r"[URMADC]{1}", line[:2]): return True return False def get_head(): """Return commit hash of current HEAD.""" |
2f9cc12bb
|
245 |
stdout = execute(["git", "rev-parse", "HEAD"]) |
b1a3f2434
|
246 247 |
return stdout.strip(' ') |
e2042a8a8
|
248 249 |
def partition(lst, size): """Partition list @lst into eveni-sized lists of size @size.""" |
7c5227af2
|
250 |
return [lst[i::size] for i in range(size)] |
e2042a8a8
|
251 252 253 254 255 |
def init_worker(): """Set signal handler to ignore SIGINT.""" signal.signal(signal.SIGINT, signal.SIG_IGN) |
36c79c7fa
|
256 |
def find_sims(symbol, ignore, defined=[]): |
1b2c84146
|
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 |
"""Return a list of max. ten Kconfig symbols that are string-similar to @symbol.""" if defined: return sorted(difflib.get_close_matches(symbol, set(defined), 10)) pool = Pool(cpu_count(), init_worker) kfiles = [] for gitfile in get_files(): if REGEX_FILE_KCONFIG.match(gitfile): kfiles.append(gitfile) arglist = [] for part in partition(kfiles, cpu_count()): arglist.append((part, ignore)) for res in pool.map(parse_kconfig_files, arglist): defined.extend(res[0]) return sorted(difflib.get_close_matches(symbol, set(defined), 10)) def get_files(): """Return a list of all files in the current git directory.""" # use 'git ls-files' to get the worklist |
2f9cc12bb
|
281 |
stdout = execute(["git", "ls-files"]) |
1b2c84146
|
282 283 284 285 286 287 288 289 290 291 292 293 294 |
if len(stdout) > 0 and stdout[-1] == " ": stdout = stdout[:-1] files = [] for gitfile in stdout.rsplit(" "): if ".git" in gitfile or "ChangeLog" in gitfile or \ ".log" in gitfile or os.path.isdir(gitfile) or \ gitfile.startswith("tools/"): continue files.append(gitfile) return files |
cf132e4a8
|
295 |
def check_symbols(ignore): |
b1a3f2434
|
296 |
"""Find undefined Kconfig symbols and return a dict with the symbol as key |
cf132e4a8
|
297 298 |
and a list of referencing files as value. Files matching %ignore are not checked for undefined symbols.""" |
e2042a8a8
|
299 300 301 302 303 304 305 306 307 308 309 310 |
pool = Pool(cpu_count(), init_worker) try: return check_symbols_helper(pool, ignore) except KeyboardInterrupt: pool.terminate() pool.join() sys.exit(1) def check_symbols_helper(pool, ignore): """Helper method for check_symbols(). Used to catch keyboard interrupts in check_symbols() in order to properly terminate running worker processes.""" |
24fe1f03e
|
311 312 |
source_files = [] kconfig_files = [] |
ef3f55438
|
313 314 |
defined_symbols = [] referenced_symbols = dict() # {file: [symbols]} |
24fe1f03e
|
315 |
|
1b2c84146
|
316 |
for gitfile in get_files(): |
24fe1f03e
|
317 318 319 |
if REGEX_FILE_KCONFIG.match(gitfile): kconfig_files.append(gitfile) else: |
e2042a8a8
|
320 321 322 |
if ignore and not re.match(ignore, gitfile): continue # add source files that do not match the ignore pattern |
24fe1f03e
|
323 |
source_files.append(gitfile) |
e2042a8a8
|
324 325 326 |
# parse source files arglist = partition(source_files, cpu_count()) for res in pool.map(parse_source_files, arglist): |
ef3f55438
|
327 |
referenced_symbols.update(res) |
24fe1f03e
|
328 |
|
e2042a8a8
|
329 330 331 332 333 |
# parse kconfig files arglist = [] for part in partition(kconfig_files, cpu_count()): arglist.append((part, ignore)) for res in pool.map(parse_kconfig_files, arglist): |
ef3f55438
|
334 335 336 |
defined_symbols.extend(res[0]) referenced_symbols.update(res[1]) defined_symbols = set(defined_symbols) |
e2042a8a8
|
337 |
|
ef3f55438
|
338 |
# inverse mapping of referenced_symbols to dict(symbol: [files]) |
e2042a8a8
|
339 |
inv_map = dict() |
ef3f55438
|
340 341 342 343 344 345 346 347 |
for _file, symbols in referenced_symbols.items(): for symbol in symbols: inv_map[symbol] = inv_map.get(symbol, set()) inv_map[symbol].add(_file) referenced_symbols = inv_map undefined = {} # {symbol: [files]} for symbol in sorted(referenced_symbols): |
cc641d552
|
348 |
# filter some false positives |
ef3f55438
|
349 350 |
if symbol == "FOO" or symbol == "BAR" or \ symbol == "FOO_BAR" or symbol == "XXX": |
cc641d552
|
351 |
continue |
ef3f55438
|
352 353 |
if symbol not in defined_symbols: if symbol.endswith("_MODULE"): |
cc641d552
|
354 |
# avoid false positives for kernel modules |
ef3f55438
|
355 |
if symbol[:-len("_MODULE")] in defined_symbols: |
24fe1f03e
|
356 |
continue |
ef3f55438
|
357 358 |
undefined[symbol] = referenced_symbols.get(symbol) return undefined, defined_symbols |
24fe1f03e
|
359 |
|
e2042a8a8
|
360 361 362 |
def parse_source_files(source_files): """Parse each source file in @source_files and return dictionary with source files as keys and lists of references Kconfig symbols as values.""" |
ef3f55438
|
363 |
referenced_symbols = dict() |
e2042a8a8
|
364 |
for sfile in source_files: |
ef3f55438
|
365 366 |
referenced_symbols[sfile] = parse_source_file(sfile) return referenced_symbols |
e2042a8a8
|
367 368 369 |
def parse_source_file(sfile): |
ef3f55438
|
370 |
"""Parse @sfile and return a list of referenced Kconfig symbols.""" |
24fe1f03e
|
371 |
lines = [] |
e2042a8a8
|
372 373 374 375 |
references = [] if not os.path.exists(sfile): return references |
7c5227af2
|
376 |
with open(sfile, "r", encoding='utf-8', errors='replace') as stream: |
24fe1f03e
|
377 378 379 |
lines = stream.readlines() for line in lines: |
36c79c7fa
|
380 |
if "CONFIG_" not in line: |
24fe1f03e
|
381 |
continue |
ef3f55438
|
382 383 384 |
symbols = REGEX_SOURCE_SYMBOL.findall(line) for symbol in symbols: if not REGEX_FILTER_SYMBOLS.search(symbol): |
24fe1f03e
|
385 |
continue |
ef3f55438
|
386 |
references.append(symbol) |
e2042a8a8
|
387 388 |
return references |
24fe1f03e
|
389 |
|
ef3f55438
|
390 391 392 |
def get_symbols_in_line(line): """Return mentioned Kconfig symbols in @line.""" return REGEX_SYMBOL.findall(line) |
24fe1f03e
|
393 |
|
e2042a8a8
|
394 395 396 397 398 399 |
def parse_kconfig_files(args): """Parse kconfig files and return tuple of defined and references Kconfig symbols. Note, @args is a tuple of a list of files and the @ignore pattern.""" kconfig_files = args[0] ignore = args[1] |
ef3f55438
|
400 401 |
defined_symbols = [] referenced_symbols = dict() |
e2042a8a8
|
402 403 404 |
for kfile in kconfig_files: defined, references = parse_kconfig_file(kfile) |
ef3f55438
|
405 |
defined_symbols.extend(defined) |
e2042a8a8
|
406 407 408 |
if ignore and re.match(ignore, kfile): # do not collect references for files that match the ignore pattern continue |
ef3f55438
|
409 410 |
referenced_symbols[kfile] = references return (defined_symbols, referenced_symbols) |
e2042a8a8
|
411 412 413 |
def parse_kconfig_file(kfile): |
ef3f55438
|
414 |
"""Parse @kfile and update symbol definitions and references.""" |
24fe1f03e
|
415 |
lines = [] |
e2042a8a8
|
416 417 |
defined = [] references = [] |
24fe1f03e
|
418 |
skip = False |
e2042a8a8
|
419 420 |
if not os.path.exists(kfile): return defined, references |
7c5227af2
|
421 |
with open(kfile, "r", encoding='utf-8', errors='replace') as stream: |
24fe1f03e
|
422 423 424 425 426 427 |
lines = stream.readlines() for i in range(len(lines)): line = lines[i] line = line.strip(' ') |
cc641d552
|
428 |
line = line.split("#")[0] # ignore comments |
24fe1f03e
|
429 430 |
if REGEX_KCONFIG_DEF.match(line): |
ef3f55438
|
431 432 |
symbol_def = REGEX_KCONFIG_DEF.findall(line) defined.append(symbol_def[0]) |
24fe1f03e
|
433 434 435 436 |
skip = False elif REGEX_KCONFIG_HELP.match(line): skip = True elif skip: |
cc641d552
|
437 |
# ignore content of help messages |
24fe1f03e
|
438 439 |
pass elif REGEX_KCONFIG_STMT.match(line): |
e2042a8a8
|
440 |
line = REGEX_QUOTES.sub("", line) |
ef3f55438
|
441 |
symbols = get_symbols_in_line(line) |
cc641d552
|
442 |
# multi-line statements |
24fe1f03e
|
443 444 445 446 447 |
while line.endswith("\\"): i += 1 line = lines[i] line = line.strip(' ') |
ef3f55438
|
448 449 450 |
symbols.extend(get_symbols_in_line(line)) for symbol in set(symbols): if REGEX_NUMERIC.match(symbol): |
0bd38ae35
|
451 452 |
# ignore numeric values continue |
ef3f55438
|
453 |
references.append(symbol) |
e2042a8a8
|
454 455 |
return defined, references |
24fe1f03e
|
456 457 458 459 |
if __name__ == "__main__": main() |