Blame view
scripts/checkkconfigsymbols.py
15.4 KB
4b6fda0b8 checkkconfigsymbo... |
1 |
#!/usr/bin/env python2 |
24fe1f03e checkkconfigsymbo... |
2 |
|
b1a3f2434 checkkconfigsymbo... |
3 |
"""Find Kconfig symbols that are referenced but not defined.""" |
24fe1f03e checkkconfigsymbo... |
4 |
|
c74556630 checkkconfigsymbo... |
5 |
# (c) 2014-2015 Valentin Rothberg <valentinrothberg@gmail.com> |
cc641d552 checkkconfigsymbo... |
6 |
# (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de> |
24fe1f03e checkkconfigsymbo... |
7 |
# |
cc641d552 checkkconfigsymbo... |
8 |
# Licensed under the terms of the GNU GPL License version 2 |
24fe1f03e checkkconfigsymbo... |
9 |
|
1b2c84146 checkkconfigsymbo... |
10 |
import difflib |
24fe1f03e checkkconfigsymbo... |
11 12 |
import os import re |
e2042a8a8 checkkconfigsymbo... |
13 |
import signal |
b1a3f2434 checkkconfigsymbo... |
14 |
import sys |
e2042a8a8 checkkconfigsymbo... |
15 |
from multiprocessing import Pool, cpu_count |
b1a3f2434 checkkconfigsymbo... |
16 |
from optparse import OptionParser |
e2042a8a8 checkkconfigsymbo... |
17 |
from subprocess import Popen, PIPE, STDOUT |
24fe1f03e checkkconfigsymbo... |
18 |
|
cc641d552 checkkconfigsymbo... |
19 20 |
# regex expressions |
24fe1f03e checkkconfigsymbo... |
21 |
OPERATORS = r"&|\(|\)|\||\!" |
cc641d552 checkkconfigsymbo... |
22 23 |
FEATURE = r"(?:\w*[A-Z0-9]\w*){2,}" DEF = r"^\s*(?:menu){,1}config\s+(" + FEATURE + r")\s*" |
24fe1f03e checkkconfigsymbo... |
24 |
EXPR = r"(?:" + OPERATORS + r"|\s|" + FEATURE + r")+" |
0bd38ae35 scripts/checkkcon... |
25 26 |
DEFAULT = r"default\s+.*?(?:if\s.+){,1}" STMT = r"^\s*(?:if|select|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR |
cc641d552 checkkconfigsymbo... |
27 |
SOURCE_FEATURE = r"(?:\W|\b)+[D]{,1}CONFIG_(" + FEATURE + r")" |
24fe1f03e checkkconfigsymbo... |
28 |
|
cc641d552 checkkconfigsymbo... |
29 |
# regex objects |
24fe1f03e checkkconfigsymbo... |
30 |
REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$") |
e2042a8a8 checkkconfigsymbo... |
31 |
REGEX_FEATURE = re.compile(r'(?!\B)' + FEATURE + r'(?!\B)') |
cc641d552 checkkconfigsymbo... |
32 33 |
REGEX_SOURCE_FEATURE = re.compile(SOURCE_FEATURE) REGEX_KCONFIG_DEF = re.compile(DEF) |
24fe1f03e checkkconfigsymbo... |
34 35 36 37 |
REGEX_KCONFIG_EXPR = re.compile(EXPR) REGEX_KCONFIG_STMT = re.compile(STMT) REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$") REGEX_FILTER_FEATURES = re.compile(r"[A-Za-z0-9]$") |
0bd38ae35 scripts/checkkcon... |
38 |
REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+") |
e2042a8a8 checkkconfigsymbo... |
39 |
REGEX_QUOTES = re.compile("(\"(.*?)\")") |
24fe1f03e checkkconfigsymbo... |
40 |
|
b1a3f2434 checkkconfigsymbo... |
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 |
def parse_options(): """The user interface of this module.""" usage = "%prog [options] " \ "Run this tool to detect Kconfig symbols that are referenced but " \ "not defined in Kconfig. The output of this tool has the " \ "format \'Undefined symbol\\tFile list\' " \ "If no option is specified, %prog will default to check your " \ "current tree. Please note that specifying commits will " \ "\'git reset --hard\' your current tree! You may save " \ "uncommitted changes to avoid losing data." parser = OptionParser(usage=usage) parser.add_option('-c', '--commit', dest='commit', action='store', default="", help="Check if the specified commit (hash) introduces " "undefined Kconfig symbols.") parser.add_option('-d', '--diff', dest='diff', action='store', default="", help="Diff undefined symbols between two commits. The " "input format bases on Git log's " "\'commmit1..commit2\'.") |
a42fa92ce checkkconfigsymbo... |
71 72 73 74 |
parser.add_option('-f', '--find', dest='find', action='store_true', default=False, help="Find and show commits that may cause symbols to be " "missing. Required to run with --diff.") |
cf132e4a8 checkkconfigsymbo... |
75 76 77 78 79 |
parser.add_option('-i', '--ignore', dest='ignore', action='store', default="", help="Ignore files matching this pattern. Note that " "the pattern needs to be a Python regex. To " "ignore defconfigs, specify -i '.*defconfig'.") |
1b2c84146 checkkconfigsymbo... |
80 81 |
parser.add_option('-s', '--sim', dest='sim', action='store', default="", help="Print a list of maximum 10 string-similar symbols.") |
b1a3f2434 checkkconfigsymbo... |
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
parser.add_option('', '--force', dest='force', action='store_true', default=False, help="Reset current Git tree even when it's dirty.") (opts, _) = parser.parse_args() if opts.commit and opts.diff: sys.exit("Please specify only one option at once.") if opts.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", opts.diff): sys.exit("Please specify valid input in the following format: " "\'commmit1..commit2\'") if opts.commit or opts.diff: if not opts.force and tree_is_dirty(): sys.exit("The current Git tree is dirty (see 'git status'). " "Running this script may delete important data since it " "calls 'git reset --hard' for some performance reasons. " " Please run this script in a clean Git tree or pass " "'--force' if you want to ignore this warning and " "continue.") |
a42fa92ce checkkconfigsymbo... |
106 107 |
if opts.commit: opts.find = False |
cf132e4a8 checkkconfigsymbo... |
108 109 110 111 112 |
if opts.ignore: try: re.match(opts.ignore, "this/is/just/a/test.c") except: sys.exit("Please specify a valid Python regex.") |
b1a3f2434 checkkconfigsymbo... |
113 |
return opts |
24fe1f03e checkkconfigsymbo... |
114 115 |
def main(): """Main function of this module.""" |
b1a3f2434 checkkconfigsymbo... |
116 |
opts = parse_options() |
1b2c84146 checkkconfigsymbo... |
117 118 119 120 121 122 123 124 125 126 127 |
if opts.sim and not opts.commit and not opts.diff: sims = find_sims(opts.sim, opts.ignore) if sims: print "%s: %s" % (yel("Similar symbols"), ', '.join(sims)) else: print "%s: no similar symbols found" % yel("Similar symbols") sys.exit(0) # dictionary of (un)defined symbols defined = {} undefined = {} |
b1a3f2434 checkkconfigsymbo... |
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 |
if opts.commit or opts.diff: head = get_head() # get commit range commit_a = None commit_b = None if opts.commit: commit_a = opts.commit + "~" commit_b = opts.commit elif opts.diff: split = opts.diff.split("..") commit_a = split[0] commit_b = split[1] undefined_a = {} undefined_b = {} # get undefined items before the commit execute("git reset --hard %s" % commit_a) |
1b2c84146 checkkconfigsymbo... |
146 |
undefined_a, _ = check_symbols(opts.ignore) |
b1a3f2434 checkkconfigsymbo... |
147 148 149 |
# get undefined items for the commit execute("git reset --hard %s" % commit_b) |
1b2c84146 checkkconfigsymbo... |
150 |
undefined_b, defined = check_symbols(opts.ignore) |
b1a3f2434 checkkconfigsymbo... |
151 152 |
# report cases that are present for the commit but not before |
e9533ae53 checkkconfigsymbo... |
153 |
for feature in sorted(undefined_b): |
b1a3f2434 checkkconfigsymbo... |
154 155 |
# feature has not been undefined before if not feature in undefined_a: |
e9533ae53 checkkconfigsymbo... |
156 |
files = sorted(undefined_b.get(feature)) |
1b2c84146 checkkconfigsymbo... |
157 |
undefined[feature] = files |
b1a3f2434 checkkconfigsymbo... |
158 159 |
# check if there are new files that reference the undefined feature else: |
e9533ae53 checkkconfigsymbo... |
160 161 |
files = sorted(undefined_b.get(feature) - undefined_a.get(feature)) |
b1a3f2434 checkkconfigsymbo... |
162 |
if files: |
1b2c84146 checkkconfigsymbo... |
163 |
undefined[feature] = files |
b1a3f2434 checkkconfigsymbo... |
164 165 166 167 168 169 |
# reset to head execute("git reset --hard %s" % head) # default to check the entire tree else: |
1b2c84146 checkkconfigsymbo... |
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 |
undefined, defined = check_symbols(opts.ignore) # now print the output for feature in sorted(undefined): print red(feature) files = sorted(undefined.get(feature)) print "%s: %s" % (yel("Referencing files"), ", ".join(files)) sims = find_sims(feature, opts.ignore, defined) sims_out = yel("Similar symbols") if sims: print "%s: %s" % (sims_out, ', '.join(sims)) else: print "%s: %s" % (sims_out, "no similar symbols found") if opts.find: print "%s:" % yel("Commits changing symbol") commits = find_commits(feature, opts.diff) if commits: for commit in commits: commit = commit.split(" ", 1) print "\t- %s (\"%s\")" % (yel(commit[0]), commit[1]) else: print "\t- no commit found" print # new line |
c74556630 checkkconfigsymbo... |
196 197 198 199 200 201 202 203 204 205 206 207 208 209 |
def yel(string): """ Color %string yellow. """ return "\033[33m%s\033[0m" % string def red(string): """ Color %string red. """ return "\033[31m%s\033[0m" % string |
b1a3f2434 checkkconfigsymbo... |
210 211 212 213 214 215 216 217 218 |
def execute(cmd): """Execute %cmd and return stdout. Exit in case of error.""" pop = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True) (stdout, _) = pop.communicate() # wait until finished if pop.returncode != 0: sys.exit(stdout) return stdout |
a42fa92ce checkkconfigsymbo... |
219 220 221 222 |
def find_commits(symbol, diff): """Find commits changing %symbol in the given range of %diff.""" commits = execute("git log --pretty=oneline --abbrev-commit -G %s %s" % (symbol, diff)) |
1b2c84146 checkkconfigsymbo... |
223 224 |
return [x for x in commits.split(" ") if x] |
a42fa92ce checkkconfigsymbo... |
225 |
|
b1a3f2434 checkkconfigsymbo... |
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 |
def tree_is_dirty(): """Return true if the current working tree is dirty (i.e., if any file has been added, deleted, modified, renamed or copied but not committed).""" stdout = execute("git status --porcelain") for line in stdout: if re.findall(r"[URMADC]{1}", line[:2]): return True return False def get_head(): """Return commit hash of current HEAD.""" stdout = execute("git rev-parse HEAD") return stdout.strip(' ') |
e2042a8a8 checkkconfigsymbo... |
241 242 243 244 245 246 247 248 |
def partition(lst, size): """Partition list @lst into eveni-sized lists of size @size.""" return [lst[i::size] for i in xrange(size)] def init_worker(): """Set signal handler to ignore SIGINT.""" signal.signal(signal.SIGINT, signal.SIG_IGN) |
1b2c84146 checkkconfigsymbo... |
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 |
def find_sims(symbol, ignore, defined = []): """Return a list of max. ten Kconfig symbols that are string-similar to @symbol.""" if defined: return sorted(difflib.get_close_matches(symbol, set(defined), 10)) pool = Pool(cpu_count(), init_worker) kfiles = [] for gitfile in get_files(): if REGEX_FILE_KCONFIG.match(gitfile): kfiles.append(gitfile) arglist = [] for part in partition(kfiles, cpu_count()): arglist.append((part, ignore)) for res in pool.map(parse_kconfig_files, arglist): defined.extend(res[0]) return sorted(difflib.get_close_matches(symbol, set(defined), 10)) def get_files(): """Return a list of all files in the current git directory.""" # use 'git ls-files' to get the worklist stdout = execute("git ls-files") if len(stdout) > 0 and stdout[-1] == " ": stdout = stdout[:-1] files = [] for gitfile in stdout.rsplit(" "): if ".git" in gitfile or "ChangeLog" in gitfile or \ ".log" in gitfile or os.path.isdir(gitfile) or \ gitfile.startswith("tools/"): continue files.append(gitfile) return files |
cf132e4a8 checkkconfigsymbo... |
288 |
def check_symbols(ignore): |
b1a3f2434 checkkconfigsymbo... |
289 |
"""Find undefined Kconfig symbols and return a dict with the symbol as key |
cf132e4a8 checkkconfigsymbo... |
290 291 |
and a list of referencing files as value. Files matching %ignore are not checked for undefined symbols.""" |
e2042a8a8 checkkconfigsymbo... |
292 293 294 295 296 297 298 299 300 301 302 303 |
pool = Pool(cpu_count(), init_worker) try: return check_symbols_helper(pool, ignore) except KeyboardInterrupt: pool.terminate() pool.join() sys.exit(1) def check_symbols_helper(pool, ignore): """Helper method for check_symbols(). Used to catch keyboard interrupts in check_symbols() in order to properly terminate running worker processes.""" |
24fe1f03e checkkconfigsymbo... |
304 305 |
source_files = [] kconfig_files = [] |
e2042a8a8 checkkconfigsymbo... |
306 307 |
defined_features = [] referenced_features = dict() # {file: [features]} |
24fe1f03e checkkconfigsymbo... |
308 |
|
1b2c84146 checkkconfigsymbo... |
309 |
for gitfile in get_files(): |
24fe1f03e checkkconfigsymbo... |
310 311 312 |
if REGEX_FILE_KCONFIG.match(gitfile): kconfig_files.append(gitfile) else: |
e2042a8a8 checkkconfigsymbo... |
313 314 315 |
if ignore and not re.match(ignore, gitfile): continue # add source files that do not match the ignore pattern |
24fe1f03e checkkconfigsymbo... |
316 |
source_files.append(gitfile) |
e2042a8a8 checkkconfigsymbo... |
317 318 319 320 |
# parse source files arglist = partition(source_files, cpu_count()) for res in pool.map(parse_source_files, arglist): referenced_features.update(res) |
24fe1f03e checkkconfigsymbo... |
321 |
|
e2042a8a8 checkkconfigsymbo... |
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 |
# parse kconfig files arglist = [] for part in partition(kconfig_files, cpu_count()): arglist.append((part, ignore)) for res in pool.map(parse_kconfig_files, arglist): defined_features.extend(res[0]) referenced_features.update(res[1]) defined_features = set(defined_features) # inverse mapping of referenced_features to dict(feature: [files]) inv_map = dict() for _file, features in referenced_features.iteritems(): for feature in features: inv_map[feature] = inv_map.get(feature, set()) inv_map[feature].add(_file) referenced_features = inv_map |
24fe1f03e checkkconfigsymbo... |
339 |
|
b1a3f2434 checkkconfigsymbo... |
340 |
undefined = {} # {feature: [files]} |
24fe1f03e checkkconfigsymbo... |
341 |
for feature in sorted(referenced_features): |
cc641d552 checkkconfigsymbo... |
342 343 344 345 |
# filter some false positives if feature == "FOO" or feature == "BAR" or \ feature == "FOO_BAR" or feature == "XXX": continue |
24fe1f03e checkkconfigsymbo... |
346 347 |
if feature not in defined_features: if feature.endswith("_MODULE"): |
cc641d552 checkkconfigsymbo... |
348 |
# avoid false positives for kernel modules |
24fe1f03e checkkconfigsymbo... |
349 350 |
if feature[:-len("_MODULE")] in defined_features: continue |
b1a3f2434 checkkconfigsymbo... |
351 |
undefined[feature] = referenced_features.get(feature) |
1b2c84146 checkkconfigsymbo... |
352 |
return undefined, defined_features |
24fe1f03e checkkconfigsymbo... |
353 |
|
e2042a8a8 checkkconfigsymbo... |
354 355 356 357 358 359 360 361 362 363 364 |
def parse_source_files(source_files): """Parse each source file in @source_files and return dictionary with source files as keys and lists of references Kconfig symbols as values.""" referenced_features = dict() for sfile in source_files: referenced_features[sfile] = parse_source_file(sfile) return referenced_features def parse_source_file(sfile): """Parse @sfile and return a list of referenced Kconfig features.""" |
24fe1f03e checkkconfigsymbo... |
365 |
lines = [] |
e2042a8a8 checkkconfigsymbo... |
366 367 368 369 |
references = [] if not os.path.exists(sfile): return references |
24fe1f03e checkkconfigsymbo... |
370 371 372 373 374 375 376 377 378 379 |
with open(sfile, "r") as stream: lines = stream.readlines() for line in lines: if not "CONFIG_" in line: continue features = REGEX_SOURCE_FEATURE.findall(line) for feature in features: if not REGEX_FILTER_FEATURES.search(feature): continue |
e2042a8a8 checkkconfigsymbo... |
380 381 382 |
references.append(feature) return references |
24fe1f03e checkkconfigsymbo... |
383 384 385 386 387 |
def get_features_in_line(line): """Return mentioned Kconfig features in @line.""" return REGEX_FEATURE.findall(line) |
e2042a8a8 checkkconfigsymbo... |
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 |
def parse_kconfig_files(args): """Parse kconfig files and return tuple of defined and references Kconfig symbols. Note, @args is a tuple of a list of files and the @ignore pattern.""" kconfig_files = args[0] ignore = args[1] defined_features = [] referenced_features = dict() for kfile in kconfig_files: defined, references = parse_kconfig_file(kfile) defined_features.extend(defined) if ignore and re.match(ignore, kfile): # do not collect references for files that match the ignore pattern continue referenced_features[kfile] = references return (defined_features, referenced_features) def parse_kconfig_file(kfile): |
24fe1f03e checkkconfigsymbo... |
408 409 |
"""Parse @kfile and update feature definitions and references.""" lines = [] |
e2042a8a8 checkkconfigsymbo... |
410 411 |
defined = [] references = [] |
24fe1f03e checkkconfigsymbo... |
412 |
skip = False |
e2042a8a8 checkkconfigsymbo... |
413 414 |
if not os.path.exists(kfile): return defined, references |
24fe1f03e checkkconfigsymbo... |
415 416 417 418 419 420 421 |
with open(kfile, "r") as stream: lines = stream.readlines() for i in range(len(lines)): line = lines[i] line = line.strip(' ') |
cc641d552 checkkconfigsymbo... |
422 |
line = line.split("#")[0] # ignore comments |
24fe1f03e checkkconfigsymbo... |
423 424 425 |
if REGEX_KCONFIG_DEF.match(line): feature_def = REGEX_KCONFIG_DEF.findall(line) |
e2042a8a8 checkkconfigsymbo... |
426 |
defined.append(feature_def[0]) |
24fe1f03e checkkconfigsymbo... |
427 428 429 430 |
skip = False elif REGEX_KCONFIG_HELP.match(line): skip = True elif skip: |
cc641d552 checkkconfigsymbo... |
431 |
# ignore content of help messages |
24fe1f03e checkkconfigsymbo... |
432 433 |
pass elif REGEX_KCONFIG_STMT.match(line): |
e2042a8a8 checkkconfigsymbo... |
434 |
line = REGEX_QUOTES.sub("", line) |
24fe1f03e checkkconfigsymbo... |
435 |
features = get_features_in_line(line) |
cc641d552 checkkconfigsymbo... |
436 |
# multi-line statements |
24fe1f03e checkkconfigsymbo... |
437 438 439 440 441 442 443 |
while line.endswith("\\"): i += 1 line = lines[i] line = line.strip(' ') features.extend(get_features_in_line(line)) for feature in set(features): |
0bd38ae35 scripts/checkkcon... |
444 445 446 |
if REGEX_NUMERIC.match(feature): # ignore numeric values continue |
e2042a8a8 checkkconfigsymbo... |
447 448 449 |
references.append(feature) return defined, references |
24fe1f03e checkkconfigsymbo... |
450 451 452 453 |
if __name__ == "__main__": main() |