blob: 4bb8f2880d8ab5edf523952b476f51adec41b96f [file] [log] [blame]
#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Support the print function in Python 2.
from __future__ import print_function
from io import StringIO
import glob
import json
import logging
import optparse
import os
import re
import subprocess
import sys
from kudu_util import get_upstream_commit, check_output, ROOT, Colors, \
init_logging, get_thirdparty_dir
import iwyu.fix_includes
from iwyu.fix_includes import ParseAndMergeIWYUOutput
_USAGE = """\
%prog [--fix] [--sort-only] [--all | --from-git | <path>...]
%prog is a wrapper around include-what-you-use that passes the appropriate
configuration and filters the output to ignore known issues. In addition,
it can automatically pipe the output back into the IWYU-provided ''
script in order to fix any reported issues.
_MAPPINGS_DIR = os.path.join(ROOT, "build-support/iwyu/mappings/")
_TOOLCHAIN_DIR = os.path.join(get_thirdparty_dir(), "clang-toolchain/bin")
_IWYU_TOOL = os.path.join(ROOT, "build-support/iwyu/")
# Matches source files that we should run on.
_RE_SOURCE_FILE = re.compile(r'\.(c|cc|h)$')
# Matches compilation errors in the output of IWYU
_RE_CLANG_ERROR = re.compile(r'^.+?:\d+:\d+:\s*'
r'(fatal )?error:', re.MULTILINE)
# Files that we don't want to ever run IWYU on because it doesn't handle them properly.
_MUTED_FILES = set([
# Flags to pass to iwyu/ for Kudu-specific style.
# Directory containing the compilation database.
_BUILD_DIR = os.path.join(ROOT, 'build/latest')
def _get_file_list_from_git():
upstream_commit = get_upstream_commit()
out = check_output(["git", "diff", "--name-only", upstream_commit]).splitlines()
return [l.decode('utf-8') for l in out if'utf-8'))]
def _get_paths_from_compilation_db():
db_path = os.path.join(_BUILD_DIR, 'compile_commands.json')
with open(db_path, 'r') as fileobj:
compilation_db = json.load(fileobj)
return [entry['file'] for entry in compilation_db]
def _run_iwyu_tool(verbose, paths):
iwyu_args = ['--max_line_length=256']
for m in glob.glob(os.path.join(_MAPPINGS_DIR, "*.imp")):
iwyu_args.append("--mapping_file=%s" % os.path.abspath(m))
cmdline = [_IWYU_TOOL, '-p', _BUILD_DIR]
if verbose:
# requires include-what-you-use on the path
env = os.environ.copy()
env['PATH'] = "%s:%s" % (_TOOLCHAIN_DIR, env['PATH'])
def crash(output):
sys.exit((Colors.RED + "Failed to run IWYU tool.\n\n" + Colors.RESET +
Colors.YELLOW + "Command line:\n" + Colors.RESET +
"%s\n\n" +
Colors.YELLOW + "Output:\n" + Colors.RESET +
"%s") % (" ".join(cmdline), output))
output = check_output(cmdline, env=env, stderr=subprocess.STDOUT).decode('utf-8')
if '\nFATAL ERROR: ' in output or \
'Assertion failed: ' in output or \
return output
except subprocess.CalledProcessError as e:
def _is_muted(path):
assert os.path.isabs(path)
rel = os.path.relpath(path, ROOT)
return not rel.startswith('src/') or rel in _MUTED_FILES
def _filter_paths(paths):
return [p for p in paths if not _is_muted(p)]
def _relativize_paths(paths):
""" Make paths relative to the build directory. """
return [os.path.relpath(p, _BUILD_DIR) for p in paths]
def _get_thirdparty_include_dirs():
return glob.glob(os.path.join(get_thirdparty_dir(), "installed", "*", "include"))
def _get_fixer_flags(flags):
args = ['--quiet',
'--source_root=%s' % os.path.join(ROOT, 'src')]
if flags.dry_run:
for d in _get_thirdparty_include_dirs():
args.extend(['--thirdparty_include_dir', d])
fixer_flags, _ = iwyu.fix_includes.ParseArgs(args)
return fixer_flags
def _do_iwyu(flags, paths):
iwyu_output = _run_iwyu_tool(flags.verbose, paths)
if flags.dump_iwyu_output:"Dumping iwyu output to %s", flags.dump_iwyu_output)
with open(flags.dump_iwyu_output, "w") as f:
print(iwyu_output, file=f)
stream = StringIO(iwyu_output)
fixer_flags = _get_fixer_flags(flags)
# Passing None as 'fix_paths' tells the fixer script to process
# all of the IWYU output, instead of just the output corresponding
# to files in 'paths'. This means that if you run this script on a
# .cc file, it will also report and fix errors in headers included
# by that .cc file.
fix_paths = None
records = ParseAndMergeIWYUOutput(stream, fix_paths, fixer_flags)
unfiltered_count = len(records)
records = [r for r in records if not _is_muted(os.path.abspath(r.filename))]
if len(records) < unfiltered_count:"Muted IWYU suggestions on %d file(s)", unfiltered_count - len(records))
return iwyu.fix_includes.FixManyFiles(records, fixer_flags)
def _do_sort_only(flags, paths):
fixer_flags = _get_fixer_flags(flags)
iwyu.fix_includes.SortIncludesInFiles(paths, fixer_flags)
def main(argv):
parser = optparse.OptionParser(usage=_USAGE)
for i, arg in enumerate(argv):
if arg.startswith('-'):
argv[i] = argv[i].replace('_', '-')
parser.add_option('--all', action='store_true',
help=('Process all files listed in the compilation database of the current '
parser.add_option('--from-git', action='store_true',
help=('Determine the list of files to run IWYU automatically based on git. '
'All files which are modified in the current working tree or in commits '
'not yet committed upstream by gerrit are processed.'))
parser.add_option('--fix', action='store_false', dest="dry_run", default=True,
help=('If this is set, fixes IWYU issues in place.'))
parser.add_option('-s', '--sort-only', action='store_true',
help=('Just sort #includes of files listed on cmdline;'
' do not add or remove any #includes'))
parser.add_option('--verbose', action='store_true',
help=('Run in verbose mode. Useful for debugging'))
parser.add_option('--dump-iwyu-output', type='str',
help=('A path to dump the raw IWYU output to. This can be useful for '
'debugging this tool.'))
(flags, paths) = parser.parse_args(argv[1:])
if bool(flags.from_git) + bool(flags.all) + (len(paths) > 0) != 1:
sys.exit('Must specify exactly one of --all, --from-git, or a list of paths')
do_filtering = True
if flags.from_git:
paths = _get_file_list_from_git()
paths = [os.path.abspath(os.path.join(ROOT, p)) for p in paths]
elif paths:
paths = [os.path.abspath(p) for p in paths]
# If paths are specified explicitly, don't filter them out.
do_filtering = False
elif flags.all:
paths = _filter_paths(_get_paths_from_compilation_db())
assert False, "Should not reach here"
if do_filtering:
orig_count = len(paths)
paths = _filter_paths(paths)
if len(paths) != orig_count:"Filtered %d paths muted by configuration in",
orig_count - len(paths))
muted_paths = [p for p in paths if _is_muted(p)]
if muted_paths:
logging.warning("%d selected path(s) are known to have IWYU issues:" % len(muted_paths))
for p in muted_paths:
logging.warning(" %s" % p)
# If we came up with an empty list (no relevant files changed in the commit)
# then we should early-exit. Otherwise, we'd end up passing an empty list to
# IWYU and it will run on every file.
if flags.from_git and not paths:"No files selected for analysis.")
# IWYU output will be relative to the compilation database which is in
# the build directory. In order for the fixer script to properly find them, we need
# to treat all paths relative to that directory and chdir into it first.
paths = _relativize_paths(paths)
# For correct results, IWYU depends on the generated header files."Ensuring IWYU dependencies are built...")
if os.path.exists('Makefile'):
subprocess.check_call(['make', 'generated-headers'])
elif os.path.exists(''):
subprocess.check_call(['ninja', 'generated-headers'])
logging.error('No Makefile or found in build directory %s',
sys.exit(1)"Checking %d file(s)...", len(paths))
if flags.sort_only:
return _do_sort_only(flags, paths)
return _do_iwyu(flags, paths)
if __name__ == "__main__":