| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| """Add or check license header |
| |
| Usuage: |
| |
| - add the default license header to source files that do not contain a valid |
| license: |
| |
| license_header.py add |
| |
| - check if every files has a license header |
| |
| license_header.py check |
| """ |
| |
| import re |
| import os |
| import argparse |
| from itertools import chain |
| import logging |
| import sys |
| import subprocess |
| |
| # the default apache license |
| _LICENSE = """Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, |
| software distributed under the License is distributed on an |
| "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| KIND, either express or implied. See the License for the |
| specific language governing permissions and limitations |
| under the License.""" |
| |
| # if a file contains any str in the list, then consider it has been licensed |
| _APACHE_LICENSE_PATTERNS = ['Licensed to the Apache Software Foundation'] |
| _OTHER_LICENSE_PATTERNS = ['THE SOFTWARE IS PROVIDED \"AS IS\"', |
| 'THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS'] |
| TOP_LEVEL_LICENSE_FILE = 'LICENSE' |
| |
| # the folders or files that will be ignored |
| _WHITE_LIST = [ |
| # Git submodules under different licenses |
| '3rdparty/ctc_include/contrib/moderngpu', |
| '3rdparty/dlpack', |
| '3rdparty/dmlc-core', |
| '3rdparty/googletest', |
| '3rdparty/mkldnn', |
| '3rdparty/nvidia_cub', |
| '3rdparty/onnx-tensorrt', |
| '3rdparty/openmp', |
| '3rdparty/ps-lite', |
| '3rdparty/tvm', |
| |
| # 3rdparty headerfiles under different licenses |
| 'include/mkldnn', |
| |
| # Docs Sphinx themes under different licenses |
| 'docs/python_docs/themes', |
| |
| # Docs Jekyll website under different licenses |
| 'docs/static_site', |
| |
| # Code shared with project by author - see file for details |
| 'src/operator/special_functions-inl.h', |
| |
| # Code generated by scala-package, checked in, and verified |
| 'scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h', |
| 'scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h', |
| |
| # Licensed under Caffe header |
| 'src/operator/nn/pool.h', |
| 'src/operator/nn/pool.cuh', |
| 'src/operator/contrib/psroi_pooling-inl.h', |
| 'src/operator/contrib/nn/deformable_im2col.h', |
| 'src/operator/contrib/nn/deformable_im2col.cuh', |
| 'src/operator/contrib/nn/modulated_deformable_im2col.h', |
| 'src/operator/contrib/nn/modulated_deformable_im2col.cuh', |
| 'src/operator/nn/im2col.h', |
| 'src/operator/nn/im2col.cuh', |
| |
| # Licenses in headers |
| 'src/operator/contrib/erfinv-inl.h', |
| 'docs/_static/searchtools_custom.js', |
| 'docs/_static/js/clipboard.js', |
| 'docs/_static/js/clipboard.min.js', |
| 'docs/static_site/src/assets/js/clipboard.js', |
| 'cmake/upstream/FindCUDAToolkit.cmake', |
| 'cmake/upstream/select_compute_arch.cmake', |
| |
| # Licensed under 2-Clause BSD in header |
| 'example/ssd/dataset/pycocotools/coco.py', |
| |
| # Licensed under the Boost Software License, Version 1.0 |
| 'cmake/Modules/FindJeMalloc.cmake', |
| |
| # Licensed under MIT license and contributed by Microsoft |
| 'src/operator/contrib/nn/modulated_deformable_im2col.cuh', |
| 'src/operator/contrib/nn/modulated_deformable_im2col.h', |
| 'src/operator/contrib/modulated_deformable_convolution-inl.h', |
| 'src/operator/contrib/modulated_deformable_convolution.cc', |
| 'src/operator/contrib/modulated_deformable_convolution.cu', |
| |
| # Licensed under Apache 2.0 license and contributed by Microsoft |
| 'src/operator/contrib/deformable_psroi_pooling.cu', |
| 'src/operator/contrib/deformable_convolution.cu', |
| 'src/operator/contrib/deformable_convolution-inl.h', |
| 'src/operator/contrib/psroi_pooling.cc', |
| 'src/operator/contrib/multi_proposal.cu', |
| 'src/operator/contrib/deformable_psroi_pooling-inl.h', |
| 'src/operator/contrib/deformable_psroi_pooling.cc', |
| 'src/operator/contrib/deformable_convolution.cc', |
| 'src/operator/contrib/psroi_pooling.cu', |
| 'src/operator/contrib/multi_proposal.cc', |
| 'src/operator/contrib/multi_proposal-inl.h', |
| |
| # Julia package metadata, generated by Pkg3.jl |
| 'julia/Project.toml', |
| |
| # Licensed under Apache 2.0 license |
| 'example/image-classification/predict-cpp/image-classification-predict.cc', |
| 'src/operator/nn/mkldnn/mkldnn_base-inl.h', |
| |
| # This file |
| 'tools/license_header.py', |
| |
| # Dual-Licensed under Apache 2.0 and Nvidia BSD-3 |
| 'python/mxnet/onnx/mx2onnx/_export_onnx.py', |
| 'python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset12.py', |
| 'python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset13.py', |
| |
| # Github template |
| '.github/ISSUE_TEMPLATE/bug_report.md', |
| '.github/ISSUE_TEMPLATE/feature_request.md', |
| '.github/ISSUE_TEMPLATE/flaky_test.md', |
| '.github/PULL_REQUEST_TEMPLATE.md' |
| ] |
| |
| # language extensions and the according commment mark |
| _LANGS = {'.cc':'*', '.h':'*', '.cu':'*', '.cuh':'*', '.py':'#', |
| '.pm':'#', '.scala':'*', '.cc':'*', '.sh':'#', '.cmake':'#', |
| '.java':'*', '.sh':'#', '.cpp':'*', '.hpp':'*', '.c':'*', |
| '.bat':'rem', '.pl':'#', '.m':'%', '.R':'#', '.mk':'#', '.cfg':'#', |
| '.t':'#', '.ps1':'#', '.jl':'#', '.clj':';;', '.pyx':'#', '.js':'*', |
| '.md':'<!---'} |
| |
| # Previous license header, which will be removed |
| _OLD_LICENSE = re.compile('.*Copyright.*by Contributors') |
| |
| |
| def get_mxnet_root(): |
| curpath = os.path.abspath(os.path.dirname(__file__)) |
| def is_mxnet_root(path: str) -> bool: |
| return os.path.exists(os.path.join(path, ".mxnet_root")) |
| while not is_mxnet_root(curpath): |
| parent = os.path.abspath(os.path.join(curpath, os.pardir)) |
| if parent == curpath: |
| raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root") |
| curpath = parent |
| return curpath |
| |
| |
| def _lines_have_old_license(lines): |
| for l in lines: |
| if _OLD_LICENSE.match(l): |
| return True |
| return False |
| |
| |
| def _lines_have_multiple_license(lines): |
| has_apache_license = False |
| has_other_license = False |
| for l in lines: |
| if any(p in l for p in _APACHE_LICENSE_PATTERNS): |
| has_apache_license = True |
| if any(p in l for p in _OTHER_LICENSE_PATTERNS): |
| has_other_license = True |
| return (has_apache_license and has_other_license) |
| |
| |
| def _lines_have_apache_license(lines): |
| return any([any([p in l for p in _APACHE_LICENSE_PATTERNS]) for l in lines]) |
| |
| |
| def _file_listed_in_top_level_license(fname): |
| with open(TOP_LEVEL_LICENSE_FILE, 'r', encoding="utf-8") as f: |
| lines = f.readlines() |
| module = os.path.split(fname)[0] + '/LICENSE' |
| return any([fname in l or module in l for l in lines]) |
| |
| |
| def file_have_valid_license(fname): |
| with open(fname, 'r', encoding="utf-8") as f: |
| lines = f.readlines() |
| if not lines: |
| return True |
| if (_lines_have_apache_license(lines) and (not _lines_have_multiple_license(lines))): |
| return True |
| elif _lines_have_multiple_license(lines): |
| logging.error("File %s has multiple licenses", fname) |
| return False |
| else: |
| if _file_listed_in_top_level_license(fname): |
| return True |
| else: |
| logging.error("File %s doesn't have a valid license", fname) |
| return False |
| |
| |
| def _get_license(comment_mark): |
| if comment_mark == '*': |
| body = '/*\n' |
| else: |
| body = '' |
| for l in _LICENSE.split('\n'): |
| if comment_mark == '*': |
| body += ' ' |
| body += comment_mark |
| if len(l): |
| body += ' ' + l |
| if comment_mark == '<!---': |
| body += ' -->' |
| body += '\n' |
| |
| if comment_mark == '*': |
| body += ' */\n' |
| body += '\n' |
| return body |
| |
| |
| def should_have_license(fname): |
| if any([l in fname for l in _WHITE_LIST]): |
| logging.debug('skip ' + fname + ', it matches the white list') |
| return False |
| _, ext = os.path.splitext(fname) |
| if ext not in _LANGS: |
| logging.debug('skip ' + fname + ', unknown file extension') |
| return False |
| return True |
| |
| |
| def file_has_license(fname): |
| if not should_have_license(fname): |
| return True |
| try: |
| return file_have_valid_license(fname) |
| except UnicodeError: |
| return True |
| return True |
| |
| |
| def file_add_license(fname): |
| if not should_have_license(fname): |
| return |
| if file_have_valid_license(fname): |
| return |
| with open(fname, 'r', encoding="utf-8") as f: |
| lines = f.readlines() |
| _, ext = os.path.splitext(fname) |
| with open(fname, 'w', encoding="utf-8") as f: |
| # shebang line |
| if lines[0].startswith('#!'): |
| f.write(lines[0].rstrip()+'\n\n') |
| del lines[0] |
| f.write(_get_license(_LANGS[ext])) |
| for l in lines: |
| if _OLD_LICENSE.match(l): |
| continue |
| f.write(l.rstrip()+'\n') |
| logging.info('added license header to ' + fname) |
| return |
| |
| |
| def under_git(): |
| return subprocess.run(['git', 'rev-parse', 'HEAD'], |
| stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 |
| |
| |
| def git_files(): |
| return list(map(os.fsdecode, |
| subprocess.check_output('git ls-tree -r HEAD --name-only -z'.split()).split(b'\0'))) |
| |
| |
| def file_generator(path: str): |
| for (dirpath, dirnames, files) in os.walk(path): |
| for file in files: |
| yield os.path.abspath(os.path.join(dirpath, file)) |
| |
| |
| def foreach(fn, iterable): |
| for x in iterable: |
| fn(x) |
| |
| |
| def script_name(): |
| """:returns: script name with leading paths removed""" |
| return os.path.split(sys.argv[0])[1] |
| |
| |
| def main(): |
| logging.basicConfig( |
| format='{}: %(levelname)s %(message)s'.format(script_name()), |
| level=os.environ.get("LOGLEVEL", "INFO")) |
| |
| parser = argparse.ArgumentParser( |
| description='Add or check source license header') |
| |
| parser.add_argument( |
| 'action', nargs=1, type=str, |
| choices=['add', 'check'], default='add', |
| help='add or check') |
| |
| parser.add_argument( |
| 'file', nargs='*', type=str, action='append', |
| help='Files to add license header to') |
| |
| args = parser.parse_args() |
| action = args.action[0] |
| files = list(chain(*args.file)) |
| if not files and action == 'check': |
| if under_git(): |
| logging.info("Git detected: Using files under version control") |
| files = git_files() |
| else: |
| logging.info("Using files under mxnet sources root") |
| files = file_generator(get_mxnet_root()) |
| |
| if action == 'check': |
| logging.info("Start to check %d files", (len(files))) |
| if False in list(map(file_has_license, files)): |
| return 1 |
| else: |
| logging.info("All known and whitelisted files have license") |
| return 0 |
| else: |
| assert action == 'add' |
| foreach(file_add_license, files) |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |