#!/usr/bin/env python # Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Checks that source files are mentioned in CMakeLists.txt. Also checks that files mentioned in CMakeLists.txt exist on the filesystem. Note that this check needs to be able to run before anything has been built, so generated files must be excluded from the check. Add a "# NOLINT(generated)" comment to any line mentioning such a file to ensure they don't falsely trigger errors. This only needs to be done once within a file. """ import argparse import collections import os import re import sys from lib import git # Directories relative to the repo root that will be scanned by default if no # arguments are passed. _DEFAULT_DIRS = [ 'Firestore/core', 'Firestore/Example/Tests' 'Firestore/Source' ] # When scanning the filesystem, look for specific files or files with these # extensions. _INCLUDE_FILES = {'CMakeLists.txt'} _INCLUDE_EXTENSIONS = {'.c', '.cc', '.h', '.m', '.mm'} # When scanning the filesystem, exclude any files or directories with these # names. _EXCLUDE_DIRS = {'third_party', 'Pods', 'Protos'} _verbose = False def main(args): global _verbose parser = argparse.ArgumentParser( description='Check CMakeLists.txt file membership.') parser.add_argument('--verbose', '-v', action='store_true', help='Run verbosely') parser.add_argument('filenames', nargs='*', metavar='file_or_dir', help='Files and directories to scan') args = parser.parse_args(args) if args.verbose: _verbose = True scan_filenames = args.filenames if not scan_filenames: scan_filenames = default_args() filenames = find_source_files(scan_filenames) groups = group_by_cmakelists(filenames) errors = find_all_errors(groups) trace('checked %d files' % len(filenames)) sys.exit(1 if errors else 0) def default_args(): """Returns a default list of directories to scan. """ toplevel = git.get_repo_root() return [os.path.join(toplevel, dirname) for dirname in _DEFAULT_DIRS] def find_source_files(roots): """Finds source files on the filesystem. Args: roots: A list of files or directories Returns: A list of filenames found in the roots, excluding those that are uninteresting. """ result = [] for root in roots: for parent, dirs, files in os.walk(root, topdown=True): # Prune directories known to be uninteresting dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS] for filename in files: if filename in _INCLUDE_FILES or is_source_file(filename): result.append(os.path.join(parent, filename)) return result _filename_pattern = re.compile(r'\b([A-Za-z0-9_/+]+\.)+(?:c|cc|h|m|mm)\b') _comment_pattern = re.compile(r'^(\s*)#') _check_pattern = re.compile(r'^\s*check_[A-Za-z0-9_]+\(.*\)$') _nolint_pattern = re.compile(r'NOLINT') def read_listed_source_files(filename): """Reads the contents of the given filename and finds all the filenames it finds in the file. Args: filename: A filename to read, typically some path to a CMakeLists.txt file. Returns: A pair of lists. The first list contains filenames mentioned in the file. The second contains files that have been ignored (by marking them NOLINT) in the file. Elements from the second list might also be present in the first list. """ found = [] ignored = [] parent = os.path.dirname(filename) with open(filename, 'r') as fd: for line in fd.readlines(): # Simple hack to exclude files mentioned in CMake checks if _check_pattern.match(line): continue ignore = bool(_nolint_pattern.search(line)) if not ignore: # Exclude comments, but only on regular lines. This allows files to # include files in comments that mark them NOLINT. m = _comment_pattern.match(line) if m: line = m.group(1) for m in _filename_pattern.finditer(line): listed_filename = os.path.join(parent, m.group(0)) if ignore: trace('ignoring %s' % listed_filename) ignored.append(listed_filename) else: found.append(listed_filename) found.sort() ignored.sort() return found, ignored class Group(object): """A comparison group. Groups include the location of a CMakeLists.txt file along with files that were found on the filesystem that should be mentioned in the file, files that were found in the CMakeLists.txt file, and any files that should be ignored. """ def __init__(self): self.list_file = None self.fs_files = [] self.list_files = [] self.ignored_files = [] def shorten(self): """Shorten filenames to make them relative to the directory containing the CMakeLists.txt file and make the file lists into sets. """ prefix = os.path.dirname(self.list_file) + '/' self.fs_files = self._remove_prefix(prefix, self.fs_files) self.list_files = self._remove_prefix(prefix, self.list_files) self.ignored_files = self._remove_prefix(prefix, self.ignored_files) def _remove_prefix(self, prefix, filenames): result = [] for filename in filenames: if not filename.startswith(prefix): raise Exception('Filename %s not in prefix %s' % (filename, prefix)) result.append(filename[len(prefix):]) return set(result) def __repr__(self): def files(items): return repr(sorted(list(items))) return """""" % (self.list_file, files(self.fs_files), files(self.list_files), files(self.ignored_files)) def group_by_cmakelists(filenames): """Groups the given filenames by the nearest CMakeLists.txt Args: filenames: A list of filenames found on the filesystem. Returns: A list of filled-out Groups for evaluation. """ filename_set = set(filenames) groups = collections.defaultdict(Group) for filename in filenames: if is_source_file(filename): for cmake_list in possible_cmake_lists_files(filename): if cmake_list in filename_set: groups[cmake_list].fs_files.append(filename) break elif os.path.basename(filename) == 'CMakeLists.txt': g = groups[filename] g.list_file = filename g.list_files, g.ignored_files = read_listed_source_files(filename) return sorted(list(groups.values()), key=lambda g: g.list_file) def find_all_errors(groups): """Finds errors in the given groups. Args: groups: A list of groups. Each group is shortened. Returns: A count of errors encountered; errors information is printed for the user. """ errors = 0 for group in groups: group.shorten() errors += find_errors(group) return errors def find_errors(group): """Evaluates whether or not a group has any errors. """ in_both = group.fs_files.intersection(group.list_files) in_both = in_both | group.ignored_files just_fs = group.fs_files - in_both just_list = group.list_files - in_both if just_fs or just_list: sys.stderr.write('%s had errors:\n' % group.list_file) for filename in sorted(just_fs): sys.stderr.write(' %s: missing from CMakeLists.txt\n' % filename) for filename in sorted(just_list): sys.stderr.write(' %s: missing from filesystem\n' % filename) return len(just_fs) + len(just_list) def is_source_file(filename): ext = os.path.splitext(filename)[1] return ext in _INCLUDE_EXTENSIONS def possible_cmake_lists_files(filename): """Finds CMakeLists.txt files that might apply to the given filename. Args: filename: A source file Yields: A sequence of CMakeLists.txt filenames that might govern the source file, starting in the directory containing the given filename and working up toward the filesystem root. The filenames may point to files that don't exist. """ while filename: filename = os.path.dirname(filename) yield os.path.join(filename, 'CMakeLists.txt') def trace(line): if _verbose: sys.stderr.write('%s\n' % line) if __name__ == '__main__': main(sys.argv[1:])