#! /usr/bin/env python3 # # Copyright 2018 - The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function """Tool for comparing 2 LTP projects to find added / removed tests & testsuites. The tool can be run in two separate steps by dumping the list of tests as a json file, i.e.: $ git checkout master $ compare_ltp_projects.py --ltp-old . --ltp-old-json ./ltp_old.json $ git checkout mymergebranch $ compare_ltp_projects.py --ltp-old-json ./ltp_old.json --ltp-new . """ import os import argparse import os.path import sys import json def scan_tests(ltp_root, suite): ''' Find all tests that are run as part of given test suite in LTP. Args: ltp_root: Path ot the LTP project. suite: Name of testsuite. Returns: List of tests names that are run as part of the given test suite. ''' tests = [] if not suite: return tests runtest_dir = ltp_root + os.path.sep + 'runtest' test_suiteFile = runtest_dir + os.path.sep + suite if not os.path.isfile(test_suiteFile): print('No tests defined for suite {}',format(suite)) return tests with open(test_suiteFile) as f: for line in f: line = line.strip() if line and not line.startswith('#'): tests.append(line.split()[0]) f.close() return tests def scan_test_suites(ltp_root, scenario): ''' Find all testuites and tests run as part of given LTP scenario Args: ltp_root: Path to the LTP project. scenario: name of the scenario (found in ltp_root/scenario_groups). E.g. "vts" Returns: List of testsuite names that are run as part of given scenario (e.g. vts). If scenario is not specified, return all testsuites. ''' runtest_dir = ltp_root + os.path.sep + 'runtest' if not os.path.isdir(runtest_dir): print('Invalid ltp_root {}, runtest directory doesnt exist'.format(ltp_root)) sys.exit(2) test_suites = [] if scenario: scenarioFile = ltp_root + os.path.sep + 'scenario_groups' + os.path.sep + scenario if not os.path.isfile(scenarioFile): return test_suites with open(scenarioFile) as f: for line in f: line = line.strip() if not line.startswith('#'): test_suites.append(line) return test_suites runtest_dir = ltp_root + os.path.sep + 'runtest' test_suites = [f for f in os.listdir(runtest_dir) if os.path.isfile(runtest_dir + os.path.sep + f)] return test_suites def scan_ltp(ltp_root, scenario): ''' scan LTP project and return all tests and testuites present. Args: ltp_root: Path to the LTP project. scenario: specific scenario we want to scan. e.g. vts Returns: Dictionary of all LTP test names keyed by testsuite they are run as part of. E.g. { 'mem': ['oom01', 'mem02',...], 'syscalls': ['open01', 'read02',...], ... } ''' if not os.path.isdir(ltp_root): print('ltp_root {} does not exist'.format(ltp_root)) sys.exit(1) test_suites = scan_test_suites(ltp_root, scenario) if not test_suites: print('No Testsuites found for scenario {}'.format(scenario)) sys.exit(3) ltp_tests = {} for suite in test_suites: ltp_tests[suite] = scan_tests(ltp_root, suite) return ltp_tests def show_diff(ltp_tests_1, ltp_tests_2): ''' Find and print diff between testcases in 2 LTP project checkouts. Args: ltp_tests_1: dictionary of tests keyed by test suite names ltp_tests_2: dictionary of tests keyed by test suite names ''' test_suites1 = set(ltp_tests_1.keys()) test_suites2 = set(ltp_tests_2.keys()) # Generate lists of deleted, added and common test suites between # LTP1 & LTP2 deleted_test_suites = sorted(test_suites1.difference(test_suites2)) added_test_suites = sorted(test_suites2.difference(test_suites1)) common_test_suites = test_suites1.intersection(test_suites2) deleted_tests = [] added_tests = [] for suite in common_test_suites: tests1 = set(ltp_tests_1[suite]) tests2 = set(ltp_tests_2[suite]) exclusive_test1 = tests1.difference(tests2) exclusive_test2 = tests2.difference(tests1) for e in exclusive_test1: deleted_tests.append(suite + '.' + e) for e in exclusive_test2: added_tests.append(suite + '.' + e) deleted_tests = sorted(deleted_tests) added_tests = sorted(added_tests) print_columns(added_test_suites, deleted_test_suites, added_tests, deleted_tests) def print_columns(added_test_suites, deleted_test_suites, added_tests, deleted_tests): DEFAULT_WIDTH = 8 # find the maximum width of a test suite name or test name # we have to print to decide the alignment. if not deleted_test_suites: width_suites = DEFAULT_WIDTH else: width_suites = max([len(x) for x in deleted_test_suites]) if not deleted_tests: width_tests = DEFAULT_WIDTH else: width_tests = max([len(x) for x in deleted_tests]) width = max(width_suites, width_tests); # total rows we have to print total_rows = max(len(deleted_test_suites), len(added_test_suites)) added_text = 'Added ({})'.format(len(added_test_suites)) deleted_text = 'Deleted ({})'.format(len(deleted_test_suites)) if total_rows > 0: print('{:*^{len}}'.format(' Tests Suites ', len=width*2+2)) print('{:<{len}} {:<{len}}'.format(added_text, deleted_text, len=width)) for i in range(total_rows): print('{:<{len}} {:<{len}}'.format('' if i >= len(added_test_suites) else str(added_test_suites[i]), '' if i >= len(deleted_test_suites) else str(deleted_test_suites[i]), len=width)) print('') # total rows we have to print total_rows = max(len(deleted_tests), len(added_tests)) added_text = 'Added ({})'.format(len(added_tests)) deleted_text = 'Deleted ({})'.format(len(deleted_tests)) if total_rows: print('{:*^{len}}'.format(' Tests ', len=width*2+2)) print('{:^{len}} {:^{len}}'.format(added_text, deleted_text, len=width)) for i in range(total_rows): print('{:<{len}} {:<{len}}'.format('' if i >= len(added_tests) else str(added_tests[i]), '' if i >= len(deleted_tests) else str(deleted_tests[i]), len=width)) def main(): arg_parser = argparse.ArgumentParser( description='Diff 2 LTP projects for supported test cases') arg_parser.add_argument('-o', '--ltp-old', dest='ltp_old', help="LTP Root Directory before merge") arg_parser.add_argument('-oj', '--ltp-old-json', dest='ltp_old_json', default='ltp_old.json', help="Old LTP parsed directory in json format") arg_parser.add_argument('-n', '--ltp-new', dest='ltp_new', help="LTP Root Directory after merge") arg_parser.add_argument('-nj', '--ltp-new-json', dest='ltp_new_json', default='ltp_new.json', help="New LTP parsed directory in json format") arg_parser.add_argument('--scenario', default=None, dest='scenario', help="LTP scenario to list tests for") args = arg_parser.parse_args() # Find tests in the "old" directory or read the json output from a prior run if args.ltp_old: ltp_tests1 = scan_ltp(args.ltp_old, args.scenario) elif args.ltp_old_json and os.path.isfile(args.ltp_old_json): with open(args.ltp_old_json) as f: ltp_tests1 = json.load(f) else: ltp_tests1 = None # Do the same for the "new" directory if args.ltp_new: ltp_tests2 = scan_ltp(args.ltp_new, args.scenario) elif args.ltp_new_json and os.path.isfile(args.ltp_new_json): with open(args.ltp_new_json) as f: ltp_tests2 = json.load(f) else: ltp_tests2 = None if ltp_tests1 and ltp_tests2: # Show the difference of the two directories if both are present show_diff(ltp_tests1, ltp_tests2) elif ltp_tests1: # If just the old directory was read, dump it as json with open(args.ltp_old_json, 'w') as f: json.dump(ltp_tests1, f) elif ltp_tests2: # If just the new directory was read, dump it as json with open(args.ltp_new_json, 'w') as f: json.dump(ltp_tests2, f) if __name__ == '__main__': main()