# Lint as: python2, python3 # Copyright 2017 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import os import unittest import common from autotest_lib.server.cros.tradefed import tradefed_utils def _load_data(filename): """Loads the test data of the given file name.""" with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', filename), 'r') as f: return f.read() class TradefedTestTest(unittest.TestCase): """Unittest for tradefed_utils.""" def test_parse_tradefed_result(self): """Test for parse_tradefed_result.""" waivers = set([ 'android.app.cts.SystemFeaturesTest#testUsbAccessory', 'android.widget.cts.GridViewTest#testSetNumColumns', ]) # b/35605415 and b/36520623 # http://pantheon/storage/browser/chromeos-autotest-results/108103986-chromeos-test/ # CTS: Tradefed may split a module to multiple chunks. # Besides, the module name may not end with "TestCases". waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsHostsideNetworkTests.txt'), waivers=waivers) self.assertEquals(0, len(waived)) # b/35530394 # http://pantheon/storage/browser/chromeos-autotest-results/108291418-chromeos-test/ # Crashed, but the automatic retry by tradefed executed the rest. waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsMediaTestCases.txt'), waivers=waivers) self.assertEquals(0, len(waived)) # b/35530394 # http://pantheon/storage/browser/chromeos-autotest-results/106540705-chromeos-test/ # Crashed in the middle, and the device didn't came back. waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsSecurityTestCases.txt'), waivers=waivers) self.assertEquals(0, len(waived)) # b/36629187 # http://pantheon/storage/browser/chromeos-autotest-results/108855595-chromeos-test/ # Crashed in the middle. Tradefed decided not to continue. waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsViewTestCases.txt'), waivers=waivers) self.assertEquals(0, len(waived)) # b/36375690 # http://pantheon/storage/browser/chromeos-autotest-results/109040174-chromeos-test/ # Mixture of real failures and waivers. waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsAppTestCases.txt'), waivers=waivers) self.assertEquals(1, len(waived)) # ... and the retry of the above failing iteration. waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsAppTestCases-retry.txt'), waivers=waivers) self.assertEquals(1, len(waived)) # http://pantheon/storage/browser/chromeos-autotest-results/116875512-chromeos-test/ # When a test case crashed during teardown, tradefed prints the "fail" # message twice. Tolerate it and still return an (inconsistent) count. waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsWidgetTestCases.txt'), waivers=waivers) self.assertEquals(1, len(waived)) # http://pantheon/storage/browser/chromeos-autotest-results/117914707-chromeos-test/ # When a test case unrecoverably crashed during teardown, tradefed # prints the "fail" and failure summary message twice. Tolerate it. waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsPrintTestCases.txt'), waivers=waivers) self.assertEquals(0, len(waived)) gts_waivers = set([ ('com.google.android.placement.gts.CoreGmsAppsTest#' + 'testCoreGmsAppsPreloaded'), ('com.google.android.placement.gts.CoreGmsAppsTest#' + 'testGoogleDuoPreloaded'), 'com.google.android.placement.gts.UiPlacementTest#testPlayStore' ]) # crbug.com/748116 # http://pantheon/storage/browser/chromeos-autotest-results/130080763-chromeos-test/ # 3 ABIS: x86, x86_64, and armeabi-v7a waived, _ = tradefed_utils.parse_tradefed_result( _load_data('GtsPlacementTestCases.txt'), waivers=gts_waivers) self.assertEquals(9, len(waived)) # b/64095702 # http://pantheon/storage/browser/chromeos-autotest-results/130211812-chromeos-test/ # The result of the last chunk not reported by tradefed. # The actual dEQP log is too big, hence the test data here is trimmed. waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsDeqpTestCases-trimmed.txt'), waivers=waivers) self.assertEquals(0, len(waived)) # b/80160772 # http://pantheon/storage/browser/chromeos-autotest-results/201962931-kkanchi/ # The newer tradefed requires different parsing to count waivers. waived, _ = tradefed_utils.parse_tradefed_result( _load_data('CtsAppTestCases_P_simplified.txt'), waivers=waivers) self.assertEquals(1, len(waived)) # b/66899135, tradefed may reported inaccuratly with `list results`. # Check if summary section shows that the result is inacurrate. _, accurate = tradefed_utils.parse_tradefed_result( _load_data('CtsAppTestCases_P_simplified.txt'), waivers=waivers) self.assertTrue(accurate) _, accurate = tradefed_utils.parse_tradefed_result( _load_data('CtsDeqpTestCases-trimmed-inaccurate.txt'), waivers=waivers) self.assertFalse(accurate) def test_get_test_result_xml_path(self): path = tradefed_utils.get_test_result_xml_path(os.path.join( os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', 'results')) self.assertEqual(path, os.path.join( os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', 'results', '2019.11.07_10.14.55', 'test_result.xml')) # assertNoRaises tradefed_utils.get_test_result_xml_path(os.path.join( os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', 'not_exist')) def test_parse_tradefed_testresults_xml_no_failure(self): waived = tradefed_utils.parse_tradefed_testresults_xml( os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', 'test_result.xml')) self.assertEquals(0, len(waived)) def test_parse_tradefed_testresult_xml_waivers(self): waived = tradefed_utils.parse_tradefed_testresults_xml( os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', 'gtsplacement_test_result.xml')) self.assertEquals(0, len(waived)) waivers = set([ 'com.google.android.media.gts.WidevineDashPolicyTests#testL1RenewalDelay5S', 'com.google.android.media.gts.MediaDrmTest#testWidevineApi28', 'com.google.android.media.gts.WidevineGenericOpsTests#testL3', 'com.google.android.media.gts.WidevineDashPolicyTests#testL3RenewalDelay5S', 'com.google.android.media.gts.WidevineH264PlaybackTests#testCbc1L3WithUHD30', 'com.google.android.media.gts.WidevineH264PlaybackTests#testCbcsL3WithUHD30', 'com.google.android.media.gts.WidevineH264PlaybackTests#testCbc1L1WithUHD30', 'com.google.android.media.gts.WidevineDashPolicyTests#testL3RenewalDelay13S', 'com.google.android.gts.backup.BackupHostTest#testGmsBackupTransportIsDefault', 'com.google.android.placement.gts.CoreGmsAppsTest#testGoogleDuoPreloaded', 'com.google.android.placement.gts.CoreGmsAppsTest#testCoreGmsAppsPreloaded', 'com.google.android.media.gts.WidevineH264PlaybackTests#testCbcsL1WithUHD30']) waived = tradefed_utils.parse_tradefed_testresults_xml(os.path.join( os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', 'gtsplacement_test_result.xml'), waivers=waivers) self.assertEquals(4, len(waived)) def test_get_perf_metrics_from_test_result_xml(self): perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml( os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', 'test_result.xml'), os.path.join('/', 'resultsdir')) expected_result = [ {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioRecordTest' '#testAudioRecordLocalMono16Bit', 'value': '7.1688596491228065', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioRecordTest' '#testAudioRecordLocalMono16BitShort', 'value': '2.5416666666666665', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioRecordTest' '#testAudioRecordLocalNonblockingStereoFloat', 'value': '1.75', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioRecordTest' '#testAudioRecordMonoFloat', 'value': '12.958881578947368', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioRecordTest' '#testAudioRecordResamplerMono8Bit', 'value': '0.0', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioRecordTest' '#testAudioRecordResamplerStereo8Bit', 'value': '3.5', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioRecordTest' '#testAudioRecordStereo16Bit', 'value': '3.5', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioTrackTest' '#testFastTimestamp', 'value': '0.1547618955373764', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioTrackTest' '#testGetTimestamp', 'value': '0.1490119844675064', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioTrack_ListenerTest' '#testAudioTrackCallback', 'value': '9.347127739984884', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioTrack_ListenerTest' '#testAudioTrackCallbackWithHandler', 'value': '7.776177955844914', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioTrack_ListenerTest' '#testStaticAudioTrackCallback', 'value': '7.776177955844914', 'higher_is_better': False}, {'units': 'ms', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.AudioTrack_ListenerTest' '#testStaticAudioTrackCallbackWithHandler', 'value': '9.514361300075587', 'higher_is_better': False}, {'units': 'count', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.DecoderTest' '#testH264ColorAspects', 'value': '1.0', 'higher_is_better': True}, {'units': 'count', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.DecoderTest' '#testH265ColorAspects', 'value': '1.0', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testAvcGoog0Perf0320x0240', 'value': '580.1607045151507', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testAvcGoog0Perf0720x0480', 'value': '244.18184010611358', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testAvcGoog0Perf1280x0720', 'value': '70.96290491279275', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testAvcGoog0Perf1920x1080', 'value': '31.299613935451564', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testAvcOther0Perf0320x0240', 'value': '1079.6843075197307', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testAvcOther0Perf0720x0480', 'value': '873.7785366761784', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testAvcOther0Perf1280x0720', 'value': '664.6463289568261', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testAvcOther0Perf1920x1080', 'value': '382.10811352923474', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testH263Goog0Perf0176x0144', 'value': '1511.3027429644353', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testHevcGoog0Perf0352x0288', 'value': '768.8737453173384', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testHevcGoog0Perf0640x0360', 'value': '353.7226028743237', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testHevcGoog0Perf0720x0480', 'value': '319.3122874170939', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testHevcGoog0Perf1280x0720', 'value': '120.89218432028369', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testMpeg4Goog0Perf0176x0144', 'value': '1851.890822618321', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp8Goog0Perf0320x0180', 'value': '1087.946513466716', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp8Goog0Perf0640x0360', 'value': '410.18461316281423', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp8Goog0Perf1920x1080', 'value': '36.26433070651982', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp8Other0Perf0320x0180', 'value': '1066.7819511702078', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp8Other0Perf0640x0360', 'value': '930.261434505189', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp8Other0Perf1280x0720', 'value': '720.4170603577236', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp8Other0Perf1920x1080', 'value': '377.55742437554915', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp9Goog0Perf0320x0180', 'value': '988.6158776121617', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp9Goog0Perf0640x0360', 'value': '409.8162085338674', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp9Goog0Perf1280x0720', 'value': '147.75847359424512', 'higher_is_better': True}, {'units': 'fps', 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 'description': 'android.media.cts.VideoDecoderPerfTest' '#testVp9Goog0Perf1920x1080', 'value': '83.95677136649255', 'higher_is_better': True} ] self.assertListEqual(list(perf_result), expected_result) perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml( os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', 'malformed_test_result.xml'), os.path.join('/', 'resultsdir')) self.assertListEqual(list(perf_result), []) # assertNoRaises tradefed_utils.get_perf_metrics_from_test_result_xml( os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tradefed_utils_unittest_data', 'not_exist'), os.path.join('/', 'resultsdir')) if __name__ == '__main__': unittest.main()