# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------
# drawElements Quality Program utilities
# --------------------------------------
#
# Copyright 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------
from ctsbuild.common import *
from ctsbuild.config import *
from ctsbuild.build import *
import os
import sys
import string
import socket
import fnmatch
from datetime import datetime
BASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly"))
BASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build")
BASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs")
BASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs")
EXECUTOR_PATH = "executor/executor"
LOG_TO_CSV_PATH = "executor/testlog-to-csv"
EXECSERVER_PATH = "execserver/execserver"
CASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data")
COMPARE_NUM_RESULTS = 4
COMPARE_REPORT_NAME = "nightly-report.html"
COMPARE_REPORT_TMPL = '''
${TITLE}
${TITLE}
'''
class NightlyRunConfig:
def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []):
self.name = name
self.buildConfig = buildConfig
self.generator = generator
self.binaryName = binaryName
self.testset = testset
self.args = args
self.exclude = exclude
self.ignore = ignore
def getBinaryPath(self, basePath):
return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath))
class NightlyBuildConfig(BuildConfig):
def __init__(self, name, buildType, args):
BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args)
class TestCaseResult:
def __init__ (self, name, statusCode):
self.name = name
self.statusCode = statusCode
class MultiResult:
def __init__ (self, name, statusCodes):
self.name = name
self.statusCodes = statusCodes
class BatchResult:
def __init__ (self, name):
self.name = name
self.results = []
def parseResultCsv (data):
lines = data.splitlines()[1:]
results = []
for line in lines:
items = line.split(",")
results.append(TestCaseResult(items[0], items[1]))
return results
def readTestCaseResultsFromCSV (filename):
return parseResultCsv(readFile(filename))
def readBatchResultFromCSV (filename, batchResultName = None):
batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename))
batchResult.results = readTestCaseResultsFromCSV(filename)
return batchResult
def getResultTimestamp ():
return datetime.now().strftime("%Y-%m-%d-%H-%M")
def getCompareFilenames (logsDir):
files = []
for file in os.listdir(logsDir):
fullPath = os.path.join(logsDir, file)
if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"):
files.append(fullPath)
files.sort()
return files[-COMPARE_NUM_RESULTS:]
def parseAsCSV (logPath, config):
args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath]
proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate()
return out
def computeUnifiedTestCaseList (batchResults):
caseList = []
caseSet = set()
for batchResult in batchResults:
for result in batchResult.results:
if not result.name in caseSet:
caseList.append(result.name)
caseSet.add(result.name)
return caseList
def computeUnifiedResults (batchResults):
def genResultMap (batchResult):
resMap = {}
for result in batchResult.results:
resMap[result.name] = result
return resMap
resultMap = [genResultMap(r) for r in batchResults]
caseList = computeUnifiedTestCaseList(batchResults)
results = []
for caseName in caseList:
statusCodes = []
for i in range(0, len(batchResults)):
result = resultMap[i][caseName] if caseName in resultMap[i] else None
statusCode = result.statusCode if result != None else 'Missing'
statusCodes.append(statusCode)
results.append(MultiResult(caseName, statusCodes))
return results
def allStatusCodesEqual (result):
firstCode = result.statusCodes[0]
for i in range(1, len(result.statusCodes)):
if result.statusCodes[i] != firstCode:
return False
return True
def computeDiffResults (unifiedResults):
diff = []
for result in unifiedResults:
if not allStatusCodesEqual(result):
diff.append(result)
return diff
def genCompareReport (batchResults, title, ignoreCases):
class TableRow:
def __init__ (self, testCaseName, innerHTML):
self.testCaseName = testCaseName
self.innerHTML = innerHTML
unifiedResults = computeUnifiedResults(batchResults)
diffResults = computeDiffResults(unifiedResults)
rows = []
# header
headerCol = '\n'
for batchResult in batchResults:
headerCol += '\n' % batchResult.name
rows.append(TableRow(None, headerCol))
# results
for result in diffResults:
col = '%s | \n' % result.name
for statusCode in result.statusCodes:
col += '%s | \n' % (statusCode, statusCode)
rows.append(TableRow(result.name, col))
tableStr = ""
for row in rows:
if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases):
tableStr += '\n%s
\n' % row.innerHTML
else:
tableStr += '\n%s
\n' % row.innerHTML
html = COMPARE_REPORT_TMPL
html = html.replace("${TITLE}", title)
html = html.replace("${RESULTS}", tableStr)
return html
def matchesAnyPattern (name, patterns):
for pattern in patterns:
if fnmatch.fnmatch(name, pattern):
return True
return False
def statusCodesMatch (refResult, resResult):
return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult
def compareBatchResults (referenceBatch, resultBatch, ignoreCases):
unifiedResults = computeUnifiedResults([referenceBatch, resultBatch])
failedCases = []
for result in unifiedResults:
if not matchesAnyPattern(result.name, ignoreCases):
refResult = result.statusCodes[0]
resResult = result.statusCodes[1]
if not statusCodesMatch(refResult, resResult):
failedCases.append(result)
return failedCases
def getUnusedPort ():
# \note Not 100%-proof method as other apps may grab this port before we launch execserver
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('localhost', 0))
addr, port = s.getsockname()
s.close()
return port
def runNightly (config):
build(config.buildConfig, config.generator)
# Run parameters
timestamp = getResultTimestamp()
logDir = os.path.join(BASE_LOGS_DIR, config.name)
testLogPath = os.path.join(logDir, timestamp + ".qpa")
infoLogPath = os.path.join(logDir, timestamp + ".txt")
csvLogPath = os.path.join(logDir, timestamp + ".csv")
compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv")
port = getUnusedPort()
if not os.path.exists(logDir):
os.makedirs(logDir)
if os.path.exists(testLogPath) or os.path.exists(infoLogPath):
raise Exception("Result '%s' already exists", timestamp)
# Paths, etc.
binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName))
workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName))
execArgs = [
config.getBinaryPath(EXECUTOR_PATH),
'--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH),
'--port=%d' % port,
'--binaryname=%s' % binaryName,
'--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "),
'--workdir=%s' % workingDir,
'--caselistdir=%s' % CASELIST_PATH,
'--testset=%s' % string.join(config.testset, ","),
'--out=%s' % testLogPath,
'--info=%s' % infoLogPath,
'--summary=no'
]
if len(config.exclude) > 0:
execArgs += ['--exclude=%s' % string.join(config.exclude, ",")]
execute(execArgs)
# Translate to CSV for comparison purposes
lastResultCsv = parseAsCSV(testLogPath, config)
writeFile(csvLogPath, lastResultCsv)
if os.path.exists(compareLogPath):
refBatchResult = readBatchResultFromCSV(compareLogPath, "reference")
else:
refBatchResult = None
# Generate comparison report
compareFilenames = getCompareFilenames(logDir)
batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames]
if refBatchResult != None:
batchResults = [refBatchResult] + batchResults
writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore))
print("Comparison report written to %s" % COMPARE_REPORT_NAME)
# Compare to reference
if refBatchResult != None:
curBatchResult = BatchResult("current")
curBatchResult.results = parseResultCsv(lastResultCsv)
failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore)
print("")
for result in failedCases:
print("MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1]))
print("")
print("%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed"))
if len(failedCases) > 0:
return False
return True
# Configurations
DEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR
DEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR
WGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl'])
ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu'])
BASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable']
CONFIGS = [
NightlyRunConfig(
name = "wgl_x64_release_gles2",
buildConfig = WGL_X64_RELEASE_BUILD_CFG,
generator = DEFAULT_WIN64_GENERATOR,
binaryName = "modules/gles2/deqp-gles2",
args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"],
exclude = [
"dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*",
"dEQP-GLES2.functional.shaders.loops.*while*only_continue*",
"dEQP-GLES2.functional.shaders.loops.*while*double_continue*",
],
ignore = []
),
NightlyRunConfig(
name = "wgl_x64_release_gles3",
buildConfig = WGL_X64_RELEASE_BUILD_CFG,
generator = DEFAULT_WIN64_GENERATOR,
binaryName = "modules/gles3/deqp-gles3",
args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"],
exclude = [
"dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*",
"dEQP-GLES3.functional.shaders.loops.*while*only_continue*",
"dEQP-GLES3.functional.shaders.loops.*while*double_continue*",
],
ignore = [
"dEQP-GLES3.functional.transform_feedback.*",
"dEQP-GLES3.functional.occlusion_query.*",
"dEQP-GLES3.functional.lifetime.*",
"dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops",
]
),
NightlyRunConfig(
name = "wgl_x64_release_gles31",
buildConfig = WGL_X64_RELEASE_BUILD_CFG,
generator = DEFAULT_WIN64_GENERATOR,
binaryName = "modules/gles31/deqp-gles31",
args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
testset = ["dEQP-GLES31.*"],
exclude = [],
ignore = [
"dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3",
"dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer",
"dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset",
"dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint",
"dEQP-GLES31.functional.blend_equation_advanced.basic.*",
"dEQP-GLES31.functional.blend_equation_advanced.srgb.*",
"dEQP-GLES31.functional.blend_equation_advanced.barrier.*",
"dEQP-GLES31.functional.uniform_location.*",
"dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv",
"dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv",
"dEQP-GLES31.functional.debug.error_filters.case_0",
"dEQP-GLES31.functional.debug.error_filters.case_2",
]
),
NightlyRunConfig(
name = "wgl_x64_release_gl3",
buildConfig = WGL_X64_RELEASE_BUILD_CFG,
generator = DEFAULT_WIN64_GENERATOR,
binaryName = "modules/gl3/deqp-gl3",
args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"],
exclude = [
"dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*",
"dEQP-GL3.functional.shaders.loops.*while*only_continue*",
"dEQP-GL3.functional.shaders.loops.*while*double_continue*",
],
ignore = [
"dEQP-GL3.functional.transform_feedback.*"
]
),
NightlyRunConfig(
name = "arm_gles3_emu_x32_egl",
buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG,
generator = DEFAULT_WIN32_GENERATOR,
binaryName = "modules/egl/deqp-egl",
args = BASE_ARGS,
testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"],
exclude = [
"dEQP-EGL.functional.sharing.gles2.multithread.*",
"dEQP-EGL.functional.multithread.*",
],
ignore = []
),
NightlyRunConfig(
name = "opencl_x64_release",
buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']),
generator = DEFAULT_WIN64_GENERATOR,
binaryName = "modules/opencl/deqp-opencl",
args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS,
testset = ["dEQP-CL.*"],
exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"],
ignore = [
"dEQP-CL.scheduler.random.*",
"dEQP-CL.language.set_kernel_arg.random_structs.*",
"dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset",
"dEQP-CL.language.call_function.arguments.random_structs.*",
"dEQP-CL.language.call_kernel.random_structs.*",
"dEQP-CL.language.inf_nan.nan.frexp.float",
"dEQP-CL.language.inf_nan.nan.lgamma_r.float",
"dEQP-CL.language.inf_nan.nan.modf.float",
"dEQP-CL.language.inf_nan.nan.sqrt.float",
"dEQP-CL.api.multithread.*",
"dEQP-CL.api.callback.random.nested.*",
"dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb",
"dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb",
"dEQP-CL.image.addressing_filtering12.1d_array.*",
"dEQP-CL.image.addressing_filtering12.2d_array.*"
]
)
]
if __name__ == "__main__":
config = None
if len(sys.argv) == 2:
cfgName = sys.argv[1]
for curCfg in CONFIGS:
if curCfg.name == cfgName:
config = curCfg
break
if config != None:
isOk = runNightly(config)
if not isOk:
sys.exit(-1)
else:
print("%s: [config]" % sys.argv[0])
print("")
print(" Available configs:")
for config in CONFIGS:
print(" %s" % config.name)
sys.exit(-1)