/*
 * Copyright (C) 2010 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.tradefed.config;

import com.android.tradefed.build.BuildRetrievalError;
import com.android.tradefed.command.CommandOptions;
import com.android.tradefed.config.proxy.TradefedDelegator;
import com.android.tradefed.config.remote.ExtendedFile;
import com.android.tradefed.config.remote.IRemoteFileResolver.ResolvedFile;
import com.android.tradefed.config.yaml.ConfigurationYamlParser;
import com.android.tradefed.invoker.tracing.CloseableTraceScope;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.error.InfraErrorIdentifier;
import com.android.tradefed.util.ClassPathScanner;
import com.android.tradefed.util.ClassPathScanner.IClassPathFilter;
import com.android.tradefed.util.DirectedGraph;
import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.StreamUtil;
import com.android.tradefed.util.SystemUtil;
import com.android.tradefed.util.keystore.DryRunKeyStore;
import com.android.tradefed.util.keystore.IKeyStoreClient;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSortedSet;

import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.regex.Pattern;

/**
 * Factory for creating {@link IConfiguration}.
 */
public class ConfigurationFactory implements IConfigurationFactory {

    /** Currently supported extensions for Tradefed configurations */
    private static final Set<String> SUPPORTED_EXTENSIONS =
            ImmutableSortedSet.of(".xml", ".config");

    private static IConfigurationFactory sInstance = null;
    private static final String CONFIG_PREFIX = "config/";
    private static final String DRY_RUN_TEMPLATE_CONFIG = "empty";
    private static final String CONFIG_ERROR_PATTERN = "(Could not find option with name )(.*)";
    // TODO(murj) generalize this to a URI matcher
    private static final String DIRECT_CONFIG_PATTERN = "^(gs|file|http|https)://.*";

    private Map<ConfigId, ConfigurationDef> mConfigDefMap;

    /**
     * A simple struct-like class that stores a configuration's name alongside
     * the arguments for any {@code <template-include>} tags it may contain.
     * Because the actual bits stored by the configuration may vary with
     * template arguments, they must be considered as essential a part of the
     * configuration's identity as the filename.
     */
    static class ConfigId {
        public String name = null;
        public Map<String, String> templateMap = new HashMap<>();

        /**
         * No-op constructor
         */
        public ConfigId() {
        }

        /**
         * Convenience constructor. Equivalent to calling two-arg constructor
         * with {@code null} {@code templateMap}.
         */
        public ConfigId(String name) {
            this(name, null);
        }

        /**
         * Two-arg convenience constructor. {@code templateMap} may be null.
         */
        public ConfigId(String name, Map<String, String> templateMap) {
            this.name = name;
            if (templateMap != null) {
                this.templateMap.putAll(templateMap);
            }
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public int hashCode() {
            return 2 * ((name == null) ? 0 : name.hashCode()) + 3 * templateMap.hashCode();
        }

        private boolean matches(Object a, Object b) {
            if (a == null && b == null) {
                return true;
            }
            if (a == null || b == null) {
                return false;
            }
            return a.equals(b);
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public boolean equals(Object other) {
            if (other == null) {
                return false;
            }
            if (!(other instanceof ConfigId)) {
                return false;
            }

            final ConfigId otherConf = (ConfigId) other;
            return matches(name, otherConf.name) && matches(templateMap, otherConf.templateMap);
        }
    }

    /**
     * A {@link IClassPathFilter} for configuration XML files.
     */
    private class ConfigClasspathFilter implements IClassPathFilter {

        private String mPrefix = null;

        public ConfigClasspathFilter(String prefix) {
            mPrefix = getConfigPrefix();
            if (prefix != null) {
                mPrefix += prefix;
            }
            CLog.d("Searching the '%s' config path", mPrefix);
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public boolean accept(String pathName) {
            // only accept entries that match the pattern, and that we don't already know about
            final ConfigId pathId = new ConfigId(pathName);
            String extension = FileUtil.getExtension(pathName);
            return pathName.startsWith(mPrefix)
                    && SUPPORTED_EXTENSIONS.contains(extension)
                    && !mConfigDefMap.containsKey(pathId);
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public String transform(String pathName) {
            // strip off CONFIG_PREFIX and config extension
            int pathStartIndex = getConfigPrefix().length();
            String extension = FileUtil.getExtension(pathName);
            int pathEndIndex = pathName.length() - extension.length();
            return pathName.substring(pathStartIndex, pathEndIndex);
        }
    }

    /**
     * A {@link Comparator} for {@link ConfigurationDef} that sorts by
     * {@link ConfigurationDef#getName()}.
     */
    private static class ConfigDefComparator implements Comparator<ConfigurationDef> {

        /**
         * {@inheritDoc}
         */
        @Override
        public int compare(ConfigurationDef d1, ConfigurationDef d2) {
            return d1.getName().compareTo(d2.getName());
        }

    }

    /**
     * Get a list of {@link File} of the test cases directories
     *
     * <p>The wrapper function is for unit test to mock the system calls.
     *
     * @return a list of {@link File} of directories of the test cases folder of build output, based
     *     on the value of environment variables.
     */
    @VisibleForTesting
    List<File> getExternalTestCasesDirs() {
        return SystemUtil.getExternalTestCasesDirs();
    }

    /**
     * Get the path to the config file for a test case.
     *
     * <p>The given name in a test config can be the name of a test case located in an out directory
     * defined in the following environment variables:
     *
     * <p>ANDROID_TARGET_OUT_TESTCASES
     *
     * <p>ANDROID_HOST_OUT_TESTCASES
     *
     * <p>This method tries to locate the test config name in these directories. If no config is
     * found, return null.
     *
     * @param name Name of a config file.
     * @return A File object of the config file for the given test case.
     */
    @VisibleForTesting
    File getTestCaseConfigPath(String name) {
        String[] possibleConfigFileNames = {name};
        if (Strings.isNullOrEmpty(FileUtil.getExtension(name))) {
            possibleConfigFileNames = new String[SUPPORTED_EXTENSIONS.size()];
            int i = 0;
            for (String supportedExtension : SUPPORTED_EXTENSIONS) {
                possibleConfigFileNames[i] = (name + supportedExtension);
                i++;
            }
        }

        for (File testCasesDir : getExternalTestCasesDirs()) {
            for (String configFileName : possibleConfigFileNames) {
                File config = FileUtil.findFile(testCasesDir, configFileName);
                if (config != null) {
                    CLog.d("Using config: %s/%s", testCasesDir.getAbsoluteFile(), configFileName);
                    return config;
                }
            }
        }
        return null;
    }

    /**
     * Implementation of {@link IConfigDefLoader} that tracks the included configurations from one
     * root config, and throws an exception on circular includes.
     */
    protected class ConfigLoader implements IConfigDefLoader {

        private final boolean mIsGlobalConfig;
        private DirectedGraph<String> mConfigGraph = new DirectedGraph<String>();

        public ConfigLoader(boolean isGlobalConfig) {
            mIsGlobalConfig = isGlobalConfig;
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public ConfigurationDef getConfigurationDef(String name, Map<String, String> templateMap)
                throws ConfigurationException {

            String configName = findConfigName(name, null);
            final ConfigId configId = new ConfigId(name, templateMap);
            ConfigurationDef def = mConfigDefMap.get(configId);

            if (def == null || def.isStale()) {
                def = new ConfigurationDef(configName);
                loadConfiguration(configName, def, null, templateMap, null);
                mConfigDefMap.put(configId, def);
            } else {
                if (templateMap != null) {
                    // Clearing the map before returning the cached config to
                    // avoid seeing them as unused.
                    templateMap.clear();
                }
            }
            return def;
        }

        /** Returns true if it is a config file found inside the classpath. */
        protected boolean isBundledConfig(String name) {
            InputStream configStream = getBundledConfigStream(name);
            return configStream != null;
        }

        /**
         * Get the absolute path of a local config file.
         *
         * @param root parent path of config file
         * @param name config file
         * @return absolute path for local config file.
         * @throws ConfigurationException
         */
        private String getAbsolutePath(String root, String name) throws ConfigurationException {
            File file = new File(name);
            if (!file.isAbsolute()) {
                if (root == null) {
                    // if root directory was not specified, get the current
                    // working directory.
                    root = System.getProperty("user.dir");
                }
                file = new File(root, name);
            }
            try {
                return file.getCanonicalPath();
            } catch (IOException e) {
                throw new ConfigurationException(
                        String.format(
                                "Failure when trying to determine local file canonical path %s", e),
                        InfraErrorIdentifier.CONFIGURATION_NOT_FOUND);
            }
        }

        /**
         * Find config's name based on its name and its parent name. This is used to properly handle
         * bundle configs and local configs.
         *
         * @param name config's name
         * @param parentName config's parent's name.
         * @return the config's full name.
         * @throws ConfigurationException
         */
        protected String findConfigName(String name, String parentName)
                throws ConfigurationException {
            if (isBundledConfig(name)) {
                return name;
            }
            if (parentName == null || isBundledConfig(parentName)) {
                // Search files for config.
                String configName = getAbsolutePath(null, name);
                File localConfig = new File(configName);
                if (!localConfig.exists()) {
                    localConfig = getTestCaseConfigPath(name);
                }
                if (localConfig != null) {
                    return localConfig.getAbsolutePath();
                }
                // Can not find local config.
                if (parentName == null) {
                    throw new ConfigurationException(
                            String.format("Can not find local config %s.", name),
                            InfraErrorIdentifier.CONFIGURATION_NOT_FOUND);

                } else {
                    throw new ConfigurationException(
                            String.format(
                                    "Bundled config '%s' is including a config '%s' that's neither "
                                            + "local nor bundled.",
                                    parentName, name),
                            InfraErrorIdentifier.CONFIGURATION_NOT_FOUND);
                }
            }
            try {
                // Local configs' include should be relative to their parent's path.
                String parentRoot = new File(parentName).getParentFile().getCanonicalPath();
                return getAbsolutePath(parentRoot, name);
            } catch (IOException e) {
                throw new ConfigurationException(
                        e.getMessage(), e.getCause(), InfraErrorIdentifier.CONFIGURATION_NOT_FOUND);
            }
        }

        /**
         * Configs that are bundled inside the tradefed.jar can only include other configs also
         * bundled inside tradefed.jar. However, local (external) configs can include both local
         * (external) and bundled configs.
         */
        @Override
        public void loadIncludedConfiguration(
                ConfigurationDef def,
                String parentName,
                String name,
                String deviceTagObject,
                Map<String, String> templateMap,
                Set<String> templateSeen)
                throws ConfigurationException {

            String config_name = findConfigName(name, parentName);
            mConfigGraph.addEdge(parentName, config_name);
            // If the inclusion of configurations is a cycle we throw an exception.
            if (!mConfigGraph.isDag()) {
                CLog.e("%s", mConfigGraph);
                throw new ConfigurationException(
                        String.format(
                                "Circular configuration include: config '%s' is already included",
                                config_name),
                        InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR);
            }
            loadConfiguration(config_name, def, deviceTagObject, templateMap, templateSeen);
        }

        /**
         * Loads a configuration.
         *
         * @param name the name of a built-in configuration to load or a file path to configuration
         *     file to load
         * @param def the loaded {@link ConfigurationDef}
         * @param deviceTagObject name of the current deviceTag if we are loading from a config
         *     inside an <include>. Null otherwise.
         * @param templateMap map from template-include names to their respective concrete
         *     configuration files
         * @param templateSeen set of template placeholder name already encountered
         * @throws ConfigurationException if a configuration with given name/file path cannot be
         *     loaded or parsed
         */
        void loadConfiguration(
                String name,
                ConfigurationDef def,
                String deviceTagObject,
                Map<String, String> templateMap,
                Set<String> templateSeen)
                throws ConfigurationException {
            BufferedInputStream bufStream = getConfigStream(name);
            String extension = FileUtil.getExtension(name);
            switch (extension) {
                case ".xml":
                case ".config":
                case "":
                    ConfigurationXmlParser parser =
                            new ConfigurationXmlParser(this, deviceTagObject);
                    parser.parse(def, name, bufStream, templateMap, templateSeen);
                    break;
                case ".tf_yaml":
                    ConfigurationYamlParser yamlParser = new ConfigurationYamlParser();
                    yamlParser.parse(def, name, bufStream, false);
                    break;
                default:
                    throw new ConfigurationException(
                            String.format("The config format for %s is not supported.", name));
            }
            trackConfig(name, def);
        }

        /**
         * Track config for dynamic loading. Right now only local files are supported.
         *
         * @param name config's name
         * @param def config's def.
         */
        protected void trackConfig(String name, ConfigurationDef def) {
            // Track local config source files
            if (!isBundledConfig(name)) {
                def.registerSource(new File(name));
            }
        }

        /**
         * Should track the config's life cycle or not.
         *
         * @param name config's name
         * @return <code>true</code> if the config is trackable, otherwise <code>false</code>.
         */
        protected boolean isTrackableConfig(String name) {
            return !isBundledConfig(name);
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public boolean isGlobalConfig() {
            return mIsGlobalConfig;
        }

    }

    protected ConfigurationFactory() {
        mConfigDefMap = new Hashtable<ConfigId, ConfigurationDef>();
    }

    /**
     * Get the singleton {@link IConfigurationFactory} instance.
     */
    public static IConfigurationFactory getInstance() {
        if (sInstance == null) {
            sInstance = new ConfigurationFactory();
        }
        return sInstance;
    }

    /**
     * Retrieve the {@link ConfigurationDef} for the given name
     *
     * @param name the name of a built-in configuration to load or a file path to configuration file
     *     to load
     * @return {@link ConfigurationDef}
     * @throws ConfigurationException if an error occurred loading the config
     */
    protected ConfigurationDef getConfigurationDef(
            String name, boolean isGlobal, Map<String, String> templateMap)
            throws ConfigurationException {
        try (CloseableTraceScope ignored = new CloseableTraceScope("getConfigurationDef")) {
            return new ConfigLoader(isGlobal).getConfigurationDef(name, templateMap);
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public IConfiguration createConfigurationFromArgs(String[] arrayArgs)
            throws ConfigurationException {
        return createConfigurationFromArgs(arrayArgs, null);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public IConfiguration createConfigurationFromArgs(String[] arrayArgs,
            List<String> unconsumedArgs) throws ConfigurationException {
        return createConfigurationFromArgs(arrayArgs, unconsumedArgs, null);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public IConfiguration createConfigurationFromArgs(String[] arrayArgs,
            List<String> unconsumedArgs, IKeyStoreClient keyStoreClient)
            throws ConfigurationException {
        if (arrayArgs.length == 0) {
            throw new ConfigurationException("Configuration to run was not specified");
        }

        List<String> listArgs = new ArrayList<String>(arrayArgs.length);
        // FIXME: Update parsing to not care about arg order.
        String[] reorderedArrayArgs = reorderArgs(arrayArgs);
        IConfiguration config =
                internalCreateConfigurationFromArgs(
                        reorderedArrayArgs, listArgs, keyStoreClient, null);
        config.setCommandLine(arrayArgs);
        if (listArgs.contains("--" + CommandOptions.DRY_RUN_OPTION)
                || listArgs.contains("--" + CommandOptions.NOISY_DRY_RUN_OPTION)) {
            // In case of dry-run, we replace the KeyStore by a dry-run one.
            CLog.w("dry-run detected, we are using a dryrun keystore");
            keyStoreClient = new DryRunKeyStore();
        }
        final List<String> tmpUnconsumedArgs =
                config.setOptionsFromCommandLineArgs(listArgs, keyStoreClient);

        if (unconsumedArgs == null && tmpUnconsumedArgs.size() > 0) {
            // (unconsumedArgs == null) is taken as a signal that the caller
            // expects all args to
            // be processed.
            throw new ConfigurationException(
                    String.format(
                            "Invalid arguments provided. Unprocessed arguments: %s",
                            tmpUnconsumedArgs),
                    InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR);
        } else if (unconsumedArgs != null) {
            // Return the unprocessed args
            unconsumedArgs.addAll(tmpUnconsumedArgs);
        }

        return config;
    }

    /** {@inheritDoc} */
    @Override
    public IConfiguration createPartialConfigurationFromArgs(
            String[] arrayArgs,
            IKeyStoreClient keyStoreClient,
            Set<String> allowedObjects,
            TradefedDelegator delegator)
            throws ConfigurationException {
        if (arrayArgs.length == 0) {
            throw new ConfigurationException(
                    "Configuration to run was not specified",
                    InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR);
        }

        List<String> listArgs = new ArrayList<String>(arrayArgs.length);
        String[] reorderedArrayArgs = reorderArgs(arrayArgs);
        IConfiguration config =
                internalCreateConfigurationFromArgs(
                        reorderedArrayArgs, listArgs, keyStoreClient, allowedObjects);
        if (delegator != null) {
            config.setConfigurationObject(TradefedDelegator.DELEGATE_OBJECT, delegator);
        }
        config.setCommandLine(arrayArgs);
        List<String> leftOver =
                config.setBestEffortOptionsFromCommandLineArgs(listArgs, keyStoreClient);
        CLog.d("Non-applied arguments: %s", leftOver);
        return config;
    }

    @VisibleForTesting
    protected boolean isDirectConfiguration(String configName) {
        return Pattern.matches(DIRECT_CONFIG_PATTERN, configName);
    }

    /**
     * Creates a {@link Configuration} from the name given in arguments.
     *
     * <p>Note will not populate configuration with values from options
     *
     * @param arrayArgs the full list of command line arguments, including the config name
     * @param optionArgsRef an empty list, that will be populated with the option arguments left to
     *     be interpreted
     * @param keyStoreClient {@link IKeyStoreClient} keystore client to use if any.
     * @param allowedObjects config object that are allowed to be created.
     * @return An {@link IConfiguration} object representing the configuration that was loaded
     * @throws ConfigurationException
     */
    private IConfiguration internalCreateConfigurationFromArgs(
            String[] arrayArgs,
            List<String> optionArgsRef,
            IKeyStoreClient keyStoreClient,
            Set<String> allowedObjects)
            throws ConfigurationException {
        final List<String> listArgs = new ArrayList<>(Arrays.asList(arrayArgs));
        // first arg is config name
        final String configName = listArgs.remove(0);

        // ATTN This section short-circuts the rest of the configuration pipeline
        if (isDirectConfiguration(configName)) {
            return internalCreateDirectConfiguration(
                    configName, listArgs, optionArgsRef, keyStoreClient, allowedObjects);
        }

        Map<String, String> uniqueMap =
                extractTemplates(configName, listArgs, optionArgsRef, keyStoreClient);
        if (allowedObjects != null && !allowedObjects.isEmpty()) {
            ConfigLoader tmpLoader = new ConfigLoader(false);
            // For partial loading be lenient about templates and let the delegate deal with it.
            // In some cases this won't be 100% correct but it's better than failing on all new
            // configs.
            for (String key : uniqueMap.keySet()) {
                try {
                    tmpLoader.findConfigName(uniqueMap.get(key), null);
                } catch (ConfigurationException e) {
                    uniqueMap.put(key, "empty");
                }
            }
        }
        ConfigurationDef configDef = getConfigurationDef(configName, false, uniqueMap);
        if (!uniqueMap.isEmpty()) {
            // remove the bad ConfigDef from the cache.
            for (ConfigId cid : mConfigDefMap.keySet()) {
                if (mConfigDefMap.get(cid) == configDef) {
                    CLog.d("Cleaning the cache for this configdef");
                    mConfigDefMap.remove(cid);
                    break;
                }
            }
            throw new ConfigurationException(
                    String.format("Unused template:map parameters: %s", uniqueMap.toString()),
                    InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR);
        }
        return configDef.createConfiguration(allowedObjects);
    }

    private Map<String, String> extractTemplates(
            String configName,
            List<String> listArgs,
            List<String> optionArgsRef,
            IKeyStoreClient keyStoreClient)
            throws ConfigurationException {
        try (CloseableTraceScope ignored = new CloseableTraceScope("extractTemplates")) {
            final String extension = FileUtil.getExtension(configName);
            switch (extension) {
                case ".xml":
                case ".config":
                case "":
                    final ConfigurationXmlParserSettings parserSettings =
                            new ConfigurationXmlParserSettings();
                    final ArgsOptionParser templateArgParser = new ArgsOptionParser(parserSettings);
                    if (keyStoreClient != null) {
                        templateArgParser.setKeyStore(keyStoreClient);
                    }
                    optionArgsRef.addAll(templateArgParser.parseBestEffort(listArgs));
                    // Check that the same template is not attempted to be loaded twice.
                    for (String key : parserSettings.templateMap.keySet()) {
                        if (parserSettings.templateMap.get(key).size() > 1) {
                            throw new ConfigurationException(
                                    String.format(
                                            "More than one template specified for key '%s'", key),
                                    InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR);
                        }
                    }
                    return parserSettings.templateMap.getUniqueMap();
                case ".tf_yaml":
                    // We parse the arguments but don't support template for YAML
                    final ArgsOptionParser allArgsParser = new ArgsOptionParser();
                    if (keyStoreClient != null) {
                        allArgsParser.setKeyStore(keyStoreClient);
                    }
                    optionArgsRef.addAll(allArgsParser.parseBestEffort(listArgs));
                    return new HashMap<>();
                default:
                    return new HashMap<>();
            }
        }
    }

    /**
     * A delegation method from the other `internalCreateConfiguration...` methods for direct
     * configs
     *
     * <p>This method encapsulates the direct configuration flow so that we can separate it from the
     * legacy flows for future refactoring.
     *
     * @param listArgs list of command arguments **not including the config name**
     * @param optionArgsRef an empty list, that will be populated with the option arguments left to
     *     be interpreted (should be populated with all non-template option arguments)
     * @param keyStoreClient {@link IKeyStoreClient} keystore client to use if any.
     * @param allowedObjects config object that are allowed to be created.
     * @return An {@link IConfiguration} object representing the configuration that was loaded
     * @throws ConfigurationException
     */
    @VisibleForTesting
    private IConfiguration internalCreateDirectConfiguration(
            String configName,
            List<String> listArgs,
            List<String> optionArgsRef,
            IKeyStoreClient keyStoreClient,
            Set<String> allowedObjects)
            throws ConfigurationException {
        // Download the file and do some error handling here
        try {
            URI configURI = new URI(configName);
            String name =
                    Arrays.stream(configURI.getPath().split("/"))
                            .reduce((first, second) -> second)
                            .orElseThrow();
            CLog.i("Determined the config name was %s", name);

            // GCS resolver doesn't respect this, but just in case others do,
            // we'd prefer them here.
            File destDir = FileUtil.createTempDir("tf-configs");

            ResolvedFile resolvedConfigFile = resolveRemoteFile(configURI, destDir.toURI());
            File configFile = resolvedConfigFile.getResolvedFile();
            if (configFile instanceof ExtendedFile) {
                ((ExtendedFile) configFile).waitForDownload();
            }

            CLog.i("Attempting to read from file: %s", configFile.getPath());
            try (BufferedInputStream configInputStream =
                    new BufferedInputStream(new FileInputStream(configFile))) {
                ConfigurationDef configDef = new ConfigurationDef(configName);

                switch (FileUtil.getExtension(configFile.getPath())) {
                    case ".xml":
                    case ".config":
                    case "":
                        // Note: this disabled config loader both prevents templates from being
                        // instantiated and allows me to use the ConfigurationXMLParser without
                        // substantial modification, for now.
                        ConfigLoader exceptionLoader = new ExceptionLoader(false);
                        ConfigurationXmlParser parser =
                                new ConfigurationXmlParser(exceptionLoader, null);

                        parser.parse(
                                configDef,
                                configName,
                                configInputStream,
                                new HashMap<String, String>(),
                                new HashSet<String>());
                        break;
                    case ".tf_yaml":
                        ConfigurationYamlParser yamlParser = new ConfigurationYamlParser();
                        yamlParser.parse(configDef, configName, configInputStream, false);
                        break;
                    default:
                        throw new ConfigurationException(
                                String.format(
                                        "The config format for %s is not supported.", configName),
                                InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR);
                }

                if (resolvedConfigFile.shouldCleanUp()) {
                    FileUtil.deleteFile(configFile);
                }
                FileUtil.recursiveDelete(destDir);

                final ConfigurationXmlParserSettings parserSettings =
                        new ConfigurationXmlParserSettings();
                final ArgsOptionParser cmdArgParser = new ArgsOptionParser(parserSettings);
                if (keyStoreClient != null) {
                    cmdArgParser.setKeyStore(keyStoreClient);
                }
                optionArgsRef.addAll(cmdArgParser.parseBestEffort(listArgs));

                return configDef.createConfiguration(allowedObjects);
            }
        } catch (FileNotFoundException e) {
            throw new ConfigurationException(e.toString(), e);
        } catch (BuildRetrievalError e) {
            throw new ConfigurationException(e.toString(), e);
        } catch (URISyntaxException e) {
            throw new ConfigurationException(
                    String.format("Invalid URI specified: %s", configName), e);
        } catch (IOException e) {
            throw new RuntimeException("Failed to create temp dir for config", e);
        }
    }

    @VisibleForTesting
    protected ResolvedFile resolveRemoteFile(URI configURI, URI destDir)
            throws BuildRetrievalError {
        return RemoteFileResolver.resolveRemoteFile(configURI, destDir);
    }

    protected class ExceptionLoader extends ConfigLoader {
        public ExceptionLoader(boolean isGlobal) {
            super(isGlobal);
        }

        @Override
        public ConfigurationDef getConfigurationDef(String name, Map<String, String> templateMap)
                throws ConfigurationException {
            throw new ConfigurationException(
                    "Templates are not allowed in direct configuration contexts");
        }

        @Override
        public boolean isBundledConfig(String name) {
            throw new RuntimeException(
                    new ConfigurationException(
                            "Templates are not allowed in direct configuration contexts"));
        }

        @Override
        protected String findConfigName(String name, String parentName) {
            throw new RuntimeException(
                    new ConfigurationException(
                            "Templates are not allowed in direct configuration contexts"));
        }

        @Override
        public void loadIncludedConfiguration(
                ConfigurationDef def,
                String parentName,
                String name,
                String deviceTagObject,
                Map<String, String> templateMap,
                Set<String> templateSeen)
                throws ConfigurationException {
            throw new ConfigurationException(
                    "Templates are not allowed in direct configuration contexts",
                    InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR);
        }

        @Override
        public void loadConfiguration(
                String name,
                ConfigurationDef def,
                String deviceTagObject,
                Map<String, String> templateMap,
                Set<String> templateSeen)
                throws ConfigurationException {
            throw new ConfigurationException(
                    "Templates are not allowed in direct configuration contexts",
                    InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR);
        }

        @Override
        protected void trackConfig(String name, ConfigurationDef def) {
            throw new RuntimeException(
                    new ConfigurationException(
                            "Templates are not allowed in direct configuration contexts"));
        }

        @Override
        protected boolean isTrackableConfig(String name) {
            throw new RuntimeException(
                    new ConfigurationException(
                            "Templates are not allowed in direct configuration contexts"));
        }

        // `private String getAbsolutePath(String root, String name)` is private
        // and so cannot be overridden

        // `public boolean isGlobalConfig()` not overridden because it is probably
        // fine to call in this context
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public IGlobalConfiguration createGlobalConfigurationFromArgs(String[] arrayArgs,
            List<String> remainingArgs) throws ConfigurationException {
        List<String> listArgs = new ArrayList<String>(arrayArgs.length);
        IGlobalConfiguration config = internalCreateGlobalConfigurationFromArgs(arrayArgs,
                listArgs);
        remainingArgs.addAll(config.setOptionsFromCommandLineArgs(listArgs));

        return config;
    }

    /**
     * Creates a {@link GlobalConfiguration} from the name given in arguments.
     * <p/>
     * Note will not populate configuration with values from options
     *
     * @param arrayArgs the full list of command line arguments, including the config name
     * @param optionArgsRef an empty list, that will be populated with the
     *            remaining option arguments
     * @return a {@link IGlobalConfiguration} created from the args
     * @throws ConfigurationException
     */
    private IGlobalConfiguration internalCreateGlobalConfigurationFromArgs(String[] arrayArgs,
            List<String> optionArgsRef) throws ConfigurationException {
        if (arrayArgs.length == 0) {
            throw new ConfigurationException("Configuration to run was not specified");
        }
        optionArgsRef.addAll(Arrays.asList(arrayArgs));
        // first arg is config name
        final String configName = optionArgsRef.remove(0);
        ConfigurationDef configDef = getConfigurationDef(configName, true, null);
        IGlobalConfiguration config = configDef.createGlobalConfiguration();
        config.setOriginalConfig(configName);
        config.setConfigurationFactory(this);
        return config;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void printHelp(PrintStream out) {
        try {
            loadAllConfigs(true);
        } catch (ConfigurationException e) {
            // ignore, should never happen
        }
        // sort the configs by name before displaying
        SortedSet<ConfigurationDef> configDefs = new TreeSet<ConfigurationDef>(
                new ConfigDefComparator());
        configDefs.addAll(mConfigDefMap.values());
        StringBuilder sb = new StringBuilder();
        for (ConfigurationDef def : configDefs) {
            sb.append(String.format("  %s: %s\n", def.getName(), def.getDescription()));
        }
        out.printf(sb.toString());
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<String> getConfigList() {
        return getConfigList(null, true);
    }

    /** {@inheritDoc} */
    @Override
    public List<String> getConfigList(String subPath, boolean loadFromEnv) {
        Set<String> configNames = getConfigSetFromClasspath(subPath);
        if (loadFromEnv) {
            // list config on variable path too
            configNames.addAll(getConfigNamesFromTestCases(subPath));
        }
        // sort the configs by name before adding to list
        SortedSet<String> configDefs = new TreeSet<String>();
        configDefs.addAll(configNames);
        List<String> configs = new ArrayList<String>();
        configs.addAll(configDefs);
        return configs;
    }

    /**
     * Private helper to get the full set of configurations.
     */
    private Set<String> getConfigSetFromClasspath(String subPath) {
        ClassPathScanner cpScanner = new ClassPathScanner();
        return cpScanner.getClassPathEntries(new ConfigClasspathFilter(subPath));
    }

    /**
     * Helper to get the test config files from test cases directories from build output.
     *
     * @param subPath where to look for configuration. Can be null.
     */
    @VisibleForTesting
    Set<String> getConfigNamesFromTestCases(String subPath) {
        return ConfigurationUtil.getConfigNamesFromDirs(subPath, getExternalTestCasesDirs());
    }

    @VisibleForTesting
    Map<String, String> getConfigSetFromClasspathFromJar(String subPath) {
        ClassPathScanner cpScanner = new ClassPathScanner();
        return cpScanner.getClassPathEntriesFromJar(new ConfigClasspathFilter(subPath));
    }

    /**
     * Loads all configurations found in classpath and test cases directories.
     *
     * @param discardExceptions true if any ConfigurationException should be ignored.
     * @throws ConfigurationException
     */
    public void loadAllConfigs(boolean discardExceptions) throws ConfigurationException {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        PrintStream ps = new PrintStream(baos);
        boolean failed = false;
        Set<String> configNames = getConfigSetFromClasspath(null);
        // TODO: split the configs into two lists, one from the jar packages and one from test
        // cases directories.
        configNames.addAll(getConfigNamesFromTestCases(null));
        for (String configName : configNames) {
            final ConfigId configId = new ConfigId(configName);
            try {
                ConfigurationDef configDef = attemptLoad(configId, null);
                mConfigDefMap.put(configId, configDef);
            } catch (ConfigurationException e) {
                ps.printf("Failed to load %s: %s", configName, e.getMessage());
                ps.println();
                failed = true;
            }
        }
        if (failed) {
            if (discardExceptions) {
                CLog.e("Failure loading configs");
                CLog.e(baos.toString());
            } else {
                throw new ConfigurationException(baos.toString());
            }
        }
    }

    /**
     * Helper to load a configuration.
     */
    private ConfigurationDef attemptLoad(ConfigId configId, Map<String, String> templateMap)
            throws ConfigurationException {
        ConfigurationDef configDef = null;
        try {
            configDef = getConfigurationDef(configId.name, false, templateMap);
            return configDef;
        } catch (TemplateResolutionError tre) {
            // When a template does not have a default, we try again with known good template
            // to make sure file formatting at the very least is fine.
            Map<String, String> fakeTemplateMap = new HashMap<String, String>();
            if (templateMap != null) {
                fakeTemplateMap.putAll(templateMap);
            }
            fakeTemplateMap.put(tre.getTemplateKey(), DRY_RUN_TEMPLATE_CONFIG);
            // We go recursively in case there are several template to dry run.
            return attemptLoad(configId, fakeTemplateMap);
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void printHelpForConfig(String[] args, boolean importantOnly, PrintStream out) {
        try {
            IConfiguration config =
                    internalCreateConfigurationFromArgs(
                            args, new ArrayList<String>(args.length), null, null);
            config.printCommandUsage(importantOnly, out);
        } catch (ConfigurationException e) {
            // config must not be specified. Print generic help
            printHelp(out);
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void dumpConfig(String configName, PrintStream out) {
        try {
            InputStream configStream = getConfigStream(configName);
            StreamUtil.copyStreams(configStream, out);
        } catch (ConfigurationException e) {
            CLog.e(e);
        } catch (IOException e) {
            CLog.e(e);
        }
    }

    /**
     * Return the path prefix of config xml files on classpath
     *
     * <p>Exposed so unit tests can mock.
     *
     * @return {@link String} path with trailing /
     */
    protected String getConfigPrefix() {
        return CONFIG_PREFIX;
    }

    /**
     * Loads an InputStream for given config name
     *
     * @param name the configuration name to load
     * @return a {@link BufferedInputStream} for reading config contents
     * @throws ConfigurationException if config could not be found
     */
    protected BufferedInputStream getConfigStream(String name) throws ConfigurationException {
        InputStream configStream = getBundledConfigStream(name);
        if (configStream == null) {
            // now try to load from file
            try {
                configStream = new FileInputStream(name);
            } catch (FileNotFoundException e) {
                throw new ConfigurationException(String.format("Could not find configuration '%s'",
                        name));
            }
        }
        // buffer input for performance - just in case config file is large
        return new BufferedInputStream(configStream);
    }

    protected InputStream getBundledConfigStream(String name) {
        String extension = FileUtil.getExtension(name);
        if (Strings.isNullOrEmpty(extension)) {
            // If the default name doesn't have an extension, search all possible extensions.
            for (String supportExtension : SUPPORTED_EXTENSIONS) {
                InputStream res =
                        getClass()
                                .getResourceAsStream(
                                        String.format(
                                                "/%s%s%s",
                                                getConfigPrefix(), name, supportExtension));
                if (res != null) {
                    return res;
                }
            }
            return null;
        }
        // Check directly with extension if it has one.
        return getClass().getResourceAsStream(String.format("/%s%s", getConfigPrefix(), name));
    }

    /**
     * Utility method that checks that all configs can be loaded, parsed, and
     * all option values set.
     * Only exposed so that depending project can validate their configs.
     * Should not be exposed in the console.
     *
     * @throws ConfigurationException if one or more configs failed to load
     */
    public void loadAndPrintAllConfigs() throws ConfigurationException {
        loadAllConfigs(false);
        boolean failed = false;
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        PrintStream ps = new PrintStream(baos);

        for (ConfigurationDef def : mConfigDefMap.values()) {
            try {
                def.createConfiguration().printCommandUsage(false,
                        new PrintStream(StreamUtil.nullOutputStream()));
            } catch (ConfigurationException e) {
                if (e.getCause() != null &&
                        e.getCause() instanceof ClassNotFoundException) {
                    ClassNotFoundException cnfe = (ClassNotFoundException) e.getCause();
                    String className = cnfe.getLocalizedMessage();
                    // Some Cts configs are shipped with Trade Federation, we exclude those from
                    // the failure since these packages are not available for loading.
                    if (className != null && className.startsWith("com.android.cts.")) {
                        CLog.w("Could not confirm %s: %s because not part of Trade Federation "
                                + "packages.", def.getName(), e.getMessage());
                        continue;
                    }
                } else if (Pattern.matches(CONFIG_ERROR_PATTERN, e.getMessage())) {
                    // If options are inside configuration object tag we are able to validate them
                    if (!e.getMessage().contains("com.android.") &&
                            !e.getMessage().contains("com.google.android.")) {
                        // We cannot confirm if an option is indeed missing since a template of
                        // option only is possible to avoid repetition in configuration with the
                        // same base.
                        CLog.w("Could not confirm %s: %s", def.getName(), e.getMessage());
                        continue;
                    }
                }
                ps.printf("Failed to print %s: %s", def.getName(), e.getMessage());
                ps.println();
                failed = true;
            }
        }
        if (failed) {
            throw new ConfigurationException(baos.toString());
        }
    }

    /**
     * Exposed for testing. Return a copy of the Map.
     */
    protected Map<ConfigId, ConfigurationDef> getMapConfig() {
        // We return a copy to ensure it is not modified outside
        return new HashMap<ConfigId, ConfigurationDef>(mConfigDefMap);
    }

    /** In some particular case, we need to clear the map. */
    @VisibleForTesting
    public void clearMapConfig() {
        mConfigDefMap.clear();
    }

    /** Reorder the args so that template:map args are all moved to the front. */
    @VisibleForTesting
    protected String[] reorderArgs(String[] args) {
        List<String> nonTemplateArgs = new ArrayList<String>();
        List<String> reorderedArgs = new ArrayList<String>();
        String[] reorderedArgsArray = new String[args.length];
        String arg;

        // First arg is the config.
        if (args.length > 0) {
            reorderedArgs.add(args[0]);
        }

        // Split out the template and non-template args so we can add
        // non-template args at the end while maintaining their order.
        for (int i = 1; i < args.length; i++) {
            arg = args[i];
            if (arg.equals("--template:map")) {
                // We need to account for these two types of template:map args.
                // --template:map tm=tm1
                // --template:map tm tm1
                reorderedArgs.add(arg);
                for (int j = i + 1; j < args.length; j++) {
                    if (args[j].startsWith("-")) {
                        break;
                    } else {
                        reorderedArgs.add(args[j]);
                        i++;
                    }
                }
            } else {
                nonTemplateArgs.add(arg);
            }
        }
        reorderedArgs.addAll(nonTemplateArgs);
        return reorderedArgs.toArray(reorderedArgsArray);
    }
}