/**
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE and NOTICE files at the root of the source
 * tree and available online at
 *
 * http://www.dspace.org/license/
 */
package org.dspace.app.statistics;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.sql.SQLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.StringUtils;
import org.dspace.core.Context;
import org.dspace.core.LogHelper;
import org.dspace.core.Utils;
import org.dspace.discovery.DiscoverQuery;
import org.dspace.discovery.SearchServiceException;
import org.dspace.discovery.SearchUtils;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;

/**
 * This class performs all the actual analysis of a given set of DSpace log
 * files.  Most input can be configured; use the -help flag for a full list
 * of usage information.
 *
 * <p>
 * The output of this file is plain text and forms an "aggregation" file which
 * can then be used for display purposes using the related ReportGenerator
 * class.
 *
 * @author Richard Jones
 */
public class LogAnalyser {

    // set up our class globals
    // FIXME: there are so many of these perhaps they should exist in a static
    // object of their own

    /////////////////
    // aggregators
    /////////////////

    /**
     * aggregator for all actions performed in the system
     */
    private static Map<String, Integer> actionAggregator;

    /**
     * aggregator for all searches performed
     */
    private static Map<String, Integer> searchAggregator;

    /**
     * aggregator for user logins
     */
    private static Map<String, Integer> userAggregator;

    /**
     * aggregator for item views
     */
    private static Map<String, Integer> itemAggregator;

    /**
     * aggregator for current archive state statistics
     */
    private static Map<String, Integer> archiveStats;

    /**
     * warning counter
     */
    private static int warnCount = 0;

    /**
     * exception counter
     */
    private static int excCount = 0;

    /**
     * log line counter
     */
    private static int lineCount = 0;

    //////////////////
    // config data
    //////////////////

    /**
     * list of actions to be included in the general summary
     */
    private static List<String> generalSummary;

    /**
     * list of words not to be aggregated
     */
    private static List<String> excludeWords;

    /**
     * list of search types to be ignored, such as "author:"
     */
    private static List<String> excludeTypes;

    /**
     * list of characters to be excluded
     */
    private static List<String> excludeChars;

    /**
     * list of item types to be reported on in the current state
     */
    private static List<String> itemTypes;

    /**
     * bottom limit to output for search word analysis
     */
    private static int searchFloor;

    /**
     * bottom limit to output for item view analysis
     */
    private static int itemFloor;

    /**
     * number of items from most popular to be looked up in the database
     */
    private static int itemLookup;

    /**
     * mode to use for user email display
     */
    private static String userEmail;

    /**
     * URL of the service being analysed
     */
    private static String url;

    /**
     * Name of the service being analysed
     */
    private static String name;

    /**
     * Name of the service being analysed
     */
    private static String hostName;

    /**
     * the average number of views per item
     */
    private static long views = 0;

    ///////////////////////
    // regular expressions
    ///////////////////////

    /**
     * Exclude characters regular expression pattern
     */
    private static Pattern excludeCharRX = null;

    /**
     * handle indicator string regular expression pattern
     */
    private static Pattern handleRX = null;

    /**
     * item id indicator string regular expression pattern
     */
    private static Pattern itemRX = null;

    /**
     * query string indicator regular expression pattern
     */
    private static Pattern queryRX = null;

    /**
     * collection indicator regular expression pattern
     */
    private static Pattern collectionRX = null;

    /**
     * community indicator regular expression pattern
     */
    private static Pattern communityRX = null;

    /**
     * results indicator regular expression pattern
     */
    private static Pattern resultsRX = null;

    /**
     * single character regular expression pattern
     */
    private static Pattern singleRX = null;

    /**
     * a pattern to match a valid version 1.3 log file line
     */
    private static Pattern valid13 = null;

    /**
     * basic log line
     */
    private static Pattern validBase = null;

    /**
     * a pattern to match a valid version 1.4 log file line
     */
    private static Pattern valid14 = null;

    /**
     * pattern to match valid log file names
     */
    private static Pattern logRegex = null;

    /**
     * pattern to match commented out lines from the config file
     */
    private static final Pattern COMMENT = Pattern.compile("^#");

    /**
     * pattern to match genuine lines from the config file
     */
    private static final Pattern REAL = Pattern.compile("^(.+)=(.+)");

    /**
     * pattern to match all search types
     */
    private static Pattern typeRX = null;

    /**
     * pattern to match all search types
     */
    private static Pattern wordRX = null;

    //////////////////////////
    // Miscellaneous variables
    //////////////////////////

    /**
     * process timing clock
     */
    private static Calendar startTime = null;

    /////////////////////////
    // command line options
    ////////////////////////

    /**
     * the log directory to be analysed
     */
    private static String logDir;

    /**
     * the regex to describe the file name format
     */
    private static String fileTemplate = "dspace\\.log.*";

    private static final ConfigurationService configurationService =
            DSpaceServicesFactory.getInstance().getConfigurationService();

    /**
     * the configuration file from which to configure the analyser
     */
    private static String configFile = configurationService.getProperty("dspace.dir")
            + File.separator + "config" + File.separator + "dstat.cfg";

    /**
     * the output file to which to write aggregation data
     */
    private static String outFile;

    /**
     * the starting date of the report
     */
    private static Date startDate = null;

    /**
     * the end date of the report
     */
    private static Date endDate = null;

    /**
     * the starting date of the report as obtained from the log files
     */
    private static Date logStartDate = null;

    /**
     * the end date of the report as obtained from the log files
     */
    private static Date logEndDate = null;

    /**
     * Default constructor
     */
    private LogAnalyser() { }

    /**
     * main method to be run from command line.  See usage information for
     * details as to how to use the command line flags (-help)
     *
     * @param argv the command line arguments given
     * @throws Exception    if error
     * @throws SQLException if database error
     */
    public static void main(String[] argv)
        throws Exception, SQLException {
        // first, start the processing clock
        startTime = new GregorianCalendar();

        // create context as super user
        Context context = new Context();
        context.turnOffAuthorisationSystem();

        // set up our command line variables
        String myLogDir = null;
        String myFileTemplate = null;
        String myConfigFile = null;
        String myOutFile = null;
        Date myStartDate = null;
        Date myEndDate = null;
        boolean myLookUp = false;

        // Define command line options.
        Options options = new Options();
        Option option;

        option = Option.builder().longOpt("log").hasArg().build();
        options.addOption(option);

        option = Option.builder().longOpt("file").hasArg().build();
        options.addOption(option);

        option = Option.builder().longOpt("cfg").hasArg().build();
        options.addOption(option);

        option = Option.builder().longOpt("out").hasArg().build();
        options.addOption(option);

        option = Option.builder().longOpt("help").build();
        options.addOption(option);

        option = Option.builder().longOpt("start").hasArg().build();
        options.addOption(option);

        option = Option.builder().longOpt("end").hasArg().build();
        options.addOption(option);

        option = Option.builder().longOpt("lookup").build();
        options.addOption(option);

        // Parse the command.
        DefaultParser cmdParser = new DefaultParser();
        CommandLine cmd = cmdParser.parse(options, argv);

        // Analyze the command.
        if (cmd.hasOption("help")) {
            LogAnalyser.usage();
            System.exit(0);
        }

        if (cmd.hasOption("log")) {
            myLogDir = cmd.getOptionValue("log");
        }

        if (cmd.hasOption("file")) {
            myFileTemplate = cmd.getOptionValue("file");
        }

        if (cmd.hasOption("cfg")) {
            myConfigFile = cmd.getOptionValue("cfg");
        }

        if (cmd.hasOption("out")) {
            myOutFile = cmd.getOptionValue("out");
        }

        if (cmd.hasOption("start")) {
            myStartDate = parseDate(cmd.getOptionValue("start"));
        }

        if (cmd.hasOption("end")) {
            myEndDate = parseDate(cmd.getOptionValue("end"));
        }

        myLookUp = cmd.hasOption("lookup");

        // now call the method which actually processes the logs
        processLogs(context, myLogDir, myFileTemplate, myConfigFile, myOutFile,
                myStartDate, myEndDate, myLookUp);
    }

    /**
     * using the pre-configuration information passed here, analyse the logs
     * and produce the aggregation file
     *
     * @param context        the DSpace context object this occurs under
     * @param myLogDir       the passed log directory.  Uses default if null
     * @param myFileTemplate the passed file name regex.  Uses default if null
     * @param myConfigFile   the DStat config file.  Uses default if null
     * @param myOutFile      the file to which to output aggregation data.  Uses default if null
     * @param myStartDate    the desired start of the analysis.  Starts from the beginning otherwise
     * @param myEndDate      the desired end of the analysis.  Goes to the end otherwise
     * @param myLookUp       force a lookup of the database
     * @return aggregate output
     * @throws IOException            if IO error
     * @throws SQLException           if database error
     * @throws SearchServiceException if search error
     */
    public static String processLogs(Context context, String myLogDir,
                                     String myFileTemplate, String myConfigFile,
                                     String myOutFile, Date myStartDate,
                                     Date myEndDate, boolean myLookUp)
        throws IOException, SQLException, SearchServiceException {
        // FIXME: perhaps we should have all parameters and aggregators put
        // together in a single aggregating object

        // if the timer has not yet been started, then start it
        startTime = new GregorianCalendar();

        //instantiate aggregators
        actionAggregator = new HashMap<>();
        searchAggregator = new HashMap<>();
        userAggregator = new HashMap<>();
        itemAggregator = new HashMap<>();
        archiveStats = new HashMap<>();

        //instantiate lists
        generalSummary = new ArrayList<>();
        excludeWords = new ArrayList<>();
        excludeTypes = new ArrayList<>();
        excludeChars = new ArrayList<>();
        itemTypes = new ArrayList<>();

        // set the parameters for this analysis
        setParameters(myLogDir, myFileTemplate, myConfigFile, myOutFile, myStartDate, myEndDate, myLookUp);

        // pre prepare our standard file readers and buffered readers
        FileReader fr = null;
        BufferedReader br = null;

        // read in the config information, throwing an error if we fail to open
        // the given config file
        readConfig(configFile);

        // assemble the regular expressions for later use (requires the file
        // template to build the regex to match it
        setRegex(fileTemplate);

        // get the log files
        File[] logFiles = getLogFiles(logDir);

        // standard loop counter
        int i = 0;

        // for every log file do analysis
        // FIXME: it is easy to implement not processing log files after the
        // dates exceed the end boundary, but is there an easy way to do it
        // for the start of the file?  Note that we can assume that the contents
        // of the log file are sequential, but can we assume the files are
        // provided in a data sequence?
        for (i = 0; i < logFiles.length; i++) {
            // check to see if this file is a log file agains the global regex
            Matcher matchRegex = logRegex.matcher(logFiles[i].getName());
            if (matchRegex.matches()) {
                // if it is a log file, open it up and lets have a look at the
                // contents.
                try {
                    fr = new FileReader(logFiles[i].toString());
                    br = new BufferedReader(fr);
                } catch (IOException e) {
                    System.out.println("Failed to read log file " + logFiles[i].toString());
                    System.exit(0);
                }

                // for each line in the file do the analysis
                // FIXME: perhaps each section needs to be dolled out to an
                // analysing class to allow pluggability of other methods of
                // analysis, and ease of code reading too - Pending further thought
                String line = null;
                while ((line = br.readLine()) != null) {
                    // get the log line object
                    LogLine logLine = getLogLine(line);

                    // if there are line segments get on with the analysis
                    if (logLine != null) {
                        // first find out if we are constraining by date and
                        // if so apply the restrictions
                        if ((startDate != null) && (!logLine.afterDate(startDate))) {
                            continue;
                        }

                        if ((endDate != null) && (!logLine.beforeDate(endDate))) {
                            break;
                        }

                        // count the number of lines parsed
                        lineCount++;

                        // if we are not constrained by date, register the date
                        // as the start/end date if it is the earliest/latest so far
                        // FIXME: this should probably have a method of its own
                        if (startDate == null) {
                            if (logStartDate != null) {
                                if (logLine.beforeDate(logStartDate)) {
                                    logStartDate = logLine.getDate();
                                }
                            } else {
                                logStartDate = logLine.getDate();
                            }
                        }

                        if (endDate == null) {
                            if (logEndDate != null) {
                                if (logLine.afterDate(logEndDate)) {
                                    logEndDate = logLine.getDate();
                                }
                            } else {
                                logEndDate = logLine.getDate();
                            }
                        }

                        // count the warnings
                        if (logLine.isLevel("WARN")) {
                            // FIXME: really, this ought to be some kind of level
                            // aggregator
                            warnCount++;
                        }
                        // count the exceptions
                        if (logLine.isLevel("ERROR")) {
                            excCount++;
                        }

                        if (null == logLine.getAction()) {
                            continue;
                        }

                        // is the action a search?
                        if (logLine.isAction("search")) {
                            // get back all the valid search words from the query
                            String[] words = analyseQuery(logLine.getParams());

                            // for each search word add to the aggregator or
                            // increment the aggregator's counter
                            for (String word : words) {
                                // FIXME: perhaps aggregators ought to be objects
                                // themselves
                                searchAggregator.put(word,
                                        increment(searchAggregator, word));
                            }
                        }

                        // is the action a login, and are we counting user logins?
                        if (logLine.isAction("login") && !userEmail.equals("off")) {
                            userAggregator.put(logLine.getUser(), increment(userAggregator, logLine.getUser()));
                        }

                        // is the action an item view?
                        if (logLine.isAction("view_item")) {
                            String handle = logLine.getParams();

                            // strip the handle string
                            Matcher matchHandle = handleRX.matcher(handle);
                            handle = matchHandle.replaceAll("");

                            // strip the item id string
                            Matcher matchItem = itemRX.matcher(handle);
                            handle = matchItem.replaceAll("").trim();

                            // either add the handle to the aggregator or
                            // increment its counter
                            itemAggregator.put(handle, increment(itemAggregator, handle));
                        }

                        // log all the activity
                        actionAggregator.put(logLine.getAction(), increment(actionAggregator, logLine.getAction()));
                    }
                }

                // close the file reading buffers
                br.close();
                fr.close();

            }
        }

        // do we want to do a database lookup?  Do so only if the start and
        // end dates are null or lookUp is true
        // FIXME: this is a kind of separate section.  Would it be worth building
        // the summary string separately and then inserting it into the real
        // summary later?  Especially if we make the archive analysis more complex
        archiveStats.put("All Items", getNumItems(context));
        for (i = 0; i < itemTypes.size(); i++) {
            archiveStats.put(itemTypes.get(i), getNumItems(context, itemTypes.get(i)));
        }

        // now do the host name and url lookup
        hostName = Utils.getHostName(configurationService.getProperty("dspace.ui.url"));
        name = configurationService.getProperty("dspace.name").trim();
        url = configurationService.getProperty("dspace.ui.url").trim();
        if ((url != null) && (!url.endsWith("/"))) {
            url = url + "/";
        }

        // do the average views analysis
        if ((archiveStats.get("All Items")) != 0) {
            // FIXME: this is dependent on their being a query on the db, which
            // there might not always be if it becomes configurable
            double avg = Math.ceil(
                (actionAggregator.get("view_item")).doubleValue() /
                    (archiveStats.get("All Items")).doubleValue());
            views = Math.round(avg);
        }

        // finally, write the output
        return createOutput();
    }


    /**
     * set the passed parameters up as global class variables.  This has to
     * be done in a separate method because the API permits for running from
     * the command line with args or calling the processLogs method statically
     * from elsewhere
     *
     * @param myLogDir       the log file directory to be analysed
     * @param myFileTemplate regex for log file names
     * @param myConfigFile   config file to use for dstat
     * @param myOutFile      file to write the aggregation into
     * @param myStartDate    requested log reporting start date
     * @param myEndDate      requested log reporting end date
     * @param myLookUp       requested look up force flag
     */
    public static void setParameters(String myLogDir, String myFileTemplate,
                                     String myConfigFile, String myOutFile,
                                     Date myStartDate, Date myEndDate,
                                     boolean myLookUp) {

        if (myLogDir != null) {
            logDir = myLogDir;
        } else {
            logDir = configurationService.getProperty("log.report.dir");
        }

        if (myFileTemplate != null) {
            fileTemplate = myFileTemplate;
        }

        if (myConfigFile != null) {
            configFile = myConfigFile;
        }

        if (myStartDate != null) {
            startDate = new Date(myStartDate.getTime());
        }

        if (myEndDate != null) {
            endDate = new Date(myEndDate.getTime());
        }

        if (myOutFile != null) {
            outFile = myOutFile;
        } else {
            outFile = configurationService.getProperty("log.report.dir") + File.separator + "dstat.dat";
        }
    }


    /**
     * generate the analyser's output to the specified out file
     *
     * @return output
     */
    public static String createOutput() {
        // start a string buffer to hold the final output
        StringBuilder summary = new StringBuilder();

        // define an iterator that will be used to go over the hashmap keys
        Iterator<String> keys = null;

        // output the number of lines parsed
        summary.append("log_lines=").append(Integer.toString(lineCount)).append("\n");

        // output the number of warnings encountered
        summary.append("warnings=").append(Integer.toString(warnCount)).append("\n");
        summary.append("exceptions=").append(Integer.toString(excCount)).append("\n");

        // set the general summary config up in the aggregator file
        for (int i = 0; i < generalSummary.size(); i++) {
            summary.append("general_summary=").append(generalSummary.get(i)).append("\n");
        }

        // output the host name
        summary.append("server_name=").append(hostName).append("\n");

        // output the service name
        summary.append("service_name=").append(name).append("\n");

        // output the date information if necessary
        SimpleDateFormat sdf = new SimpleDateFormat("dd'/'MM'/'yyyy");

        if (startDate != null) {
            summary.append("start_date=").append(sdf.format(startDate)).append("\n");
        } else if (logStartDate != null) {
            summary.append("start_date=").append(sdf.format(logStartDate)).append("\n");
        }

        if (endDate != null) {
            summary.append("end_date=").append(sdf.format(endDate)).append("\n");
        } else if (logEndDate != null) {
            summary.append("end_date=").append(sdf.format(logEndDate)).append("\n");
        }

        // write out the archive stats
        keys = archiveStats.keySet().iterator();
        while (keys.hasNext()) {
            String key = keys.next();
            summary.append("archive.").append(key).append("=").append(archiveStats.get(key)).append("\n");
        }

        // write out the action aggregation results
        keys = actionAggregator.keySet().iterator();
        while (keys.hasNext()) {
            String key = keys.next();
            summary.append("action.").append(key).append("=").append(actionAggregator.get(key)).append("\n");
        }

        // depending on the config settings for reporting on emails output the
        // login information
        summary.append("user_email=").append(userEmail).append("\n");
        int address = 1;
        keys = userAggregator.keySet().iterator();

        // for each email address either write out the address and the count
        // or alias it with an "Address X" label, to keep the data confidential
        // FIXME: the users reporting should also have a floor value
        while (keys.hasNext()) {
            String key = keys.next();
            summary.append("user.");
            if (userEmail.equals("on")) {
                summary.append(key).append("=").append(userAggregator.get(key)).append("\n");
            } else if (userEmail.equals("alias")) {
                summary.append("Address ").append(Integer.toString(address++))
                        .append("=").append(userAggregator.get(key)).append("\n");
            }
        }

        // FIXME: all values which have floors set should provide an "other"
        // record which counts how many other things which didn't make it into
        // the listing there are

        // output the search word information
        summary.append("search_floor=").append(searchFloor).append("\n");
        keys = searchAggregator.keySet().iterator();
        while (keys.hasNext()) {
            String key = keys.next();
            if ((searchAggregator.get(key)) >= searchFloor) {
                summary.append("search.").append(key).append("=")
                        .append(searchAggregator.get(key)).append("\n");
            }
        }

        // FIXME: we should do a lot more with the search aggregator
        // Possible feature list:
        //  - constrain by collection/community perhaps?
        //  - we should consider building our own aggregator class which can
        //      be full of rich data.  Perhaps this and the Stats class should
        //      be the same thing.

        // item viewing information
        summary.append("item_floor=").append(itemFloor).append("\n");
        summary.append("host_url=").append(url).append("\n");
        summary.append("item_lookup=").append(itemLookup).append("\n");

        // write out the item access information
        keys = itemAggregator.keySet().iterator();
        while (keys.hasNext()) {
            String key = keys.next();
            if ((itemAggregator.get(key)) >= itemFloor) {
                summary.append("item.").append(key).append("=")
                        .append(itemAggregator.get(key)).append("\n");
            }
        }

        // output the average views per item
        if (views > 0) {
            summary.append("avg_item_views=").append(views).append("\n");
        }

        // insert the analysis processing time information
        Calendar endTime = new GregorianCalendar();
        long timeInMillis = (endTime.getTimeInMillis() - startTime.getTimeInMillis());
        summary.append("analysis_process_time=")
                .append(Long.toString(timeInMillis / 1000)).append("\n");

        // finally write the string into the output file
        try (BufferedWriter out = new BufferedWriter(new FileWriter(outFile));) {
            out.write(summary.toString());
            out.flush();
        } catch (IOException e) {
            System.out.println("Unable to write to output file " + outFile);
            System.exit(0);
        }

        return summary.toString();
    }


    /**
     * get an array of file objects representing the passed log directory
     *
     * @param logDir the log directory in which to pick up files
     * @return an array of file objects representing the given logDir
     */
    public static File[] getLogFiles(String logDir) {
        // open the log files directory, read in the files, check that they
        // match the passed regular expression then analyse the content
        File logs = new File(logDir);

        // if log dir is not a directory throw and error and exit
        if (!logs.isDirectory()) {
            System.out.println("Passed log directory is not a directory");
            System.exit(0);
        }

        // get the files in the directory
        return logs.listFiles();
    }


    /**
     * set up the regular expressions to be used by this analyser.  Mostly this
     * exists to provide a degree of segregation and readability to the code
     * and to ensure that you only need to set up the regular expressions to
     * be used once
     *
     * @param fileTemplate the regex to be used to identify dspace log files
     */
    public static void setRegex(String fileTemplate) {
        // build the exclude characters regular expression
        StringBuilder charRegEx = new StringBuilder();
        charRegEx.append("[");
        for (int i = 0; i < excludeChars.size(); i++) {
            charRegEx.append("\\").append(excludeChars.get(i));
        }
        charRegEx.append("]");
        excludeCharRX = Pattern.compile(charRegEx.toString());

        // regular expression to find handle indicators in strings
        handleRX = Pattern.compile("handle=");

        // regular expression to find item_id indicators in strings
        itemRX = Pattern.compile(",item_id=.*$");

        // regular expression to find query indicators in strings
        queryRX = Pattern.compile("query=");

        // regular expression to find collections in strings
        collectionRX = Pattern.compile("collection_id=[0-9]*,");

        // regular expression to find communities in strings
        communityRX = Pattern.compile("community_id=[0-9]*,");

        // regular expression to find search result sets
        resultsRX = Pattern.compile(",results=(.*)");

        // regular expressions to find single characters anywhere in the string
        singleRX = Pattern.compile("( . |^. | .$)");

        // set up the standard log file line regular expression
        String logLineBase = "^(\\d\\d\\d\\d-\\d\\d\\-\\d\\d) \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d (\\w+)\\s+\\S+ @ (.*)";
        //date time LEVEL class @ whatever
        String logLine13 = "^(\\d\\d\\d\\d-\\d\\d\\-\\d\\d) \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d (\\w+)\\s+\\S+ @ ([^:]+)" +
            ":[^:]+:([^:]+):(.*)";
        String logLine14 = "^(\\d\\d\\d\\d-\\d\\d\\-\\d\\d) \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d (\\w+)\\s+\\S+ @ ([^:]+)" +
            ":[^:]+:[^:]+:([^:]+):(.*)";
        valid13 = Pattern.compile(logLine13);
        valid14 = Pattern.compile(logLine14);
        validBase = Pattern.compile(logLineBase);

        // set up the pattern for validating log file names
        logRegex = Pattern.compile(fileTemplate);

        // set up the pattern for matching any of the query types
        StringBuilder typeRXString = new StringBuilder();
        typeRXString.append("(");
        for (int i = 0; i < excludeTypes.size(); i++) {
            if (i > 0) {
                typeRXString.append("|");
            }
            typeRXString.append(excludeTypes.get(i));
        }
        typeRXString.append(")");
        typeRX = Pattern.compile(typeRXString.toString());

        // set up the pattern for matching any of the words to exclude
        StringBuilder wordRXString = new StringBuilder();
        wordRXString.append("(");
        for (int i = 0; i < excludeWords.size(); i++) {
            if (i > 0) {
                wordRXString.append("|");
            }
            wordRXString.append(" ").append(excludeWords.get(i)).append(" ");
            wordRXString.append("|");
            wordRXString.append("^").append(excludeWords.get(i)).append(" ");
            wordRXString.append("|");
            wordRXString.append(" ").append(excludeWords.get(i)).append("$");
        }
        wordRXString.append(")");
        wordRX = Pattern.compile(wordRXString.toString());
    }

    /**
     * get the current config file name
     *
     * @return The name of the config file
     */
    public static String getConfigFile() {
        return configFile;
    }

    /**
     * Read in the current config file and populate the class globals.
     *
     * @throws IOException if IO error
     */
    public static void readConfig() throws IOException {
        readConfig(configFile);
    }

    /**
     * Read in the given config file and populate the class globals.
     *
     * @param configFile the config file to read in
     * @throws IOException if IO error
     */
    public static void readConfig(String configFile) throws IOException {
        //instantiate aggregators
        actionAggregator = new HashMap<>();
        searchAggregator = new HashMap<>();
        userAggregator = new HashMap<>();
        itemAggregator = new HashMap<>();
        archiveStats = new HashMap<>();

        //instantiate lists
        generalSummary = new ArrayList<>();
        excludeWords = new ArrayList<>();
        excludeTypes = new ArrayList<>();
        excludeChars = new ArrayList<>();
        itemTypes = new ArrayList<>();

        // prepare our standard file readers and buffered readers
        FileReader fr = null;
        BufferedReader br = null;

        String record = null;
        try {
            fr = new FileReader(configFile);
            br = new BufferedReader(fr);
        } catch (IOException e) {
            System.out.println("Failed to read config file: " + configFile);
            System.exit(0);
        }

        // read in the config file and set up our instance variables
        while ((record = br.readLine()) != null) {
            // check to see what kind of line we have
            Matcher matchComment = COMMENT.matcher(record);
            Matcher matchReal = REAL.matcher(record);

            // if the line is not a comment and is real, read it in
            if (!matchComment.matches() && matchReal.matches()) {
                // lift the values out of the matcher's result groups
                String key = matchReal.group(1).trim();
                String value = matchReal.group(2).trim();

                // read the config values into our instance variables (see
                // documentation for more info on config params)
                if (key.equals("general.summary")) {
                    actionAggregator.put(value, 0);
                    generalSummary.add(value);
                }

                if (key.equals("exclude.word")) {
                    excludeWords.add(value);
                }

                if (key.equals("exclude.type")) {
                    excludeTypes.add(value);
                }

                if (key.equals("exclude.character")) {
                    excludeChars.add(value);
                }

                if (key.equals("item.type")) {
                    itemTypes.add(value);
                }

                if (key.equals("item.floor")) {
                    itemFloor = Integer.parseInt(value);
                }

                if (key.equals("search.floor")) {
                    searchFloor = Integer.parseInt(value);
                }

                if (key.equals("item.lookup")) {
                    itemLookup = Integer.parseInt(value);
                }

                if (key.equals("user.email")) {
                    userEmail = value;
                }
            }
        }

        // close the inputs
        br.close();
        fr.close();
    }

    /**
     * increment the value of the given map at the given key by one.
     *
     * @param map the map whose value we want to increase
     * @param key the key of the map whose value to increase
     * @return an integer object containing the new value
     */
    public static Integer increment(Map<String, Integer> map, String key) {
        Integer newValue = null;
        if (map.containsKey(key)) {
            // FIXME: this seems like a ridiculous way to add Integers
            newValue = (map.get(key)) + 1;
        } else {
            newValue = 1;
        }
        return newValue;
    }

    /**
     * Take the standard date string requested at the command line and convert
     * it into a Date object.  Throws and error and exits if the date does
     * not parse
     *
     * @param date the string representation of the date
     * @return a date object containing the date, with the time set to
     * 00:00:00
     */
    public static Date parseDate(String date) {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy'-'MM'-'dd");
        Date parsedDate = null;

        try {
            parsedDate = sdf.parse(date);
        } catch (ParseException e) {
            System.out.println("The date is not in the correct format");
            System.exit(0);
        }
        return parsedDate;
    }


    /**
     * Take the date object and convert it into a string of the form YYYY-MM-DD
     *
     * @param date the date to be converted
     * @return A string of the form YYYY-MM-DD
     */
    public static String unParseDate(Date date) {
        // Use SimpleDateFormat
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy'-'MM'-'dd'T'hh:mm:ss'Z'");
        sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
        return sdf.format(date);
    }


    /**
     * Take a search query string and pull out all of the meaningful information
     * from it, giving the results in the form of a String array, a single word
     * to each element
     *
     * @param query the search query to be analysed
     * @return the string array containing meaningful search terms
     */
    public static String[] analyseQuery(String query) {
        // register our standard loop counter
        int i = 0;

        // make the query string totally lower case, to ensure we don't miss out
        // on matches due to capitalisation
        query = query.toLowerCase();

        // now perform successive find and replace operations using pre-defined
        // global regular expressions
        Matcher matchQuery = queryRX.matcher(query);
        query = matchQuery.replaceAll(" ");

        Matcher matchCollection = collectionRX.matcher(query);
        query = matchCollection.replaceAll(" ");

        Matcher matchCommunity = communityRX.matcher(query);
        query = matchCommunity.replaceAll(" ");

        Matcher matchResults = resultsRX.matcher(query);
        query = matchResults.replaceAll(" ");

        Matcher matchTypes = typeRX.matcher(query);
        query = matchTypes.replaceAll(" ");

        Matcher matchChars = excludeCharRX.matcher(query);
        query = matchChars.replaceAll(" ");

        Matcher matchWords = wordRX.matcher(query);
        query = matchWords.replaceAll(" ");

        Matcher single = singleRX.matcher(query);
        query = single.replaceAll(" ");

        // split the remaining string by whitespace, trim and stuff into an
        // array to be returned
        StringTokenizer st = new StringTokenizer(query);
        String[] words = new String[st.countTokens()];
        for (i = 0; i < words.length; i++) {
            words[i] = st.nextToken().trim();
        }

        // FIXME: some single characters are still slipping through the net;
        // why? and how do we fix it?
        return words;
    }


    /**
     * split the given line into it's relevant segments if applicable (i.e. the
     * line matches the required regular expression.
     *
     * @param line the line to be segmented
     * @return a Log Line object for the given line
     */
    public static LogLine getLogLine(String line) {
        // FIXME: consider moving this code into the LogLine class.  To do this
        // we need to much more carefully define the structure and behaviour
        // of the LogLine class
        Matcher match;

        if (line.indexOf(":ip_addr") > 0) {
            match = valid14.matcher(line);
        } else {
            match = valid13.matcher(line);
        }

        if (match.matches()) {
            // set up a new log line object
            LogLine logLine = new LogLine(parseDate(match.group(1).trim()),
                                          LogHelper.unescapeLogField(match.group(2)).trim(),
                                          LogHelper.unescapeLogField(match.group(3)).trim(),
                                          LogHelper.unescapeLogField(match.group(4)).trim(),
                                          LogHelper.unescapeLogField(match.group(5)).trim());

            return logLine;
        } else {
            match = validBase.matcher(line);
            if (match.matches()) {
                LogLine logLine = new LogLine(parseDate(match.group(1).trim()),
                                              LogHelper.unescapeLogField(match.group(2)).trim(),
                                              null,
                                              null,
                                              null
                );
                return logLine;
            }
            return null;
        }
    }


    /**
     * get the number of items in the archive which were accessioned between
     * the provided start and end dates, with the given value for the DC field
     * 'type' (unqualified)
     *
     * @param context the DSpace context for the action
     * @param type    value for DC field 'type' (unqualified)
     * @return an integer containing the relevant count
     * @throws SQLException           if database error
     * @throws SearchServiceException if search error
     */
    public static Integer getNumItems(Context context, String type)
        throws SQLException, SearchServiceException {
        // FIXME: this method is clearly not optimised

        // FIXME: we don't yet collect total statistics, such as number of items
        // withdrawn, number in process of submission etc.  We should probably do
        // that

        DiscoverQuery discoverQuery = new DiscoverQuery();
        if (StringUtils.isNotBlank(type)) {
            discoverQuery.addFilterQueries("dc.type=" + type + "*");
        }
        StringBuilder accessionedQuery = new StringBuilder();
        accessionedQuery.append("dc.date.accessioned_dt:[");
        if (startDate != null) {
            accessionedQuery.append(unParseDate(startDate));
        } else {
            accessionedQuery.append("*");
        }
        accessionedQuery.append(" TO ");
        if (endDate != null) {
            accessionedQuery.append(unParseDate(endDate));
        } else {
            accessionedQuery.append("*");
        }
        accessionedQuery.append("]");
        discoverQuery.addFilterQueries(accessionedQuery.toString());
        discoverQuery.addFilterQueries("withdrawn: false");
        discoverQuery.addFilterQueries("archived: true");

        return (int) SearchUtils.getSearchService().search(context, discoverQuery).getTotalSearchResults();
    }


    /**
     * get the total number of items in the archive at time of execution,
     * ignoring all other constraints
     *
     * @param context the DSpace context the action is being performed in
     * @return an Integer containing the number of items in the
     * archive
     * @throws SQLException           if database error
     * @throws SearchServiceException if search error
     */
    public static Integer getNumItems(Context context)
        throws SQLException, SearchServiceException {
        return getNumItems(context, null);
    }


    /**
     * print out the usage information for this class to the standard out
     */
    public static void usage() {
        String usage = "Usage Information:\n" +
            "LogAnalyser [options [parameters]]\n" +
            "-log [log directory]\n" +
            "\tOptional\n" +
            "\tSpecify a directory containing log files\n" +
            "\tDefault uses [dspace.dir]/log from dspace.cfg\n" +
            "-file [file name regex]\n" +
            "\tOptional\n" +
            "\tSpecify a regular expression as the file name template.\n" +
            "\tCurrently this needs to be correctly escaped for Java string handling (FIXME)\n" +
            "\tDefault uses dspace.log*\n" +
            "-cfg [config file path]\n" +
            "\tOptional\n" +
            "\tSpecify a config file to be used\n" +
            "\tDefault uses dstat.cfg in dspace config directory\n" +
            "-out [output file path]\n" +
            "\tOptional\n" +
            "\tSpecify an output file to write results into\n" +
            "\tDefault uses dstat.dat in dspace log directory\n" +
            "-start [YYYY-MM-DD]\n" +
            "\tOptional\n" +
            "\tSpecify the start date of the analysis\n" +
            "\tIf a start date is specified then no attempt to gather \n" +
            "\tcurrent database statistics will be made unless -lookup is\n" +
            "\talso passed\n" +
            "\tDefault is to start from the earliest date records exist for\n" +
            "-end [YYYY-MM-DD]\n" +
            "\tOptional\n" +
            "\tSpecify the end date of the analysis\n" +
            "\tIf an end date is specified then no attempt to gather \n" +
            "\tcurrent database statistics will be made unless -lookup is\n" +
            "\talso passed\n" +
            "\tDefault is to work up to the last date records exist for\n" +
            "-lookup\n" +
            "\tOptional\n" +
            "\tForce a lookup of the current database statistics\n" +
            "\tOnly needs to be used if date constraints are also in place\n" +
            "-help\n" +
            "\tdisplay this usage information\n";

        System.out.println(usage);
    }
}
