/**
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE and NOTICE files at the root of the source
 * tree and available online at
 *
 * http://www.dspace.org/license/
 */
package org.dspace.ctask.general;

import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.XMLConstants;
import javax.xml.namespace.NamespaceContext;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.content.MetadataValue;
import org.dspace.core.Constants;
import org.dspace.curate.AbstractCurationTask;
import org.dspace.curate.Curator;
import org.dspace.curate.Mutative;
import org.dspace.curate.Suspendable;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

/**
 * MetadataWebService task calls a web service using metadata from
 * passed item to obtain data. Depending on configuration, this
 * data may be assigned to item metadata fields, or just recorded in the
 * task result string. Task succeeds if web service call succeeds and
 * configured updates occur, fails if task user not authorized or item
 * lacks metadata to call service, and returns error in all other cases
 * (except skip status for non-item objects).
 * Intended use: cataloging tool in workflow and general curation.
 * The task uses a URL 'template' to compose the service call, e.g.
 *
 * <p>{@code http://www.sherpa.ac.uk/romeo/api29.php?issn=\{dc.identifier.issn\}}
 *
 * <p>Task will substitute the value of the passed item's metadata field
 * in the {parameter} position. If multiple values are present in the
 * item field, the first value is used.
 *
 * <p>The task uses another property (the datamap) to determine what data
 * to extract from the service response and how to use it, e.g.
 *
 * <p>{@code //publisher/name=>dc.publisher,//romeocolour}
 *
 * <p>Task will evaluate the left-hand side (or entire token) of each
 * comma-separated token in the property as an XPath 1.0 expression into
 * the response document, and if there is a mapping symbol (e.g. {@code '=>'}) and
 * value, it will assign the response document value(s) to the named
 * metadata field in the passed item. If the response document contains
 * multiple values, they will all be assigned to the item field. The
 * mapping symbol governs the nature of metadata field assignment:
 *
 * <ul>
 * <li>{@code '->'} mapping will add to any existing values in the item field</li>
 * <li>{@code '=>'} mapping will replace any existing values in the item field</li>
 * <li>{@code '~>'} mapping will add *only* if item field has no existing values</li>
 * </ul>
 *
 * <p>Unmapped data (without a mapping symbol) will simply be added to the task
 * result string, prepended by the XPath expression (a little prettified).
 * Each label/value pair in the result string is separated by a space,
 * unless the optional 'separator' property is defined.
 *
 * <p>A very rudimentary facility for transformation of data is supported, e.g.
 *
 * <p>{@code http://www.crossref.org/openurl/?id=\{doi:dc.relation.isversionof\}&format=unixref}
 *
 * <p>The 'doi:' prefix will cause the task to look for a 'transform' with that
 * name, which is applied to the metadata value before parameter substitution
 * occurs. Transforms are defined in a task property such as the following:
 *
 * <p>{@code transform.doi = match 10. trunc 60}
 *
 * <p>This means exclude the value string up to the occurrence of '10.', then
 * truncate after 60 characters. The only transform functions currently defined:
 *
 * <ul>
 * <li>{@code 'cut' <number>} = remove number leading characters</li>
 * <li>{@code 'trunc' <number>} = remove trailing characters after number length</li>
 * <li>{@code 'match' <pattern>} = start match at pattern</li>
 * <li>{@code 'text' <characters>} = append literal characters (enclose in ' ' when whitespace needed)</li>
 * </ul>
 *
 * <p>If the transform results in an invalid state (e.g. cutting more characters
 * than are in the value), the condition will be logged and the
 * un-transformed value used.
 *
 * <p>Transforms may also be used in datamaps, e.g.
 *
 * <p>{@code //publisher/name=>shorten:dc.publisher,//romeocolour}
 *
 * <p>which would apply the 'shorten' transform to the service response value(s)
 * prior to metadata field assignment.
 *
 * <p>An optional property 'headers' may be defined to stipulate any HTTP headers
 * required in the service call. The property syntax is double-pipe separated headers:
 *
 * <p>{@code Accept: text/xml||Cache-Control: no-cache}
 *
 * @author richardrodgers
 */
@Mutative
@Suspendable
public class MetadataWebService extends AbstractCurationTask implements NamespaceContext {
    /**
     * logging category
     */
    private static final Logger log = LogManager.getLogger();
    // transform token parsing pattern
    protected Pattern ttPattern = Pattern.compile("\'([^\']*)\'|(\\S+)");
    // URL of web service with template parameters
    protected String urlTemplate = null;
    // template parameter
    protected String templateParam = null;
    // Item metadata field to use in service call
    protected String lookupField = null;
    // Optional transformation of lookupField
    protected String lookupTransform = null;
    // response data to map/record
    protected List<MetadataWebServiceDataInfo> dataList = null;
    // response document parsing tools
    protected DocumentBuilder docBuilder = null;
    // language for metadata fields assigned
    protected String lang = null;
    // field separator in result string
    protected String fieldSeparator = null;
    // optional XML namespace map
    protected Map<String, String> nsMap = new HashMap<>();
    // optional HTTP headers
    protected Map<String, String> headers = new HashMap<>();

    /**
     * Initializes task
     *
     * @param curator Curator object performing this task
     * @param taskId  the configured local name of the task
     * @throws IOException if the parser could not be configured, or passed through.
     */
    @Override
    public void init(Curator curator, String taskId) throws IOException {
        super.init(curator, taskId);
        lang = configurationService.getProperty("default.language");
        String fldSep = taskProperty("separator");
        fieldSeparator = (fldSep != null) ? fldSep : " ";
        urlTemplate = taskProperty("template");
        templateParam = urlTemplate.substring(urlTemplate.indexOf("{") + 1,
                                              urlTemplate.indexOf("}"));
        String[] parsed = parseTransform(templateParam);
        lookupField = parsed[0];
        lookupTransform = parsed[1];
        dataList = new ArrayList<>();
        for (String entry : taskArrayProperty("datamap")) {
            entry = entry.trim();
            String src = entry;
            String mapping = null;
            String field = null;
            int mapIdx = getMapIndex(entry);
            if (mapIdx > 0) {
                src = entry.substring(0, mapIdx);
                mapping = entry.substring(mapIdx, mapIdx + 2);
                field = entry.substring(mapIdx + 2);
            }
            int slIdx = src.lastIndexOf("/");
            String label = (slIdx > 0) ? src.substring(slIdx + 1) : src;
            dataList.add(new MetadataWebServiceDataInfo(this, src, label, mapping, field));
        }
        String hdrs = taskProperty("headers");
        if (hdrs != null) {
            for (String header : hdrs.split("\\|\\|")) {
                int split = header.indexOf(":");
                headers.put(header.substring(0, split).trim(), header.substring(split + 1).trim());
            }
        }
        // initialize response document parser
        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        factory.setNamespaceAware(true);
        try {
            // disallow DTD parsing to ensure no XXE attacks can occur
            // See https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html
            factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
            factory.setXIncludeAware(false);
            docBuilder = factory.newDocumentBuilder();
        } catch (ParserConfigurationException pcE) {
            log.error("caught exception: " + pcE);
            // no point in continuing
            throw new IOException(pcE.getMessage(), pcE);
        }
    }

    @Override
    public int perform(DSpaceObject dso) throws IOException {

        int status = Curator.CURATE_SKIP;
        StringBuilder resultSb = new StringBuilder();

        if (dso.getType() == Constants.ITEM) {
            Item item = (Item) dso;
            String itemId = item.getHandle();
            if (itemId == null) {
                // we are still in workflow - no handle assigned - try title
                List<MetadataValue> titleDc = itemService.getMetadata(item, "dc", "title", null, Item.ANY);
                String title = (titleDc.size() > 0) ? titleDc.get(0).getValue() : "untitled - dbId: " + item.getID();
                itemId = "Workflow item: " + title;
            } else {
                itemId = "handle: " + itemId;
            }
            resultSb.append(itemId);
            // Only proceed if item has a value for service template parameter
            List<MetadataValue> dcVals = itemService.getMetadataByMetadataString(item, lookupField);
            if (dcVals.size() > 0 && dcVals.get(0).getValue().length() > 0) {
                String value = transform(dcVals.get(0).getValue(), lookupTransform);
                status = callService(value, item, resultSb);
            } else {
                resultSb.append(" lacks metadata value required for service: ").append(lookupField);
                status = Curator.CURATE_FAIL;
            }
        } else {
            resultSb.append("Object skipped");
        }
        report(resultSb.toString());
        setResult(resultSb.toString());
        return status;
    }

    protected int callService(String value, Item item, StringBuilder resultSb) throws IOException {

        String callUrl = urlTemplate.replaceAll("\\{" + templateParam + "\\}", value);
        CloseableHttpClient client = HttpClientBuilder.create().build();
        HttpGet req = new HttpGet(callUrl);
        for (Map.Entry<String, String> entry : headers.entrySet()) {
            req.addHeader(entry.getKey(), entry.getValue());
        }
        HttpResponse resp = client.execute(req);
        int status = Curator.CURATE_ERROR;
        int statusCode = resp.getStatusLine().getStatusCode();
        if (statusCode == HttpStatus.SC_OK) {
            HttpEntity entity = resp.getEntity();
            if (entity != null) {
                // boiler-plate handling taken from Apache 4.1 javadoc
                InputStream instream = entity.getContent();
                try {
                    // This next line triggers a false-positive XXE warning from LGTM, even though we disallow DTD
                    // parsing during initialization of docBuilder in init()
                    Document doc = docBuilder.parse(instream);  // lgtm [java/xxe]
                    status = processResponse(doc, item, resultSb);
                } catch (SAXException saxE) {
                    log.error("caught exception: " + saxE);
                    resultSb.append(" unable to read response document");
                } catch (RuntimeException ex) {
                    // In case of an unexpected exception you may want to abort
                    // the HTTP request in order to shut down the underlying
                    // connection and release it back to the connection manager.
                    req.abort();
                    log.error("caught exception: " + ex);
                    throw ex;
                } finally {
                    // Closing the input stream will trigger connection release
                    instream.close();
                }
                // When HttpClient instance is no longer needed,
                // shut down the connection manager to ensure
                // immediate deallocation of all system resources
                client.close();
            } else {
                log.error(" obtained no valid service response");
                resultSb.append("no service response");
            }
        } else {
            log.error("service returned non-OK status: " + statusCode);
            resultSb.append("no service response");
        }
        return status;
    }

    protected int processResponse(Document doc, Item item, StringBuilder resultSb) throws IOException {
        boolean update = false;
        int status = Curator.CURATE_ERROR;
        List<String> values = new ArrayList<>();
        checkNamespaces(doc);
        try {
            for (MetadataWebServiceDataInfo info : dataList) {
                NodeList nodes = (NodeList) info.getExpr().evaluate(doc, XPathConstants.NODESET);
                values.clear();
                // if data found and we are mapping, check assignment policy
                if (nodes.getLength() > 0 && info.getMapping() != null) {
                    if ("=>".equals(info.getMapping())) {
                        itemService.clearMetadata(Curator.curationContext(), item, info.getSchema(), info.getElement(),
                                                  info.getQualifier(), Item.ANY);
                    } else if ("~>".equals(info.getMapping())) {
                        if (itemService
                            .getMetadata(item, info.getSchema(), info.getElement(), info.getQualifier(), Item.ANY)
                            .size() > 0) {
                            // there are values, so don't overwrite
                            continue;
                        }
                    } else {
                        for (MetadataValue dcVal : itemService
                            .getMetadata(item, info.getSchema(), info.getElement(), info.getQualifier(), Item.ANY)) {
                            values.add(dcVal.getValue());
                        }
                    }
                }
                for (int i = 0; i < nodes.getLength(); i++) {
                    Node node = nodes.item(i);
                    String tvalue = transform(node.getFirstChild().getNodeValue(), info.getTransform());
                    // assign to metadata field if mapped && not present
                    if (info.getMapping() != null && !values.contains(tvalue)) {
                        itemService.addMetadata(Curator.curationContext(), item, info.getSchema(), info.getElement(),
                                                info.getQualifier(), lang, tvalue);
                        update = true;
                    }
                    // add to result string in any case
                    resultSb.append(fieldSeparator).append(info.getLabel()).append(": ").append(tvalue);
                }
            }
            // update Item if it has changed
            if (update) {
                itemService.update(Curator.curationContext(), item);
            }
            status = Curator.CURATE_SUCCESS;
        } catch (AuthorizeException authE) {
            log.error("caught exception: " + authE);
            resultSb.append(" not authorized to update");
            status = Curator.CURATE_FAIL;
        } catch (SQLException sqlE) {
            log.error("caught exception: " + sqlE);
            resultSb.append(" error updating metadata");
        } catch (XPathExpressionException xpeE) {
            log.error("caught exception: " + xpeE);
            resultSb.append(" error reading response document");
        }
        return status;
    }

    protected String transform(String value, String transDef) {
        if (transDef == null) {
            return value;
        }
        Queue<String> tokens = tokenize(transDef);
        String retValue = value;
        while (!tokens.isEmpty()) {
            String function = tokens.poll();
            if ("cut".equals(function) || "trunc".equals(function)) {
                String argument = tokens.poll();
                int index = Integer.parseInt(argument);
                if (retValue.length() > index) {
                    if ("cut".equals(function)) {
                        retValue = retValue.substring(index);
                    } else {
                        retValue = retValue.substring(0, index);
                    }
                } else if ("cut".equals(function)) {
                    log.error("requested cut: {} exceeds value length", index);
                    return value;
                }
            } else if ("match".equals(function)) {
                String argument = tokens.poll();
                int index2 = retValue.indexOf(argument);
                if (index2 > 0) {
                    retValue = retValue.substring(index2);
                } else {
                    log.error("requested match: {} failed", argument);
                    return value;
                }
            } else if ("text".equals(function)) {
                retValue = retValue + tokens.poll();
            } else {
                log.error(" unknown transform operation: " + function);
                return value;
            }
        }
        return retValue;
    }

    protected Queue<String> tokenize(String text) {
        Matcher m = ttPattern.matcher(text);
        Queue<String> list = new ArrayDeque<>(m.groupCount());
        while (m.find()) {
            if (m.group(1) != null) {
                list.add(m.group(1));
            } else if (m.group(2) != null) {
                list.add(m.group(2));
            }
        }
        return list;
    }

    protected int getMapIndex(String mapping) {
        int index = mapping.indexOf("->");
        if (index == -1) {
            index = mapping.indexOf("=>");
        }
        if (index == -1) {
            index = mapping.indexOf("~>");
        }
        return index;
    }

    protected String[] parseTransform(String field) {
        String[] parsed = new String[2];
        parsed[0] = field;
        int txIdx = field.indexOf(":");
        if (txIdx > 0) {
            // transform specified
            String txName = field.substring(0, txIdx);
            parsed[1] = taskProperty("transform." + txName);
            if (parsed[1] == null) {
                log.error("no transform found for: " + txName);
            }
            parsed[0] = field.substring(txIdx + 1);
        }
        return parsed;
    }

    protected void checkNamespaces(Document document) throws IOException {
        // skip if already done
        if (dataList.get(0).getExpr() != null) {
            return;
        }
        try {
            XPath xpath = XPathFactory.newInstance().newXPath();
            String prefix = null;
            NamedNodeMap attrs = document.getDocumentElement().getAttributes();
            for (int i = 0; i < attrs.getLength(); i++) {
                Node n = attrs.item(i);
                String name = n.getNodeName();
                // remember if a namespace
                if (name.startsWith("xmlns")) {
                    if (!"xmlns".equals(name)) {
                        // it is a declared (non-default) namespace - capture prefix
                        nsMap.put(name.substring(name.indexOf(":") + 1), n.getNodeValue());
                    } else {
                        // it is the default name space - mint a unique prefix
                        prefix = "pre";
                        nsMap.put(prefix, n.getNodeValue());
                    }
                }
            }
            if (nsMap.size() > 0) {
                xpath.setNamespaceContext(this);
            }
            // now compile the XPath expressions
            for (MetadataWebServiceDataInfo info : dataList) {
                info.setExpr(xpath.compile(mangleExpr(info.getXpsrc(), prefix)));
            }
        } catch (XPathExpressionException xpeE) {
            log.error("caught exception: " + xpeE);
            // no point in continuing
            throw new IOException(xpeE.getMessage(), xpeE);
        }
    }

    protected String mangleExpr(String expr, String prefix) {
        if (prefix == null) {
            return expr;
        }
        // OK the drill is to prepend all node names with the prefix
        // *unless* the node name already has a prefix.
        StringBuilder sb = new StringBuilder();
        int i = 0;
        while (i < expr.length()) {
            if (expr.charAt(i) == '/') {
                sb.append("/");
                i++;
            } else {
                int next = expr.indexOf("/", i);
                String token = (next > 0) ? expr.substring(i, next) : expr.substring(i);
                if (!token.startsWith("@") && !token.contains(":")) {
                    sb.append(prefix).append(":");
                }
                sb.append(token);
                i += token.length();
            }
        }
        return sb.toString();
    }

    // ---- NamespaceContext methods ---- //

    @Override
    public String getNamespaceURI(String prefix) {
        if (prefix == null) {
            throw new NullPointerException("Null prefix");
        } else if ("xml".equals(prefix)) {
            return XMLConstants.XML_NS_URI;
        }
        String nsURI = nsMap.get(prefix);
        return (nsURI != null) ? nsURI : XMLConstants.NULL_NS_URI;
    }

    @Override
    public String getPrefix(String uri) {
        throw new UnsupportedOperationException();
    }

    @Override
    public Iterator getPrefixes(String uri) {
        throw new UnsupportedOperationException();
    }
}
