/*
 * Licensed Materials - Property of IBM Corp.
 * IBM UrbanCode Deploy
 * IBM Urbancode Release
 * (c) Copyright IBM Corporation 2016, 2017. All Rights Reserved.
 *
 * U.S. Government Users Restricted Rights - Use, duplication or disclosure restricted by
 * GSA ADP Schedule Contract with IBM Corp.
 */
package com.urbancode.air.plugin.appscan

import java.io.StringWriter
import java.io.File
import java.io.FileInputStream
import java.net.URL
import java.net.URLEncoder
import java.text.Format
import java.text.SimpleDateFormat
import java.util.Iterator
import java.util.concurrent.TimeoutException
import java.util.concurrent.TimeUnit

import javax.xml.XMLConstants
import javax.xml.namespace.NamespaceContext
import javax.xml.parsers.DocumentBuilderFactory
import javax.xml.transform.Transformer
import javax.xml.transform.TransformerFactory
import javax.xml.transform.dom.DOMSource
import javax.xml.transform.stream.StreamResult
import javax.xml.xpath.XPath
import javax.xml.xpath.XPathConstants
import javax.xml.xpath.XPathFactory

import org.apache.commons.io.FileUtils
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.methods.HttpPost
import org.apache.http.client.methods.HttpUriRequest
import org.apache.http.entity.InputStreamEntity
import org.apache.http.entity.StringEntity
import org.apache.http.HttpEntity
import org.apache.http.HttpResponse
import org.apache.http.util.EntityUtils
import org.w3c.dom.Document

import com.urbancode.air.XTrustProvider
import com.urbancode.commons.httpcomponentsutil.HttpClientBuilder

public class AppScanRestHelper {

    def user
    def password
    def baseUrl
    def disableCerts
    def client
    String sessionId
    XPath xpath

    // Retrieve Reports parameters
    def reportsFIID
    def filepath
    def applicationID
    def issuesAllowed

    // Run Scan parameters
    def rerun
    def scanFIID
    int timeout
    Boolean waitForScanComp
    Boolean stopScan

    // Create Scan parameters
    def appID
    def folderID
    def scanDescription
    def scanName
    def templateName

    // Configure Job Options parameters
    def httpAuth
    def httpPassword
    def httpUser
    def recordedTraffic
    def setAuto
    def scanLimit
    def scanPassword
    def startUrl
    def scanUser

    // Retrieve PDF Report parameters
    def port

    // AppScan Enterprise Configuration
    /**
     * Persist cookies across requests
     */
    def cookieContainer

    public AppScanRestHelper(props) {
        this.appID = props['appID']
        this.applicationID = props['applicationID']
        this.baseUrl = props['baseUrl']
        this.disableCerts = Boolean.valueOf(props['disableCerts'])
        this.filepath = props['filepath']
        this.folderID = props['folderID']
        this.httpAuth = props['httpAuth']
        this.httpUser = props['httpUser']
        this.httpPassword = props['httpPassword']
        this.issuesAllowed = props['issuesAllowed']
        this.scanLimit = props['scanLimit']
        this.password = props['password']
        this.port = props['port']
        this.recordedTraffic = props['recordedTraffic']
        this.reportsFIID = props['reportsFIID']
        if (props['rerun']) {
            this.rerun = (props['rerun'] as int)
        }
        this.scanDescription = props['scanDescription']
        this.scanFIID = props['scanFIID']
        this.scanName = props['scanName']
        this.scanPassword = props['scanPassword']
        this.scanUser = props['scanUser']
        this.setAuto = props['setAuto']
        this.startUrl = props['startUrl']
        this.templateName = props['templateName']
        this.waitForScanComp = Boolean.valueOf(props['waitForScanComp'])
        this.stopScan = Boolean.valueOf(props['stopScan'])
        if (props['timeout']) {
            this.timeout = props['timeout']
        }
        this.user = props['user']
        this.cookieContainer = ""

        // baseUrl syntax differs between steps
        if (port) {
            if (baseUrl.endsWith("/")) {
                baseUrl = baseUrl.substring(0, baseUrl.length()-1) + ":${port}/"
            }
        }
        else if (!baseUrl.endsWith("/")) {
            baseUrl += "/"
        }

        if ((filepath) && (!filepath.endsWith(File.separator))) {
            filepath += File.separator
        }

        HttpClientBuilder builder = new HttpClientBuilder()

        if (disableCerts) {
            XTrustProvider.install()
            builder.setTrustAllCerts(true)
        }

        builder.setUsername(user)
        builder.setPassword(password)
        builder.setPreemptiveAuthentication(true)

        this.client = builder.buildClient()

        System.setProperty("https.protocols", "TLSv1")

        // Initialization of XPath utilities
        XPathFactory xfactory = XPathFactory.newInstance()
        xpath = xfactory.newXPath()
        xpath.setNamespaceContext(_nsContext)

        try {
            this.sessionId = authenticate()
            println "Logged in."
        } catch (Exception e) {
            println "[Error] Failed to authenticate."
            e.printStackTrace()
            System.exit(1)
        }
    }

    public String[] retrieveReportInfo() {
        try {
            String[] outProps = getReportInfo()
            println "Retrieved Report successfully."
            return outProps
        } catch (Exception e) {
            println "[Error] Failed report retrieval."
            e.printStackTrace()
            System.exit(1)
        }
    }

    public String[] retrievePDFReportInfo() {
        try {
            getPDFReportInfo()
            println "Retrieved PDF Report successfully."
        } catch (Exception e) {
            println "[Error] Failed report retrieval."
            e.printStackTrace()
            System.exit(1)
        }
    }

    public def runScan() {
        int status = ScanState.UNKNOWN
        def result = [status, 1]
        def runNumber = 1
        def retry = true
        if (rerun) {
            runNumber += rerun
        }

        while (retry) {
            try {
                status = runScanByFIID()
                // 0 is success
                // 1 is failure
                if (!waitForScanComp && (status == ScanState.READY || status == ScanState.RUNNING)) {
                    // Success if Wait for Scan Completion is unchecked
                    println "Scan in a RUNNING or READY state."
                    result = [getKey(status), 0]
                } else if (status == ScanState.READY){
                    println "Scan and/or Report Pack finished successfully."
                    // Success if Wait for Scan Completion is checked
                    result = [getKey(status), 0]
                } else {
                    // Fail if a non-running/ready state is found
                    result = [getKey(status), 1]
                }
            } catch (Exception e) {
                println "[Error] Failed scan."
                e.printStackTrace()
            } finally {
                // If failure found, retry
                if (result[1] == 1 && runNumber > 1) {
                    status = stopScanByFIID()
                    println ""
                    println "Rerunning scan."
                    runNumber -= 1
                }
                else {
                    if (result[1] == 1 && stopScan) {
                        status = stopScanByFIID()
                        result = [getKey(status), 1]
                    }
                    retry = false
                }
            }
        }

        return result
    }

    public void listTemplates() {
        Document doc = sendGetRequest(baseUrl + "ase/services/templates")
        printOuterXml(doc)
    }

    public String[] createScan() {
        try {
            String[] outProps = createScanByTemplateID()
            println "Scan created successfully."
            return outProps
        } catch (Exception e) {
            println "[Error] Failed scan creation."
            e.printStackTrace()
            System.exit(1)
        }
    }

    public void configJobOptions() {
        try {
            configureJobOptions()
            println "Job options configured successfully."
        } catch (Exception e) {
            println "[Error] Failed configuration."
            e.printStackTrace()
            System.exit(1)
        }
    }

    private String[] createScanByTemplateID() {
        String createURL = baseUrl + "ase/services/"

        // Get templateFIID
        print("Retrieving the identifier of the " + templateName + " template...");
        Document doc = sendGetRequest(baseUrl + "ase/services/templates")
        String templateId = (String) xpath.evaluate("//content-scan-job[name='" + templateName + "']/id/text()", doc, XPathConstants.STRING);
        println "Template ID: ${templateId}"


        if (folderID) {
            createURL += "folders/${folderID}/folderitems?templateid=${templateId}"
        }
        else {
            createURL += "folderitems?templateid=${templateId}"
        }
        if (appID) {
            createURL += "&appid=${appID}"
        }

        def encodedName = URLEncoder.encode(scanName, "UTF-8")
        encodedName = encodedName.replace('.', '%2E')
        encodedName = encodedName.replace('-', '%2D')

        println "Creating ${scanName} based on the ${templateName} template."
        def postData = "name=" + encodedName + "&description=" + URLEncoder.encode(scanDescription, "UTF-8")
        doc = sendPostRequest(createURL, postData)
        println "Scan information:"

        String createdScanFIID = (String) xpath.evaluate("//folder-items/content-scan-job/id/text()", doc, XPathConstants.STRING)
        String createdReportsFIID = (String) xpath.evaluate("//folder-items/report-pack/id/text()", doc, XPathConstants.STRING)
        String createdReportsURL = (String) xpath.evaluate("//folder-items/report-pack/reports/@href", doc, XPathConstants.STRING)

        println "scanFIID: ${createdScanFIID}"
        println "reportsFIID: ${createdReportsFIID}"
        println "reportsURL: ${createdReportsURL}"

        String[] outProps = [createdScanFIID, createdReportsFIID, createdReportsURL]
        return outProps

    }

    private void configureJobOptions() {
        String scanURL = "ase/services/folderitems/" + scanFIID
        Document doc = sendGetRequest(baseUrl + scanURL)

        String optionsURL = (String) xpath.evaluate("//content-scan-job/options/@href", doc, XPathConstants.STRING)

        def postData
        // Set the starting URL
        if(startUrl) {
            println "Updating the starting URL."
            postData = "value=" + URLEncoder.encode(startUrl, "UTF-8")
            doc = sendPostRequest(optionsURL + "/" + OptionType.STARTING_URLS, postData)
        }
        // Set automatic login
        if(setAuto == "true") {
            println "Enabling Automatic Login Method."
            postData = "value=3"
            doc = sendPostRequest(optionsURL + "/" + OptionType.AUTOMATIC_LOGIN, postData)
        } else if(setAuto == "false") {
            println "Disabling Automatic Login Method."
            postData = "value=0"
            doc = sendPostRequest(optionsURL + "/" + OptionType.AUTOMATIC_LOGIN, postData)
        }
        // Set the autoform username
        if(scanUser) {
            println "Updating Automatic Login username."
            postData = "value=" + URLEncoder.encode(scanUser, "UTF-8")
            doc = sendPostRequest(optionsURL + "/" + OptionType.AUTOMATIC_LOGIN_USERNAME, postData)
        }
        // Set the autoform password
        if(scanPassword) {
            println "Updating Automatic Login password."
            postData = "value=" + URLEncoder.encode(scanPassword, "UTF-8")
            doc = sendPostRequest(optionsURL + "/" + OptionType.AUTOMATIC_LOGIN_PASSWORD, postData)
        }
        // Set the http authentication
        if(httpAuth == "true") {
            println "Enabling HTTP authentication."
            postData = "value=" + URLEncoder.encode(httpAuth, "UTF-8")
            doc = sendPostRequest(optionsURL + "/" + OptionType.HTTP_AUTHENTICATION, postData)
        } else if(httpAuth == "false") {
            println "Disabling HTTP authentication."
            postData = "value=" + URLEncoder.encode(httpAuth, "UTF-8")
            doc = sendPostRequest(optionsURL + "/" + OptionType.HTTP_AUTHENTICATION, postData)
        }
        // Set the http username
        if(httpUser) {
            println "Updating the HTTP username."
            postData = "value=" + URLEncoder.encode(httpUser, "UTF-8")
            doc = sendPostRequest(optionsURL + "/" + OptionType.HTTP_USERNAME, postData)
        }
        // Set the http password
        if(httpPassword) {
            println "Updating the HTTP username."
            postData = "value=" + URLEncoder.encode(httpPassword, "UTF-8")
            doc = sendPostRequest(optionsURL + "/" + OptionType.HTTP_PASSWORD, postData)
        }

        //Set recorded traffic
        if (recordedTraffic) {
            println "Updating the recorded traffic."
            File file = new File(recordedTraffic)

            HttpPost postRequest = new HttpPost(scanURL + "/httptrafficdata")

            FileInputStream fileInputStream = new FileInputStream(file)
            InputStreamEntity reqEntity = new InputStreamEntity(fileInputStream, file.length())
            postRequest.setEntity(reqEntity);
            reqEntity.setContentType("binary/octet-stream");

            postRequest.addHeader("Content-Type", "application/octet-stream")
            postRequest.addHeader("Accept", "application/json,application/xml")

            HttpResponse response = client.execute(request)
            doc = convertToDOM(response)
            checkForError(doc)
        }

        // Set the scan limit
        if(scanLimit) {
            println "Updating the scan limit."
            postData = "value=" + URLEncoder.encode(scanLimit, "UTF-8")
            doc = sendPostRequest(optionsURL + "/" + OptionType.SCAN_LIMIT, postData)
        }
    }

    private int runScanByFIID() throws Exception{
        int result = ScanState.UNKNOWN
        // Set scan URL for later use and pull out some information about the scan
        String reportPackURL = ""
        def reportPackLastRunDateTime = ""
        String scanURL = "ase/services/folderitems/" + scanFIID
        Document doc = sendGetRequest(baseUrl + scanURL)

        String scanLastRunDateTime = DocUtil.getContentScanJob_LastRun(doc)
        println "Scan last run: " + scanLastRunDateTime

        // Set report URL for later use and pull out some information about the report pack
        if(reportsFIID) {
            reportPackURL = "ase/services/folderitems/" + reportsFIID
            doc = sendGetRequest(baseUrl + reportPackURL)

            String reportsURL = (String) xpath.evaluate("//report-pack/reports/@href", doc, XPathConstants.STRING)
            println "Reports URL: " + reportsURL
            reportPackLastRunDateTime = xpath.evaluate("//report-pack/last-run/text()", doc, XPathConstants.STRING)
            println "Reports last run: " + reportPackLastRunDateTime
        }

        // Confirm Scan is ready to run
        Double scanState = (Double) xpath.evaluate("//state/id/text()", doc, XPathConstants.NUMBER)
        String status = (String) xpath.evaluate("//state/name/text()", doc, XPathConstants.STRING)
        println "[${new Date().toString()}] Status: " + status
        result = scanState.intValue()

        def postData
        // Stop scan if starting, running, or resuming
        if (result == ScanState.READY ) {
            // Run the scan by setting the action
            println "Running the scan."
            postData = "action=" + ScanAction.RUN
            doc = sendPostRequest(baseUrl + scanURL, postData)
        } else if (result == ScanState.RUNNING ) {
            println "Scan is already in the RUNNING state."
        } else {
            println "Scan is in an invalid '${status}' state."
        }

        if (waitForScanComp && (result == ScanState.READY || result == ScanState.RUNNING)) {
            // Wait for scan completion
            println "Waiting for scan completion..."
            result = waitForCompletion(baseUrl + scanURL, scanLastRunDateTime)
            println "Scan completed successfully."
            // Wait for report pack completion
            if(reportsFIID) {
                println "Waiting for report pack completion..."
                result = waitForCompletion(baseUrl + reportPackURL, reportPackLastRunDateTime)
                println "Reports finished successfully."
                getReportInfo()
            }
        } else {
            // Wait for running
            result = waitForRunning(baseUrl + scanURL, scanLastRunDateTime)
        }
        return result
    }

    private int stopScanByFIID() {
        int result = ScanState.UNKNOWN
        String fullURL = baseUrl + "ase/services/folderitems/" + scanFIID
        Document doc = sendGetRequest(fullURL)
        Double scanState = (Double) xpath.evaluate("//state/id/text()", doc, XPathConstants.NUMBER)
        String status = (String) xpath.evaluate("//state/name/text()", doc, XPathConstants.STRING)
        println "[${new Date().toString()}] Status: " + status
        result = scanState.intValue()
        // Stop scan if starting, running, or resuming
        if (result == ScanState.STARTING ||
            result == ScanState.RUNNING  ||
            result == ScanState.RESUMING ) {
            println "Stopping the scan."
            String postData = "action=" + ScanAction.END
            sendPostRequest(fullURL, postData)
            // Wait for scan to enter a ready state
            result = waitForReady(fullURL)
        } else {
            println "Scan is stoppped."
        }
        return result
    }

    private String[] getReportInfo() {
        // Set report URL for later use and pull out some information about the report pack
        String reportPackURL = "ase/services/folderitems/" + reportsFIID
        Document doc = sendGetRequest(baseUrl + reportPackURL)
        // Will determine whether or not the severity counts have been found
        Boolean haveSevs = false
        String highSevIssuesCount = ""
        String mediumSevIssuesCount = ""

        String reportsURL = (String) xpath.evaluate("//report-pack/reports/@href", doc, XPathConstants.STRING)
        println reportsURL
        def reportPackLastRunDateTime = xpath.evaluate("//report-pack/last-run/text()", doc, XPathConstants.STRING)
        println "Report Pack Last Run: ${reportPackLastRunDateTime}"

        // Get report
        println "Retrieving report summary information..."
        doc = sendGetRequest(reportsURL)

        printOuterXml(doc)

        String secIssuesDataURL = (String) xpath.evaluate("//report[name='Security Issues']/data/@href", doc, XPathConstants.STRING)
        println secIssuesDataURL

        // Try finding the high and medium severity issue counts for ASE version 9, then try for earlier versions
        try {
            highSevIssuesCount = (String) xpath.evaluate("//report[name='Broken Links']/issue-counts-severity/issue-count[severity/name='High']/count/text()", doc, XPathConstants.STRING)
            mediumSevIssuesCount = (String) xpath.evaluate("//report[name='Broken Links']/issue-counts-severity/issue-count[severity/name='Medium']/count/text()", doc, XPathConstants.STRING)
            if (highSevIssuesCount && mediumSevIssuesCount) {
                haveSevs = true
                println "Acquired High and Medium Severity Issues Count."
            }
            if (haveSevs == false) {
                try {
                    highSevIssuesCount = (String) xpath.evaluate("//report[name='Security Issues']/issue-counts-severity/issue-count[severity/name='High']/count/text()", doc, XPathConstants.STRING)
                    mediumSevIssuesCount = (String) xpath.evaluate("//report[name='Security Issues']/issue-counts-severity/issue-count[severity/name='Medium']/count/text()", doc, XPathConstants.STRING)
                    if (highSevIssuesCount && mediumSevIssuesCount) {
                        haveSevs = true
                    }
                }
                finally {
                    if (haveSevs == true) {
                        println "Acquired High and Medium Severity Issues Count."
                    }
                }
            }
        }
        finally {
            if (haveSevs == true) {
                println "High Severity Issues: ${highSevIssuesCount}"
                println "Medium Severity Issues: ${mediumSevIssuesCount}"
            }
            else {
                println "[Warning] Could not acquire High and Medium Severity Issues Count."
            }
        }

        Format formatter = new SimpleDateFormat("ddMMMyyyy-HHmmss");
        String date = formatter.format(new Date());

        def filename = reportsFIID + "-" + date + ".txt"
        File file = new File(filepath + filename)
        def fullFileName = "${filepath}${filename}"
        def output = new BufferedWriter(new FileWriter(file))
        // Retrieve all data
        String secIssuesAllDataURL = secIssuesDataURL + "?mode=all"
        println "Retrieving report detail information..."
        doc = sendGetRequest(secIssuesAllDataURL)

        String text
        if (filepath) {
            text = getXmlString(doc)
            output.write(text)
        }
        else {
            printOuterXml(doc)
        }

        // Get the schema for this report
        String secIssuesSchemaURL = secIssuesDataURL + "?metadata=schema"
        println "Retrieving report schema information..."
        doc = sendGetRequest(secIssuesSchemaURL)

        if (filepath) {
            text = getXmlString(doc)
            output.write(text)
            output.close()
            println "Saved report detail as and schema information as ${fullFileName}"
        }
        else {
            printOuterXml(doc)
        }

        def isReadyForDeployment = ""
        if (highSevIssuesCount && mediumSevIssuesCount) {
            isReadyForDeployment = ((highSevIssuesCount.toInteger() + mediumSevIssuesCount.toInteger()) > issuesAllowed.toInteger()) ? false : true
        }
        String[] outProps = [fullFileName, highSevIssuesCount, mediumSevIssuesCount, isReadyForDeployment]
        return outProps
    }

    private String[] getPDFReportInfo() {

        String body = ""
        if (scanName) {
            body = "{\"config\": {\"executiveSummaryIncluded\": false,\"advisoriesIncluded\": false,\"pdfPageBreakOnIssue\": false,\"sortByURL\": false,\"applicationAttributeConfig\": {\"showEmptyValues\": false,\"attributeLookups\": [\"\"]},\"issueConfig\": {\"includeAdditionalInfo\": false,\"variantConfig\": {\"variantLimit\": 0,\"requestResponseIncluded\": false,\"differencesIncluded\": false},\"issueAttributeConfig\": {\"showEmptyValues\": false,\"attributeLookups\": [\"\"]}}},\"layout\": {\"reportOptionLayoutCoverPage\": {\"companyLogo\": \"\",\"additionalLogo\": \"\",\"includeDate\": false,\"includeReportType\": false,\"reportTitle\": \"\",\"description\": \"\"},\"reportOptionLayoutBody\": {\"header\": \"\",\"footer\": \"\"},\"includeTableOfContents\": false},\"reportFileType\": \"PDF\",\"issueIdsAndQueries\": [\"Scan Name=${scanName}\"]}"
            println body
        }
        else {
            body = "{\"config\": {\"executiveSummaryIncluded\": false,\"advisoriesIncluded\": false,\"pdfPageBreakOnIssue\": false,\"sortByURL\": false,\"applicationAttributeConfig\": {\"showEmptyValues\": false,\"attributeLookups\": [\"\"]},\"issueConfig\": {\"includeAdditionalInfo\": false,\"variantConfig\": {\"variantLimit\": 0,\"requestResponseIncluded\": false,\"differencesIncluded\": false},\"issueAttributeConfig\": {\"showEmptyValues\": false,\"attributeLookups\": [\"\"]}}},\"layout\": {\"reportOptionLayoutCoverPage\": {\"companyLogo\": \"\",\"additionalLogo\": \"\",\"includeDate\": false,\"includeReportType\": false,\"reportTitle\": \"\",\"description\": \"\"},\"reportOptionLayoutBody\": {\"header\": \"\",\"footer\": \"\"},\"includeTableOfContents\": false},\"reportFileType\": \"PDF\",\"issueIdsAndQueries\": [\"\"]}"
        }
        StringEntity params = new StringEntity(body)


        def securityDetailURL = "ase/api/issues/reports/securitydetails?appId=" + applicationID

        HttpPost securityDetailRequest = new HttpPost(baseUrl + securityDetailURL)
        HttpResponse response = doRequest(securityDetailRequest, params)

        def statusLine = response.getStatusLine()
        def statusCode = statusLine.getStatusCode()

        if (statusCode != 202) {
            println("[Error] Generation failed with an Http response code of ${statusCode}: ${statusLine.getReasonPhrase()}")
            throw new RuntimeException(response.entity?.content?.text)
        }
        else {
            println("[OK] Report generation was successful.")
        }
        def locations = response.getHeaders("Location")
        String location = locations[0]
        location = location.substring(10)
        println ("report status location: " + location)

        HttpGet statusRequest = new HttpGet(location)
        // Get report status
        while (statusCode != 201) {

            response = doRequest(statusRequest)
            statusLine = response.getStatusLine()
            statusCode = statusLine.getStatusCode()

            if (statusCode == 201) {
                println("[OK] Report ready.")
            }
            else if (statusCode != 200) {
                println("[Error] Report failure with an Http response code of ${statusCode}: ${statusLine.getReasonPhrase()}")
                throw new RuntimeException(response.entity?.content?.text)
            }
            else {
                println "In progress"
            }
            TimeUnit.SECONDS.sleep(5)
        }
        locations = response.getHeaders("Location")
        location = locations[0]
        location = location.substring(10)
        println ("report location: " + location)

        // Get report
        HttpGet reportRequest = new HttpGet(location)
        response = doPDFRequest(reportRequest)
        statusLine = response.getStatusLine()
        statusCode = statusLine.getStatusCode()

        if (statusCode != 200) {
            println("[Error] Download failed with an Http response code of ${statusCode}: ${statusLine.getReasonPhrase()}")
            throw new RuntimeException(response.entity?.content?.text)
        }
        else {
            println("")
        }

        // Create file
        Format formatter = new SimpleDateFormat("ddMMMyyyy-HHmmss");
        String date = formatter.format(new Date());
        def filename = "AppScanReportOutput-" + date + ".zip"
        File file = new File(filepath + filename)

        outputZipFile(response, file)
    }

    private void outputZipFile(def response, File file) {
        // Download file in a stream
        InputStream input = null
        FileOutputStream output = null
        byte[] buffer = new byte[1024]
        try {
            input = response.getEntity().getContent()
            output = new FileOutputStream(file)
            for (int length; (length = input.read(buffer)) > 0;) {
                output.write(buffer, 0, length)
            }
        } finally {
            if (output != null) try { output.close(); } catch (IOException logOrIgnore) {}
            if (input != null) try { input.close(); } catch (IOException logOrIgnore) {}
        }
    }

    private static String getXmlString(Document response) throws Exception {
        Transformer transformer = TransformerFactory.newInstance().newTransformer()
        StringWriter writer = new StringWriter()
        transformer.transform(new DOMSource(response.getDocumentElement()), new StreamResult(writer))
        return writer.toString()
    }

    // creates a new LW-SSO token
    private String authenticate() {
        HttpPost loginRequest
        HttpResponse response
        def statusLine
        def statusCode
        String sessionId = null
        // Authentication headers tacked on by HttpClientBuilder
        if (port) {
            String auth = "{\"userId\":\"${user.replace("\\","\\\\")}\",\"password\":\"${password}\",\"featureKey\":\"AppScanEnterpriseUser\"}"
            StringEntity params = new StringEntity(auth)

            loginRequest = new HttpPost(baseUrl + "ase/api/login")
            response = doRequest(loginRequest, params)
            statusLine = response.getStatusLine()
            statusCode = statusLine.getStatusCode()

            if (statusCode != 200) {
                println("[Error] Authentication failed with an Http response code of ${statusCode}: ${statusLine.getReasonPhrase()}")
                throw new RuntimeException(response.entity?.content?.text)
            }
            else {
                println("[OK] Authentication was successful.")
            }

            HttpEntity r_entity = response.getEntity()
            def ent = EntityUtils.toString(r_entity)
            String [] resp = ent.split("\"")
            sessionId = resp[5]
            println "session id: " + sessionId

        }
        else {
            def loginPostData = "userid=" + user + "&password=" + password
            loginRequest = new HttpPost(baseUrl + "ase/services/login")
            response = doRequest(loginRequest, loginPostData)
            statusLine = response.getStatusLine()
            statusCode = statusLine.getStatusCode()

            if (statusCode != 200) {
                println("[Error] Authentication failed with an Http response code of ${statusCode}: ${statusLine.getReasonPhrase()}")
                throw new RuntimeException(response.entity?.content?.text)
            }
            else {
                println("[OK] Authentication was successful.")
            }
        }
        return sessionId
    }

    // Execute a general http request
    private HttpResponse doRequest(HttpUriRequest request) {
        return doRequest(request, null)
    }

    // Execute a general http request
    private HttpResponse doPDFRequest(HttpUriRequest request) {
        request.addHeader("Content-Type", "application/octet-stream")
        request.addHeader("Accept", "application/json,application/xml,application/octet-stream")

        if (sessionId) {
            request.addHeader("asc_xsrf_token", sessionId)
        }
        HttpResponse response = client.execute(request)

        return response
    }

    // Specify a json string to provide a string entity to the http request
    private HttpResponse doRequest(HttpUriRequest request, String contentString) {
        request.addHeader("Content-Type", "application/x-www-form-urlencoded")
        request.addHeader("Accept", "application/json,application/xml")

        if (sessionId) {
            request.addHeader("asc_xsrf_token", sessionId)
        }

        if (contentString) {
            StringEntity input

            try {
                input = new StringEntity(contentString)
            }
            catch(UnsupportedEncodingException ex) {
                println("Unsupported characters in http request content: ${contentString}")
                System.exit(1)
            }

            request.setEntity(input)
        }

        HttpResponse response = client.execute(request)

        return response
    }

    // Specify a json string to provide a string entity to the http request
    private HttpResponse doRequest(HttpUriRequest request, StringEntity contentEntity) {
        request.addHeader("Content-Type", "application/json")
        request.addHeader("Accept", "application/json,application/xml")

        if (sessionId) {
            request.addHeader("asc_xsrf_token", sessionId)
        }

        request.setEntity(contentEntity);

        HttpResponse response = client.execute(request)

        return response
    }

    private Document sendGetRequest(String url) {
        HttpGet getRequest = new HttpGet(url)
        def response = doRequest(getRequest)
        Document doc = convertToDOM(response)
        checkForError(doc)
        return doc
    }

    private Document sendPostRequest(String url, def postData) {
        HttpPost postRequest = new HttpPost(url)
        def response = doRequest(postRequest, postData)
        Document doc = convertToDOM(response)
        checkForError(doc)
        return doc
    }

    private int waitForRunning(String url, String lastRunDate) throws Exception {
        int result = ScanState.UNKNOWN
        String prevStatus = ""
        String lastRunDateTime
        long startTime = System.currentTimeMillis()
        while (System.currentTimeMillis() < startTime + timeout) {
            Document doc = sendGetRequest(url)

            // Get the status and print it
            String status = (String) xpath.evaluate("//state/name/text()", doc, XPathConstants.STRING)
            if (!status.equals(prevStatus)) {
                println "[${new Date().toString()}] Status: " + status
                prevStatus = status
            }

            Double scanState = (Double) xpath.evaluate("//state/id/text()", doc, XPathConstants.NUMBER)
            result = scanState.intValue()
            // If manually stopped, user must "Save Current Results and Stop".
            // Otherwise, last runtime is not saved and infinite loop occurs.
            if (result == ScanState.READY) {
                // If scan marked ready, check last run time to verify completion
                // This means the scan fully ran within a 10 second sleep time...
                lastRunDateTime = (String) xpath.evaluate("//last-run/text()", doc, XPathConstants.STRING)
                if (lastRunDateTime.compareTo(lastRunDate) != 0) {
                    println "Completed successfully."
                    break
                }
            } else if (result == ScanState.STARTING) {
                // Scan is STARTING, wait for RUNNING
            } else if (result == ScanState.RUNNING) {
                println "Scan is running."
                break
            } else {
                result = scanState.intValue()
                break
            }
            Thread.sleep(10000)
        }
        if (result != ScanState.RUNNING ||
            (result == ScanState.READY && (lastRunDateTime.compareTo(lastRunDate) == 0))) {
            println ("[Error] The scan runtime has exceeded the specified timeout while waiting for RUNNING.")
        }
        return result
    }

    private int waitForCompletion(String url, String lastRunDate) throws Exception {
        int result = ScanState.UNKNOWN
        String prevStatus = ""
        long startTime = System.currentTimeMillis()
        while (System.currentTimeMillis() < startTime + timeout) {
            Document doc = sendGetRequest(url)

            // Get the status and print it
            String status = DocUtil.getState_Name(doc)
            if (!status.equals(prevStatus)) {
                println "[${new Date().toString()}] Status: " + status
                prevStatus = status
            }

            Double scanState = DocUtil.getState_Id(doc)
            result = scanState.intValue()
            // If manually stopped, user must "Save Current Results and Stop".
            // Otherwise, last runtime is not saved and infinite loop occurs.
            if (result == ScanState.READY) {
                // If scan marked ready, check last run time to verify completion
                String lastRunDateTime = DocUtil.getLastRun(doc)

                if (lastRunDateTime.compareTo(lastRunDate) != 0) {
                    println "Completed successfully."
                    break
                }

            } else if (result == ScanState.SUSPENDED) {
                // An error has occurred
                println "Object has suspended. Check AppScan Enterprise Console for more information on the scan: "
                if(baseUrl.endsWith('ase/')) {
                    print baseUrl + "Jobs/JobStatistics.aspx?fiid=" + scanFIID
                }
                break
            } else if (result == ScanState.CANCELING) {
                // The scan was cancelled
                println "Object is canceling."
                break
            }

            Thread.sleep(calculateSleep(startTime + timeout))
        }
        if (result != ScanState.READY) {
            println ("[Error] The scan runtime has exceeded the specified timeout while waiting for COMPLETION.")
        }
        return result
    }

    // Sleep a different amount based on distance from timeout
    private long calculateSleep(endTime) {
        long result = 10000
        long remTime = endTime - System.currentTimeMillis()
        float percRem = remTime / timeout
        // Greater than 95% remaining
        if (percRem > 0.95) {
            result = 15000
        // Greater than 90% remaining
        } else if (percRem > 0.90) {
            result = 60000
        // Greater than 10% remaining
        } else if (percRem > 0.1) {
            result = 120000
        } else {
            result = 60000
        }
        return result
    }

    private int waitForReady(String url) throws Exception {
        int result = ScanState.UNKNOWN
        int retries = 0
        String prevStatus = ""
        while (retries < 12) {
            Document doc = sendGetRequest(url)

            // Get the status and print it
            String status = (String) xpath.evaluate("//state/name/text()", doc, XPathConstants.STRING)
            if (!status.equals(prevStatus)) {
                println "[${new Date().toString()}] Status: " + status
                prevStatus = status
                retries = 0 // reset retries if new status is found
            }

            Double scanState = (Double) xpath.evaluate("//state/id/text()", doc, XPathConstants.NUMBER)
            result = scanState.intValue()
            if (result == ScanState.READY) {
                println "Scan is in ready state."
                break
            } else if (result == ScanState.SUSPENDED) {
                // An error has occurred
                println "Object has suspended. Check AppScan Enterprise Console for more information on the scan: "
                if(baseUrl.endsWith('ase/')) {
                    print baseUrl + "Jobs/JobStatistics.aspx?fiid=" + scanFIID
                }
                break
            }
            retries++
            Thread.sleep(5000)
        }
        if (result != ScanState.READY) {
            println ("[Error] The scan runtime has exceeded the specified timeout while waiting for READY.")
        }
        return result
    }

    // Checks response for errors
    private void checkForError(Document doc) throws Exception {
        if((String) xpath.evaluate("//error/text()", doc, XPathConstants.STRING)) {
            println "*** Error Occurred ***"

            String code = (String) xpath.evaluate("//code/text()", doc, XPathConstants.STRING)
            if(code) {
                println code
            }
            String message = (String) xpath.evaluate("//message/text()", doc, XPathConstants.STRING)
            if(message) {
                println message
            }
            String help = (String) xpath.evaluate("//error/help/@weblink", doc, XPathConstants.STRING)
            if(help) {
                println help
            }
            throw new Exception("Unexpected error.")
        }
    }

    private static void printOuterXml(Document response) throws Exception {
        Transformer transformer = TransformerFactory.newInstance().newTransformer()
        transformer.setOutputProperty("omit-xml-declaration", "yes")

        StringWriter writer = new StringWriter()
        transformer.transform(new DOMSource(response.getDocumentElement()), new StreamResult(writer))
        println writer.toString()
    }

    private static Document convertToDOM(HttpResponse response) {
        HttpEntity r_entity = response.getEntity()
        def ent = EntityUtils.toString(r_entity)
        Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new StringBufferInputStream(ent)) //here
        return doc
    }

    private static NamespaceContext _nsContext
    static {
        _nsContext = new NamespaceContext() {
            public String getNamespaceURI(String prefix) {
                if (prefix.equalsIgnoreCase("ase"))
                    return "http://www.ibm.com/Rational/AppScanEnterprise"
                return XMLConstants.NULL_NS_URI
            }

            public String getPrefix(String arg0) {
                return null
            }

            public Iterator<?> getPrefixes(String arg0) {
                return null
            }
        }
    }

    private class ScanAction {
        public static int RUN = 2
        public static int SUSPEND = 3
        public static int CANCEL = 4
        public static int END = 5
    }
    private class OptionType {
        public static String STARTING_URLS = "epcsCOTListOfStartingUrls"
        public static String AUTOMATIC_LOGIN = "elCOTLoginSequenceType"
        public static String AUTOMATIC_LOGIN_USERNAME = "esCOTAutoFormFillUserNameValue"
        public static String AUTOMATIC_LOGIN_PASSWORD = "esCOTAutoFormFillPasswordValue"
        public static String HTTP_AUTHENTICATION = "ebCOTHttpAuthentication"
        public static String HTTP_USERNAME = "esCOTHttpUser"
        public static String HTTP_PASSWORD = "esCOTHttpPassword"
        public static String SCAN_LIMIT = "elCOTScanLimit"
    }

    private Map<String, Integer> ScanState = [
        READY: 1,
        STARTING: 2,
        RUNNING: 3,
        RESUMING: 6,
        CANCELING: 7,
        SUSPENDING: 8,
        SUSPENDED: 9,
        POSTPROCESSING: 10,
        ENDING: 12,
        UNKNOWN: -1 // Not a real AppScan ScanState
    ]

    private String getKey(int value){
        for(String key:ScanState.keySet()){
            if(ScanState.get(key).equals(value)) {
                return key
            }
        }
        return "UNKNOWN";
      }
}
