site/build.gradle

import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import groovy.text.markup.MarkupTemplateEngine
import groovy.text.markup.TemplateConfiguration
import org.gradle.internal.logging.ConsoleRenderer
import com.yahoo.platform.yui.compressor.CssCompressor
import com.yahoo.platform.yui.compressor.JavaScriptCompressor
import org.apache.tools.ant.filters.BaseFilterReader

buildscript {
    repositories {
        jcenter()
    }
    dependencies {
        classpath 'org.apache.httpcomponents:httpclient:4.5.1'
    }
}

ext.watchmode = project.hasProperty('watchmode')?project.getProperty('watchmode'):'false'

apply plugin: 'base'

task copyAssets(type:Copy) {
    from file('src/site/assets')
    into file("$buildDir/site")
    filesMatching('**/*.css') { f->
        if (!f.name.contains('.min.')) {
            filter(CssFilter)
        }
    }
    filesMatching('**/*.js') { f->
        if (!f.name.contains('.min.')) {
            filter(JsFilter)
        }
    }
}

task generateSite(type:JavaExec) {

    description = 'Generates the Groovy Website'
    dependsOn copyAssets

    ext.sources = file('src/site')
    ext.outputDir = file("$buildDir/site")

    inputs.files fileTree(sources)
    outputs.files fileTree(outputDir)

    classpath = project(':generator').sourceSets.main.runtimeClasspath
    main = 'generator.SiteGenerator'
    args = [sources, outputDir, project.watchmode]
    systemProperties.docs_baseurl = System.getProperty('docs_baseurl')
}

task checkDeadLinks(dependsOn: generateSite) {
    description = "Checks for dead links in the generated Groovy website"

    ext.outputDir = file("$buildDir/reports")
    ext.reportFile = file("$outputDir/deadlinks.html")

    inputs.files fileTree(generateSite.outputDir)
    outputs.file reportFile

    // a collection of links which have either been validated
    // or are dummy links
    def excludeFromChecks = [
            'http://issues.apache.org/jira',
            'https://issues.apache.org/jira',
            'target.html',
            'foo.html',
            'http://www.acme.com/cars',
            'http://localhost:8080/groovy/hello.groovy'
    ]

    doLast {
        def deadLinks = [:]

        def currentPath
        def isDead = [:].withDefault { String link ->
            if (excludeFromChecks.any { link.startsWith(it) }) {
                // skip checking those links because they dramatically increase build time
                // while being most likely ok because generated through changelog parsing
                return false
            }

            try {
                URL url
                try {
                    url = URI.create(link).toURL()
                } catch (e) {
                    if (e.message.contains('URI is not absolute')) {
                        url = URI.create("file://${file("${generateSite.outputDir}/$currentPath/$link").absoluteFile}").toURL()
                    }
                }
                logger.debug("Checking URL: $url")
                def cx = url.openConnection()
                if (cx instanceof HttpURLConnection) {
                    CloseableHttpClient httpclient = HttpClients.createDefault()
                    RequestConfig requestConfig = RequestConfig.custom()
                            .setSocketTimeout(5_000)
                            .setConnectTimeout(5_000)
                            .setConnectionRequestTimeout(5_000)
                            .build()
                    HttpGet httpget = new HttpGet(link)
                    httpget.config = requestConfig
                    CloseableHttpResponse response
                    try {
                        response = httpclient.execute(httpget)
                        if (response.statusLine.statusCode == 404) {
                            return true
                        }
                    } finally {
                        response.close()
                    }
                }
            } catch (e) {
                return true
            }
            return false
        }

        def checkLink = { List dead, int line, String link ->
            if (isDead[link]) {
                dead << [line:line, link:link]
            }
        }

        def checkPage = { File f ->
            currentPath = GFileUtils.relativePath(generateSite.outputDir, f.parentFile)
            f.eachLine('utf-8') { String line, int nb ->
                def dead = []
                [/\shref=['"](.+?)['"]/, /src=['"](.+?)['"]/].each { regex ->
                    def matcher = line =~ regex
                    if (matcher) {
                        matcher.each {
                            def path = it[1]
                            checkLink(dead, nb, path)
                        }
                    }
                }
                if (dead) {
                    deadLinks[f] = dead
                }
            }
        }

        file(generateSite.outputDir).eachFileRecurse {
            if (it.name.endsWith('.html')) {
                checkPage(it)
            }
        }

        outputDir.mkdirs()
        def tplConf = new TemplateConfiguration()
        tplConf.with {
            autoIndent = true
            autoNewLine = true
        }
        def tplEngine = new MarkupTemplateEngine(this.class.classLoader, file('gradle/templates'), tplConf)

        def report = tplEngine.createTemplateByPath("deadlinks.groovy").make(deadLinks: deadLinks).toString()

        reportFile.write(report, 'utf-8')
        def reportURL = new ConsoleRenderer().asClickableFileUrl(reportFile)
        logger.lifecycle "Dead links report written at $reportURL"
    }
}

task webzip(type:Zip, dependsOn: checkDeadLinks) {
    description = "Creates a zip with the generated website and the deadlink report"
    destinationDir = file("$buildDir/distributions")

    baseName = 'groovy'
    appendix = 'website'
    from(generateSite.outputDir) {
        into 'site'
    }
    from (checkDeadLinks.outputDir) {
        into 'reports'
    }
}

// Resource filtering classes

class JsFilter extends BaseFilterReader {
    Writer writer
    Thread worker

    JsFilter(Reader reader) {
        super(new PipedReader())
        writer = new PipedWriter(this.@in)
        def compressor = new JavaScriptCompressor(reader, null)
        reader.close()
        worker = Thread.start {
            compressor.compress(writer, -1, true, false, false, false)
            writer.close()
        }
    }

    void close() {
        worker.join()
        super.close()
    }

}

class CssFilter extends BaseFilterReader {
    Writer writer
    Thread worker

    CssFilter(Reader reader) {
        super(new PipedReader())
        writer = new PipedWriter(this.@in)
        def compressor = new CssCompressor(reader)
        reader.close()
        worker = Thread.start {
            compressor.compress(writer, -1)
            writer.close()
        }
    }

    void close() {
        worker.join()
        super.close()
    }

}