build.gradle

import java.security.MessageDigest
import java.text.SimpleDateFormat
import org.apache.commons.net.ftp.*
import org.eclipse.jgit.storage.file.FileRepositoryBuilder
import org.eclipse.jgit.lib.Repository
import org.eclipse.jgit.lib.ObjectId
import org.gradle.api.*
buildscript{
    repositories{
        mavenCentral()
    }
    dependencies{
        classpath localGroovy()
        classpath 'com.netflix.nebula:gradle-ospackage-plugin:11.1.0'
        classpath 'commons-net:commons-net:3.3'
        classpath 'org.eclipse.jgit:org.eclipse.jgit:4.6.0.201612231935-r'
        classpath group: 'org.ccil.cowan.tagsoup', name: 'tagsoup', version: '1.2.1'
    }
}

repositories{
    mavenCentral()
}


void checkGitVersion(String version){
    logger.info("checking for git tag $version")
    FileRepositoryBuilder builder=new FileRepositoryBuilder()
    Repository repo=builder.readEnvironment().findGitDir(project.buildDir).setMustExist(true).build()
    ObjectId head=repo.resolve("HEAD")
    logger.info "current git commit=$head.name"
    def tags=repo.getTags()
    def tagCommit=tags.get(version)
    assert tagCommit,"unable to find tag $version"
    ObjectId tagResolved=repo.resolve(tagCommit.getName())
    assert tagResolved.getName() == head.getName(),"found $version ($tagResolved.name) but this is not the current HEAD"
    logger.info "version $version resolves to $tagCommit"
}

if (project.hasProperty("avnavRelease")){
    project.ext.avnavRelease=true
    logger.lifecycle "setting avnavRelease"
}
else {
    project.ext.avnavRelease=false
    def cmdLineTasks=project.gradle.getStartParameter().getTaskNames()
    cmdLineTasks.each { String t ->
        if (t.matches(/.*[rR]elease.*/)) {
            project.ext.avnavRelease = true
            logger.lifecycle "seeting avnavRelease due to task $t"
        }
    }
}
if (! project.hasProperty('packageVersion')){
    SimpleDateFormat f=new SimpleDateFormat('yyyyMMdd')
    project.ext.avnavVersion=f.format(new Date())
    logger.lifecycle("creating version from date: ${project.avnavVersion}")
}
else{
    project.ext.avnavVersion=project.packageVersion
}

boolean buildAndroid=false
if (System.getenv("ANDROID_HOME")){
    buildAndroid=true
}
else{
    logger.warn("ANDROID_HOME not set, cannot build android")
}


description="NMEA multiplexer and Browser based navigation"
apply plugin: 'java'

apply plugin: 'com.netflix.nebula.ospackage-base'

task testGit{
    doLast() {
        try {
            checkGitVersion((project.avnavRelease ? "release-" : "") + project.avnavVersion)
        } catch (Error t) {
            if (!project.hasProperty("avnavIgnoreGit")) throw t;
            logger.lifecycle "### ignoring failed GIT check ###"
        }
    }
}

def taskgroup="avnav"
['assembleRelease','assembleDebug','assembleBeta','clean'].each { tt ->
    tasks.create(name:"${tt}Android",type: GradleBuild) {
        group taskgroup
        setDir("android")
        setTasks([tt])
    }
}
clean.dependsOn cleanAndroid
build.dependsOn assembleReleaseAndroid

["debug","release","clean"].each { tt ->
    def vt=tasks.create(name: "${tt}Viewer",type: GradleBuild) { GradleBuild b->
        b.group taskgroup
        b.setDir("viewer")
        b.setTasks([tt])

    }
    if (tt == "clean"){
        clean.dependsOn vt
    }
    else{
        build.dependsOn vt
    }
}

def guiVersionFile=new File(project.buildDir,"avnav_gui_version.py")
def serverVersionFile=new File(project.buildDir,"avnav_server_version.py")


task pkgVersion{
    doLast() {
        ospackage.version = project.avnavVersion
        println "package version $ospackage.version"
        [guiVersionFile, serverVersionFile].each {
            if (!it.getParentFile().isDirectory()) it.getParentFile().mkdirs()
            it.withWriter { wr ->
                wr.println("AVNAV_VERSION=\"$ospackage.version\"")
            }
        }
    }
}

def markerFile=new File(project.buildDir,"__avnav_software_archive__")
task zipMarker{
    doLast{
        if (!markerFile.getParentFile().isDirectory()) markerFile.getParentFile().mkdirs();
        markerFile.withWriter{ wr->
            wr.println(project.avnavVersion)
        }
    }
}


task windowsBuild(type: org.gradle.api.tasks.GradleBuild){
    dir="windows"
    tasks=["build"]
}

def emptyBase=new File(project.buildDir,"empty")
ospackage {
    //release='3'
    os = LINUX // only applied to RPM
    packageGroup='misc'
    packageName='avnav'
    requires('python3-pip')
    requires('python3-pyudev')
    requires('python3-imaging').or('python3-pil')
    requires('python3-netifaces')
    user='root'
    into ('/usr/lib/avnav') {
        from('viewer/build/release') {
            into 'viewer'
        }
        from('server') {
            include "*.py"
            into "server"
            fileMode 0755
        }
        from('sounds'){
            into "sounds"
            include "*.mp3"
            include "*.txt"
        }
        from('server/handler') {
            include "*.py"
            into "server/handler"
            fileMode 0755
        }
        from('server/plugins'){
            into 'server/plugins'
            exclude '**/__pycache__/**'
            fileMode 0644
        }
        from('chartconvert') {
            include "*.py"
            into 'chartconvert'
            fileMode 0755
        }
        from('chartconvert/tiler_tools') {
            include "*html"
            include "*csv"
            into 'chartconvert/tiler_tools'
            fileMode 0644
        }
        from('chartconvert/tiler_tools') {
            include "*.py"
            into 'chartconvert/tiler_tools'
            fileMode 0755
        }
        from('libraries') {
            include 'gpxpy*/**'
            exclude '**/__pycache__/**'
            into 'libraries'
        }
        from('linux') {
            include "avnav_template.xml"
        }
        from('linux') {
            include "avnav"
            fileMode 0755
        }
        from('linux') {
            include "plugin.sh"
            include 'patchServerConfig.py'
            include 'avnavservice'
            fileMode 0755
        }
        from('linux') {
            include "avnav_gui*"
            fileMode 0755
            into "gui"
        }
        from(guiVersionFile.getParentFile()){
            into "gui"
            include guiVersionFile.getName()
        }
        from(serverVersionFile.getParentFile()){
            into "server"
            include serverVersionFile.getName()
        }
    }
    into ('/usr/lib/systemd/system'){
        from('linux'){
            include 'avnav.service'
        }
    }
    into('/usr/lib/systemd/user/'){
        from('linux'){
            include 'avnavuser.service'
            rename('avnavuser.service','avnav.service')
        }
    }
    postInstall file('linux/postinstall')

    link('/usr/bin/avnav','/usr/lib/avnav/avnav')
    link('/usr/bin/avnavservice','/usr/lib/avnav/avnavservice')
}


task buildZip(type: Zip){
    //archiveName="avnav.zip"
    archiveVersion=project.avnavVersion
    dependsOn 'releaseViewer',pkgVersion, zipMarker
    group taskgroup
    includeEmptyDirs=false
    with ospackage.delegateCopySpec
    eachFile{ d->
        if (!d.getPath().matches("^usr/lib/avnav.*")) {
            d.exclude()
            return
        }
        d.setPath(d.getPath().replaceAll("^usr/lib/avnav/",""))
    }
    into('usr/lib/avnav/windows') {
        from('windows'){
            include("avnav_server_win.xml")
            include("avnav.cmd")
            include("postinstall.ps1")
            include("sound.ps1")
            include("sound.cmd")
        }
    }
    into('usr/lib/avnav'){
        from(markerFile.getParentFile()){
            include markerFile.getName()
        }
    }
}

task raspiDeb(type: GradleBuild){
    group taskgroup
    buildFile="raspi.gradle"
    tasks=['raspiDeb']
}



task releaseRpm(type: Rpm) {
    group taskgroup
    arch = NOARCH
    dependsOn 'releaseViewer',pkgVersion
    requires('gcc')
    requires('python3-devel')
    requires('bluez-devel')
    requires('python3-websocket-client')
    requires('python3-pybluez').or('python3-bluez')
    requires('python3-pydbus')
    requires('python3-pyserial')
    requires('python3-GDAL').or('python3-gdal')
}

task releaseDeb(type: Deb) {
    group taskgroup
    dependsOn 'releaseViewer',pkgVersion
    requires('bluetooth')
    requires('libbluetooth-dev')
    requires('python3-websocket')
    requires('python3-bluez')
    requires('python3-dbus')
    requires('python3-serial')
    requires('python3-gdal')
}

task release{
    group taskgroup
    description "build all avnav release stuff"
    doLast {
        println "all release packages have been build"
    }
    dependsOn testGit, releaseRpm, releaseDeb,raspiDeb
    dependsOn windowsBuild, buildZip
    if (buildAndroid){
        dependsOn assembleReleaseAndroid
    }
}
task beta{
    group taskgroup
    description "build all avnav beta stuff"
    doLast {
        println "all beta packages have been build"
    }
    dependsOn releaseRpm, releaseDeb,raspiDeb
    dependsOn windowsBuild, buildZip
    if (buildAndroid){
        dependsOn assembleBetaAndroid
    }
}


def UPLOAD_GROUP="avnavUpload"


task uploadDeb(type: UploadTask){
    group=UPLOAD_GROUP
    inputFile=file("$buildDir/distributions/avnav_${project.avnavVersion}_all.deb")
    dependsOn releaseDeb
    baseDir="daily/${project.avnavVersion}"
}
task uploadRaspi(type: UploadTask){
    group=UPLOAD_GROUP
    inputFile=file("$buildDir/distributions/avnav-raspi_${project.avnavVersion}_all.deb")
    dependsOn raspiDeb
    baseDir="daily/${project.avnavVersion}"
}
task uploadZip(type: UploadTask){
    group=UPLOAD_GROUP
    inputFile=file("$buildDir/distributions/avnav-${project.avnavVersion}.zip")
    dependsOn buildZip
    baseDir="daily/${project.avnavVersion}"
}
task uploadRpm(type: UploadTask){
    group=UPLOAD_GROUP
    inputFile=file("$buildDir/distributions/avnav-${project.avnavVersion}.noarch.rpm")
    dependsOn releaseRpm
    baseDir="daily/${project.avnavVersion}"
}

task uploadWindowsNet(type: UploadTask){
    group=UPLOAD_GROUP
    def windowsVersion=project.avnavVersion
    inputFile=file("windows/build/avnav-service-${windowsVersion}.exe")
    dependsOn windowsBuild
    baseDir="daily/${project.avnavVersion}"
}

task uploadRepoDaily(type: UploadSftp){
    inputFiles=fileTree("$buildDir/distributions",{
        include "avnav_${project.avnavVersion}_all.deb"
        include "avnav-raspi_${project.avnavVersion}_all.deb"
    })
    server='free-x.de'
    base=''
    baseDir="daily/avnav/${project.avnavVersion}"
    user="wellenvogel"
    doFirst{
        assert System.getenv('AVNAV_REPO_KEY')
    }
    dependsOn 'beta'
}

task uploadRepoRelease(type: UploadSftp){
    inputFiles=fileTree("$buildDir/distributions",{
        include "avnav_${project.avnavVersion}_all.deb"
        include "avnav-raspi_${project.avnavVersion}_all.deb"
    })
    server='free-x.de'
    base=''
    baseDir="release/avnav/${project.avnavVersion}"
    user="wellenvogel"
    doFirst{
        assert System.getenv('AVNAV_REPO_KEY')
    }
    dependsOn 'release'
}

def ks=System.getenv("ANDROID_KEYSTORE")
def sign="-unsigned"
if (ks) sign=""
task uploadAndroidRelease(type: UploadTask){
    inputFile=file("android/build/outputs/apk/release/avnav-release$sign-${avnavVersion}.apk")
    dependsOn assembleReleaseAndroid
    baseDir="release/${project.avnavVersion}"
}
task uploadAndroidBeta(type: UploadTask){
    inputFile=file("android/build/outputs/apk/beta/avnav-beta-${avnavVersion}.apk")
    dependsOn assembleBetaAndroid
    baseDir="daily/${project.avnavVersion}"
}

task uploadViewer(type: UploadTask){
    useHashes=true
    inputFiles=fileTree(dir:"viewer/build/release")
    inputFiles+=fileTree("viewer/demo",{
        include "demo-osm/**"
    })
    inputFiles+=fileTree("viewer/demo",{
        include "demo-eniro/**"
    })
    inputFiles+=fileTree("viewer/demo",{
        include "demo-bsh/**"
    })
    baseDir="../viewern"
    //deleteTargetDir=true
    dependsOn releaseViewer
}

task uploadViewerRelease(type: UploadTask){
    useHashes=true
    inputFiles=fileTree(dir:"viewer/build/release")
    inputFiles+=fileTree("viewer/demo",{
        include "demo-osm/**"
    })
    inputFiles+=fileTree("viewer/demo",{
        include "demo-eniro/**"
    })
    inputFiles+=fileTree("viewer/demo",{
        include "demo-bsh/**"
    })
    baseDir="../viewern"
    //deleteTargetDir=true
    dependsOn releaseViewer
}

def docTree=fileTree("docs",{
    exclude "viewerimages"
    include "**/*html"
    include "**/*css"
    include "**/*pdf"
    include "**/*PDF"
    include "**/*json"
    include "**/*js"
    include "**/.htaccess"
})
task addImagesToDoc(type: DefaultTask){
    doFirst(){
        def imageFiles=findImages(docTree)
        imageFiles.each { img->
            logger.info("add image file $img")
            docTree.include(img)
        }
    }
}
task uploadDownloadIndex(type: UploadTask){
    inputFile=file("docs/downloads/index.json")
    deleteTargetDir=false
    baseDir=""
    useHashes=false
}

task uploadDoc(type: UploadTask){
    inputFiles=docTree
    deleteTargetDir=false
    baseDir="../docs"
    useHashes=true
    dependsOn addImagesToDoc, 'buildDocMap', uploadDownloadIndex
}

task computeDocHashes(type:DefaultTask){
    dependsOn addImagesToDoc
    doFirst(){
        def hashes=UploadTask.computeHashes(docTree)
        hashes.each{ n,v->
            println "${n}=${v}"
        }
    }
}


import groovy.util.XmlSlurper
def findImages(fileTree){
    def result=[]
    fileTree.visit{ FileVisitDetails element ->
          if (! element.getFile().isFile()) return;
          if (! element.getName().matches('.*html$')) return;  
          def tagsoupParser = new org.ccil.cowan.tagsoup.Parser()
          def page = new XmlSlurper(tagsoupParser).parse(element.getFile())
          page.'**'.findAll().each{ node->
            if (node.name() == "img" ){
                if (node.@src.toString().startsWith("http")) return;
                def imgFile=RelativePath.parse(true,node.@src.toString())
                def completePath=element.getRelativePath().getParent().append(imgFile)
                def segments=new ArrayList()
                for (String s : completePath.getSegments()){
                    if (s == ".."){
                        if (segments.size() > 0 && segments[segments.size()-1] != ".." ){
                            segments.remove(segments.size()-1)
                        }
                        else{
                            segments.add(s)
                        }
                    }
                    else{
                        segments.add(s)
                    }
                }
                def completeName=new RelativePath(true,segments.toArray(new String[segments.size()])).getPathString();
                if (completeName.matches(".*viewerimages/.*")) return
                result.add(completeName)
            }
          }  
    }
    return result
}

def headingToAnchor(tag,txt){
    if (txt == null || txt.isEmpty()) return null;
    return tag+":"+txt.replaceAll("[^a-zA-Z0-9_-]","");
}
def findHeadings(fileTree,rlang){
    def result=[]
    def tagList=['h1','h2','h3','h4']
    fileTree.visit{ FileVisitDetails element ->
          if (! element.getFile().isFile()) return;
          if (! element.getName().matches('.*html$')) return;  
          def tagsoupParser = new org.ccil.cowan.tagsoup.Parser()
          def page = new XmlSlurper(tagsoupParser).parse(element.getFile())
          def pagePath=element.getPath()
          def relPath=element.getRelativePath().getParent().getPathString()
          def pageTitle=null
          //first get the h1
          page.'**'.findAll({node->node.name()=='h1'}).each { h1 ->
              if (pageTitle == null && !h1.text().isEmpty()) pageTitle = h1.text();
          }
          def bodyElem=page.body
          def pageOrder=bodyElem['@data-order']

          if (pageTitle == null) return;
          def pageName=element.getName()
          def lang='de';
          if (pageName.startsWith('en_')){
              lang='en';
              pageName=pageName.replaceAll("^en_","");
          }
          if (lang != rlang) return
          def prefix=relPath
          if (prefix != "") prefix+="/"
          page.'**'.findAll().each{ node->
            if (node.name() in tagList ){
                if (node.text() =~ '^ *$' ) return;
                def entry=[tag:node.name(),
                    lang: lang,
                    href:prefix+pageName+"#"+headingToAnchor(node.name(),node.text()),
                    path:pagePath,
                    relPath:relPath,
                    page: pageTitle,
                    text: node.text(),
                    depth: element.getRelativePath().getSegments().length,
                    ]
                if (pageOrder.toString() != "") {
                    entry['pageOrder']=pageOrder.toInteger()
                }
                result.add(entry)
            }
          }  
    }
    result.sort {a,b -> 
        if (a.depth < b.depth) return -1;
        if (a.depth > b.depth) return 1;
        return 0
    }
    return result
}

import groovy.json.*

task buildDocMap(type: DefaultTask){
    def enTarget=file("docs/en_docmapdata.json")
    def deTarget=file("docs/de_docmapdata.json")
    outputs.file enTarget
    outputs.file deTarget
    def list=fileTree("docs",{
                    include "**/*html"
                    exclude "**/*docmap.html"
                    exclude "**/*onepage.html"
            })
    inputs.files list        
    doLast(){
        def rt=findHeadings(list,"de")
        deTarget.withWriter('utf-8'){ wr->
            wr.writeLine(JsonOutput.toJson(rt))

        } 
        rt=findHeadings(list,"en")
        enTarget.withWriter('utf-8'){ wr->
            wr.write(JsonOutput.toJson(rt))

        }
    }
}

//https://stackoverflow.com/questions/31241519/xmlslurper-obtain-the-line-number-of-groovy-util-slurpersupport-nodechild-in-or
import org.xml.sax.*;
import org.xml.sax.ext.Attributes2Impl;
import javax.xml.parsers.*;

class MySlurper extends XmlSlurper{
    Locator locator
    static def LINE_NUM_ATTR='__line'
    public MySlurper(XMLReader parser) throws ParserConfigurationException, SAXException {
        super(parser);
    }
    @Override
    public void setDocumentLocator(Locator locator) {
        this.locator = locator;
    }
    @Override
    public void startElement(String uri, String localName, String qName, Attributes attrs) throws SAXException {
        Attributes2Impl newAttrs = new Attributes2Impl(attrs); 
        newAttrs.addAttribute("", LINE_NUM_ATTR, LINE_NUM_ATTR, "ENTITY", "" + locator.getLineNumber());        
        super.startElement(uri, localName, qName, newAttrs);
    }
}

task findBrokenLinks(type: DefaultTask){
    def baseDir="docs"
    def list=fileTree(baseDir,{
                    include "**/*html"
            })
    inputs.files list 
    doLast(){
        def anchorMap=[:]
        def requestedAnchors=[]
        list.visit{ FileVisitDetails element ->
          if (! element.getFile().isFile()) return;
          if (! element.getName().matches('.*html$')) return;
          def prefix="";
          if (element.getName().startsWith("en_")) prefix="en_";  
          def pageName=element.getPath();
          def tagsoupParser = new org.ccil.cowan.tagsoup.Parser()
          def page = new MySlurper(tagsoupParser).parse(element.getFile())
          def relPath=element.getRelativePath().getParent().getPathString()
          if (! relPath.isEmpty()) relPath+="/"
          page.'**'.findAll().each{ node->
            def line=node["@"+MySlurper.LINE_NUM_ATTR]
            if (node.name() == 'a'){
                def anchorName=node.@name
                if (!anchorName.isEmpty()){
                    def anchorlink=pageName+"#"+anchorName;
                    logger.info("adding achor $anchorlink")
                    anchorMap[anchorlink]=[page:pageName,anchor:anchorName,line:line]
                }
            }
            def link=node.@src
            if (link.isEmpty()) link=node.@href
            if (link.isEmpty()) return;
            link=link.toString()
            if (link.startsWith("http")) return; //TODO: check online links?
            if (link.startsWith("mailto:")) return;
            link=link.replaceAll("[?].*","")
            def parts=link.split('#')
            def pageLink=parts[0]
            if (pageLink.endsWith("html")){
                pageLink=pageLink.replaceAll("/([^/]*)\$","/"+prefix+'$1')
            }
            if (!pageLink.isEmpty()){        
                def target=file("$baseDir/$relPath$pageLink")
                if (!target.exists()){
                    println("##$pageName:$line:$relPath$pageLink")
                }
            }
            if (parts.length > 1){
                def lpage=parts[0].isEmpty()?pageName:(relPath+parts[0])
                lpage=lpage.replaceAll("[^/]+/\\.\\./*","")
                requestedAnchors.add([link:lpage+"#"+parts[1],source:pageName,line:line])
            }
            
          }  
          
        }
        requestedAnchors.each { anchor ->
            def destination=anchorMap[anchor.link]
            if (! destination){
                println("##@@${anchor.source}:${anchor.line}:${anchor.link}")
            }
        }
    }
}


task findImages(type: DefaultTask){
    doFirst(){
            def list=fileTree("docs",{
                    include "**/*html"
            })
        def rt=findImages(list)
        rt.each { img ->
            println "image: ${img}"
        }
    }
}

task findUnusedImages(type: DefaultTask){
    doFirst(){
            def list=fileTree("docs",{
                    include "**/*html"
            })
        def rt=findImages(list)
        def images=fileTree("docs",{
                include "**/*png"
                include "**/*PNG"
                include "**/*jpg"
                include "**/*JPG"
        })
        images.visit{ FileVisitDetails element ->
            if (! element.getFile().isFile()) return;
            def path=element.getRelativePath().getPathString()
            if (path.startsWith("viewerimages")) return
            if (! rt.contains(path)){
                println "unused: $path"
            }
            else{
                println "used: $path"
            }
        }
    }
}

task getTimeZones{
    doLast{
        def url="https://raw.githubusercontent.com/vvo/tzdb/master/time-zones-names.json"
        def tzfile=file('raspberry/configui/timezones.json')
        new URL(url).withInputStream{ i -> tzfile.withOutputStream{ it << i }}
    }
}

task uploadConfigGen(type: UploadTask){
    inputFiles=fileTree(
        'raspberry/configui',{
            include '*html'
            include '*js'
            include '*css'
            include '*conf'
            include '*.json'
        }
    )
    deleteTargetDir=false
    baseDir="../configGen"
    useHashes=true
    dependsOn getTimeZones
}


task upload{
    doLast {
        println "###uploaded artifacts to version dir $project.avnavVersion###"
    }
    dependsOn uploadDeb,uploadRaspi,uploadRpm,uploadViewer, uploadZip
    dependsOn uploadWindowsNet
    if (buildAndroid){
        dependsOn uploadAndroidBeta
    }
}

task uploadRelease{
    doLast {
        println "###upload release artifacts to version dir $project.avnavVersion###"
    }
    dependsOn testGit,uploadDeb,uploadRaspi,uploadRpm,uploadViewerRelease,uploadZip
    dependsOn uploadWindowsNet
    if (buildAndroid){
        dependsOn uploadAndroidRelease
    }
}

afterEvaluate{
    tasks.withType(org.gradle.api.tasks.GradleBuild.class){GradleBuild b->
        b.dependsOn pkgVersion
        if (project.avnavRelease) b.dependsOn testGit
        b.doFirst{
            startParameter.projectProperties.put('packageVersion',project.avnavVersion)
        }
    }
    if (project.avnavRelease) {
        tasks.withType(UploadTask.class) { UploadTask task ->
            if (task.group != UPLOAD_GROUP) return;
            def baseDir = "release/${project.avnavVersion}"
            logger.info("setting baseDir for ${task.name} to $baseDir")
            task.baseDir=baseDir

        }
    }
}