build.xml

<?xml version="1.0"?>
<!-- Build file for Joshua. -->
<project name="Joshua Machine Translation Toolkit" default="all" basedir="." xmlns:doxygen="antlib:org.doxygen.tools" xmlns:ivy="antlib:org.apache.ivy.ant">

  <!-- ~~~~~ Properties, Paths, etc ~~~~~~~~~~~~~~~~~~~~~~~~~~ -->

  <property environment="env" />

  <property name="JOSHUA" value="${basedir}" />
  <property name="src" value="${JOSHUA}/src" />
  <property name="test" value="${JOSHUA}/test" />
  <property name="build" value="${JOSHUA}/class" />
  <property name="lib" value="${JOSHUA}/lib" />
  <property name="thraxlib" value="${JOSHUA}/thrax/bin" />
  <property name="doc" value="${JOSHUA}/doc" />

  <!-- TODO: should these be made into 'path's instead of 'property's? -->
  <property name="testng" value="${lib}/testng-6.7.jar" />
  <!-- <property name="bdb"    value="${lib}/je-3.2.23.jar"/> -->

  <property name="cli" value="commons-cli-1.2.jar"/>

  <path id="compile.all.classpath">
    <fileset dir="${lib}">
      <include name="${cli}" />
      <!-- Jung uses a BSD-like license, see ./lib/LICENSES/LICENSE-jung.txt -->
      <include name="jung-api-2.0.jar" />
      <include name="jung-graph-impl-2.0.jar" />
      <include name="jung-algorithms-2.0.jar" />
      <include name="jung-visualization-2.0.jar" />
      <include name="collections-generic-4.01.jar" />
      <include name="berkeleylm.jar" />
      <include name="junit-4.10.jar" />
    </fileset>
    <fileset dir="${thraxlib}">
      <include name="thrax.jar" />
    </fileset>
  </path>

  <!-- ~~~~~ Init build task ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->

  <!-- Create the build directory for compiled class files -->
  <target name="init" depends="check-joshua-home, resolve, download-thrax, download-hadoop" 
    description="--> Check for environment variables, retrieve dependencies with ivy, download Thrax and Hadoop is required">
  </target>

  <!-- Check for environment variables -->
  <target name="check-joshua-home" description="--> Check for environment variables">
    <echo message="JOSHUA = ${JOSHUA} basedir = ${basedir}" />
    <!-- <if> -->
    <!--   <not> -->
    <!--     <equals arg1="${JOSHUA}" arg2="${basedir}" /> -->
    <!--   </not> -->
    <!--   <then> -->
    <!--     <fail message="WARNING: $JOSHUA environment variable overrident to ${basedir}" /> -->
    <!--   </then> -->
    <!-- </if> -->
  </target>

  <!-- Set git version -->
  <target name="git-version" description="--> Store latest git commit ID">
    <exec executable="git">
      <arg value="rev-parse"/>
      <arg value="--short"/>
      <arg value="HEAD"/>
    </exec>
  </target>

  <target name="set-joshua-version" unless="env.JOSHUA_VERSION" description="--> Prompt user to set Joshua version environment variable">
    <fail message="Please set the $JOSHUA_VERSION environment variable." />
  </target>

  <target name="set-java-home" unless="env.JAVA_HOME" description="--> Prompt user to set JAVA_HOME environment variable">
    <fail message="Please set the $JAVA_HOME environment variable." />
    <!-- TODO: add suggestion to use /System/Library/Frameworks/JavaVM.framework/Home/ iff on OSX -->
  </target>

  <target name="kenlm" depends="check-joshua-home" description="--> 'Make' the kenlm sofrtware in ${src}/joshua/decoder/ff/lm/kenlm/">
    <exec executable="make">
      <arg value="-j" />
      <arg value="4" />
      <arg value="-C" />
      <arg value="${src}/joshua/decoder/ff/lm/kenlm/" />
      <arg value="install" />
    </exec>
  </target>

  <target name="giza" depends="check-joshua-home" description="--> 'Make' the giza software in scripts/training/giza-pp/">
    <exec executable="make">
      <arg value="-j" />
      <arg value="4" />
      <arg value="-C" />
      <arg value="scripts/training/giza-pp/" />
      <arg value="all" />
      <arg value="install" />
    </exec>
    <exec executable="make">
      <arg value="-C" />
      <arg value="scripts/training/symal/" />
      <arg value="all" />
    </exec>
    <!-- 	  <exec executable="./configure" dir="scripts/training/MGIZA" /> -->
    <!-- 	  <exec executable="make"> -->
    <!-- 		<arg value="-C scripts/training/MGIZA/" /> -->
    <!--       </exec> -->
    <!-- 	  <exec executable="cp"> -->
    <!-- 		<arg value="-C scripts/training/MGIZA/" /> -->
    <!--       </exec> -->
  </target>

  <target name="parallelize" depends="check-joshua-home" description="--> 'Make' the parallelize software in scripts/training/parallelize/">
    <exec executable="make">
      <arg value="-j" />
      <arg value="4" />
      <arg value="-C" />
      <arg value="scripts/training/parallelize/" />
    </exec>
  </target>

  <target name="download-thrax" depends="check-joshua-home" description="--> Download the Thrax software">
    <exec executable="git">
      <arg value="submodule" />
      <arg value="init" />
    </exec>
    <exec executable="git">
      <arg value="submodule" />
      <arg value="update" />
    </exec>
  </target>

  <target name="thrax" depends="check-joshua-home" description="--> Build Thrax">
    <subant buildpath="thrax">
      <property name="env.HADOOP" value="${lib}" />
      <property name="env.HADOOP_VERSION" value="0.20.203.0" />
      <property name="env.AWS_SDK" value="${JOSHUA}" />
      <property name="env.AWS_VERSION" value="1.1.3" />
    </subant>
  </target>

  <!-- Download the hadoop tool tarball (not jar library), which gets used by
       pipeline.pl and test/hadoop/ -->
  <target name="download-hadoop" depends="check-joshua-home" description="--> Download the Hadoop software">
    <get
      src="http://archive.apache.org/dist/hadoop/core/hadoop-0.20.2/hadoop-0.20.2.tar.gz"
      dest="${JOSHUA}/lib"
      usetimestamp="true"
      skipexisting="true" />
  </target>

    <!-- Determines whether the "devel" target needs to be built, which
       is conditioned on the presence of the .git subdirectory -->
  <condition property="devel-env">
    <and>
      <available file=".git"/>
    </and>
  </condition>

  <!-- ~~~~~ Java build tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->

  <target name="all" depends="setup, jar, giza, kenlm, parallelize" description="--> Default target Java build tasks">
  </target>

  <!-- The setup target does some downloading of files. These files
       are packaged into releases, so this step only needs to be run if
       we're in a development environment -->
  <target name="setup" if="devel-env">
    <antcall inheritAll="true" inheritRefs="true" target="init"/>
    <antcall inheritAll="true" inheritRefs="true" target="thrax"/>
  </target>

  <!-- Compile the Java code. -->
  <target name="java" depends="check-joshua-home" description="--> Compile the Java code">
    <mkdir dir="${build}" />
    <javac compiler="javac1.7" srcdir="${src}" destdir="${build}" classpathref="compile.all.classpath" debug="on" encoding="utf8" sourcepath="" includeantruntime="false">
      <!-- We nullify the sourcepath in order to disable Ant's usual resolution mechanism. This makes it an error for our basic code to call into code that has external dependencies, rather than auto-including those files and then having a classpath error. -->
      <include name="**/*.java" />

      <!-- This should be on, so people see it. But it can be quite verbose...
      <compilerarg value="-deprecation"/>
      -->
    </javac>
  </target>

  <!-- Create a JAR file -->
  <target name="jar" depends="java,check-joshua-home" description="--> Create a JAR file of compiles classes">
    <jar destfile="${build}/joshua.jar" index="true">
      <fileset dir="${build}">
        <include name="**/*.class" />
      </fileset>
      <manifest>
        <attribute name="Main-Class" value="joshua.decoder.JoshuaDecoder" />
      </manifest>
    </jar>
  </target>

  <target name="tree_visualizer" depends="java,check-joshua-home" description="--> Runs the Tree Visualizer software in examples/tree_visualizer/tree_visualizer.jar">
    <jar destfile="examples/tree_visualizer/tree_visualizer.jar" index="true">
      <zipgroupfileset dir="lib" includes="*.jar" />
      <fileset dir="${build}">
        <include name="**/*.class" />
      </fileset>
      <indexjars>
        <fileset dir="${lib}" />
      </indexjars>
      <manifest>
        <attribute name="Class-Path" value="tree_visualizer.jar" />
        <attribute name="Main-Class" value="joshua.ui.tree_visualizer.browser.Browser" />
      </manifest>
    </jar>
  </target>

  <!-- Create a versioned release -->
  <target name="release" depends="set-joshua-version, devel-clean, init, thrax" description="--> Create a versioned release">
    <exec executable="./scripts/support/make-release.sh">
      <arg value="${env.JOSHUA_VERSION}" />
    </exec>
  </target>

  <!-- Create a JAR file of the source code -->
  <target name="source-jar" description="--> Create a JAR file of the source code">
    <jar destfile="${build}/joshua-src.jar">
      <fileset dir="${build}">
        <include name="**/*.java" />
      </fileset>
    </jar>
  </target>

  <!-- ~~~~~ Cleaning tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->

  <!-- Delete the compiled files -->
  <target name="clean" depends="clean-thrax,clean-java" description="--> Clean (remove) all build directories">
    <exec executable="make">
      <arg value="-C" />
      <arg value="scripts/training/giza-pp" />
      <arg value="clean" />
    </exec>
    <exec executable="/bin/bash" dir="src/joshua/decoder/ff/lm/kenlm">
      <arg value="clean" />
    </exec>
    <delete file="${lib}/libken.so" />
    <delete file="tree_visualizer/tree_visualizer.jar" />
    <delete dir="doc/html" />
    <delete dir="doc/latex" />
  </target>

  <!-- Delete just the java files -->
  <target name="clean-java" description="--> Delete just the java files">
    <delete verbose="true" quiet="true">
      <fileset dir="${build}">
        <include name="**/*.class" />
      </fileset>
    </delete>
  </target>

  <target name="clean-thrax" description="--> Delete just the thrax files">
      <delete verbose="true" quiet="true">
      <fileset dir="${thraxlib}">
        <include name="**/*.class"/>
      </fileset>
    </delete>
  </target>

  <!-- EXPERIMENTAL: Delete *all* generated files -->
  <target name="distclean" depends="clean" description="--> EXPERIMENTAL: Delete *all* generated files">
    <!-- BUG: this doesn't delete empty folders (neither ${build} itself, nor the class dirs (the latter makes sense since we don't traverse them)) -->
    <delete verbose="true" quiet="true" includeEmptyDirs="true">
      <fileset dir="${build}">
        <include name="joshua.jar" />
        <include name="joshua-ui.jar" />
      </fileset>
    </delete>

    <!-- HACK: these two work perfectly, but maybe problematic if people reset ${test} or ${doc} -->
    <delete verbose="true" quiet="true" includeEmptyDirs="true">
      <fileset dir="${test}-output" />
    </delete>

    <delete verbose="true" quiet="true" includeEmptyDirs="true">
      <fileset dir="${doc}" includes="**/*.tmp" />
      <fileset dir="${doc}/html" />
      <fileset dir="${doc}/latex" />
    </delete>

    <delete verbose="true" quiet="true" file="joshua-fb.xml" />

    <delete verbose="true" quiet="true" includeEmptyDirs="true">
      <fileset dir="tree_visualizer">
        <include name="tree_visualizer.jar" />
      </fileset>
    </delete>
  </target>


  <!-- Delete *all* generated files,
       including files and directories not tracked by git -->
  <target name="devel-clean" description="--> Delete *all* generated files, including files and directories not tracked by git">

    <input
      message="WARNING: all untracked, ignored files will be removed. Continue? (y/n) "
      validargs="y,n"
      addproperty="do.delete"
    />
    <condition property="do.abort">
      <equals arg1="n" arg2="${do.delete}"/>
    </condition>
    <fail if="do.abort">Build aborted by user.</fail>

    <!-- Delete untracked, ignored files and directories. -->
    <exec executable="git">
      <arg value="clean" />
      <arg value="-fdX" />
    </exec>
  </target>

  <!-- ~~~~~ Javadoc ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->

  <!-- Compile the Javadocs into web pages -->
  <target name="javadoc" description="--> Compile the Javadocs into web pages">
    <mkdir dir="${doc}" />
    <javadoc packagenames="joshua.*" classpath="${cli}" sourcepath="${src}" destdir="${doc}" author="true" version="true" charset="utf-8" overview="src/overview.html">
      <link href="http://docs.oracle.com/javase/7/docs/api/" />
      <link href="http://commons.apache.org/cli/api-release" />
    </javadoc>
  </target>

  <!-- ~~~~~ Doxygen ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->

  <taskdef uri="antlib:org.doxygen.tools" resource="org/doxygen/tools/antlib.xml" classpath="${lib}/ant-doxygen-1.6.1.jar" />

  <!-- Compile the Javadocs and Markdown documentation into web pages -->
  <target name="documentation" depends="check-joshua-home" description="--> Compile the Javadocs and Markdown documentation into web pages">
    <doxygen:doxygen configFilename="${doc}/Doxyfile" />
  </target>

  <!-- ~~~~~ Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->

  <target name="test" depends="all" description="--> Run tests">
    <exec dir="test" executable="/bin/bash" failonerror="false">
      <arg line="run-all-tests.sh" />
    </exec>
  </target>

  <!-- ~~~~~ TestNG ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->

  <!-- Define the unit testing task -->
  <taskdef resource="testngtasks" classpath="${testng}" />


  <!-- Compile the unit test code -->
  <!-- FIXME: why is this broken out from the 'test' target? -->
  <target name="compile-tests" depends="java" description="--> Compile the unit test code">
    <javac compiler="javac1.5" srcdir="${test}/joshua/ui" destdir="${build}" classpath="${testng}:${build}" debug="on" encoding="utf8" />
  </target>


  <!-- Run the unit tests -->
  <target name="testng" depends="all,compile-tests" description="--> Run the unit tests">
    <testng classpath="${build}" sourcedir="${test}">
      <jvmarg value="-Dfile.encoding=UTF8" />
      <jvmarg value="-Xms256m" />
      <jvmarg value="-Xmx256m" />
      <xmlfileset dir="${test}" includes="testng.xml" />
    </testng>
  </target>

  <property name="ivy.install.version" value="2.2.0" />
  <condition property="ivy.home" value="${env.IVY_HOME}">
    <isset property="env.IVY_HOME" />
  </condition>
  <property name="ivy.home" value="${JOSHUA}" />
  <property name="ivy.jar.dir" value="${ivy.home}/lib" />
  <property name="ivy.jar.file" value="${ivy.jar.dir}/ivy.jar" />
  <target name="download-ivy" unless="offline">

    <!-- Test for Ant Version.-->
    <fail message="Ivy requires Ant version 1.8.0 or greater. Please upgrade to the latest version.">
      <condition>
        <not>
          <antversion atleast="1.8.0"/>
        </not>
      </condition>
    </fail>

    <property name="ivy.default.ivy.user.dir" value="${JOSHUA}/lib" />
    <mkdir dir="${ivy.jar.dir}" />
    <!-- download Ivy from web site so that it can be used even without any special installation -->
    <get src="http://repo2.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/ivy-${ivy.install.version}.jar" dest="${ivy.jar.file}" usetimestamp="true" skipexisting="true" />
  </target>

  <target name="init-ivy" depends="download-ivy" description="--> Try to load and initialize Ivy here from Ivy Home">
    <!-- try to load ivy here from ivy home, in case the user has not already dropped
            it into ant's lib dir (note that the latter copy will always take precedence).
            We will not fail as long as local lib dir exists (it may be empty) and
            ivy is in at least one of ant's lib dir or the local lib dir. -->
    <path id="ivy.lib.path">
      <fileset dir="${ivy.jar.dir}" includes="*.jar" />
    </path>
    <taskdef resource="org/apache/ivy/ant/antlib.xml" uri="antlib:org.apache.ivy.ant" classpathref="ivy.lib.path" />
  </target>

  <!-- ================================= 
          target: resolve              
       ================================= -->
  <target name="resolve" description="--> retrieve dependencies with ivy" depends="init-ivy">
    <property name="ivy.default.ivy.user.dir" value="${JOSHUA}/lib" />
    <ivy:configure file="${JOSHUA}/lib/ivysettings.xml" />
    <ivy:resolve file="${JOSHUA}/lib/ivy.xml" />
    <ivy:retrieve type="jar" />
  </target>

</project>