added script to facilitate easy testing of language packs
It just takes a language pack directory and a test set, runs the language pack on the test set,
and writes the results to a subdirectory LP/evaluation/TEST/*, where LP is the language pack and
TEST is the basename of the test set.
diff --git a/scripts/language-pack/test_lp.sh b/scripts/language-pack/test_lp.sh
new file mode 100755
index 0000000..962ef1f
--- /dev/null
+++ b/scripts/language-pack/test_lp.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Strings together the preprocessing scripts
+
+# This script takes
+# - a source and target language
+# - a language pack directory (which should contain scripts 'prepare.sh' and 'joshua')
+# - a prefix point to a test set (e.g., a line from ~mpost/hybrid16/parallel/$lang/test.txt)
+#
+# It then decodes the test set with the specified model. It assumes the
+# model pointed to in the tuned config is the unfiltered, fully packed
+# model, which is nice, because you don't have to filter and re-pack.
+#
+# e.g.,
+# qsub test.sh es en apache-joshua-es-en ~mpost/hybrid16/parallel/ar/test/globalvoices-test
+#
+# The results will be written to a directory evaluation/TESTSET within the language pack.
+# The output file will be stored in a file named "output" and the BLEU score in a file
+# named "bleu"
+
+# QSUB PARAMETERS
+#$ -S /bin/bash -V
+#$ -cwd
+#$ -j y -o logs
+#$ -l mem_free=32G,h_rt=4:00:00,num_proc=8
+#$ -m aes
+
+. ~/.bashrc
+
+: ${TMP=/scratch}
+
+# Adjust this to your own model root! Assumes $lang/$rundir
+sourcelang=$1
+targetlang=$2
+lpdir=$3
+testset=$4
+
+if [[ -z $4 ]]; then
+ echo "Usage: test-lp SOURCE TARGET LANGUAGE_PACK TEST_PREFIX"
+ echo "where"
+ echo " SOURCE is the source language extension (e.g., es)"
+ echo " TARGET is the target language extension (e.g., en)"
+ echo " LANGUAGE_PACK points to a language pack directory"
+ echo " TEST_PREFIX is the path prefix to a test set"
+ exit
+fi
+
+set -u
+
+# Ensure test set exists
+if [[ ! -s $testset.$sourcelang ]]; then
+ echo "* FATAL: can't find test set '$testset.$sourcelang'"
+ exit
+fi
+
+test_name=$(basename $testset)
+
+rundir=$lpdir/evaluation/$test_name
+[[ ! -d $rundir ]] && mkdir -p $rundir
+
+# Make sure there's a config file
+binary=$lpdir/joshua
+config=$lpdir/joshua.config
+prepare=$lpdir/prepare.sh
+for file in $binary $config $prepare; do
+ if [[ ! -s "$file" ]]; then
+ echo "* FATAL: '$lpdir' doesn't look like a language pack (can't find $file)."
+ exit
+ fi
+done
+
+# Decode
+echo "Decoding $test_name with $lpdir..."
+for file in $testset.{$sourcelang,$targetlang}; do
+ [[ ! -e $rundir/$(basename $file) ]] && cp $file $rundir/$(basename $file)
+done
+cat $rundir/$test_name.$sourcelang | $lpdir/prepare.sh | $lpdir/joshua > $rundir/out 2> $rundir/log
+
+tokenize() {
+ cat $1 | lang=$2 $prepare | $lpdir/scripts/lowercase.pl > $3
+}
+
+if [[ -x $JOSHUA/bin/bleu ]]; then
+ echo -n "Scoring with BLEU..."
+ tokenize $rundir/out $targetlang $rundir/out.tok
+ tokenize $rundir/$test_name.$targetlang $targetlang $rundir/ref.tok
+ $JOSHUA/bin/bleu $rundir/out.tok $rundir/ref.tok > $rundir/bleu
+ cat $rundir/bleu | grep "BLEU =" | awk '{print $NF}'
+fi
+
