src/joshua/discriminative/training/risk_annealer/nbest/NbestMinRiskDAMert.java - joshua - Git at Google

 package joshua.discriminative.training.risk_annealer.nbest;

 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Logger;

 import joshua.discriminative.FileUtilityOld;
 import joshua.discriminative.training.NbestMerger;
 import joshua.discriminative.training.risk_annealer.AbstractMinRiskMERT;
 import joshua.discriminative.training.risk_annealer.DeterministicAnnealer;
 import joshua.discriminative.training.risk_annealer.GradientComputer;
 import joshua.discriminative.training.risk_annealer.hypergraph.MRConfig;
 import joshua.util.FileUtility;


 /**
 * @author Zhifei Li, <zhifei.work@gmail.com>
 * @version $LastChangedDate: 2008-10-20 00:12:30 -0400  $
 */
 public abstract class NbestMinRiskDAMert extends AbstractMinRiskMERT {
 	int totalNumHyp = 0;

 	String nbestPrefix;
 	boolean useShortestRef;

 	private static Logger logger = Logger.getLogger(NbestMinRiskDAMert.class.getSimpleName());


 	public NbestMinRiskDAMert(boolean useShortestRef, String decoderConfigFile, int numSentInTrainSet, String[] refFiles, String nbestPrefix) {
 		super(decoderConfigFile, numSentInTrainSet, refFiles);
 		this.nbestPrefix = nbestPrefix;
 		this.useShortestRef = useShortestRef;

 		initialize();
 	}

 	public abstract void writeConfigFile(double[] weights, String configFileTemplate, String configOutFile);

 	public void mainLoop(){
         for(int iter=1; iter<=MRConfig.maxNumIter; iter++){
         	//#re-normalize weights
         	normalizeWeightsByFirstFeature(lastWeightVector,0);

         	//############decoding
         	writeConfigFile(lastWeightVector, configFile, configFile+"." + iter);
         	String curNbestFile = nbestPrefix +"." + iter;
         	decodingTestSet(lastWeightVector, curNbestFile); //call decoder to produce an nbest using the new weight vector

         	//##############merge nbest and check convergency
         	String oldNbestMergedFile = nbestPrefix +".merged." + (iter-1);
         	String newNbestMergedFile = nbestPrefix +".merged." + (iter);
         	if(iter ==1){
         		copyNbest(curNbestFile, newNbestMergedFile);
         	}else{
         		boolean haveNewHyp = true;
         		if(true){
         			int newTotalNumHyp = NbestMerger.mergeNbest(oldNbestMergedFile, curNbestFile, newNbestMergedFile);
         			if(newTotalNumHyp!=totalNumHyp)
         				haveNewHyp = true;
         			totalNumHyp = newTotalNumHyp;
         		}else{
         			haveNewHyp = mergeNbest(oldNbestMergedFile, curNbestFile, newNbestMergedFile);
         		}

 	            if(haveNewHyp==false) {
 	            	System.out.println("No new hypotheses generated at iteration " + iter);
 	            	break;
 	            }
             }

         	//String f_nbest_merged_new = "C:/Users/zli/Documents/minriskannealer.nbest.merged.17";//????????????
         	//String f_nbest_merged_new = "C:/Users/zli/Documents/minriskannealer.nbest.merged.1";//????????????
         	GradientComputer gradientComputer = new NbestRiskGradientComputer(newNbestMergedFile, referenceFiles, useShortestRef, numTrainingSentence,
         			numPara, MRConfig.gainFactor, 1.0, 0.0, true, MRConfig.linearCorpusGainThetas);
         	annealer = new DeterministicAnnealer( numPara,  lastWeightVector, MRConfig.isMinimizer, gradientComputer,
         			MRConfig.useL2Regula, MRConfig.varianceForL2, MRConfig.useModelDivergenceRegula, MRConfig.lambda, MRConfig.printFirstN);

         	if(MRConfig.annealingMode==0)//do not anneal
         		lastWeightVector = annealer.runWithoutAnnealing(MRConfig.isScalingFactorTunable, MRConfig.startScaleAtNoAnnealing, MRConfig.temperatureAtNoAnnealing);
         	else if(MRConfig.annealingMode==1)
         		lastWeightVector = annealer.runQuenching(1.0);
         	else if(MRConfig.annealingMode==2)
         		lastWeightVector = annealer.runDAAndQuenching();
         	else{
         		logger.severe("unsorported anneal mode, " + MRConfig.annealingMode);
         		System.exit(0);
         	}
         	//last_weight_vector will be used intial weights in the next iteration
         }

         //final output
         normalizeWeightsByFirstFeature(lastWeightVector,0);
         writeConfigFile(lastWeightVector, configFile, configFile+".final");
         System.out.println("#### Final weights are: ");
         annealer.getLBFGSRunner().printStatistics(-1, -1, null, lastWeightVector);
 	}


 	//return false: if the nbest does not add any new hyp
 	//TODO: decide converged if the number of new hyp generate is very small
 	//TODO: terminate decoding when the weights does not change much; this one makes more sense, as if the weights do not change much; then new hypotheses will be rare
 	public static boolean mergeNbest(String oldMergedNbestFile, String newNbestFile, String newMergedNbestFile){
 		boolean haveNewHyp =false;
 		BufferedReader oldMergedNbestReader = FileUtilityOld.getReadFileStream(oldMergedNbestFile);
 		BufferedReader newNbestReader = FileUtilityOld.getReadFileStream(newNbestFile);
 		BufferedWriter newMergedNbestReader =	FileUtilityOld.getWriteFileStream(newMergedNbestFile);

 		int oldSentID=-1;
 		String line;
 		String previousLineInNewNbest = FileUtilityOld.readLineLzf(newNbestReader);;
 		HashMap<String, String> oldNbests = new HashMap<String, String>();//key: hyp itself, value: remaining fds exlcuding sent_id
 		while((line=FileUtilityOld.readLineLzf(oldMergedNbestReader))!=null){
 			String[] fds = line.split("\\s+\\|{3}\\s+");
 			int newSentID = new Integer(fds[0]);
 			if(oldSentID!=-1 && oldSentID!=newSentID){
 				boolean[] t_have_new_hyp = new boolean[1];
 				previousLineInNewNbest = processNbest(newNbestReader, newMergedNbestReader, oldSentID, oldNbests, previousLineInNewNbest, t_have_new_hyp);
 				if(t_have_new_hyp[0]==true)
 					haveNewHyp = true;
 			}
 			oldSentID = newSentID;
 			oldNbests.put(fds[1], fds[2]);//last field is not needed
 		}
 		//last nbest
 		boolean[] t_have_new_hyp = new boolean[1];
 		previousLineInNewNbest= processNbest(newNbestReader, newMergedNbestReader, oldSentID, oldNbests, previousLineInNewNbest, t_have_new_hyp);
 		if(previousLineInNewNbest!=null){
 			System.out.println("last line is not null, must be wrong");
 			System.exit(0);
 		}
 		if(t_have_new_hyp[0]==true)
 			haveNewHyp = true;

 		FileUtilityOld.closeReadFile(oldMergedNbestReader);
 		FileUtilityOld.closeReadFile(newNbestReader);
 		FileUtilityOld.closeWriteFile(newMergedNbestReader);
 		return haveNewHyp;
 	}

 	private static String processNbest(BufferedReader newNbestReader, BufferedWriter newMergedNbestReader, int oldSentID, HashMap<String, String> oldNbests,
 			String previousLine, boolean[] have_new_hyp){
 		have_new_hyp[0] = false;
 		String previousLineInNewNbest = previousLine;
 //		#### read new nbest and merge into nbests
 		while(true){
 			String[] t_fds = previousLineInNewNbest.split("\\s+\\|{3}\\s+");
 			int t_new_id = new Integer(t_fds[0]);
 			if( t_new_id == oldSentID){
 				if(oldNbests.containsKey(t_fds[1])==false){//new hyp
 					have_new_hyp[0] = true;
 					oldNbests.put(t_fds[1], t_fds[2]);//merge into nbests
 				}
 			}else{
 				break;
 			}
 			previousLineInNewNbest = FileUtilityOld.readLineLzf(newNbestReader);
 			if(previousLineInNewNbest==null)
 				break;
 		}
 		//#### print the nbest: order is not important; and the last field is ignored
 		for (Map.Entry<String, String> entry : oldNbests.entrySet()){
 		    FileUtilityOld.writeLzf(newMergedNbestReader, oldSentID + " ||| " + entry.getKey() + " ||| " + entry.getValue() + "\n");
 		}
 		oldNbests.clear();
 		return previousLineInNewNbest;
 	}

 	//return false: if the nbest does not add any new hyp
 	public static void copyNbest(String newNbestFile, String newMergedNbestFile){
 		/*
 		BufferedReader newNbestReader = FileUtilityOld.getReadFileStream(newNbestFile);
 		BufferedWriter newMergedNbestReader =	FileUtilityOld.getWriteFileStream(newMergedNbestFile);

 		String line;
 		while((line=FileUtilityOld.readLineLzf(newNbestReader))!=null){
 			String[] fds = line.split("\\s+\\|{3}\\s+");
 		    FileUtilityOld.writeLzf(newMergedNbestReader, fds[0] + " ||| " + fds[1] + " ||| " + fds[2] + "\n");
 		}
 		FileUtilityOld.closeReadFile(newNbestReader);
 		FileUtilityOld.closeWriteFile(newMergedNbestReader);*/
 		try {
 			FileUtility.copyFile(newNbestFile, newMergedNbestFile);
 		} catch (IOException e) {
 			// TODO Auto-generated catch block
 			e.printStackTrace();
 		}
 	}


 //	set lastWeightVector and google linear corpus
 	private void initialize() {
 		//===== read configurations
 		MRConfig.readConfigFile(this.configFile);


 		logger.info("intilize features and weights");
 		//== get the weights
 		List<Double> weights = readBaselineFeatureWeights(this.configFile);

 		/**initialize the weights*/
 		int numPara=weights.size();
 		this.lastWeightVector = new double[numPara];
 		for(int i=0; i<lastWeightVector.length; i++){
 			lastWeightVector[i] = weights.get(i);
 			logger.info("weight: " + lastWeightVector[i]);
 		}
 		this.numPara = this.lastWeightVector.length;
 	}

 }
	package joshua.discriminative.training.risk_annealer.nbest;

	import java.io.BufferedReader;
	import java.io.BufferedWriter;
	import java.io.IOException;
	import java.util.HashMap;
	import java.util.List;
	import java.util.Map;
	import java.util.logging.Logger;

	import joshua.discriminative.FileUtilityOld;
	import joshua.discriminative.training.NbestMerger;
	import joshua.discriminative.training.risk_annealer.AbstractMinRiskMERT;
	import joshua.discriminative.training.risk_annealer.DeterministicAnnealer;
	import joshua.discriminative.training.risk_annealer.GradientComputer;
	import joshua.discriminative.training.risk_annealer.hypergraph.MRConfig;
	import joshua.util.FileUtility;


	/**
	* @author Zhifei Li, <zhifei.work@gmail.com>
	* @version $LastChangedDate: 2008-10-20 00:12:30 -0400 $
	*/
	public abstract class NbestMinRiskDAMert extends AbstractMinRiskMERT {
	int totalNumHyp = 0;

	String nbestPrefix;
	boolean useShortestRef;

	private static Logger logger = Logger.getLogger(NbestMinRiskDAMert.class.getSimpleName());


	public NbestMinRiskDAMert(boolean useShortestRef, String decoderConfigFile, int numSentInTrainSet, String[] refFiles, String nbestPrefix) {
	super(decoderConfigFile, numSentInTrainSet, refFiles);
	this.nbestPrefix = nbestPrefix;
	this.useShortestRef = useShortestRef;

	initialize();
	}

	public abstract void writeConfigFile(double[] weights, String configFileTemplate, String configOutFile);

	public void mainLoop(){
	for(int iter=1; iter<=MRConfig.maxNumIter; iter++){
	//#re-normalize weights
	normalizeWeightsByFirstFeature(lastWeightVector,0);

	//############decoding
	writeConfigFile(lastWeightVector, configFile, configFile+"." + iter);
	String curNbestFile = nbestPrefix +"." + iter;
	decodingTestSet(lastWeightVector, curNbestFile); //call decoder to produce an nbest using the new weight vector

	//##############merge nbest and check convergency
	String oldNbestMergedFile = nbestPrefix +".merged." + (iter-1);
	String newNbestMergedFile = nbestPrefix +".merged." + (iter);
	if(iter ==1){
	copyNbest(curNbestFile, newNbestMergedFile);
	}else{
	boolean haveNewHyp = true;
	if(true){
	int newTotalNumHyp = NbestMerger.mergeNbest(oldNbestMergedFile, curNbestFile, newNbestMergedFile);
	if(newTotalNumHyp!=totalNumHyp)
	haveNewHyp = true;
	totalNumHyp = newTotalNumHyp;
	}else{
	haveNewHyp = mergeNbest(oldNbestMergedFile, curNbestFile, newNbestMergedFile);
	}

	if(haveNewHyp==false) {
	System.out.println("No new hypotheses generated at iteration " + iter);
	break;
	}
	}

	//String f_nbest_merged_new = "C:/Users/zli/Documents/minriskannealer.nbest.merged.17";//????????????
	//String f_nbest_merged_new = "C:/Users/zli/Documents/minriskannealer.nbest.merged.1";//????????????
	GradientComputer gradientComputer = new NbestRiskGradientComputer(newNbestMergedFile, referenceFiles, useShortestRef, numTrainingSentence,
	numPara, MRConfig.gainFactor, 1.0, 0.0, true, MRConfig.linearCorpusGainThetas);
	annealer = new DeterministicAnnealer( numPara, lastWeightVector, MRConfig.isMinimizer, gradientComputer,
	MRConfig.useL2Regula, MRConfig.varianceForL2, MRConfig.useModelDivergenceRegula, MRConfig.lambda, MRConfig.printFirstN);

	if(MRConfig.annealingMode==0)//do not anneal
	lastWeightVector = annealer.runWithoutAnnealing(MRConfig.isScalingFactorTunable, MRConfig.startScaleAtNoAnnealing, MRConfig.temperatureAtNoAnnealing);
	else if(MRConfig.annealingMode==1)
	lastWeightVector = annealer.runQuenching(1.0);
	else if(MRConfig.annealingMode==2)
	lastWeightVector = annealer.runDAAndQuenching();
	else{
	logger.severe("unsorported anneal mode, " + MRConfig.annealingMode);
	System.exit(0);
	}
	//last_weight_vector will be used intial weights in the next iteration
	}

	//final output
	normalizeWeightsByFirstFeature(lastWeightVector,0);
	writeConfigFile(lastWeightVector, configFile, configFile+".final");
	System.out.println("#### Final weights are: ");
	annealer.getLBFGSRunner().printStatistics(-1, -1, null, lastWeightVector);
	}


	//return false: if the nbest does not add any new hyp
	//TODO: decide converged if the number of new hyp generate is very small
	//TODO: terminate decoding when the weights does not change much; this one makes more sense, as if the weights do not change much; then new hypotheses will be rare
	public static boolean mergeNbest(String oldMergedNbestFile, String newNbestFile, String newMergedNbestFile){
	boolean haveNewHyp =false;
	BufferedReader oldMergedNbestReader = FileUtilityOld.getReadFileStream(oldMergedNbestFile);
	BufferedReader newNbestReader = FileUtilityOld.getReadFileStream(newNbestFile);
	BufferedWriter newMergedNbestReader = FileUtilityOld.getWriteFileStream(newMergedNbestFile);

	int oldSentID=-1;
	String line;
	String previousLineInNewNbest = FileUtilityOld.readLineLzf(newNbestReader);;
	HashMap<String, String> oldNbests = new HashMap<String, String>();//key: hyp itself, value: remaining fds exlcuding sent_id
	while((line=FileUtilityOld.readLineLzf(oldMergedNbestReader))!=null){
	String[] fds = line.split("\\s+\\\|{3}\\s+");
	int newSentID = new Integer(fds[0]);
	if(oldSentID!=-1 && oldSentID!=newSentID){
	boolean[] t_have_new_hyp = new boolean[1];
	previousLineInNewNbest = processNbest(newNbestReader, newMergedNbestReader, oldSentID, oldNbests, previousLineInNewNbest, t_have_new_hyp);
	if(t_have_new_hyp[0]==true)
	haveNewHyp = true;
	}
	oldSentID = newSentID;
	oldNbests.put(fds[1], fds[2]);//last field is not needed
	}
	//last nbest
	boolean[] t_have_new_hyp = new boolean[1];
	previousLineInNewNbest= processNbest(newNbestReader, newMergedNbestReader, oldSentID, oldNbests, previousLineInNewNbest, t_have_new_hyp);
	if(previousLineInNewNbest!=null){
	System.out.println("last line is not null, must be wrong");
	System.exit(0);
	}
	if(t_have_new_hyp[0]==true)
	haveNewHyp = true;

	FileUtilityOld.closeReadFile(oldMergedNbestReader);
	FileUtilityOld.closeReadFile(newNbestReader);
	FileUtilityOld.closeWriteFile(newMergedNbestReader);
	return haveNewHyp;
	}

	private static String processNbest(BufferedReader newNbestReader, BufferedWriter newMergedNbestReader, int oldSentID, HashMap<String, String> oldNbests,
	String previousLine, boolean[] have_new_hyp){
	have_new_hyp[0] = false;
	String previousLineInNewNbest = previousLine;
	// #### read new nbest and merge into nbests
	while(true){
	String[] t_fds = previousLineInNewNbest.split("\\s+\\\|{3}\\s+");
	int t_new_id = new Integer(t_fds[0]);
	if( t_new_id == oldSentID){
	if(oldNbests.containsKey(t_fds[1])==false){//new hyp
	have_new_hyp[0] = true;
	oldNbests.put(t_fds[1], t_fds[2]);//merge into nbests
	}
	}else{
	break;
	}
	previousLineInNewNbest = FileUtilityOld.readLineLzf(newNbestReader);
	if(previousLineInNewNbest==null)
	break;
	}
	//#### print the nbest: order is not important; and the last field is ignored
	for (Map.Entry<String, String> entry : oldNbests.entrySet()){
	FileUtilityOld.writeLzf(newMergedNbestReader, oldSentID + " \|\|\| " + entry.getKey() + " \|\|\| " + entry.getValue() + "\n");
	}
	oldNbests.clear();
	return previousLineInNewNbest;
	}

	//return false: if the nbest does not add any new hyp
	public static void copyNbest(String newNbestFile, String newMergedNbestFile){
	/*
	BufferedReader newNbestReader = FileUtilityOld.getReadFileStream(newNbestFile);
	BufferedWriter newMergedNbestReader = FileUtilityOld.getWriteFileStream(newMergedNbestFile);

	String line;
	while((line=FileUtilityOld.readLineLzf(newNbestReader))!=null){
	String[] fds = line.split("\\s+\\\|{3}\\s+");
	FileUtilityOld.writeLzf(newMergedNbestReader, fds[0] + " \|\|\| " + fds[1] + " \|\|\| " + fds[2] + "\n");
	}
	FileUtilityOld.closeReadFile(newNbestReader);
	FileUtilityOld.closeWriteFile(newMergedNbestReader);*/
	try {
	FileUtility.copyFile(newNbestFile, newMergedNbestFile);
	} catch (IOException e) {
	// TODO Auto-generated catch block
	e.printStackTrace();
	}
	}


	// set lastWeightVector and google linear corpus
	private void initialize() {
	//===== read configurations
	MRConfig.readConfigFile(this.configFile);


	logger.info("intilize features and weights");
	//== get the weights
	List<Double> weights = readBaselineFeatureWeights(this.configFile);

	/*initialize the weights/
	int numPara=weights.size();
	this.lastWeightVector = new double[numPara];
	for(int i=0; i<lastWeightVector.length; i++){
	lastWeightVector[i] = weights.get(i);
	logger.info("weight: " + lastWeightVector[i]);
	}
	this.numPara = this.lastWeightVector.length;
	}

	}