/* * Utils.java * * Created on 7 maart 2006, 13:34 * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package DOPParser; /** * * @author gideon */ import java.util.*; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileReader; import java.io.FileWriter; public class Utils { public String[][] randomSentences; /** Creates a new instance of Utils */ public Utils() { } public static void printoutTerminalsNonTerminalsAndRules(Grammar myGrammar) { String[] myTerminals = myGrammar.GetListofTerminals(); String[] myNonTerminals = myGrammar.GetListofNonTerminals(); System.out.println("The Terminals of the grammar are {" + arrayToSentence(myTerminals, 0) + "} " ); System.out.println(""); System.out.println("The nonTerminals of the grammar are {" + arrayToSentence(myNonTerminals, 0) + "} " ); System.out.println(""); System.out.println("The rules of the grammar are:"); for (int k=0; k < myNonTerminals.length; k++){ //String[][] myRules = myGrammar.GetListofRules(myNonTerminals[k]); ArrayList> myRules = myGrammar.GetListofRulesPlusProbabilities(myNonTerminals[k]); //iterate over rules //for (int j=0; j < myRules.length; j++){ for (ArrayList oneRuleArray : myRules) { String[] oneRule = oneRuleArray.toArray(new String[0]); System.out.println(oneRule[0] + " --> " + arrayToSentence(oneRule, 1) ); } } } /* * generate random sentences from artificial grammar */ public void generateTrainingSentences(Grammar aGrammar, int howMany, int maxDepth, int minLength, int maxLength) { //String[][] randomSentences = new String[howMany][]; ArrayList randomSentencesArray = new ArrayList(); int j = 0; //for (int i=0; i < howMany; i++) { while (j < howMany) { String[] tempSentence = aGrammar.GenerateRandomSentence( maxDepth, minLength, maxLength) ; if (!(tempSentence == null)) { //this could be the case if maxDepth is exceeed //check if it is not duplicate boolean bSentenceExists = false; for (String[] testSentence : randomSentencesArray) { if (Arrays.equals(testSentence, tempSentence)) bSentenceExists = true; } if (! bSentenceExists) { randomSentencesArray.add(tempSentence); if (Main.PRINT_OUTPUT) System.out.println("Sentence " + j + ": " + arrayToSentence(tempSentence, 0) ); j++; } } } //copy to array String[][] randomSentences = randomSentencesArray.toArray(new String[0][]); if (Main.PRINT_OUTPUT) System.out.println(""); } public static void printChartInHTMLFormat(CYKParser myParser, String[] mySentence) { //print the chart in HTML format int k, iRow, jColumn; StringBuffer chartHTMLFmt = new StringBuffer(); chartHTMLFmt.append(""); for (iRow =0; iRow<=mySentence.length; iRow++){ chartHTMLFmt.append(" "); for (jColumn =0; jColumn<=mySentence.length; jColumn++){ chartHTMLFmt.append(" "); } chartHTMLFmt.append(" "); } chartHTMLFmt.append("

"); if (jColumn>iRow){ //list all entries of the chart cell, separated by \linebreak //k = 0; double maxProb = 0; String bestEntry = ""; StringBuffer bestRHS = new StringBuffer(); StringBuffer strChartCellEntries = new StringBuffer(); //HashMap> getSetOfNonTerminalArrays(){ if (Main.DOGOODMANDOP) { for (String key : myParser.myChart[iRow][jColumn].getSetOfNonTerminalArrays().keySet()){ //you want the best of each of the NonTerminalArrays (=TreeMap of NonTInChart) //create an arraylist of entries, only the max probable of each nonterminal if (myParser.myChart[iRow][jColumn].getSetOfNonTerminalArrays().get(key).firstKey().doubleValue()>maxProb) { maxProb = myParser.myChart[iRow][jColumn].getSetOfNonTerminalArrays().get(key).firstKey().doubleValue(); bestEntry = key; bestRHS = new StringBuffer(); for (Constituent myConstituent : myParser.myChart[iRow][jColumn].getSetOfNonTerminalArrays().get(key).get(myParser.myChart[iRow][jColumn].getSetOfNonTerminalArrays().get(key).firstKey()).getRule().getRightHandSide()){ bestRHS.append(" " + myConstituent.getName()); } } //if(k>0) chartHTMLFmt.append(" \\linebreak "); //chartHTMLFmt.append(" " + key + " "); //k++; } if (!bestEntry.equals("")) chartHTMLFmt.append(" " + bestEntry + " → " + bestRHS.toString() + " "); } else { strChartCellEntries = new StringBuffer(); for (String key : myParser.myChart[iRow][jColumn].getSetOfNonTerminalArrays().keySet()){ strChartCellEntries.append(key + " → "); for (Constituent myConstituent : myParser.myChart[iRow][jColumn].getSetOfNonTerminalArrays().get(key).get(myParser.myChart[iRow][jColumn].getSetOfNonTerminalArrays().get(key).firstKey()).getRule().getRightHandSide()){ strChartCellEntries.append(" " + myConstituent.getName()); } strChartCellEntries.append("
"); } chartHTMLFmt.append(" " + strChartCellEntries.toString() + " "); } } chartHTMLFmt.append("

"); System.out.println("The chart is: " + chartHTMLFmt.toString()); /* * ZOMAAR WAT TESTZINNEN *String[] mySentence = {"u_", "wil_", "smorgens_", "naar_", "sinterklaas_", "amersfoort_", "tegen_"}; //String[] mySentence = {"ik_", "hoeft_", "u_", "vanaf_", "over_", "twintig_", "zaterdag_", "zes_", "komen_", "tegen_", "hoek_", "van_", "holland_", "baarn_" }; //String[] mySentence = {"u_", "moet_", "vanuit_", "over_", "den_", "bosch_", "zijn_"}; //String[] mySentence = {"van_", "naar_", "tien_", "uur_"}; //String[] mySentence = {"ja_", "dat_", "klopt_"}; //String[] mySentence = {"van_", "amsterdam_", "naar_", "utrecht_"}; //String[] mySentence = {"vandaag_", "om_", "utrecht_", "tien_", "uur_"}; //String[] mySentence = {"ja_"}; //String[] mySentence = {"van_", "sittard_", "naar_", "heerlen_"}; //String[] mySentence = {"nee_", "bedankt_"}; //String[] mySentence = {"ergens_", "in_", "de_", "avond_"}; //String[] mySentence = {"ik_", "wil_", "van_", "voorschoten_", "naar_", "den_", "haag_", "centraal_", "station_"}; //String[] mySentence = {"ik_", "wil_", "op_", "een_", "ander_", "traject_", "reizen_"}; */ } public void readTargetGrammar() throws Exception{ Target_Grammar myTargetGrammar = new Target_Grammar("d:/grammar_page458.txt"); if (Main.PRINT_OUTPUT) { printoutTerminalsNonTerminalsAndRules(myTargetGrammar); //generate training sentences from CNF grammar generateTrainingSentences(myTargetGrammar, 20, 18, 3, 18); } } public static int computeUniqueBrackets(ArrayList listOfNodes) { //int nrConstituentsOfParse = 0; HashSet listOfUniqueBrackets = new HashSet(); for (Node myNode : listOfNodes) { if (myNode.getType()==Main.NONTERMINAL && (myNode.getRightSpan()-myNode.getLeftSpan()>1)) listOfUniqueBrackets.add(myNode.getLeftSpan() + "-" + myNode.getRightSpan()); } return listOfUniqueBrackets.size(); //nrConstituentsOfParse; } /** Converts array of words to a printable sentence. */ public static String arrayToSentence(String[] sentence, int iStart){ StringBuffer buff = new StringBuffer(); //for(String word : sentence){ for (int i= iStart; i < sentence.length; i++){ //buff.append(word).append(' '); buff.append(sentence[i]).append(' '); } return buff.toString(); } public static ArrayList getListOfWSJFiles() throws Exception { ArrayList listOfFiles = new ArrayList(); File myDir = new File("/scratch/wsj_combined/train"); File mySubDir; for (String myDirName : myDir.list()) { mySubDir = new File("/scratch/wsj_combined/train/" + myDirName); for (String myFile : mySubDir.list()) { //System.out.println(myFile ); listOfFiles.add("/scratch/wsj_combined/train/" + myDirName + "/" + myFile); } } //temp //if (!Main.READ_WSJ_DIRECTORY && Main.WSJ_TRAINING_FILE != null) { // listOfFiles.clear(); // listOfFiles.add(Main.WSJ_TRAINING_FILE); //} return listOfFiles; } public static void doFindTreebankParsesForUnlabeledViterbi(String unlabeledViterbiFile, String unlabeledFromTreebank, String goldStandardTreebank) throws Exception { //lees de unlabeledViterbiFile, unlabeledTestFile en goldStandardTestFile in in ArrayLists //voor elke zin vd unlabeledViterbiFile, zoek de index van zelfde zin in unlabeledTestFile, en schrijf zin met dezelfde index uit goldStandardTestFile naar nwe ArrayList BufferedReader buffUnlabeledViterbi = new BufferedReader(new FileReader(unlabeledViterbiFile)); BufferedReader buffUnlabeledComplete = new BufferedReader(new FileReader(unlabeledFromTreebank)); BufferedReader buffLabeled = new BufferedReader(new FileReader(goldStandardTreebank)); String mySentence; ArrayList unlabeledViterbiSentences = new ArrayList(); ArrayList unlabeledCompleteSentences = new ArrayList (); ArrayList parsedGoldStandardSentences = new ArrayList (); ArrayList parsedViterbiSentences = new ArrayList (); while ((mySentence = buffUnlabeledViterbi.readLine()) !=null){ unlabeledViterbiSentences.add(mySentence); } while ((mySentence = buffUnlabeledComplete.readLine()) !=null){ unlabeledCompleteSentences.add(mySentence); } while ((mySentence = buffLabeled.readLine()) !=null){ parsedGoldStandardSentences.add(mySentence); } int myIndex = 0; for (String unlabeledViterbiSentence : unlabeledViterbiSentences) { myIndex = unlabeledCompleteSentences.indexOf(unlabeledViterbiSentence); parsedViterbiSentences.add(parsedGoldStandardSentences.get(myIndex)); } //write parsed Viterbi file BufferedWriter out_ViterbiParses = new BufferedWriter(new FileWriter("Input_for_evaluation/Viterbi_Treebank_parses.txt")); parseTree viterbiParseTree; for (String labeledViterbiSentence : parsedViterbiSentences) { //Note! you want it in WSJ-format!! //TEMP XXX may be it is already in WSJ-format! if( Main.REWRITE_OVIS_PARSES_IN_WSJ_FORMAT) { //$$$$$$$$$$$$$$$$$ viterbiParseTree = Grammar.ExtractParseFromOVISText(labeledViterbiSentence); viterbiParseTree.calculateNodeDepthAndLabelNodes(); viterbiParseTree.calculateNodeSpans(); StringBuffer unLabeledSentence = new StringBuffer(); for (Node myNode : viterbiParseTree.getNodes()) { if (myNode.getType()==Main.TERMINAL) { //take off the _ at end of word myNode.setName(myNode.getName().substring(0, myNode.getName().length()-1 )); } //if (myNode.getType()==Main.NONTERMINAL) { //take off the _ at end of word if (Main.TAKE_OFF_SBJ && myNode.getName().contains("-")) myNode.setName(myNode.getName().split("-")[0]); //} } //write parseTree in WSJformat out_ViterbiParses.write(viterbiParseTree.printWSJFormat()); } //if( Main.REWRITE_OVIS_PARSES_IN_WSJ_FORMAT) else { if (Main.TAKE_OFF_SBJ) { viterbiParseTree = Grammar.ExtractParseFromWSJText(labeledViterbiSentence, false); viterbiParseTree.calculateNodeDepthAndLabelNodes(); viterbiParseTree.calculateNodeSpans(); StringBuffer unLabeledSentence = new StringBuffer(); for (Node myNode : viterbiParseTree.getNodes()) { if (myNode.getType()==Main.NONTERMINAL) { //take off the _ at end of word if (myNode.getName().contains("-")) myNode.setName(myNode.getName().split("-")[0]); } } //write parseTree in WSJformat out_ViterbiParses.write(viterbiParseTree.printWSJFormat()); } else out_ViterbiParses.write(labeledViterbiSentence); } out_ViterbiParses.newLine(); } out_ViterbiParses.flush(); out_ViterbiParses.close(); //now do UP/UR and write associations file if (!Main.OTHERWAYROUND) { Testing myTest = new Testing("Input_for_evaluation/Viterbi_parses.txt", "Input_for_evaluation/Viterbi_Treebank_parses.txt");} else { Testing myTest = new Testing("Input_for_evaluation/Viterbi_Treebank_parses.txt", "Input_for_evaluation/Viterbi_parses.txt"); } //dit kan evt meteen erachteraan?Testing myTest = new Testing(parseFile1, parseFile2); //dan moet je ook weer output geven waarin alle labels in Viterbi_parsefile vervangen zijn!! + labelAssociations //if (Main.ASSOCIATE_VITERBI_UNLABELED_FILE_WITH_LABELED && Main.DO_LABELED_PRECISION_AND_RECALL) {} } //for testing purposes public void doCYKParse(Grammar myGrammar){ //String[] mySentence = {"ik_", "wil_", "van_", "voorschoten_", "naar_", "den_", "haag_", "centraal_", "station_"}; String[] mySentence = {"NN", "CC", "NN", "VBD"}; //System.out.println("Starting a new parse..."); CYKParser myParser = new CYKParser(myGrammar, mySentence); Utils.printChartInHTMLFormat(myParser, mySentence); //System.out.println("Starting Readout..."); // myParser.readoutViterbiParse(); } }