Ñò P±Nc@sWddklZddklZddklZlZlZlZlZl Z l Z l Z l Z ddk lZlZlZddklZlZlZlZddklZddklZddklZdd klZdd klZdd k l!Z!l"Z"l#Z#dd k$l%Z%l&Z&e'Z(d Z)e*a+e ƒZ,da-dd„Z.e/ddd„Z0d„Z1dfd„ƒYZ2e'e/d„Z3dd„Z4d„Z5d„Z6d„Z7d„Z8d„Z9d„Z:d„Z;d „Z<d!„Z=d"„Z>d#„Z?d$„Z@d%„ZAd&„ZBd'„ZCd(„ZDd)„ZEd*„ZFd+„ZGd,„ZHd-„ZIe*d.„ZJdd/„ZKd0„ZLe*e*e*d1„ZMd2„ZNd3„ZOd4„ZPd5„ZQd6„ZReSd7jo“dd8kTZTeTiUd9e*d:eTiVeTiWBƒ\ZXZYeYoeX o d;eYGHneoedoedd<iZƒjoe[edd=ƒnnd8S(>iÿÿÿÿ(tBitParChartParser(tmemoize( tTreetWeightedProductiontWeightedGrammart edit_distancet ViterbiParsertFreqDisttWordNetLemmatizert Nonterminalt ImmutableTree(t f_measuret precisiontrecall(tchaint combinationstproductt permutations(tlog(tmul(t defaultdict(tsample(targv(theappushtheappoptnlargest(tstdtmeani ittopcs¯d„}d„}gg}}ddt|gƒf}\‰}} |iˆ|| fƒx3t|ƒD]%\} } x||||ƒoït|ƒ}x܇fd†|DƒD]Ä} ˆ| iƒ} t| t| ƒƒ}|djoq¬n| iƒd| jo=|| d}|||ƒ}| ||f}t ||ƒq¬| }|||ƒ}| ||f}t ||ƒq¬WqvWg}||}}qgWt|ƒ\}}}||fS(NcSs|ddjotSdS(Nig-Cëâ6 ¿(tTrue(thtCtN((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytabove_thresholdscSsdS(Ngð¿((tt1tw1((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytLAPsiÿÿÿÿc3s5x.|]'}|iƒˆ djo |VqqWdS(iN(tprob(t.0tx(tq(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys %s ii( Rtappendt enumerateRR%tjointproduction_to_treetNonetrhsR(tgrammartsenttrootR!R$RR RtrtttitwR'tq1R"R#tr1th1R%tfoottree((R(s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt topdownparses:   %   iccsÀ|djotdgƒ}n||jo t‚ng}|iƒD]'}t|ƒdjo||iqHqH~}|ot|dƒ}g} |i|D] }| |q¡~ } gg} } xE| D]=}|iƒ|i ƒjo| i |ƒqË| i |ƒqËWxÀ| D]¸} t |t | ƒƒ}|i ƒ|t|i ƒƒ joqn|i|i| ƒ|i|i | ƒxHt||t|i ƒƒ||ƒD]!\}}|| iƒ|fVq¦WqWxê| D]¼} t |t | ƒƒ}|i ƒ|t|i ƒƒ joqÖn|i|i| ƒ|i|i | ƒxLt||t|i ƒƒ||dƒD]!\}}|| iƒ|fVqmWqÖWn#|i ƒ|jo|dfVndS(NRii(R-Rt StopIterationtsubtreestlentnodeR t _lhs_indextlhsR.R)R+R,tleavestremovetmytopdownparseR%(R/R0R3tdeptht depthlimitt_[1]tatsubstitution_sitestleftmostt_[2]t candidatestrectnonrecR'R"tp((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRD8sL  A( # # !cCs‚x{|iƒD]m}t||tƒoS||i|ijo<t||ƒdjo%|itƒ}||i|ƒ|Sq WdS(Ni(t treepositionst isinstanceRR?R>tcopyRtextend(RHtbtctd((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR+Ws  BtTransformationDOPcBszeZd„Zdd„Zed„Zd„Zd„Zee edd„Z edd„Z ed „Z eed „Z RS( cCs7ttƒ|_ttƒ|_tƒ|_h|_dS(N(RRt grammardictt mygrammardicttfdtmangled(tself((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt__init__`s tleftc CsãdGHx×|D]Ï\}}}}|djo||}}n|iƒ}t|ƒ}t|ƒ}td„|iƒƒd}|i|i||f|ƒ|i|iƒi||f|ƒ|ii|i ƒt |ƒƒq WdS(Nsadding to grammartrightcSst|iƒƒdjS(i(R>R.(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytosi( tfreezettuplet my_flattentfiltert productionsRXtincRYRZRAtfloat( R\tmlststsourcetlefttreet righttreetlinkstcounttflattened_treetindex((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytadd_to_grammarfs     #cCsw|o5td„|iiƒDƒdd„dtƒ|_n8ttd„|iiƒDƒƒdd„dtƒ|_dS(Ncssx|] }|VqWdS(N((R&RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys xs tkeycSst|iƒƒS((R>RP(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`zstreversecss%x|]}t|ƒiƒVqWdS(N(tundecorate_with_idsRa(R&RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys }s cSst|iƒƒS((R>RP(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`s(tsortedRYtkeysRtmygrammarsortedtset(R\twithids((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt sort_grammarus    c Cs&x|D]\}}td„|iƒDƒƒ}xì|iƒD]Þ\}}ti|dƒ}ti|dƒ|jo&t||d||dgƒ}nt||gƒ}t||gƒ} | iƒd} |i| i|dfƒ|i| i|dfƒ|i i| i ƒdƒq<WqWdS(Ncss7x0|])\}}ti|dƒ||ffVqWdS(tvN(twnlt lemmatize(R&R5RO((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys †s Rziigð?((( tdicttposR{R|R ReRXRfRYRZRA( R\tcorpusRHRTtblemtwordR~tlemR_R^R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt extendlex‚s  &cCsrxk|iiƒD]Z\}}d|GHxA|iƒD]3\\}}}d|GHdt|ƒGHd|GHq2WHqWdS(NsSource rule: %s s Tree: %ss Links: %ss Count: %d (RXtitemstrepr(R\RqtvalueR:RlRm((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt print_grammar’s     tSc Csæg}|iiƒD]D\}}|t|iƒ|iƒd|d„|iƒDƒƒƒq~}|ov|oðg}tƒ} xÍ|iiƒD]¼\}}||iƒƒ} tt|ƒƒ} | i ƒ| i | i ƒƒxl| i ƒD]^} t | iƒƒoE|idt| iƒƒditt| iƒƒƒf| fƒqèqèWqŽW|t| ƒfSg} |iiƒD]K\}}| t|iƒ|iƒd||iƒƒ|i|iƒƒqs~ }tt|ƒ|ƒSn|SdS(NR%css%x|]\\}}}|VqWdS(N((R&R:RlRm((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys Ÿs s%s %ss (RXR„RRAR.RwtvaluestforceposR,tchomsky_normal_formtupdateR~ReR>R)tstrR+tmaptlistRZRR (R\tfreqfnR%tbitparR1RGRqR†R/tlexiconRmttmpRHRK((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt get_grammar›s,Z    5acCsÓtƒ}tttd„|iDƒƒƒ}xà|iiƒD]Ï\}}t|iƒƒ}ti |ƒ}|ot |ƒi ƒnx„|i ƒD]v} t | iƒƒo]|oB|idt| iƒƒditt| iƒƒƒf|ƒq|i| |ƒqqWq;W|o|iƒ|fStƒ} x0|iƒD]"\} } | i| iƒ| ƒq<Wtt|ƒg} |iƒD]J\} } | t| iƒ| iƒd| t| i| iƒ| ƒƒƒq~ ƒS(Ncssx|]}|iƒVqWdS(N(RB(R&R:((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys Ás s%s %ss R%(RRtreduceRRYR„tmaxR‰RtconvertRŠR‹ReR>R.RfRRAR+RŽRR RRgtget(R\R‘R1R/R’RqR†RmR“RHRZtkRzRG((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytget_my_grammar¿s2   *   c+#sCd„}d„}t}t}xã|iD]Ø}|ˆ|ƒoÂt}x¹|i|D]¦\}} |i||| f} |iit|iƒ| ƒ} ti |ƒ} t t |ƒƒ} t t |ƒƒ}| p| | | fVt}qRng}x5| D]-}|i t |i ˆ|d|ƒƒƒqöWt t|Œƒ}x¼|D]´}ti |ƒ} xœt|| ƒD]V\\}}}||d}t p|i| |ijo|| ||dR(R:t candidatetidx((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytmatchÛs '  cs¦d}t|iƒƒ}h}x9t|iƒddtƒD]}yàt||tƒo#||||jo|d7}n7t||tƒo"|||iƒjo dhfS||ig}||d D]}||iqÎ~jo|d7}n||i||ijo|d7}ng}t ||d ƒD].\‰}|i||ijo |ˆq;q;~} g} t ||d ƒD].\‰}|i||ijo | ˆq‹q‹~ } x[| D]S‰t ‡fd†| Dƒƒd} |d ˆf||d | f<| i | ƒqÉWWnnXt|ƒdjo|djo dhfSq;W|t|iƒƒ}|t|iƒƒ} d|| || }||fS( NgiRqiiÿÿÿÿgà?c3s#x|]}|ˆ|fVqWdS(N((R&tm(tn(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys þs i( RgtheightRtRPR>RQRRBR?R*tminRC(R:RœtsimRRlRRGRHRKtmatchest_[3]t competingtnearestRTtresult((R s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt similarityås@  0PP   icssx|]}|dVqWdS(iN((R&RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ,s RtstsqtvpRric3s%x|]}|ˆiƒjVqWdS(N(RB(R&RH(R:(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys As RqcSst|dƒS(i(R>(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`Jsiÿÿÿÿsmd vbz vbp vbd vbc3sx|]}|ˆjVqWdS(N((R&RH(tnodes(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys bs c3sx|]}|ˆjVqWdS(N((R&RH(tcovered(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys bs (stopRªssqR¬( R›RvRRYRZR˜R R?RR—Rtfrontier_nodesR)t my_mlt_derivRtziptCHECK_SUBSTITUTIONR•RR>Rtt NUM_PARTIALtanyRBtsortRPRQtsplittmungeR[RaRb(+R\R:t allowpartialRžR©tyieldedtmatchedRœRkRlRmtlhscountttargett lfrontierst frontierstnew_subtree_forestRHt new_subtreestsubtreetfreqRoRR%RGtpartialtscoretnewtreet notcoveredRKtfnodesR¥tnnodest_[4]tindicest_[5]R?toldidxt_[6]t_[7]t_[8]t verbheadstderivR¨((R­R®R:s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR°Ùsœ &  +$ (Z #  %F+<(   M(8 ; (c#s–|iƒd}|ˆijoˆi|iƒ}n†ˆodˆo dG|GHnt|ƒg}td„|DƒƒD]\}}||qv~fdfg}nˆo dG|GHnt‚|iƒˆijoˆi|iƒ} ntd|iƒƒ‚t t ‡‡‡fd†|DƒŒƒ} ˆoGdGHx?| D]3}|pq:ndGx|D] } | GHqYWd GHq:Wnt } xh|D]`\\} }}t || ƒ}|djpt ‚t| ƒ}x| D]}ti| ƒ}xût|ƒD]Â\}}||di|||d ij}t p|o:||d|||d <ˆod G||dGHqšn3ˆo*||dGd G|Gd G|GHdG|GHnP|p|d8}qíqíW|td„|Dƒƒ7}||fVt} qËWq‚Wˆo¥| ot|ƒg}td„|DƒƒD]\}}||q~d} }}t || ƒ}|djpt ‚t| ƒ}x| D]}ti| ƒ}xût|ƒD]Â\}}||di|||d ij}t p|o:||d|||d <ˆod G||dGHqFn3ˆo*||dGd G|Gd G|GHdG|GHnP|p|d8}q™q™W|td„|Dƒƒ7}||fVt} qwWndS(Nis not foundcss-x&|]}t|tƒo |VqqWdS(N(RQR(R&RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys us gð?s %s not in fd c3s<x5|].}t|tƒoˆi|ˆˆƒVqqWdS(N(RQRtget_mlt_deriv_multi(R&RH(tverboseR\t smoothing(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ~s ssubtree foresttit substitutedsdoes not fit withRiRLicssx|]\}}|VqWdS(N((R&RHR%((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys œs css-x&|]}t|tƒo |VqqWdS(N(RQR(R&RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys  s gà?cssx|]\}}|VqWdS(N((R&RHR%((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ´s (ReRXR„R,R*R<RARZt ValueErrorRRR›RtAssertionErrorR¯RR—R?R²tsumR(R\R:RÔRÓttop_productionRLRGR RHR»R¿RTR¹RkRlRmR%R¾RÀR¼RoRžRK((RÓR\RÔs5/home/andreas/ai/sslp/dop-transformations/treelink.pyRÒns” P (    &  K  &  (t__name__t __module__R]RpR›RyRƒR‡RÚRR”RšR°RÒ(((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRW_s    $ •c s|p td„|iƒDƒƒ}n‡fd†‰t}g}g}|itƒ}|itƒ}xô|oìd}d}d} x¨t|ƒD]š\} } } xˆt|ƒD]z\} }}t| tƒo[ˆ| |ƒoKtt | ƒƒ|jo.tt | ƒƒ}| || | | |f}q%q«q«WqWt oþ|djoñxît|ƒD]Ü\} } } t| tƒo½t| ƒdjoªt | dƒt jo“xt|ƒD]~\} }}t|tƒo_t|ƒdjoLt |dƒt jo5| i ddjo|i ddjp | i djoq|i djodti| ddƒti|ddƒjpk| ddjo&|ddjo| d|djp7| ddjot|ddjoc| d|djoN| d|djond G| dGdG|dGH| || | | |f} q"q¤q¤WqNqNWn|oh|\} }} } } }|ot| |ƒ\} }nt| i gƒ| | ½s cst|ƒt|ƒjoýt|tƒo³t|tƒo£|iidƒd|iidƒdjp:|iidƒdd joQ|iidƒdd jo4d|ijp#t‡fd †t||ƒDƒƒStSqt|tƒo&t|tƒo|i ƒ|i ƒjSntSdS( Nt@iÿÿÿÿt!iRªR«RˆtSQc3s(x!|]\}}ˆ||ƒVqWdS(N((R&RHRT(tideq(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys Æs (RªssqRˆRà(RªssqRˆRà( R>RQRR?R¶tallR±R›Rtlower(ttree1ttree2(Rá(s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRá¾s -4:iitvVtmdtMDRztiss'stnotsn'tt lemmatizeds<=>(smdRè(smdRè(siss's(siss's(snotsn't(snotsn't(R}RBRRRR-t my_subtreesRQRR>tleaves_and_frontier_nodestUSE_LEMMATIZATIONttypeRR?R{R|tdecorate_with_idst decorate_pairR)R›R±ReR.R,RS(RäRåtdecoratet alignmentstshared_subtreestlinked_subtreest equivalentstmax_shared_subtree_sizetmax_shared_subtreetlemmatized_equivalentstparent1tnum1R4tparent2tnum2tjtminimal_subtreesRHRT((Rás5/home/andreas/ai/sslp/dop-transformations/treelink.pytminimal_linked_subtrees¸s‚     ( : :( -770    % iècsJd„}dGHg}xȈD]À\}}td„t|ƒƒ}td„t|ƒƒ}g}|D]5} ||i| ƒ||| iidƒdƒfqb~} g} | D]} | | dq«~ } |i||| fƒqWdGHg} xY|D]Q\}}} |G|G| GHtt|ƒƒ}td„t|ƒDƒƒ}g}|iƒD]I}|idƒd |jo)|||||idƒd |fqLqL~}x¡t t |ƒƒD]\}} |o||joPng}|D](}|| jo|t |d ƒqáqá~}|i t ƒ}|i t ƒ}xK| D]C\}}}t||iƒ||_t||iƒ||_q7W| i||| tt‡fd †|Dƒdƒfƒd|ijo{|i t ƒ}|i t ƒ}t|iƒ|_t|iƒ|_| i||| tt‡fd †|Dƒdƒfƒq±q±WqñW| S( NcSsBx/t|ƒD]!\}}||ijo|Sq Wtdƒ‚dS(Nsmyindex(x): x not in list(R*R?RØ(tlR'R RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytmyindexs   s Phase 1...cSst|tƒo d|ijS(RÞ(RQRR?(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`scSst|tƒo d|ijS(RÞ(RQRR?(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR` sRÞis Phase 2...css2x+|]$\}}|idƒd|fVqWdS(RÞiÿÿÿÿN(R¶(R&RHRT((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys 0s iÿÿÿÿic3s"x|]}t|ˆƒVqWdS(N(Rm(R&tleaf(Rõ(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys =s c3s"x|]}t|ˆƒVqWdS(N(Rm(R&R(Rõ(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys Ds (RdRíRoR?R¶R)R}R¯RuR*tsublistsRRRRtrmidR•R(Rõtlimit_subtreesRtlinked_subtrees2R"tt2tl1tl2RGR'RHRKtnewtreesRlt leafindex1t leafindex2R¥Rt indices12R RÉRTRBtnewtree1tnewtree2R2Rt newtree1at newtree2a((Rõs5/home/andreas/ai/sslp/dop-transformations/treelink.pyt&linked_subtrees_to_probabilistic_rulessN  I% c <&2cstiˆƒ‰ˆiƒ}x]|oU|idƒ}tˆ|tƒo.ˆ|iƒgjoˆ|=ˆiƒ}qqWx™|o‘|d}g}ˆiƒD]0}tˆ|tƒoˆ|o ||q q ~}|idtƒg} |D]}| ˆ|iqô~ ‰d}||d i} t|ƒdjo| djo d} n+t|ƒdjo| djo d} n| ˆjoAt|ƒdt|ˆi | ƒƒjo|ˆi | ƒ}nÊt ‡fd†||d Dƒƒo¤g} ||d D]} | | iq~ ‰t ‡‡fd †|Dƒƒ} g}| D]*}t|ƒt|ƒjo ||qEqE~}|o|iƒ}q“n|djoéxæt dt|ƒƒD]Ë}||| i} t|| ƒdjo| djo d} n0t|| ƒdjo| djo d} n| ˆjoBt|ƒ|t|ˆi | ƒƒjo|ˆi | ƒ}Pq¶d }q¶Wn|o'titˆ|||ƒƒˆ|bs c3s5x.|]'}ˆ|iˆjo |d VqqWdS(iÿÿÿÿN(R?(R&R'(RÑtsiblings(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys es ((RR—RPtpopRQRBRµR>R?R-RoR´Rtranget guessorder(RÑR:RÆttpRtncRGRHRÊRKtparentR¥tyR“RÉR RËR'((RR­RÑs5/home/andreas/ai/sslp/dop-transformations/treelink.pyR·GsZ + J(  7%,> % % 7'Acs£d„}|g}|iƒD]\}}||q~g}||iƒD]\}}||qK~ƒ\‰‰|g}|iƒD]\}}||q„~g} ||iƒD]\}}| |q²~ ƒ\‰‰g} x¡tt|ƒdƒD]‰} td„|| DƒŒ} td„|| DƒŒ} | it‡‡‡fd†| Dƒƒt‡‡‡fd†| Dƒƒ| fƒqôW|it| ƒd||ƒ|S(NcSshxUtt|ƒƒD]A}|||t|ƒ!|jo|| ||t|ƒfSqWtdƒ‚dS(Nsmiddle not in list(RR>RØ(RtmiddleR ((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt partitionys !icss3x,|]%}t|tƒo|iƒVqqWdS(N(RQRR~(R&R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ƒs css3x,|]%}t|tƒo|iƒVqqWdS(N(RQRR~(R&R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys „s c3sJxC|]<\}}|ˆjp|ˆjo|ˆjo dVqqWdS(iN((R&RHRT(tleftpR_R^(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys †s c3sJxC|]<\}}|ˆjp|ˆjo|ˆjo dVqqWdS(iN((R&RHRT(trightpR_R^(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ‡s iÿÿÿÿ(R~RR>RR)RÚtinsertR–(RÑR:RÆRRGRHRTRKR¥RÉt positionsR tdlefttdright((R_RRR^s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRxs gg% cCsbg}xU|D]M}t|tƒo*t|ƒdjo|it|ƒƒq |i|ƒq W|S(Ni(RQRR>RSRíR)(R:RBtchild((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRí’s#cCst|it|ƒƒS(N(RR?Rí(R:((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRc›sc sbx[ˆD]S\}}|i|jo7tt‡fd†td„t|ƒDƒƒDƒdƒSqWdS(Nc3s&x|]}t|ˆƒdVqWdS(iN(Rm(R&RU(Rõ(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ¢s cssx|]\}}|VqWdS(N((R&R'R((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys £s ii(R?R•RRwR¯(tour_nodeRõRHRT((Rõs5/home/andreas/ai/sslp/dop-transformations/treelink.pyRmŸs  )cCsŸt|ƒo|idfgSt|ƒtjogSg}xXt|ƒD]J\}}|g}t|ƒD] \}}|||f|fqi~7}qIW|SdS(N((t frontier_nodeR?RïRR*R¯(R:RÇR~tstreeRGtfnodeR2((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR¯§s  BcCs t|tƒot|ƒdjS(Ni(RQRR>(R:((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR&¸sc Cs‰d}x|td|dƒD]g}|tttd|dƒdƒtttd|dƒdƒtttd||dƒdƒ7}qW|S(Nii(RR•R(R R¨R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytcomb»s ec s;tdtt‡fd†ttˆƒddƒDƒdƒƒS(Nc3s"x|]}tˆ|ƒVqWdS(N(R(R&RH(R(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys Ãs iiÿÿÿÿ(((((RR•RR>(R((Rs5/home/andreas/ai/sslp/dop-transformations/treelink.pyRÁscCs|idƒdS(NRÞi(R¶(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRÇscCsž|itƒ}|itƒ}xst|iƒ|iƒƒD]V\}}d|ijo:d|itf|_d|itf|_td7aq:q:W||fS(NRÞs%s@%di(RRRR±R=R?t current_id(RäRåtutree1tutree2RHRT((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRðÊs cCsOti|ƒ}x9|idd„ƒD]"}|i|iidƒ |_q%W|S(NRdcSs d|ijS(RÞ(R?(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`×sRÞ(RR—R=R?Ro(R:RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRsÕs  cCs^|itƒ}|itƒ}d|itf|_d|itf|_td7a||fS(Ns%s@%di(RRRR?R*(RäRåR+R,((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRñÛs  cCs/t|ƒtjott|ƒgƒS|SdS(N(RïR RR(R?((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyttreeifyäscCs@tt|iƒƒg}|iƒD]}|t|ƒq#~ƒS(N(RRRAR.R-(t productionRGR2((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR,êsccs_xXt|ƒD]J\}}|||fVt|tƒo xt|ƒD] }|VqDWq q WdS(N(R*RQRRì(R:R R$RÁ((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRìís  cCs­tdƒ}tdƒ}t||ƒ}x|D]\}}|G|GHq.Wt|ƒ}HHx%|D]}x|D] }|GHqiWHq\Wtƒ}|i|ƒ|iƒ|iƒ}|S(Ns2(S (NP John) (VP (V bought) (NP (DET a) (N car))))s9(S (VBZ did) (NP John) (VP (V buy) (NP (DET a) (N car))))(RRRRWRpR‡R”(RäRåR3RHRTRRUtgr((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyttestôs(         c Csqtdƒ}tdƒ}t||ƒ}dGHx|D]\}}|G|GHq3WdGHt|ƒ}dGHx%|D]}x|D] }|GHqvWHqiWtƒ}|i|ƒ|iƒdGH|iƒtdƒ}tdƒ}|i||fgƒt|i d d ƒƒ} d GH|i |d t ƒ\}} | i ƒGH| G|GHy7|i | i d iƒƒd t ƒ\}} | G|GHWntj o} | GHnXdGHtdƒ}x;tt|i|ƒƒdd„ƒD]\}} | G|GHq½Wt|id d ƒƒ} | i ƒGHyW| i d iƒƒ}x;tt|i|ƒƒdd„ƒD]\}} | G|GHq6WWntj o} | GHnXdS(NsC(TOP (SQ (VBD Did) (NP (PRP I)) (VP (VB buy) (NP (PRP it))) (. ?)))s<(TOP (S (NP (PRP I)) (VP (VBD bought) (NP (PRP it))) (. .)))s minimal linked subtreessend s' linked subtrees to probabilistic ruless DOT grammarsi(TOP Did (NP (NNP Mr.) (NNP Freeman)) (VP (VB have) (NP (NP (VB notice)) (PP (IN of) (NP (DT this))))) ?)sx(TOP (SQ (VBD Did) (NP (NNP Mr.) (NNP Freeman)) (VP (VB have) (NP (NP (VB notice)) (PP (IN of) (NP (DT this))))) (. ?)))R1tTOPt1RÔs%Did Mr. Freeman have notice of this ?t2RqcSs|dS(i((R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`%scSs|dS(i((R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`+s(RRRRWRpRyR‡RƒRR”t get_mlt_derivRR/tparseR¶t ExceptionRtRR°Rš( RäRåR3RHRTRRUttdopR:tparserROte((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyttest2s`             *   %   % cCs|itƒ}xi|idƒD]X}t||d ƒdjo7td||d i||f||gƒ||RR?(R:R¨RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRŠ0s ;cCs[|itƒ}xE|idƒD]4}d||d ijo||||d (R?(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`Bss!<>t(R=R?treplace(R:RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt remcnfmarksAscsS|o dg‰n diƒ‰x,|id‡fd†ƒD]}d|_q<WdS(NRstop s sq sinv smain vp npRdcs&|iƒdjo|iiƒˆjS(i(R¡R?Rã(R'(texclude(s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`GstX(R¶R=R?(R:RâRH((R@s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt foldlabelsDs   cCsì|diiƒdjod|d_nxº|iƒD]¬}t||tƒo’||iƒdjo{||}|iiƒdjo.t|ƒ|jo||d id7_n|iiƒd iƒjo ti |diƒd ƒd jo=t|ƒ|jo*|id7_||d id7_qàti |diƒd ƒd jo=t|ƒ|jo*|id7_||d id7_qàti |diƒd ƒd jo&t|ƒdjo|id7_qàqäq8q8WdS(NiRªtsinvtSmainiRçiÿÿÿÿs-auxsvb vbz vbd vbpRztbethavetdos-do(RªRC( R?RãRPRQRR¡R>R¶R{R|(R:tvbdepthRHt preterminal((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt mark_be_doIs" + )666cCsöd}d}tt|iƒiƒƒ}t|ddd…|ddd…ƒ}dGHx/|D]'\}}dt|ƒt|ƒfGHq\Wtƒ}xX|D]P\}}t||ƒ}t|ƒ}|i |ƒ|i tt||ƒƒƒq—W|i ƒ|i dt ƒ\}} t || dd d d ƒ} |id d ƒ}t|ƒ} d GHx¤t oœdGtƒ}d} d} y<tt| i|iƒƒƒdƒ} | iƒdG| GHWntj o}|GHnXy"| i|iƒƒ} dG| GHWntj o}|GHnXyWxP|i| dtƒD]9\}}dt|ƒ|fGHdGdi|iƒƒGHqWWntj o}|GHnX| dgfjoqNnxVt|i| dt ƒƒD]9\}}dt|ƒ|fGHdGdi|iƒƒGHq°WqNWdS(NsÀ(S (NP John) (VP (V bought) (NP (DT a) (N car)))) (S (VP (VBZ did)) (NP John) (VP (V buy) (NP (DT a) (N car)))) (S (NP Mary) (VP (VBZ is) (ADJP (JJ happy)))) (S (VBZ is) (NP Mary) (ADJP (JJ happy))) (S (NP (NP (DT the) (NN man)) (SBAR (WHNP (WP who)) (S (VP (VBZ is) (VP (VBG talking)))))) (VP (VBZ is) (VP (VBG walking)))) (S (VBZ is) (NP (NP (DT the) (NN man)) (SBAR (WHNP (WP who)) (S (VP (VBZ is) (VP (VBG talking)))))) (VP (VBG walking)))sD(S (NP Mary) (VP (VBZ is) (ADJP (JJ happy)))) (S (VBZ is) (NP Mary) (ADJP (JJ happy))) (S (NP (NP John) (SBAR (WHNP (WP who)) (S (VP (VBZ mumbles))))) (VP (VBZ dreams) (PP (IN about) (NP unicorns)))) (S (VBZ Does) (NP (NP John) (SBAR (WHNP (WP who)) (S (VP (VBZ mumbles))))) (VP (VB dream) (PP (IN about) (NP unicorns))))iiscorpus:s < %s, %s >R‘t rootsymbolRªR ièR1tdones sentence:is viterbi1:s viterbi2:RÔstransformed1 (prob=%s): %sswords:t R¸stransformed2 (prob=%s): %s(RŽRRãt splitlinesR±RRWRRRpRyR”RRRšRt raw_inputR-R<Rt nbest_parseR¶tun_chomsky_normal_formR6R5RÒR›R…R+RBR°(RRHRTR7RäRåRŸRR/R’R8tmyparsert parsetreet myparsetreeR9t transformedR%((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt interfaceZsf)         %         c%s¡ |pk|o|idtƒ\}}n|idtdtƒ\}}t||dtddddƒ} dGHng} g} d } xt|ƒD]ö\‰} |o3ˆGd Gd i| iƒƒGHhd | iƒ6}nÔˆGd G| GHyt | i | i ƒƒƒ}Wn(t j o}g}ˆGd G|GHnXt ƒ}x@|D]8}|iƒ|itit|ƒƒd|iƒƒq>WdGt|ƒGHt|ƒd jo| d 7} nt ƒ}d }x>t|ƒD]0\}}|oxot|i|ƒƒD]X\}\}}|id i|iƒƒd|||ƒ|d 7}|djoPqïqïWtoqÆdGHxut|i|dtƒƒD]X\}\}}|d 7}|id i|iƒƒd|||ƒ|djoPqwqwWqÆnxt|i|dtdtƒƒD]ö\}\}}|oË|o2|it|ƒiƒd|||pdƒn…||jo d jno||G|GdG|G|GHn|it|ƒiƒtd„|iƒDƒƒfdt||ƒ|ƒ|d 7}n|djoPqüqüWqÆW| o@| o8|o1t|iƒd ƒiƒ|jox t|it|iƒd ƒdtdtƒƒD]Ô\}\}}|o©t|ƒiƒ|jo|o2|it|ƒiƒd|||pdƒnJ|it|ƒiƒtd„|iƒDƒƒfdt||ƒ|ƒ|d 7}n|djoPqfqfWndG|GH| o‡|o€|oqÚtd„|Dƒƒ}t|ƒG|Gt ƒ}x0|iƒD]"\\}}}|i||ƒqŸW|}t|ƒGHnd„‰|oG|o@| idi‡‡fd†|iƒDƒƒdƒ| dGHnJ|o| o| id| ƒn&| o| o| id | ƒn| i|ƒq‘Wd!GHd"„‰d#„‰|o_‡fd$†}d%„}t|d&ƒi t!|| ƒƒt|d'd&ƒi t!|| ƒƒdSg}|D]} |ˆ| i"ƒi ƒƒq~}dC\} }g}g} g}!xBtt#| |ƒƒD]+\‰\}"‰ˆ|"ƒoJˆˆˆ|"ƒƒi"ƒi ƒƒˆjo| d 7} qØ|!iˆƒnˆ‡‡fd(†|"iƒDƒjo|d 7}nˆ|"ƒo[|it$‡‡‡fd)†|"Dƒƒƒ| it%ˆˆˆˆ|"ƒƒi ƒƒƒƒqo|itˆƒƒ| itˆƒƒqoWt‡‡fd*†t#| |ƒDƒƒ}#d+|#t|ƒt&|#ƒt|ƒd| t|ƒt&| ƒt|ƒd|t|ƒt&|ƒt|ƒdt't(|ƒt(‡‡‡fd,†| Dƒƒƒt)t(|ƒt(‡‡‡fd-†| Dƒƒƒt*t(|ƒt(‡‡‡fd.†| Dƒƒƒt+|ƒtd/„| Dƒƒt,| ƒt,|ƒt,|!ƒf}$|p|$d0| 7}$n|$d1td2„| Dƒƒ7}$|$GH| i|$ƒt|d&ƒi | ƒh t&|#ƒt|ƒdd36t&| ƒt|ƒdd46t&|ƒt|ƒdd56t't(|ƒt(‡‡‡fd6†| Dƒƒƒpd7d86t)t(|ƒt(‡‡‡fd9†| Dƒƒƒpd7d:6t*t(|ƒt(‡‡‡fd;†| Dƒƒƒpd7d<6t+|ƒd=6td>„| Dƒƒd?6| d@6tdA„| DƒƒdB6}$|$S(DNR‘RtcleanupRKRR ids grammar doneissource:RMisparsing failedRms parsetrees:ièstrying partialR¸RÔRÓg¬÷N’~hs =>css-x&|]}d|ijo dVqqWdS(RÞiN(R?(R&RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys Ïs css-x&|]}d|ijo dVqqWdS(RÞiN(R?(R&RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys Ús stransformations: cssx|]\}}|VqWdS(N((R&R'R((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ås cSsdi|iƒƒS(NRM(R+RB(R:((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytwordsîss c3s8x1|]*\}}dˆt|ƒˆ|ƒfVqWdS(s %d. [p=%s] %sN(R…(R&R:R%(R RX(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ñs iÿÿÿÿsnot transformed: %s snot parsed: %s RLcSsPd„}|o8tg}|D]}|ti||ƒdƒq~ƒS|SdS(NcSs(|djodS|djodS|S(Nsn'tRês'sRé((RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytfùs   Rz(RbR{R|(R0RYRGRH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR‚øs 8cSs)|ot|iƒdd„ƒdSdS(NRqcSs|\}}|S(((R&R™Rz((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`si(R–R„(RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytfmaxscs:|o+ˆ|ƒoˆ|ƒidddƒdSdSdS(NR=s()is (t _pprint_flat(RH(RZ(s5/home/andreas/ai/sslp/dop-transformations/treelink.pytstrtreescSs+|odid„|DƒƒdSdSdS(Ns css3x,|]%}|o|idddƒVqqWdS(R=s()iN(R[(R&R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys s s (R+(RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytstrtreessR5Râc3s1x*|]#}ˆˆ|ƒiƒiƒƒVqWdS(N(RãR¶(R&R'(R‚RX(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys s c3s4x-|]&}tˆˆˆ|ƒiƒƒƒVqWdS(N(RR¶(R&R'(R‚RXR0(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys s c3sCx<|]5\}}|o"ˆˆ|ƒƒ|jo dVqqWdS(iN((R&RHRT(RZRX(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys s sNexact match: %d of %d => %.2f %% match ranked first: %d of %d => %.2f %% match of any rank: %d of %d => %.2f %% f-measure: %.2f precision: %.2f recall: %.2f average edit distance (of best matches): %.2f sentences with edit distance < 1: %s distances of first matches: %s distances of best matches: %s indices of mistakes: %s c3sHxA|]:}ˆ|ƒo'ˆˆˆ|ƒƒiƒiƒƒVqqWdS(N(RãR¶(R&R'(RZR‚RX(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys 0s c3sHxA|]:}ˆ|ƒo'ˆˆˆ|ƒƒiƒiƒƒVqqWdS(N(RãR¶(R&R'(RZR‚RX(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys 1s c3sHxA|]:}ˆ|ƒo'ˆˆˆ|ƒƒiƒiƒƒVqqWdS(N(RãR¶(R&R'(RZR‚RX(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys 2s css*x#|]}|djo dVqqWdS(iN((R&R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys 3s ssentences with no parse: %d s%sentences with no transformation: %d css$x|]}|p dVqqWdS(iN((R&R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys 7s s exact match:smatch ranked first:smatch of any rank:c3sHxA|]:}ˆ|ƒo'ˆˆˆ|ƒƒiƒiƒƒVqqWdS(N(RãR¶(R&R'(RZR‚RX(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys >s gs f-measure:c3sHxA|]:}ˆ|ƒo'ˆˆˆ|ƒƒiƒiƒƒVqqWdS(N(RãR¶(R&R'(RZR‚RX(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ?s s precision:c3sHxA|]:}ˆ|ƒo'ˆˆˆ|ƒƒiƒiƒƒVqqWdS(N(RãR¶(R&R'(RZR‚RX(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys @s srecall:s(average edit distance (of best matches):css*x#|]}|djo dVqqWdS(iN((R&R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys Bs s!sentences with edit distance < 1:ssentences with no parse:css$x|]}|p dVqqWdS(iN((R&R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys Ds s!sentences with no transformation:(ii(-RšRR”RÚRR*R+RBRaRRPR¶R6RRQRfR R—R<R%R>R°R›RÒRsR=RRuR–R„R)topent writelinesRŽRãR±R¢RRgR RwR R RR…(%R7t sentsortreestgoldt resultsfilettreestmyt bootstraptrulesR’R8tresultst resultfdstnoparseRHt parsetreest parsetrees1R9RTtresultfdR3RŸtnnR¨R%tnewfdR:tmmR\R]RGtlgoldtdisttdist1twrongRZtexactcnttstats((RZR0R‚RXR s5/home/andreas/ai/sslp/dop-transformations/treelink.pytrun™s*!     ,  *   * "2 :2 2     3    #3  +& )2( ###..." 999 cCs3ddk}tdƒiƒiƒiƒ}tdƒiƒiƒiƒ}td„tdƒƒ}td„tdƒƒ}tdƒiƒiƒiƒ}td ƒiƒiƒiƒ}td „td ƒƒ}td „td ƒƒ}td„tdƒƒ} td„tdƒƒ} tt||||ƒƒ} g} x4| D],\} }t| dt ƒt|dt ƒq>Wg}xb| D]Z} | i t ƒ}| i ddddddƒ|i ddddddƒ|i |ƒq{Wt }g}|otƒ}xSt| d | d ƒD]9\}\}}|GH|itt||ƒddƒƒqW|idd„ƒd}xq|D]i\} \}}\}}}}||djo7d | |||t|ƒfGH|GH|GH|d7}qkqkW|GH|it||ƒdƒd!GHnt ad"GtGHt||d|dd#ƒd$GHdS(%Niÿÿÿÿscorpus/sentences-interr3.txtscorpus/sentences-decl3.txtcSst|iƒƒS((RRã(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`Lsscorpus/trees-interr3.txtcSst|iƒƒS((RRã(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`Msscorpus/trees-decl3.txtscorpus/sentences-interr1.txtscorpus/sentences-decl1.txtcSst|iƒƒS((RRã(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`Psscorpus/trees-decl1.txtcSst|iƒƒS((RRã(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`Qsscorpus/trees-interr1.txtcSst|iƒƒS((RRã(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`Ssscorpus/trees-decl4.txtcSstt|iƒƒƒS((RsRRã(R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`Tss trees.txtRâtfactorR^t horzMarkovit childCharRßR_iìÿÿÿRièRqcSs|d|dS(ii((R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`‡sis%d. %d / %d = %fs training doneR²s results1.txts testing done(tcPickleR^treadRãRNRŽRR±RBR›RRRR‹R)RWR*RpRRRµRgRƒR²Rv(Rzt sentsintert sentsdeclt treesintert treesdeclt newsentsintert newsentsdeclt newtreesdeclt newtreesinterR ttrees_tdop_parsedRt rightcorpusRHRTt newtreesrightta1ttrainRhR7R RäRåR'tb1tc1RURVR9((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytrunexpGsd    !  cCs˜g}x‹|D]ƒ}|diiƒdjo d}nd}d|jot|ƒn"d|jot|dtƒnd|jot||ƒnd |jo |id d d dd dƒn“d|jo |id dd dd dƒnfd|joX|itƒ}|id dd dd dƒ|id d d dd dƒ|i|ƒnd|jo-t|ƒd|jot|dƒqq q W|S(NiR«iitfoldmosttfoldallRâtmarkauxtrightbranchingRwR_RxRyRßt leftbranchingR^t bothbranchingt stripcnfmarksiÿÿÿÿ( R?RãRBRRJR‹RRR)R?(RctoptionstadditionaltreesRHt verbdepthRT((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt preprocess°s4          c'„Csqtdƒiƒiƒiƒ}tdƒiƒiƒiƒ}td„tdƒƒ}td„tdƒƒ}|tdƒiƒiƒiƒ7}|tdƒiƒiƒiƒ7}|td „td ƒƒ7}|td „td ƒƒ7}d ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~dd€dd‚dƒd„d…d†d‡dˆd‰dŠd‹dŒddŽddg„}d‘d’d“fd”d•fd–d—d˜d™fdšd›ff}tdœ„t|Œƒ}tdiƒƒtdžiƒƒf}h}x–|D]Ž}g}|D]} || i t ƒq~} g} |D]} | | i t ƒq1~ } t | |ƒ} t | |ƒ}| | | |f||i?i@iAiBiCiDiEiFiHiIiJiKiMiPiQiTiWiYiZi[i\i]i^i_i`iaibidieifigihiiijilinioipiqirisitiuiviwixi{i|ii€ii‚iƒi„i…i‡iˆi‰iŠi‹iiŽiii‘i’i“i•i—i˜i™iši›iœiži i¡i¢i£i¥i¦i§i©iªi«i¬i­i®i¯i°i±i²i³i´iµi¶i·RŒRtnofoldRŽtnomarksRRR‘tnobinarizationR’tcnfmarkscSsd|jp d|jS(R™Rš((R'((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`Ùss(foldmost nomarks rightbranching cnfmarkss'foldall markaux rightbranching cnfmarkscss-x&|]}|hgd6gd6fVqWdS(s decl->inters inter->declN((R&tparam((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys ãs RiÐs training donesresults/%s-d%d.txtt-RcRds inter->declsresults/%s-i%d.txts decl->inters (R^R{RãRNRŽRdRRbR¶RRRR–R}RR>RRwt intersectionRÙRWR*RpRRRƒR+RvR›R)t printstats('R|R}R~Rtfilteredt parameterst proccorpusR›RGRHtdeclRKtintertrdecltrinterRgRÊt samplesizetfoldR0R¥Rˆt rtreesdeclt rtreesinterRÉRRËR7R RäRåRÍtfilenameRÎRÏRut_[9]t_[10]t_[11]((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyttenfoldÄs~""ÿ“** 2/I/7  &6_  &6_  cCsùd„}ddk}td„ƒ}xP|idƒD]?}t|iid|ƒƒiƒiƒd}g}tdƒD]"}|djo|||qvqv~}g}|D]}||i d ƒq¬~}x;td ƒD]-}||d i d ƒd d ||d x7|]0}|g}ˆD]}|||q~fVqWdS(N((R&R™RGRV(tdicts(s5/home/andreas/ai/sslp/dop-transformations/treelink.pys s i(R}(R¯((R¯s5/home/andreas/ai/sslp/dop-transformations/treelink.pytcollate siÿÿÿÿcSs ttƒS((RR(((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyR`sRgióÿÿÿi ii i t:iis=>iþÿÿÿRœis10 foldss RMs %s %s %f (%f)i(s results.csvR5t,cssx|]}t|ƒVqWdS(N(R…(R&RH((s5/home/andreas/ai/sslp/dop-transformations/treelink.pys $s R R4s s, (ii i (tosRtlistdirR^tpathR+R{RNRR¶RgtrsplitR)R}R„R>RRtwriteRuR…RŽR‰tclose(R°R³RgRHtlinesRGR'RKR¥R™RzR›t directionRqR†tout((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyRž s@  +<* +3#  7G dcCsÏddk}|itddƒƒ}x¤|iƒD]–\\}}}}dG|GHdG|GHdG|GHdG|GHdGd i|iƒƒGHt|||ƒ}||jo%d G|GHd Gd i|iƒƒGHnHq1WdS( Niÿÿÿÿsmangled.pickletrbRÑR:RÆs former resultsformer leaves:RMscurrent resultscurrent leaves:(RztloadR^R„R+RBR·(RzR[RÑR:RÆR¨t newresult((s5/home/andreas/ai/sslp/dop-transformations/treelink.pyt mungetest+s        t__main__NRÓt optionflagss%d doctests succeeded!s8printstats runexp tenfold interface mungetest test test2s()(\R‘RtnltkRRRRRRRRR R t nltk.metricsR R R t itertoolsRRRRtmathRtoperatorRt collectionsRtrandomRtsysRtheapqRRRtnumpyRRRRîR³R›R²R{R*R;R-RDR+RWRRR·RRíRcRmR¯R&R)RRRðRsRñR-R,RìR0R:RŠR<R?RBRJRVRvR‹R–R®RžR¿RÜtdoctestttestmodtNORMALIZE_WHITESPACEtELLIPSIStfailt attemptedR¶teval(((s5/home/andreas/ai/sslp/dop-transformations/treelink.pytsz@"   ÿZ] 2 1              +     ?® i  H     )