Generated by Cython 0.26
Yellow lines hint at Python interaction.
Click on a line that starts with a "+" to see the C code that Cython generated for it.
Raw output: _fragments.cpp
+0001: """Fragment extraction with tree kernels.
__pyx_t_3 = PyDict_New(); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (PyDict_SetItem(__pyx_t_3, __pyx_kp_u_completebitsets_line_575, __pyx_kp_u_Generate_bitsets_corresponding_t) < 0) __PYX_ERR(0, 1, __pyx_L1_error) if (PyDict_SetItem(__pyx_t_3, __pyx_kp_u_pygetsent_line_813, __pyx_kp_u_Wrapper_of_getsent_to_make_docte) < 0) __PYX_ERR(0, 1, __pyx_L1_error) if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_3) < 0) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
0002:
0003: Implements:
0004:
0005: - van Cranenburgh. 2014. Extraction of Phrase-Structure Fragments
0006: with a Linear Average Time Tree-Kernel.
0007: http://www.clinjournal.org/sites/default/files/01-Cranenburgh-CLIN2014.pdf
0008: - Sangati et al. 2010. Efficiently extract recurring tree fragments from
0009: large treebanks. http://lrec-conf.org/proceedings/lrec2010/pdf/613_Paper.pdf
0010: - Moschitti. 2006. Making Tree Kernels practical for Natural Language Learning.
0011: http://aclweb.org/anthology/E06-1015
0012: """
0013:
0014: from __future__ import print_function
+0015: import re
__pyx_t_2 = __Pyx_Import(__pyx_n_s_re, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 15, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_re, __pyx_t_2) < 0) __PYX_ERR(0, 15, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0016: import io
__pyx_t_2 = __Pyx_Import(__pyx_n_s_io, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_io, __pyx_t_2) < 0) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0017: import os
__pyx_t_2 = __Pyx_Import(__pyx_n_s_os, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_os, __pyx_t_2) < 0) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0018: import sys
__pyx_t_2 = __Pyx_Import(__pyx_n_s_sys, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 18, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_sys, __pyx_t_2) < 0) __PYX_ERR(0, 18, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0019: from collections import Counter
__pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_Counter); __Pyx_GIVEREF(__pyx_n_s_Counter); PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_Counter); __pyx_t_3 = __Pyx_Import(__pyx_n_s_collections, __pyx_t_2, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_n_s_Counter); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_Counter, __pyx_t_2) < 0) __PYX_ERR(0, 19, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+0020: from functools import partial
__pyx_t_3 = PyList_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_n_s_partial); __Pyx_GIVEREF(__pyx_n_s_partial); PyList_SET_ITEM(__pyx_t_3, 0, __pyx_n_s_partial); __pyx_t_2 = __Pyx_Import(__pyx_n_s_functools, __pyx_t_3, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_partial); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (PyDict_SetItem(__pyx_d, __pyx_n_s_partial, __pyx_t_3) < 0) __PYX_ERR(0, 20, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0021: from itertools import islice
__pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 21, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_islice); __Pyx_GIVEREF(__pyx_n_s_islice); PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_islice); __pyx_t_3 = __Pyx_Import(__pyx_n_s_itertools, __pyx_t_2, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 21, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_n_s_islice); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 21, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_islice, __pyx_t_2) < 0) __PYX_ERR(0, 21, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+0022: from array import array
__pyx_t_3 = PyList_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_n_s_array); __Pyx_GIVEREF(__pyx_n_s_array); PyList_SET_ITEM(__pyx_t_3, 0, __pyx_n_s_array); __pyx_t_2 = __Pyx_Import(__pyx_n_s_array, __pyx_t_3, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0023: from roaringbitmap import RoaringBitmap
__pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 23, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_RoaringBitmap); __Pyx_GIVEREF(__pyx_n_s_RoaringBitmap); PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_RoaringBitmap); __pyx_t_3 = __Pyx_Import(__pyx_n_s_roaringbitmap, __pyx_t_2, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 23, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_n_s_RoaringBitmap); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 23, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_RoaringBitmap, __pyx_t_2) < 0) __PYX_ERR(0, 23, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+0024: from .tree import Tree, escape
__pyx_t_3 = PyList_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_n_s_Tree); __Pyx_GIVEREF(__pyx_n_s_Tree); PyList_SET_ITEM(__pyx_t_3, 0, __pyx_n_s_Tree); __Pyx_INCREF(__pyx_n_s_escape); __Pyx_GIVEREF(__pyx_n_s_escape); PyList_SET_ITEM(__pyx_t_3, 1, __pyx_n_s_escape); __pyx_t_2 = __Pyx_Import(__pyx_n_s_tree, __pyx_t_3, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Tree); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (PyDict_SetItem(__pyx_d, __pyx_n_s_Tree, __pyx_t_3) < 0) __PYX_ERR(0, 24, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_escape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (PyDict_SetItem(__pyx_d, __pyx_n_s_escape, __pyx_t_3) < 0) __PYX_ERR(0, 24, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0025: from .util import openread
__pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_openread); __Pyx_GIVEREF(__pyx_n_s_openread); PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_openread); __pyx_t_3 = __Pyx_Import(__pyx_n_s_util, __pyx_t_2, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_n_s_openread); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_openread, __pyx_t_2) < 0) __PYX_ERR(0, 25, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+0026: from .grammar import lcfrsproductions, printrule
__pyx_t_3 = PyList_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 26, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_n_s_lcfrsproductions); __Pyx_GIVEREF(__pyx_n_s_lcfrsproductions); PyList_SET_ITEM(__pyx_t_3, 0, __pyx_n_s_lcfrsproductions); __Pyx_INCREF(__pyx_n_s_printrule); __Pyx_GIVEREF(__pyx_n_s_printrule); PyList_SET_ITEM(__pyx_t_3, 1, __pyx_n_s_printrule); __pyx_t_2 = __Pyx_Import(__pyx_n_s_grammar, __pyx_t_3, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 26, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_lcfrsproductions); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 26, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (PyDict_SetItem(__pyx_d, __pyx_n_s_lcfrsproductions, __pyx_t_3) < 0) __PYX_ERR(0, 26, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_printrule); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 26, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (PyDict_SetItem(__pyx_d, __pyx_n_s_printrule, __pyx_t_3) < 0) __PYX_ERR(0, 26, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0027: from .treetransforms import binarize, handledisc
__pyx_t_2 = PyList_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 27, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_binarize); __Pyx_GIVEREF(__pyx_n_s_binarize); PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_binarize); __Pyx_INCREF(__pyx_n_s_handledisc); __Pyx_GIVEREF(__pyx_n_s_handledisc); PyList_SET_ITEM(__pyx_t_2, 1, __pyx_n_s_handledisc); __pyx_t_3 = __Pyx_Import(__pyx_n_s_treetransforms, __pyx_t_2, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 27, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_n_s_binarize); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 27, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_binarize, __pyx_t_2) < 0) __PYX_ERR(0, 27, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_n_s_handledisc); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 27, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_handledisc, __pyx_t_2) < 0) __PYX_ERR(0, 27, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
0028:
0029: cimport cython
0030: from libc.stdlib cimport malloc, realloc, free
0031: from libc.string cimport memset, memcpy
0032: from libc.stdint cimport uint8_t, uint32_t, uint64_t, SIZE_MAX
0033: from cpython.array cimport array, clone, extend_buffer, resize
0034: from .containers cimport Node, NodeArray, Ctrees, Vocabulary, Rule, \
0035: yieldranges, termidx
0036: from .bit cimport iteratesetbits, abitcount, subset, setunioninplace
0037:
0038: cdef extern from "macros.h":
0039: int BITNSLOTS(int nb) nogil
0040: void SETBIT(uint64_t a[], int b) nogil
0041: void CLEARBIT(uint64_t a[], int b) nogil
0042: uint64_t TESTBIT(uint64_t a[], int b) nogil
0043:
0044: # a template to create arrays of this type
+0045: cdef array ulongarray = array('L')
__pyx_t_3 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_7cpython_5array_array), __pyx_tuple__24, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 45, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_XGOTREF(((PyObject *)__pyx_v_8discodop_10_fragments_ulongarray)); __Pyx_DECREF_SET(__pyx_v_8discodop_10_fragments_ulongarray, ((arrayobject *)__pyx_t_3)); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; /* … */ __pyx_tuple__24 = PyTuple_Pack(1, __pyx_n_u_L); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(0, 45, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__24); __Pyx_GIVEREF(__pyx_tuple__24);
+0046: cdef array uintarray = array('I')
__pyx_t_3 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_7cpython_5array_array), __pyx_tuple__25, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 46, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_XGOTREF(((PyObject *)__pyx_v_8discodop_10_fragments_uintarray)); __Pyx_DECREF_SET(__pyx_v_8discodop_10_fragments_uintarray, ((arrayobject *)__pyx_t_3)); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; /* … */ __pyx_tuple__25 = PyTuple_Pack(1, __pyx_n_u_I); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(0, 46, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__25); __Pyx_GIVEREF(__pyx_tuple__25);
+0047: cdef array intarray = array('i')
__pyx_t_3 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_7cpython_5array_array), __pyx_tuple__26, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 47, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_XGOTREF(((PyObject *)__pyx_v_8discodop_10_fragments_intarray)); __Pyx_DECREF_SET(__pyx_v_8discodop_10_fragments_intarray, ((arrayobject *)__pyx_t_3)); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; /* … */ __pyx_tuple__26 = PyTuple_Pack(1, __pyx_n_u_i); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(0, 47, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__26); __Pyx_GIVEREF(__pyx_tuple__26);
+0048: cdef array shortarray = array('h')
__pyx_t_3 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_7cpython_5array_array), __pyx_tuple__27, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_XGOTREF(((PyObject *)__pyx_v_8discodop_10_fragments_shortarray)); __Pyx_DECREF_SET(__pyx_v_8discodop_10_fragments_shortarray, ((arrayobject *)__pyx_t_3)); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; /* … */ __pyx_tuple__27 = PyTuple_Pack(1, __pyx_n_u_h); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(0, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__27); __Pyx_GIVEREF(__pyx_tuple__27);
0049:
+0050: FRONTIERORTERMRE = re.compile(r' ([0-9]+)(?:=|:[0-9]+\b)') # all leaves
__pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_re); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 50, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_compile); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 50, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__28, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 50, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (PyDict_SetItem(__pyx_d, __pyx_n_s_FRONTIERORTERMRE, __pyx_t_3) < 0) __PYX_ERR(0, 50, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; /* … */ __pyx_tuple__28 = PyTuple_Pack(1, __pyx_kp_u_0_9_0_9_b); if (unlikely(!__pyx_tuple__28)) __PYX_ERR(0, 50, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__28); __Pyx_GIVEREF(__pyx_tuple__28);
+0051: TERMINDICESRE = re.compile(r'\([^(]+ ([0-9]+)=[^ ()]+\)') # term. indices
__pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_re); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 51, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_compile); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 51, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__29, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 51, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (PyDict_SetItem(__pyx_d, __pyx_n_s_TERMINDICESRE, __pyx_t_3) < 0) __PYX_ERR(0, 51, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; /* … */ __pyx_tuple__29 = PyTuple_Pack(1, __pyx_kp_u_0_9); if (unlikely(!__pyx_tuple__29)) __PYX_ERR(0, 51, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__29); __Pyx_GIVEREF(__pyx_tuple__29);
+0052: FRONTIERRE = re.compile(r' ([0-9]+):([0-9]+)\b') # non-terminal frontiers
__pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_re); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_compile); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__30, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (PyDict_SetItem(__pyx_d, __pyx_n_s_FRONTIERRE, __pyx_t_3) < 0) __PYX_ERR(0, 52, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; /* … */ __pyx_tuple__30 = PyTuple_Pack(1, __pyx_kp_u_0_9_0_9_b_2); if (unlikely(!__pyx_tuple__30)) __PYX_ERR(0, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__30); __Pyx_GIVEREF(__pyx_tuple__30);
+0053: TREEPARSEMSG = [
__pyx_t_3 = PyList_New(8); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 53, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_kp_u_0_nodes_found); __Pyx_GIVEREF(__pyx_kp_u_0_nodes_found); PyList_SET_ITEM(__pyx_t_3, 0, __pyx_kp_u_0_nodes_found); __Pyx_INCREF(__pyx_kp_u_unexpected_end_of_string_after_o); __Pyx_GIVEREF(__pyx_kp_u_unexpected_end_of_string_after_o); PyList_SET_ITEM(__pyx_t_3, 1, __pyx_kp_u_unexpected_end_of_string_after_o); __Pyx_INCREF(__pyx_kp_u_unexpected_end_of_string_after_l); __Pyx_GIVEREF(__pyx_kp_u_unexpected_end_of_string_after_l); PyList_SET_ITEM(__pyx_t_3, 2, __pyx_kp_u_unexpected_end_of_string_after_l); __Pyx_INCREF(__pyx_kp_u_unexpected_end_of_string_after_c); __Pyx_GIVEREF(__pyx_kp_u_unexpected_end_of_string_after_c); PyList_SET_ITEM(__pyx_t_3, 3, __pyx_kp_u_unexpected_end_of_string_after_c); __Pyx_INCREF(__pyx_kp_u_unexpected_data_after_end_of_tre); __Pyx_GIVEREF(__pyx_kp_u_unexpected_data_after_end_of_tre); PyList_SET_ITEM(__pyx_t_3, 4, __pyx_kp_u_unexpected_data_after_end_of_tre); __Pyx_INCREF(__pyx_kp_u_unexpected_data_after_terminal); __Pyx_GIVEREF(__pyx_kp_u_unexpected_data_after_terminal); PyList_SET_ITEM(__pyx_t_3, 5, __pyx_kp_u_unexpected_data_after_terminal); __Pyx_INCREF(__pyx_kp_u_missing_terminal_index); __Pyx_GIVEREF(__pyx_kp_u_missing_terminal_index); PyList_SET_ITEM(__pyx_t_3, 6, __pyx_kp_u_missing_terminal_index); __Pyx_INCREF(__pyx_kp_u_stack_not_empty_after_end_of_str); __Pyx_GIVEREF(__pyx_kp_u_stack_not_empty_after_end_of_str); PyList_SET_ITEM(__pyx_t_3, 7, __pyx_kp_u_stack_not_empty_after_end_of_str); if (PyDict_SetItem(__pyx_d, __pyx_n_s_TREEPARSEMSG, __pyx_t_3) < 0) __PYX_ERR(0, 53, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
0054: '0 nodes found', # 0
0055: 'unexpected end of string after opening paren', # 1
0056: 'unexpected end of string after label', # 2
0057: 'unexpected end of string after closing paren', # 3
0058: 'unexpected data after end of tree', # 4
0059: 'unexpected data after terminal', # 5
0060: 'missing terminal index', # 6
0061: 'stack not empty after end of string', # 7
0062: ]
0063:
0064:
0065: # bitsets representing fragments are uint64_t arrays with the number of
0066: # elements determined by SLOTS. While SLOTS is not a constant nor a global,
0067: # it is set just once for a treebank to fit its largest tree.
0068: # After SLOTS elements, this struct follows:
+0069: cdef packed struct BitsetTail:
#if defined(__SUNPRO_C)
#pragma pack(1)
#elif !defined(__GNUC__)
#pragma pack(push, 1)
#endif
struct __Pyx_PACKED __pyx_t_8discodop_10_fragments_BitsetTail {
uint32_t id;
short root;
};
#if defined(__SUNPRO_C)
#pragma pack()
#elif !defined(__GNUC__)
#pragma pack(pop)
#endif
0070: uint32_t id # tree no. of which this fragment was extracted
0071: short root # index of the root of the fragment in the NodeArray
0072:
0073:
0074: # we wrap bitsets in bytes objects, because these can be (1) used in
0075: # dictionaries and (2) passed between processes.
0076: # we leave one byte for NUL-termination:
0077: # data (SLOTS * 8), id (4), root (2), NUL (1)
+0078: cdef inline bytes wrap(uint64_t *data, short SLOTS):
static CYTHON_INLINE PyObject *__pyx_f_8discodop_10_fragments_wrap(uint64_t *__pyx_v_data, short __pyx_v_SLOTS) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("wrap", 0);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_AddTraceback("discodop._fragments.wrap", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0079: """Wrap bitset in bytes object for handling in Python space."""
+0080: return (<char *>data)[:SLOTS * sizeof(uint64_t) + sizeof(BitsetTail)]
__Pyx_XDECREF(__pyx_r); __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(((char *)__pyx_v_data) + 0, ((__pyx_v_SLOTS * (sizeof(uint64_t))) + (sizeof(struct __pyx_t_8discodop_10_fragments_BitsetTail))) - 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; goto __pyx_L0;
0081:
0082:
0083: # use getters & setters because a cdef class would incur overhead & indirection
0084: # of a python object, and with a struct the root & id fields must be in front
0085: # which seems to lead to bad hashing behavior (?)
+0086: cdef inline uint64_t *getpointer(object wrapper):
static CYTHON_INLINE uint64_t *__pyx_f_8discodop_10_fragments_getpointer(PyObject *__pyx_v_wrapper) {
uint64_t *__pyx_r;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("getpointer", 0);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_WriteUnraisable("discodop._fragments.getpointer", __pyx_clineno, __pyx_lineno, __pyx_filename, 1, 0);
__pyx_r = 0;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0087: """Get pointer to bitset from wrapper."""
+0088: return <uint64_t *><char *><bytes>wrapper
__pyx_t_1 = __Pyx_PyObject_AsWritableString(__pyx_v_wrapper); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) __PYX_ERR(0, 88, __pyx_L1_error) __pyx_r = ((uint64_t *)((char *)__pyx_t_1)); goto __pyx_L0;
0089:
0090:
+0091: cdef inline uint32_t getid(uint64_t *data, short SLOTS):
static CYTHON_INLINE uint32_t __pyx_f_8discodop_10_fragments_getid(uint64_t *__pyx_v_data, short __pyx_v_SLOTS) {
uint32_t __pyx_r;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("getid", 0);
/* … */
/* function exit code */
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0092: """Get id of fragment in a bitset."""
+0093: return (<BitsetTail *>&data[SLOTS]).id
__pyx_r = ((struct __pyx_t_8discodop_10_fragments_BitsetTail *)(&(__pyx_v_data[__pyx_v_SLOTS])))->id; goto __pyx_L0;
0094:
0095:
+0096: cdef inline short getroot(uint64_t *data, short SLOTS):
static CYTHON_INLINE short __pyx_f_8discodop_10_fragments_getroot(uint64_t *__pyx_v_data, short __pyx_v_SLOTS) {
short __pyx_r;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("getroot", 0);
/* … */
/* function exit code */
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0097: """Get root of fragment in a bitset."""
+0098: return (<BitsetTail *>&data[SLOTS]).root
__pyx_r = ((struct __pyx_t_8discodop_10_fragments_BitsetTail *)(&(__pyx_v_data[__pyx_v_SLOTS])))->root; goto __pyx_L0;
0099:
0100:
+0101: cdef inline void setrootid(uint64_t *data, short root, uint32_t id,
static CYTHON_INLINE void __pyx_f_8discodop_10_fragments_setrootid(uint64_t *__pyx_v_data, short __pyx_v_root, uint32_t __pyx_v_id, short __pyx_v_SLOTS) {
struct __pyx_t_8discodop_10_fragments_BitsetTail *__pyx_v_tail;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("setrootid", 0);
/* … */
/* function exit code */
__Pyx_RefNannyFinishContext();
}
0102: short SLOTS):
0103: """Set root and id of fragment in a bitset."""
+0104: cdef BitsetTail *tail = <BitsetTail *>&data[SLOTS]
__pyx_v_tail = ((struct __pyx_t_8discodop_10_fragments_BitsetTail *)(&(__pyx_v_data[__pyx_v_SLOTS])));
+0105: tail.id = id
__pyx_v_tail->id = __pyx_v_id;
+0106: tail.root = root
__pyx_v_tail->root = __pyx_v_root;
0107:
0108:
+0109: cpdef extractfragments(Ctrees trees1, int start1, int end1, Vocabulary vocab,
static PyObject *__pyx_pw_8discodop_10_fragments_1extractfragments(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static PyObject *__pyx_f_8discodop_10_fragments_extractfragments(struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees1, int __pyx_v_start1, int __pyx_v_end1, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, CYTHON_UNUSED int __pyx_skip_dispatch, struct __pyx_opt_args_8discodop_10_fragments_extractfragments *__pyx_optional_args) {
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_9);
__Pyx_XDECREF(__pyx_t_10);
__Pyx_AddTraceback("discodop._fragments.extractfragments", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_asent);
__Pyx_XDECREF(__pyx_v_fragments);
__Pyx_XDECREF(__pyx_v_inter);
__Pyx_XDECREF(__pyx_v_contentwordprods);
__Pyx_XDECREF(__pyx_v_lexicalprods);
__Pyx_XDECREF(__pyx_v_tmp);
__Pyx_XDECREF(__pyx_v_contentword);
__Pyx_XDECREF((PyObject *)__pyx_v_trees2);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_1extractfragments(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static char __pyx_doc_8discodop_10_fragments_extractfragments[] = "extractfragments(Ctrees trees1, int start1, int end1, Vocabulary vocab, Ctrees trees2=None, int start2=0, int end2=0, bool approx=True, bool debug=False, bool disc=False, unicode twoterms=None, bool adjacent=False)\nFind the largest fragments in treebank(s) with the fast tree kernel.\n\n\t- scenario 1: recurring fragments in single treebank, use::\n\t\textractfragments(trees1, start1, end1, vocab)\n\t- scenario 2: common fragments in two treebanks::\n\t\textractfragments(trees1, start1, end1, vocab, trees2)\n\n\t:param start1, end1: specify slice of treebank to be used; can be used to\n\t\tdivide the work over multiple processes; they are indices of ``trees1``\n\t\tto work on (pass 0 for both to use all trees).\n\t:param start2, end2: idem for trees2.\n\t:param approx: return approximate counts instead of bitsets.\n\t:param debug: if True, a table of common productions is printed for each\n\t\tpair of trees\n\t:param disc: if True, return trees with indices as leaves.\n\t:param twoterms: only return fragments with at least two terminals,\n\t\tone of which has a POS tag matching the given regex.\n\t:param adjacent: only extract fragments from sentences with adjacent\n\t\tindices.\n\t:returns: a dictionary; keys are fragments as strings; values are\n\t\teither counts (if approx=True), or bitsets describing fragments of\n\t\t``trees1``.\n\t";
static PyObject *__pyx_pw_8discodop_10_fragments_1extractfragments(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees1 = 0;
int __pyx_v_start1;
int __pyx_v_end1;
struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab = 0;
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2 = 0;
int __pyx_v_start2;
int __pyx_v_end2;
int __pyx_v_approx;
int __pyx_v_debug;
int __pyx_v_disc;
PyObject *__pyx_v_twoterms = 0;
int __pyx_v_adjacent;
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("extractfragments (wrapper)", 0);
{
static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_trees1,&__pyx_n_s_start1,&__pyx_n_s_end1,&__pyx_n_s_vocab,&__pyx_n_s_trees2,&__pyx_n_s_start2,&__pyx_n_s_end2,&__pyx_n_s_approx,&__pyx_n_s_debug,&__pyx_n_s_disc,&__pyx_n_s_twoterms,&__pyx_n_s_adjacent,0};
PyObject* values[12] = {0,0,0,0,0,0,0,0,0,0,0,0};
/* … */
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__pyx_r = NULL;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_8discodop_10_fragments_extractfragments(CYTHON_UNUSED PyObject *__pyx_self, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees1, int __pyx_v_start1, int __pyx_v_end1, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2, int __pyx_v_start2, int __pyx_v_end2, int __pyx_v_approx, int __pyx_v_debug, int __pyx_v_disc, PyObject *__pyx_v_twoterms, int __pyx_v_adjacent) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("extractfragments", 0);
__Pyx_XDECREF(__pyx_r);
__pyx_t_2.__pyx_n = 8;
__pyx_t_2.trees2 = __pyx_v_trees2;
__pyx_t_2.start2 = __pyx_v_start2;
__pyx_t_2.end2 = __pyx_v_end2;
__pyx_t_2.approx = __pyx_v_approx;
__pyx_t_2.debug = __pyx_v_debug;
__pyx_t_2.disc = __pyx_v_disc;
__pyx_t_2.twoterms = __pyx_v_twoterms;
__pyx_t_2.adjacent = __pyx_v_adjacent;
__pyx_t_1 = __pyx_f_8discodop_10_fragments_extractfragments(__pyx_v_trees1, __pyx_v_start1, __pyx_v_end1, __pyx_v_vocab, 0, &__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 109, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_r = __pyx_t_1;
__pyx_t_1 = 0;
goto __pyx_L0;
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_AddTraceback("discodop._fragments.extractfragments", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
struct __pyx_opt_args_8discodop_10_fragments_extractfragments {
int __pyx_n;
struct __pyx_obj_8discodop_10containers_Ctrees *trees2;
int start2;
int end2;
int approx;
int debug;
int disc;
PyObject *twoterms;
int adjacent;
};
+0110: Ctrees trees2=None, int start2=0, int end2=0,
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2 = ((struct __pyx_obj_8discodop_10containers_Ctrees *)Py_None);
int __pyx_v_start2 = ((int)0);
int __pyx_v_end2 = ((int)0);
/* … */
values[4] = (PyObject *)((struct __pyx_obj_8discodop_10containers_Ctrees *)Py_None);
+0111: bint approx=True, bint debug=False,
int __pyx_v_approx = ((int)1);
int __pyx_v_debug = ((int)0);
/* … */
__pyx_v_approx = ((int)1);
}
if (values[8]) {
__pyx_v_debug = __Pyx_PyObject_IsTrue(values[8]); if (unlikely((__pyx_v_debug == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 111, __pyx_L3_error)
} else {
__pyx_v_debug = ((int)0);
}
if (values[9]) {
__pyx_v_disc = __Pyx_PyObject_IsTrue(values[9]); if (unlikely((__pyx_v_disc == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 112, __pyx_L3_error)
} else {
+0112: bint disc=False, str twoterms=None, bint adjacent=False):
int __pyx_v_disc = ((int)0); PyObject *__pyx_v_twoterms = ((PyObject*)Py_None); int __pyx_v_adjacent = ((int)0); int __pyx_v_n; int __pyx_v_m; short __pyx_v_minterms; short __pyx_v_SLOTS; uint64_t *__pyx_v_matrix; uint64_t *__pyx_v_scratch; struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_a; struct __pyx_t_8discodop_10containers_Node *__pyx_v_anodes; PyObject *__pyx_v_asent = 0; PyObject *__pyx_v_fragments = 0; PyObject *__pyx_v_inter = 0; PyObject *__pyx_v_contentwordprods = 0; PyObject *__pyx_v_lexicalprods = 0; PyObject *__pyx_v_tmp = 0; PyObject *__pyx_v_contentword = NULL; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("extractfragments", 0); if (__pyx_optional_args) { if (__pyx_optional_args->__pyx_n > 0) { __pyx_v_trees2 = __pyx_optional_args->trees2; if (__pyx_optional_args->__pyx_n > 1) { __pyx_v_start2 = __pyx_optional_args->start2; if (__pyx_optional_args->__pyx_n > 2) { __pyx_v_end2 = __pyx_optional_args->end2; if (__pyx_optional_args->__pyx_n > 3) { __pyx_v_approx = __pyx_optional_args->approx; if (__pyx_optional_args->__pyx_n > 4) { __pyx_v_debug = __pyx_optional_args->debug; if (__pyx_optional_args->__pyx_n > 5) { __pyx_v_disc = __pyx_optional_args->disc; if (__pyx_optional_args->__pyx_n > 6) { __pyx_v_twoterms = __pyx_optional_args->twoterms; if (__pyx_optional_args->__pyx_n > 7) { __pyx_v_adjacent = __pyx_optional_args->adjacent; } } } } } } } } } __Pyx_INCREF((PyObject *)__pyx_v_trees2); /* … */ values[10] = ((PyObject*)Py_None); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); CYTHON_FALLTHROUGH; case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); CYTHON_FALLTHROUGH; case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); CYTHON_FALLTHROUGH; case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); CYTHON_FALLTHROUGH; case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); CYTHON_FALLTHROUGH; case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); CYTHON_FALLTHROUGH; case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); CYTHON_FALLTHROUGH; case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); CYTHON_FALLTHROUGH; case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_trees1)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_start1)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("extractfragments", 0, 4, 12, 1); __PYX_ERR(0, 109, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_end1)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("extractfragments", 0, 4, 12, 2); __PYX_ERR(0, 109, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 3: if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_vocab)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("extractfragments", 0, 4, 12, 3); __PYX_ERR(0, 109, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 4: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_trees2); if (value) { values[4] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 5: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_start2); if (value) { values[5] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 6: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_end2); if (value) { values[6] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 7: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_approx); if (value) { values[7] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 8: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_debug); if (value) { values[8] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 9: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_disc); if (value) { values[9] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 10: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_twoterms); if (value) { values[10] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 11: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_adjacent); if (value) { values[11] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "extractfragments") < 0)) __PYX_ERR(0, 109, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); CYTHON_FALLTHROUGH; case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); CYTHON_FALLTHROUGH; case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); CYTHON_FALLTHROUGH; case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); CYTHON_FALLTHROUGH; case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); CYTHON_FALLTHROUGH; case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); CYTHON_FALLTHROUGH; case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); CYTHON_FALLTHROUGH; case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); CYTHON_FALLTHROUGH; case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); values[2] = PyTuple_GET_ITEM(__pyx_args, 2); values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_trees1 = ((struct __pyx_obj_8discodop_10containers_Ctrees *)values[0]); __pyx_v_start1 = __Pyx_PyInt_As_int(values[1]); if (unlikely((__pyx_v_start1 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 109, __pyx_L3_error) __pyx_v_end1 = __Pyx_PyInt_As_int(values[2]); if (unlikely((__pyx_v_end1 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 109, __pyx_L3_error) __pyx_v_vocab = ((struct __pyx_obj_8discodop_10containers_Vocabulary *)values[3]); __pyx_v_trees2 = ((struct __pyx_obj_8discodop_10containers_Ctrees *)values[4]); if (values[5]) { __pyx_v_start2 = __Pyx_PyInt_As_int(values[5]); if (unlikely((__pyx_v_start2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 110, __pyx_L3_error) } else { __pyx_v_start2 = ((int)0); } if (values[6]) { __pyx_v_end2 = __Pyx_PyInt_As_int(values[6]); if (unlikely((__pyx_v_end2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 110, __pyx_L3_error) } else { __pyx_v_end2 = ((int)0); } if (values[7]) { __pyx_v_approx = __Pyx_PyObject_IsTrue(values[7]); if (unlikely((__pyx_v_approx == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 111, __pyx_L3_error) } else { /* … */ __pyx_v_disc = ((int)0); } __pyx_v_twoterms = ((PyObject*)values[10]); if (values[11]) { __pyx_v_adjacent = __Pyx_PyObject_IsTrue(values[11]); if (unlikely((__pyx_v_adjacent == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 112, __pyx_L3_error) } else { __pyx_v_adjacent = ((int)0); } } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("extractfragments", 0, 4, 12, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 109, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("discodop._fragments.extractfragments", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_trees1), __pyx_ptype_8discodop_10containers_Ctrees, 1, "trees1", 0))) __PYX_ERR(0, 109, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_vocab), __pyx_ptype_8discodop_10containers_Vocabulary, 1, "vocab", 0))) __PYX_ERR(0, 109, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_trees2), __pyx_ptype_8discodop_10containers_Ctrees, 1, "trees2", 0))) __PYX_ERR(0, 110, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_twoterms), (&PyUnicode_Type), 1, "twoterms", 1))) __PYX_ERR(0, 112, __pyx_L1_error) __pyx_r = __pyx_pf_8discodop_10_fragments_extractfragments(__pyx_self, __pyx_v_trees1, __pyx_v_start1, __pyx_v_end1, __pyx_v_vocab, __pyx_v_trees2, __pyx_v_start2, __pyx_v_end2, __pyx_v_approx, __pyx_v_debug, __pyx_v_disc, __pyx_v_twoterms, __pyx_v_adjacent);
0113: """Find the largest fragments in treebank(s) with the fast tree kernel.
0114:
0115: - scenario 1: recurring fragments in single treebank, use::
0116: extractfragments(trees1, start1, end1, vocab)
0117: - scenario 2: common fragments in two treebanks::
0118: extractfragments(trees1, start1, end1, vocab, trees2)
0119:
0120: :param start1, end1: specify slice of treebank to be used; can be used to
0121: divide the work over multiple processes; they are indices of ``trees1``
0122: to work on (pass 0 for both to use all trees).
0123: :param start2, end2: idem for trees2.
0124: :param approx: return approximate counts instead of bitsets.
0125: :param debug: if True, a table of common productions is printed for each
0126: pair of trees
0127: :param disc: if True, return trees with indices as leaves.
0128: :param twoterms: only return fragments with at least two terminals,
0129: one of which has a POS tag matching the given regex.
0130: :param adjacent: only extract fragments from sentences with adjacent
0131: indices.
0132: :returns: a dictionary; keys are fragments as strings; values are
0133: either counts (if approx=True), or bitsets describing fragments of
0134: ``trees1``.
0135: """
0136: cdef:
0137: int n, m
+0138: short minterms = 2 if twoterms else 0
__pyx_t_2 = (__pyx_v_twoterms != Py_None) && (__Pyx_PyUnicode_IS_TRUE(__pyx_v_twoterms) != 0);
if (__pyx_t_2) {
__pyx_t_1 = 2;
} else {
__pyx_t_1 = 0;
}
__pyx_v_minterms = __pyx_t_1;
0139: short SLOTS # the number of uint32_ts needed to cover the largest tree
+0140: uint64_t *matrix = NULL # bit matrix of common productions in tree pair
__pyx_v_matrix = NULL;
0141: uint64_t *scratch
0142: NodeArray a
0143: Node *anodes
0144: list asent
+0145: dict fragments = {}
__pyx_t_3 = PyDict_New(); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_v_fragments = ((PyObject*)__pyx_t_3); __pyx_t_3 = 0;
+0146: set inter = set(), contentwordprods = None, lexicalprods = None
__pyx_t_3 = PySet_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_v_inter = ((PyObject*)__pyx_t_3); __pyx_t_3 = 0; __Pyx_INCREF(Py_None); __pyx_v_contentwordprods = ((PyObject*)Py_None); __Pyx_INCREF(Py_None); __pyx_v_lexicalprods = ((PyObject*)Py_None);
+0147: list tmp = []
__pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 147, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_v_tmp = ((PyObject*)__pyx_t_3); __pyx_t_3 = 0;
+0148: if twoterms:
__pyx_t_2 = (__pyx_v_twoterms != Py_None) && (__Pyx_PyUnicode_IS_TRUE(__pyx_v_twoterms) != 0);
if (__pyx_t_2) {
/* … */
}
+0149: contentword = re.compile(twoterms)
__pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_re); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_compile); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_5))) { __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); if (likely(__pyx_t_4)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); __Pyx_INCREF(__pyx_t_4); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_5, function); } } if (!__pyx_t_4) { __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_v_twoterms); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_twoterms}; __pyx_t_3 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 149, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_3); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_twoterms}; __pyx_t_3 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 149, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_3); } else #endif { __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; __Pyx_INCREF(__pyx_v_twoterms); __Pyx_GIVEREF(__pyx_v_twoterms); PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_v_twoterms); __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_6, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } } __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_v_contentword = __pyx_t_3; __pyx_t_3 = 0;
+0150: contentwordprods = {n for n in range(len(vocab.prods))
{ /* enter inner scope */
int __pyx_7genexpr__pyx_v_n;
__pyx_t_3 = PySet_New(NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 150, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__pyx_t_5 = __pyx_v_vocab->prods;
__Pyx_INCREF(__pyx_t_5);
if (unlikely(__pyx_t_5 == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 150, __pyx_L1_error)
}
__pyx_t_7 = PyDict_Size(__pyx_t_5); if (unlikely(__pyx_t_7 == -1)) __PYX_ERR(0, 150, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) {
__pyx_7genexpr__pyx_v_n = __pyx_t_8;
/* … */
__pyx_t_5 = __Pyx_PyInt_From_int(__pyx_7genexpr__pyx_v_n); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 150, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
if (unlikely(PySet_Add(__pyx_t_3, (PyObject*)__pyx_t_5))) __PYX_ERR(0, 150, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+0151: if contentword.match(vocab.getlabel(n))}
__pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_contentword, __pyx_n_s_match); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_4 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getlabel(__pyx_v_vocab, __pyx_7genexpr__pyx_v_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_9 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_6))) { __pyx_t_9 = PyMethod_GET_SELF(__pyx_t_6); if (likely(__pyx_t_9)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); __Pyx_INCREF(__pyx_t_9); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_6, function); } } if (!__pyx_t_9) { __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_5); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_9, __pyx_t_4}; __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_9, __pyx_t_4}; __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } else #endif { __pyx_t_10 = PyTuple_New(1+1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_GIVEREF(__pyx_t_9); PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_t_9); __pyx_t_9 = NULL; __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_10, 0+1, __pyx_t_4); __pyx_t_4 = 0; __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_10, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; } } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 151, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_2) { /* … */ } } } /* exit inner scope */ __Pyx_DECREF_SET(__pyx_v_contentwordprods, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0;
+0152: lexicalprods = {n for n in range(len(vocab.prods))
{ /* enter inner scope */
int __pyx_8genexpr1__pyx_v_n;
__pyx_t_3 = PySet_New(NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 152, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__pyx_t_5 = __pyx_v_vocab->prods;
__Pyx_INCREF(__pyx_t_5);
if (unlikely(__pyx_t_5 == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 152, __pyx_L1_error)
}
__pyx_t_7 = PyDict_Size(__pyx_t_5); if (unlikely(__pyx_t_7 == -1)) __PYX_ERR(0, 152, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) {
__pyx_8genexpr1__pyx_v_n = __pyx_t_8;
/* … */
__pyx_t_5 = __Pyx_PyInt_From_int(__pyx_8genexpr1__pyx_v_n); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 152, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
if (unlikely(PySet_Add(__pyx_t_3, (PyObject*)__pyx_t_5))) __PYX_ERR(0, 152, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+0153: if vocab.islexical(n)}
__pyx_t_2 = (((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->islexical(__pyx_v_vocab, __pyx_8genexpr1__pyx_v_n) != 0);
if (__pyx_t_2) {
/* … */
}
}
} /* exit inner scope */
__Pyx_DECREF_SET(__pyx_v_lexicalprods, ((PyObject*)__pyx_t_3));
__pyx_t_3 = 0;
+0154: if trees2 is None:
__pyx_t_2 = (((PyObject *)__pyx_v_trees2) == Py_None);
__pyx_t_11 = (__pyx_t_2 != 0);
if (__pyx_t_11) {
/* … */
}
+0155: trees2 = trees1
__Pyx_INCREF(((PyObject *)__pyx_v_trees1)); __Pyx_DECREF_SET(__pyx_v_trees2, __pyx_v_trees1);
+0156: SLOTS = BITNSLOTS(max(trees1.maxnodes, trees2.maxnodes) + 1)
__pyx_t_1 = __pyx_v_trees2->maxnodes;
__pyx_t_12 = __pyx_v_trees1->maxnodes;
if (((__pyx_t_1 > __pyx_t_12) != 0)) {
__pyx_t_13 = __pyx_t_1;
} else {
__pyx_t_13 = __pyx_t_12;
}
__pyx_v_SLOTS = BITNSLOTS((__pyx_t_13 + 1));
+0157: matrix = <uint64_t *>malloc(trees2.maxnodes * SLOTS * sizeof(uint64_t))
__pyx_v_matrix = ((uint64_t *)malloc(((__pyx_v_trees2->maxnodes * __pyx_v_SLOTS) * (sizeof(uint64_t)))));
+0158: scratch = <uint64_t *>malloc((SLOTS + 2) * sizeof(uint64_t))
__pyx_v_scratch = ((uint64_t *)malloc(((__pyx_v_SLOTS + 2) * (sizeof(uint64_t)))));
+0159: if matrix is NULL or scratch is NULL:
__pyx_t_2 = ((__pyx_v_matrix == NULL) != 0);
if (!__pyx_t_2) {
} else {
__pyx_t_11 = __pyx_t_2;
goto __pyx_L12_bool_binop_done;
}
__pyx_t_2 = ((__pyx_v_scratch == NULL) != 0);
__pyx_t_11 = __pyx_t_2;
__pyx_L12_bool_binop_done:;
if (__pyx_t_11) {
/* … */
}
+0160: raise MemoryError('allocation error')
__pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(0, 160, __pyx_L1_error) /* … */ __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_u_allocation_error); if (unlikely(!__pyx_tuple_)) __PYX_ERR(0, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_);
+0161: end2 = min(end2 or trees2.len, trees2.len)
__pyx_t_8 = __pyx_v_trees2->len;
if (!__pyx_v_end2) {
} else {
__pyx_t_14 = __pyx_v_end2;
goto __pyx_L14_bool_binop_done;
}
__pyx_t_14 = __pyx_v_trees2->len;
__pyx_L14_bool_binop_done:;
if (((__pyx_t_8 < __pyx_t_14) != 0)) {
__pyx_t_15 = __pyx_t_8;
} else {
__pyx_t_15 = __pyx_t_14;
}
__pyx_v_end2 = __pyx_t_15;
0162: # loop over tree pairs to extract fragments from
+0163: for n in range(start1, min(end1 or trees1.len, trees1.len)):
__pyx_t_15 = __pyx_v_trees1->len;
if (!__pyx_v_end1) {
} else {
__pyx_t_8 = __pyx_v_end1;
goto __pyx_L16_bool_binop_done;
}
__pyx_t_8 = __pyx_v_trees1->len;
__pyx_L16_bool_binop_done:;
if (((__pyx_t_15 < __pyx_t_8) != 0)) {
__pyx_t_14 = __pyx_t_15;
} else {
__pyx_t_14 = __pyx_t_8;
}
__pyx_t_15 = __pyx_t_14;
for (__pyx_t_14 = __pyx_v_start1; __pyx_t_14 < __pyx_t_15; __pyx_t_14+=1) {
__pyx_v_n = __pyx_t_14;
+0164: a = trees1.trees[n]
__pyx_v_a = (__pyx_v_trees1->trees[__pyx_v_n]);
+0165: asent = trees1.extractsent(n, vocab)
__pyx_t_5 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_trees1), __pyx_n_s_extractsent); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 165, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = __Pyx_PyInt_From_int(__pyx_v_n); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 165, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_10 = NULL; __pyx_t_8 = 0; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_5))) { __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_5); if (likely(__pyx_t_10)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); __Pyx_INCREF(__pyx_t_10); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_5, function); __pyx_t_8 = 1; } } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[3] = {__pyx_t_10, __pyx_t_6, ((PyObject *)__pyx_v_vocab)}; __pyx_t_3 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_8, 2+__pyx_t_8); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 165, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[3] = {__pyx_t_10, __pyx_t_6, ((PyObject *)__pyx_v_vocab)}; __pyx_t_3 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_8, 2+__pyx_t_8); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 165, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { __pyx_t_4 = PyTuple_New(2+__pyx_t_8); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 165, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); if (__pyx_t_10) { __Pyx_GIVEREF(__pyx_t_10); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_10); __pyx_t_10 = NULL; } __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_4, 0+__pyx_t_8, __pyx_t_6); __Pyx_INCREF(((PyObject *)__pyx_v_vocab)); __Pyx_GIVEREF(((PyObject *)__pyx_v_vocab)); PyTuple_SET_ITEM(__pyx_t_4, 1+__pyx_t_8, ((PyObject *)__pyx_v_vocab)); __pyx_t_6 = 0; __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 165, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (!(likely(PyList_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "list", Py_TYPE(__pyx_t_3)->tp_name), 0))) __PYX_ERR(0, 165, __pyx_L1_error) __Pyx_XDECREF_SET(__pyx_v_asent, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0;
+0166: anodes = &trees1.nodes[a.offset]
__pyx_v_anodes = (&(__pyx_v_trees1->nodes[__pyx_v_a.offset]));
+0167: if adjacent:
__pyx_t_11 = (__pyx_v_adjacent != 0);
if (__pyx_t_11) {
/* … */
goto __pyx_L20;
}
+0168: m = n + 1
__pyx_v_m = (__pyx_v_n + 1);
+0169: if m < trees2.len:
__pyx_t_11 = ((__pyx_v_m < __pyx_v_trees2->len) != 0);
if (__pyx_t_11) {
/* … */
}
+0170: extractfrompair(a, anodes, trees2, n, m, debug,
__pyx_t_3 = __pyx_f_8discodop_10_fragments_extractfrompair(__pyx_v_a, __pyx_v_anodes, __pyx_v_trees2, __pyx_v_n, __pyx_v_m, __pyx_v_debug, __pyx_v_vocab, __pyx_v_inter, __pyx_v_minterms, __pyx_v_matrix, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 170, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
0171: vocab, inter, minterms, matrix, scratch, SLOTS)
+0172: elif twoterms:
__pyx_t_11 = (__pyx_v_twoterms != Py_None) && (__Pyx_PyUnicode_IS_TRUE(__pyx_v_twoterms) != 0);
if (__pyx_t_11) {
/* … */
goto __pyx_L20;
}
+0173: for m in twoterminals(a, anodes, trees2,
__pyx_t_3 = __pyx_f_8discodop_10_fragments_twoterminals(__pyx_v_a, __pyx_v_anodes, __pyx_v_trees2, __pyx_v_contentwordprods, __pyx_v_lexicalprods); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 173, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
if (likely(PyList_CheckExact(__pyx_t_3)) || PyTuple_CheckExact(__pyx_t_3)) {
__pyx_t_5 = __pyx_t_3; __Pyx_INCREF(__pyx_t_5); __pyx_t_7 = 0;
__pyx_t_16 = NULL;
} else {
__pyx_t_7 = -1; __pyx_t_5 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 173, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
__pyx_t_16 = Py_TYPE(__pyx_t_5)->tp_iternext; if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 173, __pyx_L1_error)
}
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
for (;;) {
if (likely(!__pyx_t_16)) {
if (likely(PyList_CheckExact(__pyx_t_5))) {
if (__pyx_t_7 >= PyList_GET_SIZE(__pyx_t_5)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_3 = PyList_GET_ITEM(__pyx_t_5, __pyx_t_7); __Pyx_INCREF(__pyx_t_3); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(0, 173, __pyx_L1_error)
#else
__pyx_t_3 = PySequence_ITEM(__pyx_t_5, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 173, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
#endif
} else {
if (__pyx_t_7 >= PyTuple_GET_SIZE(__pyx_t_5)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_5, __pyx_t_7); __Pyx_INCREF(__pyx_t_3); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(0, 173, __pyx_L1_error)
#else
__pyx_t_3 = PySequence_ITEM(__pyx_t_5, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 173, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
#endif
}
} else {
__pyx_t_3 = __pyx_t_16(__pyx_t_5);
if (unlikely(!__pyx_t_3)) {
PyObject* exc_type = PyErr_Occurred();
if (exc_type) {
if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
else __PYX_ERR(0, 173, __pyx_L1_error)
}
break;
}
__Pyx_GOTREF(__pyx_t_3);
}
__pyx_t_8 = __Pyx_PyInt_As_int(__pyx_t_3); if (unlikely((__pyx_t_8 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 173, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__pyx_v_m = __pyx_t_8;
/* … */
__pyx_L22_continue:;
}
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
0174: contentwordprods, lexicalprods):
+0175: if trees1 is trees2 and m <= n:
__pyx_t_2 = (__pyx_v_trees1 == __pyx_v_trees2);
__pyx_t_17 = (__pyx_t_2 != 0);
if (__pyx_t_17) {
} else {
__pyx_t_11 = __pyx_t_17;
goto __pyx_L25_bool_binop_done;
}
__pyx_t_17 = ((__pyx_v_m <= __pyx_v_n) != 0);
__pyx_t_11 = __pyx_t_17;
__pyx_L25_bool_binop_done:;
if (__pyx_t_11) {
/* … */
}
+0176: continue
goto __pyx_L22_continue;
+0177: elif start2 > m or m > end2:
__pyx_t_17 = ((__pyx_v_start2 > __pyx_v_m) != 0);
if (!__pyx_t_17) {
} else {
__pyx_t_11 = __pyx_t_17;
goto __pyx_L27_bool_binop_done;
}
__pyx_t_17 = ((__pyx_v_m > __pyx_v_end2) != 0);
__pyx_t_11 = __pyx_t_17;
__pyx_L27_bool_binop_done:;
if (__pyx_t_11) {
/* … */
}
+0178: continue
goto __pyx_L22_continue;
+0179: elif m < 0 or m >= trees2.len:
__pyx_t_17 = ((__pyx_v_m < 0) != 0);
if (!__pyx_t_17) {
} else {
__pyx_t_11 = __pyx_t_17;
goto __pyx_L29_bool_binop_done;
}
__pyx_t_17 = ((__pyx_v_m >= __pyx_v_trees2->len) != 0);
__pyx_t_11 = __pyx_t_17;
__pyx_L29_bool_binop_done:;
if (__pyx_t_11) {
/* … */
}
+0180: raise ValueError('illegal index %d' % m)
__pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_m); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 180, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyUnicode_Format(__pyx_kp_u_illegal_index_d, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 180, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 180, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 180, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(0, 180, __pyx_L1_error)
+0181: extractfrompair(a, anodes, trees2, n, m, debug,
__pyx_t_4 = __pyx_f_8discodop_10_fragments_extractfrompair(__pyx_v_a, __pyx_v_anodes, __pyx_v_trees2, __pyx_v_n, __pyx_v_m, __pyx_v_debug, __pyx_v_vocab, __pyx_v_inter, __pyx_v_minterms, __pyx_v_matrix, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 181, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_4);
__Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
0182: vocab, inter, minterms, matrix, scratch, SLOTS)
0183: else: # all pairs
+0184: if trees1 is trees2:
/*else*/ {
__pyx_t_11 = (__pyx_v_trees1 == __pyx_v_trees2);
__pyx_t_17 = (__pyx_t_11 != 0);
if (__pyx_t_17) {
/* … */
}
+0185: start2 = max(n + 1, start2)
__pyx_t_8 = __pyx_v_start2;
__pyx_t_18 = (__pyx_v_n + 1);
if (((__pyx_t_8 > __pyx_t_18) != 0)) {
__pyx_t_19 = __pyx_t_8;
} else {
__pyx_t_19 = __pyx_t_18;
}
__pyx_v_start2 = __pyx_t_19;
+0186: for m in range(start2, end2):
__pyx_t_8 = __pyx_v_end2;
for (__pyx_t_20 = __pyx_v_start2; __pyx_t_20 < __pyx_t_8; __pyx_t_20+=1) {
__pyx_v_m = __pyx_t_20;
+0187: extractfrompair(a, anodes, trees2, n, m,
__pyx_t_5 = __pyx_f_8discodop_10_fragments_extractfrompair(__pyx_v_a, __pyx_v_anodes, __pyx_v_trees2, __pyx_v_n, __pyx_v_m, __pyx_v_debug, __pyx_v_vocab, __pyx_v_inter, __pyx_v_minterms, __pyx_v_matrix, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 187, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
}
}
__pyx_L20:;
0188: debug, vocab, inter, minterms, matrix,
0189: scratch, SLOTS)
+0190: collectfragments(fragments, inter, anodes, asent, vocab,
__pyx_t_5 = __pyx_f_8discodop_10_fragments_collectfragments(__pyx_v_fragments, __pyx_v_inter, __pyx_v_anodes, __pyx_v_asent, __pyx_v_vocab, __pyx_v_disc, __pyx_v_approx, 0, __pyx_v_tmp, __pyx_v_SLOTS); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 190, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
}
0191: disc, approx, False, tmp, SLOTS)
+0192: free(matrix)
free(__pyx_v_matrix);
+0193: free(scratch)
free(__pyx_v_scratch);
+0194: return fragments
__Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_fragments); __pyx_r = __pyx_v_fragments; goto __pyx_L0;
0195:
0196:
+0197: cdef inline extractfrompair(NodeArray a, Node *anodes, Ctrees trees2,
static CYTHON_INLINE PyObject *__pyx_f_8discodop_10_fragments_extractfrompair(struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_a, struct __pyx_t_8discodop_10containers_Node *__pyx_v_anodes, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2, int __pyx_v_n, int __pyx_v_m, int __pyx_v_debug, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, PyObject *__pyx_v_inter, short __pyx_v_minterms, uint64_t *__pyx_v_matrix, uint64_t *__pyx_v_scratch, short __pyx_v_SLOTS) {
struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_b;
struct __pyx_t_8discodop_10containers_Node *__pyx_v_bnodes;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("extractfrompair", 0);
/* … */
/* function exit code */
__pyx_r = Py_None; __Pyx_INCREF(Py_None);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_AddTraceback("discodop._fragments.extractfrompair", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0198: int n, int m, bint debug, Vocabulary vocab, set inter,
0199: short minterms, uint64_t *matrix, uint64_t *scratch, short SLOTS):
0200: """Extract the bitsets of maximal overlapping fragments for a tree pair."""
+0201: cdef NodeArray b = trees2.trees[m]
__pyx_v_b = (__pyx_v_trees2->trees[__pyx_v_m]);
+0202: cdef Node *bnodes = &trees2.nodes[b.offset]
__pyx_v_bnodes = (&(__pyx_v_trees2->nodes[__pyx_v_b.offset]));
0203: # initialize table
+0204: memset(<void *>matrix, 0, b.len * SLOTS * sizeof(uint64_t))
memset(((void *)__pyx_v_matrix), 0, ((__pyx_v_b.len * __pyx_v_SLOTS) * (sizeof(uint64_t))));
0205: # fill table
+0206: fasttreekernel(anodes, bnodes, a.len, b.len, matrix, SLOTS)
__pyx_f_8discodop_10_fragments_fasttreekernel(__pyx_v_anodes, __pyx_v_bnodes, __pyx_v_a.len, __pyx_v_b.len, __pyx_v_matrix, __pyx_v_SLOTS);
0207: # dump table
+0208: if debug:
__pyx_t_1 = (__pyx_v_debug != 0);
if (__pyx_t_1) {
/* … */
}
+0209: print(n, m)
__pyx_t_2 = __Pyx_PyInt_From_int(__pyx_v_n); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 209, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_m); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 209, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 209, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_2); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_3); __pyx_t_2 = 0; __pyx_t_3 = 0; __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 209, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+0210: dumpmatrix(matrix, a, b, anodes, bnodes, vocab, scratch, SLOTS)
__pyx_t_3 = __pyx_f_8discodop_10_fragments_dumpmatrix(__pyx_v_matrix, __pyx_v_a, __pyx_v_b, __pyx_v_anodes, __pyx_v_bnodes, __pyx_v_vocab, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 210, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
0211: # extract results
+0212: extractbitsets(matrix, anodes, bnodes, b.root, n,
__pyx_t_3 = __pyx_f_8discodop_10_fragments_extractbitsets(__pyx_v_matrix, __pyx_v_anodes, __pyx_v_bnodes, __pyx_v_b.root, __pyx_v_n, __pyx_v_inter, __pyx_v_minterms, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 212, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
0213: inter, minterms, scratch, SLOTS)
0214:
0215:
+0216: cdef inline collectfragments(dict fragments, set inter, Node *anodes,
static CYTHON_INLINE PyObject *__pyx_f_8discodop_10_fragments_collectfragments(PyObject *__pyx_v_fragments, PyObject *__pyx_v_inter, struct __pyx_t_8discodop_10containers_Node *__pyx_v_anodes, PyObject *__pyx_v_asent, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, int __pyx_v_disc, int __pyx_v_approx, int __pyx_v_indices, PyObject *__pyx_v_tmp, short __pyx_v_SLOTS) {
uint64_t *__pyx_v_bitset;
PyObject *__pyx_v_wrapper = NULL;
PyObject *__pyx_v_frag = NULL;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("collectfragments", 0);
/* … */
/* function exit code */
__pyx_r = Py_None; __Pyx_INCREF(Py_None);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_7);
__Pyx_XDECREF(__pyx_t_8);
__Pyx_XDECREF(__pyx_t_9);
__Pyx_XDECREF(__pyx_t_10);
__Pyx_XDECREF(__pyx_t_13);
__Pyx_AddTraceback("discodop._fragments.collectfragments", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_wrapper);
__Pyx_XDECREF(__pyx_v_frag);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0217: list asent, Vocabulary vocab, bint disc, bint approx,
0218: bint indices, list tmp, short SLOTS):
0219: """Collect string representations of fragments given as bitsets."""
0220: cdef uint64_t *bitset
+0221: for wrapper in inter:
__pyx_t_1 = PyObject_GetIter(__pyx_v_inter); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 221, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = Py_TYPE(__pyx_t_1)->tp_iternext; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 221, __pyx_L1_error) for (;;) { { __pyx_t_3 = __pyx_t_2(__pyx_t_1); if (unlikely(!__pyx_t_3)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 221, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_3); } __Pyx_XDECREF_SET(__pyx_v_wrapper, __pyx_t_3); __pyx_t_3 = 0; /* … */ } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+0222: bitset = getpointer(wrapper)
__pyx_v_bitset = __pyx_f_8discodop_10_fragments_getpointer(__pyx_v_wrapper);
+0223: getsubtree(tmp, anodes, bitset, vocab,
__pyx_t_3 = __pyx_f_8discodop_10_fragments_getsubtree(__pyx_v_tmp, __pyx_v_anodes, __pyx_v_bitset, __pyx_v_vocab, __pyx_v_disc, __pyx_f_8discodop_10_fragments_getroot(__pyx_v_bitset, __pyx_v_SLOTS)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 223, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
0224: disc, getroot(bitset, SLOTS))
+0225: try:
{
/*try:*/ {
/* … */
}
__Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
__Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
__Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
goto __pyx_L12_try_end;
__pyx_L5_error:;
__Pyx_PyThreadState_assign
__Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
__Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
__Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
/* … */
__Pyx_PyThreadState_assign
__Pyx_XGIVEREF(__pyx_t_4);
__Pyx_XGIVEREF(__pyx_t_5);
__Pyx_XGIVEREF(__pyx_t_6);
__Pyx_ExceptionReset(__pyx_t_4, __pyx_t_5, __pyx_t_6);
goto __pyx_L1_error;
__pyx_L12_try_end:;
}
+0226: frag = getsent(''.join(tmp)) if disc else ''.join(tmp)
if ((__pyx_v_disc != 0)) {
__pyx_t_7 = PyUnicode_Join(__pyx_kp_u__2, __pyx_v_tmp); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 226, __pyx_L5_error)
__Pyx_GOTREF(__pyx_t_7);
__pyx_t_8 = __pyx_f_8discodop_10_fragments_getsent(((PyObject*)__pyx_t_7)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 226, __pyx_L5_error)
__Pyx_GOTREF(__pyx_t_8);
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
__pyx_t_3 = __pyx_t_8;
__pyx_t_8 = 0;
} else {
__pyx_t_8 = PyUnicode_Join(__pyx_kp_u__2, __pyx_v_tmp); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 226, __pyx_L5_error)
__Pyx_GOTREF(__pyx_t_8);
__pyx_t_3 = __pyx_t_8;
__pyx_t_8 = 0;
}
__Pyx_XDECREF_SET(__pyx_v_frag, __pyx_t_3);
__pyx_t_3 = 0;
+0227: except:
/*except:*/ {
__Pyx_AddTraceback("discodop._fragments.collectfragments", __pyx_clineno, __pyx_lineno, __pyx_filename);
if (__Pyx_GetException(&__pyx_t_3, &__pyx_t_8, &__pyx_t_7) < 0) __PYX_ERR(0, 227, __pyx_L7_except_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_GOTREF(__pyx_t_8);
__Pyx_GOTREF(__pyx_t_7);
+0228: print(asent)
__pyx_t_9 = PyTuple_New(1); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 228, __pyx_L7_except_error) __Pyx_GOTREF(__pyx_t_9); __Pyx_INCREF(__pyx_v_asent); __Pyx_GIVEREF(__pyx_v_asent); PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_v_asent); __pyx_t_10 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_9, NULL); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 228, __pyx_L7_except_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
+0229: print(tmp)
__pyx_t_10 = PyTuple_New(1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 229, __pyx_L7_except_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_INCREF(__pyx_v_tmp); __Pyx_GIVEREF(__pyx_v_tmp); PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_v_tmp); __pyx_t_9 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_10, NULL); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 229, __pyx_L7_except_error) __Pyx_GOTREF(__pyx_t_9); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+0230: raise
__Pyx_GIVEREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_8); __Pyx_XGIVEREF(__pyx_t_7); __Pyx_ErrRestoreWithState(__pyx_t_3, __pyx_t_8, __pyx_t_7); __pyx_t_3 = 0; __pyx_t_8 = 0; __pyx_t_7 = 0; __PYX_ERR(0, 230, __pyx_L7_except_error) } __pyx_L7_except_error:;
+0231: del tmp[:]
if (unlikely(__pyx_v_tmp == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 231, __pyx_L1_error)
}
if (__Pyx_PyObject_DelSlice(__pyx_v_tmp, 0, 0, NULL, NULL, NULL, 0, 0, 0) < 0) __PYX_ERR(0, 231, __pyx_L1_error)
+0232: if approx:
__pyx_t_11 = (__pyx_v_approx != 0);
if (__pyx_t_11) {
/* … */
goto __pyx_L15;
}
+0233: if frag not in fragments:
if (unlikely(__pyx_v_fragments == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 233, __pyx_L1_error)
}
__pyx_t_11 = (__Pyx_PyDict_ContainsTF(__pyx_v_frag, __pyx_v_fragments, Py_NE)); if (unlikely(__pyx_t_11 < 0)) __PYX_ERR(0, 233, __pyx_L1_error)
__pyx_t_12 = (__pyx_t_11 != 0);
if (__pyx_t_12) {
/* … */
}
+0234: fragments[frag] = 0
if (unlikely(__pyx_v_fragments == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 234, __pyx_L1_error)
}
if (unlikely(PyDict_SetItem(__pyx_v_fragments, __pyx_v_frag, __pyx_int_0) < 0)) __PYX_ERR(0, 234, __pyx_L1_error)
+0235: fragments[frag] += 1
if (unlikely(__pyx_v_fragments == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 235, __pyx_L1_error)
}
__Pyx_INCREF(__pyx_v_fragments);
__pyx_t_13 = __pyx_v_fragments;
__Pyx_INCREF(__pyx_v_frag);
__pyx_t_7 = __pyx_v_frag;
if (unlikely(__pyx_t_13 == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 235, __pyx_L1_error)
}
__pyx_t_8 = __Pyx_PyDict_GetItem(__pyx_t_13, __pyx_t_7); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 235, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_8);
__pyx_t_3 = __Pyx_PyInt_AddObjC(__pyx_t_8, __pyx_int_1, 1, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 235, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
if (unlikely(__pyx_t_13 == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 235, __pyx_L1_error)
}
if (unlikely(PyDict_SetItem(__pyx_t_13, __pyx_t_7, __pyx_t_3) < 0)) __PYX_ERR(0, 235, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
+0236: elif indices:
__pyx_t_12 = (__pyx_v_indices != 0);
if (__pyx_t_12) {
/* … */
goto __pyx_L15;
}
+0237: if frag not in fragments:
if (unlikely(__pyx_v_fragments == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 237, __pyx_L1_error)
}
__pyx_t_12 = (__Pyx_PyDict_ContainsTF(__pyx_v_frag, __pyx_v_fragments, Py_NE)); if (unlikely(__pyx_t_12 < 0)) __PYX_ERR(0, 237, __pyx_L1_error)
__pyx_t_11 = (__pyx_t_12 != 0);
if (__pyx_t_11) {
/* … */
}
+0238: fragments[frag] = Counter()
__pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_Counter); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 238, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_8 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_8)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); __Pyx_INCREF(__pyx_t_8); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_3, function); } } if (__pyx_t_8) { __pyx_t_7 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_8); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 238, __pyx_L1_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } else { __pyx_t_7 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 238, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (unlikely(__pyx_v_fragments == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); __PYX_ERR(0, 238, __pyx_L1_error) } if (unlikely(PyDict_SetItem(__pyx_v_fragments, __pyx_v_frag, __pyx_t_7) < 0)) __PYX_ERR(0, 238, __pyx_L1_error) __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+0239: fragments[frag][getid(bitset, SLOTS)] += 1
if (unlikely(__pyx_v_fragments == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 239, __pyx_L1_error)
}
__pyx_t_7 = __Pyx_PyDict_GetItem(__pyx_v_fragments, __pyx_v_frag); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 239, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__pyx_t_14 = __pyx_f_8discodop_10_fragments_getid(__pyx_v_bitset, __pyx_v_SLOTS);
__pyx_t_3 = __Pyx_GetItemInt(__pyx_t_7, __pyx_t_14, uint32_t, 0, __Pyx_PyInt_From_uint32_t, 0, 0, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 239, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__pyx_t_8 = __Pyx_PyInt_AddObjC(__pyx_t_3, __pyx_int_1, 1, 1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 239, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_8);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
if (unlikely(__Pyx_SetItemInt(__pyx_t_7, __pyx_t_14, __pyx_t_8, uint32_t, 0, __Pyx_PyInt_From_uint32_t, 0, 0, 0) < 0)) __PYX_ERR(0, 239, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+0240: elif frag not in fragments: # FIXME: is this condition useful?
if (unlikely(__pyx_v_fragments == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 240, __pyx_L1_error)
}
__pyx_t_11 = (__Pyx_PyDict_ContainsTF(__pyx_v_frag, __pyx_v_fragments, Py_NE)); if (unlikely(__pyx_t_11 < 0)) __PYX_ERR(0, 240, __pyx_L1_error)
__pyx_t_12 = (__pyx_t_11 != 0);
if (__pyx_t_12) {
/* … */
}
__pyx_L15:;
+0241: fragments[frag] = wrapper
if (unlikely(__pyx_v_fragments == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 241, __pyx_L1_error)
}
if (unlikely(PyDict_SetItem(__pyx_v_fragments, __pyx_v_frag, __pyx_v_wrapper) < 0)) __PYX_ERR(0, 241, __pyx_L1_error)
+0242: inter.clear()
if (unlikely(__pyx_v_inter == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "clear");
__PYX_ERR(0, 242, __pyx_L1_error)
}
__pyx_t_15 = PySet_Clear(__pyx_v_inter); if (unlikely(__pyx_t_15 == -1)) __PYX_ERR(0, 242, __pyx_L1_error)
0243:
0244:
+0245: cdef inline void fasttreekernel(Node *a, Node *b, int alen, int blen,
static CYTHON_INLINE void __pyx_f_8discodop_10_fragments_fasttreekernel(struct __pyx_t_8discodop_10containers_Node *__pyx_v_a, struct __pyx_t_8discodop_10containers_Node *__pyx_v_b, int __pyx_v_alen, int __pyx_v_blen, uint64_t *__pyx_v_matrix, short __pyx_v_SLOTS) {
int __pyx_v_i;
int __pyx_v_j;
int __pyx_v_ii;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("fasttreekernel", 0);
/* … */
/* function exit code */
__pyx_L0:;
__Pyx_RefNannyFinishContext();
}
0246: uint64_t *matrix, short SLOTS):
0247: """Fast Tree Kernel (average case linear time).
0248:
0249: Expects trees to be sorted according to their productions (in descending
0250: order, with terminals as -1). This algorithm is from the pseudocode in
0251: Moschitti (2006): Making Tree Kernels practical for Natural Language
0252: Learning."""
0253: # i is an index to a, j to b, and ii is a temp index starting at i.
+0254: cdef int i = 0, j = 0, ii = 0
__pyx_v_i = 0; __pyx_v_j = 0; __pyx_v_ii = 0;
+0255: while True:
while (1) {
+0256: if a[i].prod < b[j].prod:
__pyx_t_1 = (((__pyx_v_a[__pyx_v_i]).prod < (__pyx_v_b[__pyx_v_j]).prod) != 0);
if (__pyx_t_1) {
/* … */
goto __pyx_L5;
}
+0257: i += 1
__pyx_v_i = (__pyx_v_i + 1);
+0258: if i >= alen:
__pyx_t_1 = ((__pyx_v_i >= __pyx_v_alen) != 0);
if (__pyx_t_1) {
/* … */
}
+0259: return
goto __pyx_L0;
+0260: elif a[i].prod > b[j].prod:
__pyx_t_1 = (((__pyx_v_a[__pyx_v_i]).prod > (__pyx_v_b[__pyx_v_j]).prod) != 0);
if (__pyx_t_1) {
/* … */
goto __pyx_L5;
}
+0261: j += 1
__pyx_v_j = (__pyx_v_j + 1);
+0262: if j >= blen:
__pyx_t_1 = ((__pyx_v_j >= __pyx_v_blen) != 0);
if (__pyx_t_1) {
/* … */
}
+0263: return
goto __pyx_L0;
0264: else:
+0265: while a[i].prod == b[j].prod:
/*else*/ {
while (1) {
__pyx_t_1 = (((__pyx_v_a[__pyx_v_i]).prod == (__pyx_v_b[__pyx_v_j]).prod) != 0);
if (!__pyx_t_1) break;
+0266: ii = i
__pyx_v_ii = __pyx_v_i;
+0267: while a[ii].prod == b[j].prod:
while (1) {
__pyx_t_1 = (((__pyx_v_a[__pyx_v_ii]).prod == (__pyx_v_b[__pyx_v_j]).prod) != 0);
if (!__pyx_t_1) break;
+0268: SETBIT(&matrix[j * SLOTS], ii)
SETBIT((&(__pyx_v_matrix[(__pyx_v_j * __pyx_v_SLOTS)])), __pyx_v_ii);
+0269: ii += 1
__pyx_v_ii = (__pyx_v_ii + 1);
+0270: if ii >= alen:
__pyx_t_1 = ((__pyx_v_ii >= __pyx_v_alen) != 0);
if (__pyx_t_1) {
/* … */
}
}
__pyx_L11_break:;
+0271: break
goto __pyx_L11_break;
+0272: j += 1
__pyx_v_j = (__pyx_v_j + 1);
+0273: if j >= blen:
__pyx_t_1 = ((__pyx_v_j >= __pyx_v_blen) != 0);
if (__pyx_t_1) {
/* … */
}
}
}
__pyx_L5:;
}
+0274: return
goto __pyx_L0;
0275:
0276:
+0277: cdef inline extractbitsets(uint64_t *matrix, Node *a, Node *b, short j, int n,
static CYTHON_INLINE PyObject *__pyx_f_8discodop_10_fragments_extractbitsets(uint64_t *__pyx_v_matrix, struct __pyx_t_8discodop_10containers_Node *__pyx_v_a, struct __pyx_t_8discodop_10containers_Node *__pyx_v_b, short __pyx_v_j, int __pyx_v_n, PyObject *__pyx_v_results, int __pyx_v_minterms, uint64_t *__pyx_v_scratch, short __pyx_v_SLOTS) {
uint64_t *__pyx_v_bitset;
uint64_t __pyx_v_cur;
int __pyx_v_idx;
int __pyx_v_terms;
int __pyx_v_i;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("extractbitsets", 0);
/* … */
/* function exit code */
__pyx_r = Py_None; __Pyx_INCREF(Py_None);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_2);
__Pyx_AddTraceback("discodop._fragments.extractbitsets", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0278: set results, int minterms, uint64_t *scratch, short SLOTS):
0279: """Visit nodes of ``b`` top-down and store bitsets of common nodes.
0280:
0281: Stores bitsets of connected subsets of ``a`` as they are encountered,
0282: following the common nodes specified in bit matrix. ``j`` is the node in
0283: ``b`` to start with, which changes with each recursive step. ``n`` is the
0284: identifier of this tree which is stored with extracted fragments."""
+0285: cdef uint64_t *bitset = &matrix[j * SLOTS]
__pyx_v_bitset = (&(__pyx_v_matrix[(__pyx_v_j * __pyx_v_SLOTS)]));
+0286: cdef uint64_t cur = bitset[0]
__pyx_v_cur = (__pyx_v_bitset[0]);
+0287: cdef int idx = 0, terms
__pyx_v_idx = 0;
+0288: cdef int i = iteratesetbits(bitset, SLOTS, &cur, &idx)
__pyx_v_i = __pyx_f_8discodop_3bit_iteratesetbits(__pyx_v_bitset, __pyx_v_SLOTS, (&__pyx_v_cur), (&__pyx_v_idx));
+0289: while i != -1:
while (1) {
__pyx_t_1 = ((__pyx_v_i != -1L) != 0);
if (!__pyx_t_1) break;
+0290: memset(<void *>scratch, 0, SLOTS * sizeof(uint64_t))
memset(((void *)__pyx_v_scratch), 0, (__pyx_v_SLOTS * (sizeof(uint64_t))));
+0291: terms = extractat(matrix, scratch, a, b, i, j, SLOTS)
__pyx_v_terms = __pyx_f_8discodop_10_fragments_extractat(__pyx_v_matrix, __pyx_v_scratch, __pyx_v_a, __pyx_v_b, __pyx_v_i, __pyx_v_j, __pyx_v_SLOTS);
+0292: if terms >= minterms:
__pyx_t_1 = ((__pyx_v_terms >= __pyx_v_minterms) != 0);
if (__pyx_t_1) {
/* … */
}
+0293: setrootid(scratch, i, n, SLOTS)
__pyx_f_8discodop_10_fragments_setrootid(__pyx_v_scratch, __pyx_v_i, __pyx_v_n, __pyx_v_SLOTS);
+0294: results.add(wrap(scratch, SLOTS))
if (unlikely(__pyx_v_results == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "add");
__PYX_ERR(0, 294, __pyx_L1_error)
}
__pyx_t_2 = __pyx_f_8discodop_10_fragments_wrap(__pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 294, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_3 = PySet_Add(__pyx_v_results, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(0, 294, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0295: i = iteratesetbits(bitset, SLOTS, &cur, &idx)
__pyx_v_i = __pyx_f_8discodop_3bit_iteratesetbits(__pyx_v_bitset, __pyx_v_SLOTS, (&__pyx_v_cur), (&__pyx_v_idx)); }
+0296: if b[j].left >= 0:
__pyx_t_1 = (((__pyx_v_b[__pyx_v_j]).left >= 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0297: extractbitsets(matrix, a, b, b[j].left, n,
__pyx_t_2 = __pyx_f_8discodop_10_fragments_extractbitsets(__pyx_v_matrix, __pyx_v_a, __pyx_v_b, (__pyx_v_b[__pyx_v_j]).left, __pyx_v_n, __pyx_v_results, __pyx_v_minterms, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 297, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
0298: results, minterms, scratch, SLOTS)
+0299: if b[j].right >= 0:
__pyx_t_1 = (((__pyx_v_b[__pyx_v_j]).right >= 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0300: extractbitsets(matrix, a, b, b[j].right, n,
__pyx_t_2 = __pyx_f_8discodop_10_fragments_extractbitsets(__pyx_v_matrix, __pyx_v_a, __pyx_v_b, (__pyx_v_b[__pyx_v_j]).right, __pyx_v_n, __pyx_v_results, __pyx_v_minterms, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 300, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
0301: results, minterms, scratch, SLOTS)
0302:
0303:
+0304: cdef inline int extractat(uint64_t *matrix, uint64_t *result, Node *a, Node *b,
static CYTHON_INLINE int __pyx_f_8discodop_10_fragments_extractat(uint64_t *__pyx_v_matrix, uint64_t *__pyx_v_result, struct __pyx_t_8discodop_10containers_Node *__pyx_v_a, struct __pyx_t_8discodop_10containers_Node *__pyx_v_b, short __pyx_v_i, short __pyx_v_j, short __pyx_v_SLOTS) {
int __pyx_v_terms;
int __pyx_r;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("extractat", 0);
/* … */
/* function exit code */
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0305: short i, short j, short SLOTS):
0306: """Traverse tree ``a`` and ``b`` in parallel to extract a connected subset.
0307: """
+0308: cdef int terms = 0
__pyx_v_terms = 0;
+0309: SETBIT(result, i)
SETBIT(__pyx_v_result, __pyx_v_i);
+0310: CLEARBIT(&matrix[j * SLOTS], i)
CLEARBIT((&(__pyx_v_matrix[(__pyx_v_j * __pyx_v_SLOTS)])), __pyx_v_i);
+0311: if a[i].left < 0:
__pyx_t_1 = (((__pyx_v_a[__pyx_v_i]).left < 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0312: return 1
__pyx_r = 1;
goto __pyx_L0;
+0313: elif TESTBIT(&matrix[b[j].left * SLOTS], a[i].left):
__pyx_t_1 = (TESTBIT((&(__pyx_v_matrix[((__pyx_v_b[__pyx_v_j]).left * __pyx_v_SLOTS)])), (__pyx_v_a[__pyx_v_i]).left) != 0);
if (__pyx_t_1) {
/* … */
}
+0314: terms += extractat(matrix, result, a, b, a[i].left, b[j].left, SLOTS)
__pyx_v_terms = (__pyx_v_terms + __pyx_f_8discodop_10_fragments_extractat(__pyx_v_matrix, __pyx_v_result, __pyx_v_a, __pyx_v_b, (__pyx_v_a[__pyx_v_i]).left, (__pyx_v_b[__pyx_v_j]).left, __pyx_v_SLOTS));
+0315: if a[i].right < 0:
__pyx_t_1 = (((__pyx_v_a[__pyx_v_i]).right < 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0316: return 0
__pyx_r = 0;
goto __pyx_L0;
+0317: elif TESTBIT(&matrix[b[j].right * SLOTS], a[i].right):
__pyx_t_1 = (TESTBIT((&(__pyx_v_matrix[((__pyx_v_b[__pyx_v_j]).right * __pyx_v_SLOTS)])), (__pyx_v_a[__pyx_v_i]).right) != 0);
if (__pyx_t_1) {
/* … */
}
+0318: terms += extractat(matrix, result, a, b, a[i].right, b[j].right, SLOTS)
__pyx_v_terms = (__pyx_v_terms + __pyx_f_8discodop_10_fragments_extractat(__pyx_v_matrix, __pyx_v_result, __pyx_v_a, __pyx_v_b, (__pyx_v_a[__pyx_v_i]).right, (__pyx_v_b[__pyx_v_j]).right, __pyx_v_SLOTS));
+0319: return terms
__pyx_r = __pyx_v_terms; goto __pyx_L0;
0320:
0321:
+0322: cpdef exactcounts(Ctrees trees1, Ctrees trees2, list bitsets,
static PyObject *__pyx_pw_8discodop_10_fragments_3exactcounts(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static PyObject *__pyx_f_8discodop_10_fragments_exactcounts(struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees1, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2, PyObject *__pyx_v_bitsets, CYTHON_UNUSED int __pyx_skip_dispatch, struct __pyx_opt_args_8discodop_10_fragments_exactcounts *__pyx_optional_args) {
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_8);
__Pyx_XDECREF(__pyx_t_10);
__Pyx_XDECREF(__pyx_t_11);
__Pyx_XDECREF(__pyx_t_12);
__Pyx_AddTraceback("discodop._fragments.exactcounts", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF((PyObject *)__pyx_v_counts);
__Pyx_XDECREF((PyObject *)__pyx_v_treenums);
__Pyx_XDECREF((PyObject *)__pyx_v_nodenums);
__Pyx_XDECREF(__pyx_v_theindices);
__Pyx_XDECREF(__pyx_v_candidates);
__Pyx_XDECREF(__pyx_v_wrapper);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_3exactcounts(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static char __pyx_doc_8discodop_10_fragments_2exactcounts[] = "exactcounts(Ctrees trees1, Ctrees trees2, list bitsets, int indices=False, maxnodes=None)\nGet exact counts or indices of occurrence for fragments.\n\n\t:param trees1, bitsets: ``bitsets`` defines fragments of trees in\n\t\t``trees1`` to search for (the needles).\n\t:param trees2: the trees to search in (haystack); may be equal\n\t\tto ``trees2``.\n\t:param indices: whether to collect indices or counts of fragments.\n\n\t\t:0: return a single count per fragment.\n\t\t:1: collect the indices (sentence numbers) in which fragments occur.\n\t\t:2: collect both sentence numbers, and node numbers of fragments.\n\n\t:param maxnodes: the maximum number of nodes in a single tree to fix the\n\t\tbitset size; use the same value as the function that generated these\n\t\tbitsets. For ``extractfragments``, it is the maximum value across both\n\t\ttreebanks, which is also the default here.\n\t:returns: depending on ``indices``:\n\n\t\t:0: an array of counts, corresponding to ``bitsets``.\n\t\t:1: a list of arrays, each array being a sorted sequence of indices\n\t\t\tfor the corresponding bitset; multiple occurrences of a fragment in\n\t\t\tthe same tree are reflected as multiple occurrences of the same\n\t\t\tindex.\n\t\t:2: a list of pairs of arrays, tree indices paired with node numbers.\n\t";
static PyObject *__pyx_pw_8discodop_10_fragments_3exactcounts(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees1 = 0;
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2 = 0;
PyObject *__pyx_v_bitsets = 0;
int __pyx_v_indices;
PyObject *__pyx_v_maxnodes = 0;
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("exactcounts (wrapper)", 0);
{
static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_trees1,&__pyx_n_s_trees2,&__pyx_n_s_bitsets,&__pyx_n_s_indices,&__pyx_n_s_maxnodes,0};
PyObject* values[5] = {0,0,0,0,0};
/* … */
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__pyx_r = NULL;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_8discodop_10_fragments_2exactcounts(CYTHON_UNUSED PyObject *__pyx_self, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees1, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2, PyObject *__pyx_v_bitsets, int __pyx_v_indices, PyObject *__pyx_v_maxnodes) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("exactcounts", 0);
__Pyx_XDECREF(__pyx_r);
__pyx_t_2.__pyx_n = 2;
__pyx_t_2.indices = __pyx_v_indices;
__pyx_t_2.maxnodes = __pyx_v_maxnodes;
__pyx_t_1 = __pyx_f_8discodop_10_fragments_exactcounts(__pyx_v_trees1, __pyx_v_trees2, __pyx_v_bitsets, 0, &__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 322, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_r = __pyx_t_1;
__pyx_t_1 = 0;
goto __pyx_L0;
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_AddTraceback("discodop._fragments.exactcounts", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
struct __pyx_opt_args_8discodop_10_fragments_exactcounts {
int __pyx_n;
int indices;
PyObject *maxnodes;
};
+0323: int indices=False, maxnodes=None):
int __pyx_v_indices = ((int)0); PyObject *__pyx_v_maxnodes = ((PyObject *)Py_None); arrayobject *__pyx_v_counts = 0; arrayobject *__pyx_v_treenums = 0; arrayobject *__pyx_v_nodenums = 0; PyObject *__pyx_v_theindices = 0; PyObject *__pyx_v_candidates = 0; short __pyx_v_i; short __pyx_v_j; short __pyx_v_SLOTS; uint32_t __pyx_v_n; uint32_t __pyx_v_m; uint32_t *__pyx_v_countsp; struct __pyx_t_8discodop_10containers_NodeArray *__pyx_v_a; struct __pyx_t_8discodop_10containers_NodeArray *__pyx_v_b; struct __pyx_t_8discodop_10containers_Node *__pyx_v_anodes; struct __pyx_t_8discodop_10containers_Node *__pyx_v_bnodes; uint64_t *__pyx_v_bitset; PyObject *__pyx_v_wrapper = NULL; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("exactcounts", 0); if (__pyx_optional_args) { if (__pyx_optional_args->__pyx_n > 0) { __pyx_v_indices = __pyx_optional_args->indices; if (__pyx_optional_args->__pyx_n > 1) { __pyx_v_maxnodes = __pyx_optional_args->maxnodes; } } } /* … */ values[4] = ((PyObject *)Py_None); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); CYTHON_FALLTHROUGH; case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_trees1)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_trees2)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("exactcounts", 0, 3, 5, 1); __PYX_ERR(0, 322, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_bitsets)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("exactcounts", 0, 3, 5, 2); __PYX_ERR(0, 322, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 3: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_indices); if (value) { values[3] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 4: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_maxnodes); if (value) { values[4] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "exactcounts") < 0)) __PYX_ERR(0, 322, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); CYTHON_FALLTHROUGH; case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_trees1 = ((struct __pyx_obj_8discodop_10containers_Ctrees *)values[0]); __pyx_v_trees2 = ((struct __pyx_obj_8discodop_10containers_Ctrees *)values[1]); __pyx_v_bitsets = ((PyObject*)values[2]); if (values[3]) { __pyx_v_indices = __Pyx_PyInt_As_int(values[3]); if (unlikely((__pyx_v_indices == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 323, __pyx_L3_error) } else { __pyx_v_indices = ((int)0); } __pyx_v_maxnodes = values[4]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("exactcounts", 0, 3, 5, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 322, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("discodop._fragments.exactcounts", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_trees1), __pyx_ptype_8discodop_10containers_Ctrees, 1, "trees1", 0))) __PYX_ERR(0, 322, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_trees2), __pyx_ptype_8discodop_10containers_Ctrees, 1, "trees2", 0))) __PYX_ERR(0, 322, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bitsets), (&PyList_Type), 1, "bitsets", 1))) __PYX_ERR(0, 322, __pyx_L1_error) __pyx_r = __pyx_pf_8discodop_10_fragments_2exactcounts(__pyx_self, __pyx_v_trees1, __pyx_v_trees2, __pyx_v_bitsets, __pyx_v_indices, __pyx_v_maxnodes);
0324: """Get exact counts or indices of occurrence for fragments.
0325:
0326: :param trees1, bitsets: ``bitsets`` defines fragments of trees in
0327: ``trees1`` to search for (the needles).
0328: :param trees2: the trees to search in (haystack); may be equal
0329: to ``trees2``.
0330: :param indices: whether to collect indices or counts of fragments.
0331:
0332: :0: return a single count per fragment.
0333: :1: collect the indices (sentence numbers) in which fragments occur.
0334: :2: collect both sentence numbers, and node numbers of fragments.
0335:
0336: :param maxnodes: the maximum number of nodes in a single tree to fix the
0337: bitset size; use the same value as the function that generated these
0338: bitsets. For ``extractfragments``, it is the maximum value across both
0339: treebanks, which is also the default here.
0340: :returns: depending on ``indices``:
0341:
0342: :0: an array of counts, corresponding to ``bitsets``.
0343: :1: a list of arrays, each array being a sorted sequence of indices
0344: for the corresponding bitset; multiple occurrences of a fragment in
0345: the same tree are reflected as multiple occurrences of the same
0346: index.
0347: :2: a list of pairs of arrays, tree indices paired with node numbers.
0348: """
0349: cdef:
+0350: array counts = None
__Pyx_INCREF(Py_None);
__pyx_v_counts = ((arrayobject *)Py_None);
+0351: array treenums = None
__Pyx_INCREF(Py_None);
__pyx_v_treenums = ((arrayobject *)Py_None);
+0352: array nodenums = None
__Pyx_INCREF(Py_None);
__pyx_v_nodenums = ((arrayobject *)Py_None);
+0353: list theindices = None
__Pyx_INCREF(Py_None);
__pyx_v_theindices = ((PyObject*)Py_None);
0354: object candidates # RoaringBitmap()
0355: short i, j, SLOTS
0356: uint32_t n, m
+0357: uint32_t *countsp = NULL
__pyx_v_countsp = NULL;
0358: NodeArray *a
0359: NodeArray *b
0360: Node *anodes
0361: Node *bnodes
0362: uint64_t *bitset
+0363: if maxnodes:
__pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_maxnodes); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(0, 363, __pyx_L1_error)
if (__pyx_t_1) {
/* … */
goto __pyx_L3;
}
+0364: SLOTS = BITNSLOTS(maxnodes + 1)
__pyx_t_2 = __Pyx_PyInt_AddObjC(__pyx_v_maxnodes, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 364, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __Pyx_PyInt_As_int(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 364, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_SLOTS = BITNSLOTS(__pyx_t_3);
0365: else:
+0366: SLOTS = BITNSLOTS(max(trees1.maxnodes, trees2.maxnodes) + 1)
/*else*/ {
__pyx_t_4 = __pyx_v_trees2->maxnodes;
__pyx_t_5 = __pyx_v_trees1->maxnodes;
if (((__pyx_t_4 > __pyx_t_5) != 0)) {
__pyx_t_6 = __pyx_t_4;
} else {
__pyx_t_6 = __pyx_t_5;
}
__pyx_v_SLOTS = BITNSLOTS((__pyx_t_6 + 1));
}
__pyx_L3:;
+0367: if indices == 0:
switch (__pyx_v_indices) {
case 0:
/* … */
break;
+0368: counts = clone(uintarray, len(bitsets), True)
__pyx_t_2 = ((PyObject *)__pyx_v_8discodop_10_fragments_uintarray);
__Pyx_INCREF(__pyx_t_2);
if (unlikely(__pyx_v_bitsets == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 368, __pyx_L1_error)
}
__pyx_t_7 = PyList_GET_SIZE(__pyx_v_bitsets); if (unlikely(__pyx_t_7 == -1)) __PYX_ERR(0, 368, __pyx_L1_error)
__pyx_t_8 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_2), __pyx_t_7, 1)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 368, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_8);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__Pyx_DECREF_SET(__pyx_v_counts, ((arrayobject *)__pyx_t_8));
__pyx_t_8 = 0;
+0369: countsp = counts.data.as_uints
__pyx_t_9 = __pyx_v_counts->data.as_uints;
__pyx_v_countsp = __pyx_t_9;
+0370: elif indices == 1:
case 1:
/* … */
break;
+0371: theindices = [clone(uintarray, 0, False) for _ in bitsets]
{ /* enter inner scope */
CYTHON_UNUSED PyObject *__pyx_8genexpr2__pyx_v__ = NULL;
__pyx_t_8 = PyList_New(0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 371, __pyx_L6_error)
__Pyx_GOTREF(__pyx_t_8);
if (unlikely(__pyx_v_bitsets == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 371, __pyx_L6_error)
}
__pyx_t_2 = __pyx_v_bitsets; __Pyx_INCREF(__pyx_t_2); __pyx_t_7 = 0;
for (;;) {
if (__pyx_t_7 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_10 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_7); __Pyx_INCREF(__pyx_t_10); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(0, 371, __pyx_L6_error)
#else
__pyx_t_10 = PySequence_ITEM(__pyx_t_2, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 371, __pyx_L6_error)
__Pyx_GOTREF(__pyx_t_10);
#endif
__Pyx_XDECREF_SET(__pyx_8genexpr2__pyx_v__, __pyx_t_10);
__pyx_t_10 = 0;
__pyx_t_10 = ((PyObject *)__pyx_v_8discodop_10_fragments_uintarray);
__Pyx_INCREF(__pyx_t_10);
__pyx_t_11 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_10), 0, 0)); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 371, __pyx_L6_error)
__Pyx_GOTREF(__pyx_t_11);
__Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
if (unlikely(__Pyx_ListComp_Append(__pyx_t_8, (PyObject*)__pyx_t_11))) __PYX_ERR(0, 371, __pyx_L6_error)
__Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__Pyx_XDECREF(__pyx_8genexpr2__pyx_v__);
goto __pyx_L9_exit_scope;
__pyx_L6_error:;
__Pyx_XDECREF(__pyx_8genexpr2__pyx_v__);
goto __pyx_L1_error;
__pyx_L9_exit_scope:;
} /* exit inner scope */
__Pyx_DECREF_SET(__pyx_v_theindices, ((PyObject*)__pyx_t_8));
__pyx_t_8 = 0;
+0372: elif indices == 2:
case 2:
/* … */
break;
default: break;
}
+0373: theindices = [(clone(uintarray, 0, False), clone(shortarray, 0, False))
{ /* enter inner scope */
CYTHON_UNUSED PyObject *__pyx_8genexpr3__pyx_v__ = NULL;
__pyx_t_8 = PyList_New(0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 373, __pyx_L12_error)
__Pyx_GOTREF(__pyx_t_8);
/* … */
__pyx_t_11 = ((PyObject *)__pyx_v_8discodop_10_fragments_uintarray);
__Pyx_INCREF(__pyx_t_11);
__pyx_t_10 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_11), 0, 0)); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 373, __pyx_L12_error)
__Pyx_GOTREF(__pyx_t_10);
__Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
__pyx_t_11 = ((PyObject *)__pyx_v_8discodop_10_fragments_shortarray);
__Pyx_INCREF(__pyx_t_11);
__pyx_t_12 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_11), 0, 0)); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 373, __pyx_L12_error)
__Pyx_GOTREF(__pyx_t_12);
__Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
__pyx_t_11 = PyTuple_New(2); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 373, __pyx_L12_error)
__Pyx_GOTREF(__pyx_t_11);
__Pyx_GIVEREF(__pyx_t_10);
PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_10);
__Pyx_GIVEREF(__pyx_t_12);
PyTuple_SET_ITEM(__pyx_t_11, 1, __pyx_t_12);
__pyx_t_10 = 0;
__pyx_t_12 = 0;
if (unlikely(__Pyx_ListComp_Append(__pyx_t_8, (PyObject*)__pyx_t_11))) __PYX_ERR(0, 373, __pyx_L12_error)
__Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
+0374: for _ in bitsets]
if (unlikely(__pyx_v_bitsets == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 374, __pyx_L12_error)
}
__pyx_t_2 = __pyx_v_bitsets; __Pyx_INCREF(__pyx_t_2); __pyx_t_7 = 0;
for (;;) {
if (__pyx_t_7 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_11 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_7); __Pyx_INCREF(__pyx_t_11); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(0, 374, __pyx_L12_error)
#else
__pyx_t_11 = PySequence_ITEM(__pyx_t_2, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 374, __pyx_L12_error)
__Pyx_GOTREF(__pyx_t_11);
#endif
__Pyx_XDECREF_SET(__pyx_8genexpr3__pyx_v__, __pyx_t_11);
__pyx_t_11 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__Pyx_XDECREF(__pyx_8genexpr3__pyx_v__);
goto __pyx_L15_exit_scope;
__pyx_L12_error:;
__Pyx_XDECREF(__pyx_8genexpr3__pyx_v__);
goto __pyx_L1_error;
__pyx_L15_exit_scope:;
} /* exit inner scope */
__Pyx_DECREF_SET(__pyx_v_theindices, ((PyObject*)__pyx_t_8));
__pyx_t_8 = 0;
0375: # compare one bitset to each tree for each unique fragment.
+0376: for n, wrapper in enumerate(bitsets):
__pyx_t_13 = 0; __pyx_t_8 = __pyx_v_bitsets; __Pyx_INCREF(__pyx_t_8); __pyx_t_7 = 0; for (;;) { if (__pyx_t_7 >= PyList_GET_SIZE(__pyx_t_8)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_2 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_7); __Pyx_INCREF(__pyx_t_2); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(0, 376, __pyx_L1_error) #else __pyx_t_2 = PySequence_ITEM(__pyx_t_8, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 376, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #endif __Pyx_XDECREF_SET(__pyx_v_wrapper, __pyx_t_2); __pyx_t_2 = 0; __pyx_v_n = __pyx_t_13; __pyx_t_13 = (__pyx_t_13 + 1); /* … */ __pyx_L16_continue:; } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+0377: bitset = getpointer(wrapper)
__pyx_v_bitset = __pyx_f_8discodop_10_fragments_getpointer(__pyx_v_wrapper);
+0378: a = &(trees1.trees[getid(bitset, SLOTS)]) # fragment is from this tree
__pyx_v_a = (&(__pyx_v_trees1->trees[__pyx_f_8discodop_10_fragments_getid(__pyx_v_bitset, __pyx_v_SLOTS)]));
+0379: anodes = &trees1.nodes[a.offset]
__pyx_v_anodes = (&(__pyx_v_trees1->nodes[__pyx_v_a->offset]));
+0380: candidates = getcandidates(anodes, bitset, trees2, a.len,
__pyx_t_2 = __pyx_f_8discodop_10_fragments_getcandidates(__pyx_v_anodes, __pyx_v_bitset, __pyx_v_trees2, __pyx_v_a->len, 0, 0, __pyx_v_SLOTS); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 380, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_XDECREF_SET(__pyx_v_candidates, __pyx_t_2);
__pyx_t_2 = 0;
0381: 0, 0, SLOTS)
+0382: if candidates is None: # ran across unseen production
__pyx_t_1 = (__pyx_v_candidates == Py_None);
__pyx_t_14 = (__pyx_t_1 != 0);
if (__pyx_t_14) {
/* … */
}
+0383: continue
goto __pyx_L16_continue;
+0384: if indices == 1:
switch (__pyx_v_indices) {
case 1:
/* … */
break;
+0385: treenums = theindices[n]
if (unlikely(__pyx_v_theindices == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 385, __pyx_L1_error)
}
if (!(likely(((PyList_GET_ITEM(__pyx_v_theindices, __pyx_v_n)) == Py_None) || likely(__Pyx_TypeTest(PyList_GET_ITEM(__pyx_v_theindices, __pyx_v_n), __pyx_ptype_7cpython_5array_array))))) __PYX_ERR(0, 385, __pyx_L1_error)
__pyx_t_2 = PyList_GET_ITEM(__pyx_v_theindices, __pyx_v_n);
__Pyx_INCREF(__pyx_t_2);
__Pyx_DECREF_SET(__pyx_v_treenums, ((arrayobject *)__pyx_t_2));
__pyx_t_2 = 0;
+0386: elif indices == 2:
case 2:
/* … */
break;
default: break;
}
+0387: treenums, nodenums = theindices[n]
if (unlikely(__pyx_v_theindices == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 387, __pyx_L1_error)
}
__pyx_t_2 = PyList_GET_ITEM(__pyx_v_theindices, __pyx_v_n);
__Pyx_INCREF(__pyx_t_2);
if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) {
PyObject* sequence = __pyx_t_2;
#if !CYTHON_COMPILING_IN_PYPY
Py_ssize_t size = Py_SIZE(sequence);
#else
Py_ssize_t size = PySequence_Size(sequence);
#endif
if (unlikely(size != 2)) {
if (size > 2) __Pyx_RaiseTooManyValuesError(2);
else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
__PYX_ERR(0, 387, __pyx_L1_error)
}
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
if (likely(PyTuple_CheckExact(sequence))) {
__pyx_t_11 = PyTuple_GET_ITEM(sequence, 0);
__pyx_t_12 = PyTuple_GET_ITEM(sequence, 1);
} else {
__pyx_t_11 = PyList_GET_ITEM(sequence, 0);
__pyx_t_12 = PyList_GET_ITEM(sequence, 1);
}
__Pyx_INCREF(__pyx_t_11);
__Pyx_INCREF(__pyx_t_12);
#else
__pyx_t_11 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 387, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_11);
__pyx_t_12 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 387, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_12);
#endif
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
} else {
Py_ssize_t index = -1;
__pyx_t_10 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 387, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_10);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__pyx_t_15 = Py_TYPE(__pyx_t_10)->tp_iternext;
index = 0; __pyx_t_11 = __pyx_t_15(__pyx_t_10); if (unlikely(!__pyx_t_11)) goto __pyx_L19_unpacking_failed;
__Pyx_GOTREF(__pyx_t_11);
index = 1; __pyx_t_12 = __pyx_t_15(__pyx_t_10); if (unlikely(!__pyx_t_12)) goto __pyx_L19_unpacking_failed;
__Pyx_GOTREF(__pyx_t_12);
if (__Pyx_IternextUnpackEndCheck(__pyx_t_15(__pyx_t_10), 2) < 0) __PYX_ERR(0, 387, __pyx_L1_error)
__pyx_t_15 = NULL;
__Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
goto __pyx_L20_unpacking_done;
__pyx_L19_unpacking_failed:;
__Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
__pyx_t_15 = NULL;
if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);
__PYX_ERR(0, 387, __pyx_L1_error)
__pyx_L20_unpacking_done:;
}
if (!(likely(((__pyx_t_11) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_11, __pyx_ptype_7cpython_5array_array))))) __PYX_ERR(0, 387, __pyx_L1_error)
if (!(likely(((__pyx_t_12) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_12, __pyx_ptype_7cpython_5array_array))))) __PYX_ERR(0, 387, __pyx_L1_error)
__Pyx_DECREF_SET(__pyx_v_treenums, ((arrayobject *)__pyx_t_11));
__pyx_t_11 = 0;
__Pyx_DECREF_SET(__pyx_v_nodenums, ((arrayobject *)__pyx_t_12));
__pyx_t_12 = 0;
+0388: i = getroot(bitset, SLOTS) # root of fragment in tree 'a'
__pyx_v_i = __pyx_f_8discodop_10_fragments_getroot(__pyx_v_bitset, __pyx_v_SLOTS);
+0389: for m in candidates:
if (likely(PyList_CheckExact(__pyx_v_candidates)) || PyTuple_CheckExact(__pyx_v_candidates)) { __pyx_t_2 = __pyx_v_candidates; __Pyx_INCREF(__pyx_t_2); __pyx_t_16 = 0; __pyx_t_17 = NULL; } else { __pyx_t_16 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_candidates); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 389, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_17 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_17)) __PYX_ERR(0, 389, __pyx_L1_error) } for (;;) { if (likely(!__pyx_t_17)) { if (likely(PyList_CheckExact(__pyx_t_2))) { if (__pyx_t_16 >= PyList_GET_SIZE(__pyx_t_2)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_12 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_16); __Pyx_INCREF(__pyx_t_12); __pyx_t_16++; if (unlikely(0 < 0)) __PYX_ERR(0, 389, __pyx_L1_error) #else __pyx_t_12 = PySequence_ITEM(__pyx_t_2, __pyx_t_16); __pyx_t_16++; if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 389, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_12); #endif } else { if (__pyx_t_16 >= PyTuple_GET_SIZE(__pyx_t_2)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_12 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_16); __Pyx_INCREF(__pyx_t_12); __pyx_t_16++; if (unlikely(0 < 0)) __PYX_ERR(0, 389, __pyx_L1_error) #else __pyx_t_12 = PySequence_ITEM(__pyx_t_2, __pyx_t_16); __pyx_t_16++; if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 389, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_12); #endif } } else { __pyx_t_12 = __pyx_t_17(__pyx_t_2); if (unlikely(!__pyx_t_12)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 389, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_12); } __pyx_t_18 = __Pyx_PyInt_As_uint32_t(__pyx_t_12); if (unlikely((__pyx_t_18 == ((uint32_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 389, __pyx_L1_error) __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; __pyx_v_m = __pyx_t_18; /* … */ } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0390: b = &(trees2.trees[m])
__pyx_v_b = (&(__pyx_v_trees2->trees[__pyx_v_m]));
+0391: bnodes = &trees2.nodes[b.offset]
__pyx_v_bnodes = (&(__pyx_v_trees2->nodes[__pyx_v_b->offset]));
+0392: for j in range(b.len):
__pyx_t_19 = __pyx_v_b->len;
for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_19; __pyx_t_6+=1) {
__pyx_v_j = __pyx_t_6;
+0393: if anodes[i].prod == bnodes[j].prod:
__pyx_t_14 = (((__pyx_v_anodes[__pyx_v_i]).prod == (__pyx_v_bnodes[__pyx_v_j]).prod) != 0);
if (__pyx_t_14) {
/* … */
goto __pyx_L25;
}
+0394: if containsbitset(anodes, bnodes, bitset, i, j):
__pyx_t_14 = (__pyx_f_8discodop_10_fragments_containsbitset(__pyx_v_anodes, __pyx_v_bnodes, __pyx_v_bitset, __pyx_v_i, __pyx_v_j) != 0);
if (__pyx_t_14) {
/* … */
}
+0395: if indices == 0:
switch (__pyx_v_indices) {
case 0:
/* … */
break;
+0396: countsp[n] += 1
__pyx_t_18 = __pyx_v_n;
(__pyx_v_countsp[__pyx_t_18]) = ((__pyx_v_countsp[__pyx_t_18]) + 1);
+0397: elif indices == 1:
case 1:
/* … */
break;
+0398: treenums.append(m)
__pyx_t_12 = __Pyx_PyInt_From_uint32_t(__pyx_v_m); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 398, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_12);
__pyx_t_20 = __Pyx_PyObject_Append(((PyObject *)__pyx_v_treenums), __pyx_t_12); if (unlikely(__pyx_t_20 == -1)) __PYX_ERR(0, 398, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0;
+0399: elif indices == 2:
case 2:
/* … */
break;
default: break;
}
+0400: treenums.append(m)
__pyx_t_12 = __Pyx_PyInt_From_uint32_t(__pyx_v_m); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 400, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_12);
__pyx_t_20 = __Pyx_PyObject_Append(((PyObject *)__pyx_v_treenums), __pyx_t_12); if (unlikely(__pyx_t_20 == -1)) __PYX_ERR(0, 400, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0;
+0401: nodenums.append(j)
__pyx_t_12 = __Pyx_PyInt_From_short(__pyx_v_j); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 401, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_12); __pyx_t_20 = __Pyx_PyObject_Append(((PyObject *)__pyx_v_nodenums), __pyx_t_12); if (unlikely(__pyx_t_20 == -1)) __PYX_ERR(0, 401, __pyx_L1_error) __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0;
+0402: elif anodes[i].prod < bnodes[j].prod:
__pyx_t_14 = (((__pyx_v_anodes[__pyx_v_i]).prod < (__pyx_v_bnodes[__pyx_v_j]).prod) != 0);
if (__pyx_t_14) {
/* … */
}
__pyx_L25:;
}
__pyx_L24_break:;
+0403: break
goto __pyx_L24_break;
+0404: return theindices if indices else counts
__Pyx_XDECREF(__pyx_r); if ((__pyx_v_indices != 0)) { __Pyx_INCREF(__pyx_v_theindices); __pyx_t_8 = __pyx_v_theindices; } else { __Pyx_INCREF(((PyObject *)__pyx_v_counts)); __pyx_t_8 = ((PyObject *)__pyx_v_counts); } __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L0;
0405:
0406:
+0407: cpdef exactcountsslice(Ctrees trees1, Ctrees trees2, list bitsets,
static PyObject *__pyx_pw_8discodop_10_fragments_5exactcountsslice(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static PyObject *__pyx_f_8discodop_10_fragments_exactcountsslice(struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees1, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2, PyObject *__pyx_v_bitsets, CYTHON_UNUSED int __pyx_skip_dispatch, struct __pyx_opt_args_8discodop_10_fragments_exactcountsslice *__pyx_optional_args) {
int __pyx_v_indices = ((int)0);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_10);
__Pyx_XDECREF(__pyx_t_12);
__Pyx_XDECREF(__pyx_t_13);
__Pyx_XDECREF(__pyx_t_14);
__Pyx_AddTraceback("discodop._fragments.exactcountsslice", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF((PyObject *)__pyx_v_counts);
__Pyx_XDECREF((PyObject *)__pyx_v_candidatesarray);
__Pyx_XDECREF((PyObject *)__pyx_v_tmp);
__Pyx_XDECREF((PyObject *)__pyx_v_tmp2);
__Pyx_XDECREF(__pyx_v_theindices);
__Pyx_XDECREF(__pyx_v_candidates);
__Pyx_XDECREF(__pyx_v_wrapper);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_5exactcountsslice(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static char __pyx_doc_8discodop_10_fragments_4exactcountsslice[] = "exactcountsslice(Ctrees trees1, Ctrees trees2, list bitsets, int indices=0, maxnodes=None, start=None, end=None, maxresults=None)\nGet counts of fragments in a slice of the treebank.\n\n\tVariant of exactcounts() that releases the GIL in the inner loop and is\n\tintended for searching in subsets of ``trees2``.\n\n\t:param start, end: only search through this interval of trees from\n\t\t``trees2`` (defaults to all trees).\n\t:param maxresults: stop searching after this number of matchs.\n\t:returns: depending on ``indices``:\n\n\t\t:0: an array of counts, corresponding to ``bitsets``.\n\t\t:1: a list of arrays, each array being a sequence of indices\n\t\t\tfor the corresponding bitset.\n\t\t:2: a list of pairs of arrays, tree indices paired with node numbers.\n\t";
static PyObject *__pyx_pw_8discodop_10_fragments_5exactcountsslice(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees1 = 0;
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2 = 0;
PyObject *__pyx_v_bitsets = 0;
int __pyx_v_indices;
PyObject *__pyx_v_maxnodes = 0;
PyObject *__pyx_v_start = 0;
PyObject *__pyx_v_end = 0;
PyObject *__pyx_v_maxresults = 0;
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("exactcountsslice (wrapper)", 0);
{
static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_trees1,&__pyx_n_s_trees2,&__pyx_n_s_bitsets,&__pyx_n_s_indices,&__pyx_n_s_maxnodes,&__pyx_n_s_start,&__pyx_n_s_end,&__pyx_n_s_maxresults,0};
PyObject* values[8] = {0,0,0,0,0,0,0,0};
/* … */
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__pyx_r = NULL;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_8discodop_10_fragments_4exactcountsslice(CYTHON_UNUSED PyObject *__pyx_self, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees1, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2, PyObject *__pyx_v_bitsets, int __pyx_v_indices, PyObject *__pyx_v_maxnodes, PyObject *__pyx_v_start, PyObject *__pyx_v_end, PyObject *__pyx_v_maxresults) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("exactcountsslice", 0);
__Pyx_XDECREF(__pyx_r);
__pyx_t_2.__pyx_n = 5;
__pyx_t_2.indices = __pyx_v_indices;
__pyx_t_2.maxnodes = __pyx_v_maxnodes;
__pyx_t_2.start = __pyx_v_start;
__pyx_t_2.end = __pyx_v_end;
__pyx_t_2.maxresults = __pyx_v_maxresults;
__pyx_t_1 = __pyx_f_8discodop_10_fragments_exactcountsslice(__pyx_v_trees1, __pyx_v_trees2, __pyx_v_bitsets, 0, &__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 407, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_r = __pyx_t_1;
__pyx_t_1 = 0;
goto __pyx_L0;
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_AddTraceback("discodop._fragments.exactcountsslice", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
struct __pyx_opt_args_8discodop_10_fragments_exactcountsslice {
int __pyx_n;
int indices;
PyObject *maxnodes;
PyObject *start;
PyObject *end;
PyObject *maxresults;
};
+0408: int indices=0, maxnodes=None, start=None, end=None, maxresults=None):
PyObject *__pyx_v_maxnodes = ((PyObject *)Py_None); PyObject *__pyx_v_start = ((PyObject *)Py_None); PyObject *__pyx_v_end = ((PyObject *)Py_None); PyObject *__pyx_v_maxresults = ((PyObject *)Py_None); arrayobject *__pyx_v_counts = 0; arrayobject *__pyx_v_candidatesarray = 0; arrayobject *__pyx_v_tmp = 0; arrayobject *__pyx_v_tmp2 = 0; PyObject *__pyx_v_theindices = 0; PyObject *__pyx_v_candidates = 0; short __pyx_v_i; short __pyx_v_SLOTS; size_t __pyx_v_nummatches; size_t __pyx_v_allocated; size_t __pyx_v_maxresults_; long __pyx_v_cnt; uint32_t __pyx_v_n; uint32_t __pyx_v_numcandidates; uint32_t *__pyx_v_countsp; uint32_t *__pyx_v_treenums; short *__pyx_v_nodenums; struct __pyx_t_8discodop_10containers_NodeArray *__pyx_v_a; struct __pyx_t_8discodop_10containers_Node *__pyx_v_anodes; uint64_t *__pyx_v_bitset; int __pyx_v_start_; int __pyx_v_end_; PyObject *__pyx_v_wrapper = NULL; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("exactcountsslice", 0); if (__pyx_optional_args) { if (__pyx_optional_args->__pyx_n > 0) { __pyx_v_indices = __pyx_optional_args->indices; if (__pyx_optional_args->__pyx_n > 1) { __pyx_v_maxnodes = __pyx_optional_args->maxnodes; if (__pyx_optional_args->__pyx_n > 2) { __pyx_v_start = __pyx_optional_args->start; if (__pyx_optional_args->__pyx_n > 3) { __pyx_v_end = __pyx_optional_args->end; if (__pyx_optional_args->__pyx_n > 4) { __pyx_v_maxresults = __pyx_optional_args->maxresults; } } } } } } /* … */ values[4] = ((PyObject *)Py_None); values[5] = ((PyObject *)Py_None); values[6] = ((PyObject *)Py_None); values[7] = ((PyObject *)Py_None); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); CYTHON_FALLTHROUGH; case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); CYTHON_FALLTHROUGH; case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); CYTHON_FALLTHROUGH; case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); CYTHON_FALLTHROUGH; case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_trees1)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_trees2)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("exactcountsslice", 0, 3, 8, 1); __PYX_ERR(0, 407, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_bitsets)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("exactcountsslice", 0, 3, 8, 2); __PYX_ERR(0, 407, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 3: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_indices); if (value) { values[3] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 4: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_maxnodes); if (value) { values[4] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 5: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_start); if (value) { values[5] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 6: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_end); if (value) { values[6] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 7: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_maxresults); if (value) { values[7] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "exactcountsslice") < 0)) __PYX_ERR(0, 407, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); CYTHON_FALLTHROUGH; case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); CYTHON_FALLTHROUGH; case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); CYTHON_FALLTHROUGH; case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); CYTHON_FALLTHROUGH; case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_trees1 = ((struct __pyx_obj_8discodop_10containers_Ctrees *)values[0]); __pyx_v_trees2 = ((struct __pyx_obj_8discodop_10containers_Ctrees *)values[1]); __pyx_v_bitsets = ((PyObject*)values[2]); if (values[3]) { __pyx_v_indices = __Pyx_PyInt_As_int(values[3]); if (unlikely((__pyx_v_indices == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 408, __pyx_L3_error) } else { __pyx_v_indices = ((int)0); } __pyx_v_maxnodes = values[4]; __pyx_v_start = values[5]; __pyx_v_end = values[6]; __pyx_v_maxresults = values[7]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("exactcountsslice", 0, 3, 8, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 407, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("discodop._fragments.exactcountsslice", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_trees1), __pyx_ptype_8discodop_10containers_Ctrees, 1, "trees1", 0))) __PYX_ERR(0, 407, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_trees2), __pyx_ptype_8discodop_10containers_Ctrees, 1, "trees2", 0))) __PYX_ERR(0, 407, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_bitsets), (&PyList_Type), 1, "bitsets", 1))) __PYX_ERR(0, 407, __pyx_L1_error) __pyx_r = __pyx_pf_8discodop_10_fragments_4exactcountsslice(__pyx_self, __pyx_v_trees1, __pyx_v_trees2, __pyx_v_bitsets, __pyx_v_indices, __pyx_v_maxnodes, __pyx_v_start, __pyx_v_end, __pyx_v_maxresults);
0409: """Get counts of fragments in a slice of the treebank.
0410:
0411: Variant of exactcounts() that releases the GIL in the inner loop and is
0412: intended for searching in subsets of ``trees2``.
0413:
0414: :param start, end: only search through this interval of trees from
0415: ``trees2`` (defaults to all trees).
0416: :param maxresults: stop searching after this number of matchs.
0417: :returns: depending on ``indices``:
0418:
0419: :0: an array of counts, corresponding to ``bitsets``.
0420: :1: a list of arrays, each array being a sequence of indices
0421: for the corresponding bitset.
0422: :2: a list of pairs of arrays, tree indices paired with node numbers.
0423: """
0424: cdef:
+0425: array counts = None
__Pyx_INCREF(Py_None);
__pyx_v_counts = ((arrayobject *)Py_None);
+0426: array candidatesarray = None
__Pyx_INCREF(Py_None);
__pyx_v_candidatesarray = ((arrayobject *)Py_None);
+0427: array tmp = None, tmp2 = None
__Pyx_INCREF(Py_None); __pyx_v_tmp = ((arrayobject *)Py_None); __Pyx_INCREF(Py_None); __pyx_v_tmp2 = ((arrayobject *)Py_None);
+0428: list theindices = None
__Pyx_INCREF(Py_None);
__pyx_v_theindices = ((PyObject*)Py_None);
0429: object candidates # RoaringBitmap
0430: short i, SLOTS
+0431: size_t nummatches = 0, allocated = 1024
__pyx_v_nummatches = 0; __pyx_v_allocated = 0x400;
+0432: size_t maxresults_ = maxresults or SIZE_MAX
__pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_maxresults); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 432, __pyx_L1_error) if (!__pyx_t_2) { } else { __pyx_t_3 = __Pyx_PyInt_As_size_t(__pyx_v_maxresults); if (unlikely((__pyx_t_3 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 432, __pyx_L1_error) __pyx_t_1 = __pyx_t_3; goto __pyx_L3_bool_binop_done; } __pyx_t_1 = SIZE_MAX; __pyx_L3_bool_binop_done:; __pyx_v_maxresults_ = __pyx_t_1;
+0433: long cnt = 0
__pyx_v_cnt = 0;
0434: uint32_t n, numcandidates
+0435: uint32_t *countsp = NULL
__pyx_v_countsp = NULL;
+0436: uint32_t *treenums = NULL
__pyx_v_treenums = NULL;
+0437: short *nodenums = NULL
__pyx_v_nodenums = NULL;
0438: NodeArray *a
0439: Node *anodes
0440: uint64_t *bitset
+0441: int start_ = start or 0, end_ = end or trees2.len
__pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_start); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 441, __pyx_L1_error) if (!__pyx_t_2) { } else { __pyx_t_5 = __Pyx_PyInt_As_int(__pyx_v_start); if (unlikely((__pyx_t_5 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 441, __pyx_L1_error) __pyx_t_4 = __pyx_t_5; goto __pyx_L5_bool_binop_done; } __pyx_t_4 = 0; __pyx_L5_bool_binop_done:; __pyx_v_start_ = __pyx_t_4; __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_end); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 441, __pyx_L1_error) if (!__pyx_t_2) { } else { __pyx_t_5 = __Pyx_PyInt_As_int(__pyx_v_end); if (unlikely((__pyx_t_5 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 441, __pyx_L1_error) __pyx_t_4 = __pyx_t_5; goto __pyx_L7_bool_binop_done; } __pyx_t_4 = __pyx_v_trees2->len; __pyx_L7_bool_binop_done:; __pyx_v_end_ = __pyx_t_4;
+0442: if maxnodes:
__pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_maxnodes); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 442, __pyx_L1_error)
if (__pyx_t_2) {
/* … */
goto __pyx_L9;
}
+0443: SLOTS = BITNSLOTS(maxnodes + 1)
__pyx_t_6 = __Pyx_PyInt_AddObjC(__pyx_v_maxnodes, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 443, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_4 = __Pyx_PyInt_As_int(__pyx_t_6); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 443, __pyx_L1_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_v_SLOTS = BITNSLOTS(__pyx_t_4);
0444: else:
+0445: SLOTS = BITNSLOTS(max(trees1.maxnodes, trees2.maxnodes) + 1)
/*else*/ {
__pyx_t_7 = __pyx_v_trees2->maxnodes;
__pyx_t_8 = __pyx_v_trees1->maxnodes;
if (((__pyx_t_7 > __pyx_t_8) != 0)) {
__pyx_t_9 = __pyx_t_7;
} else {
__pyx_t_9 = __pyx_t_8;
}
__pyx_v_SLOTS = BITNSLOTS((__pyx_t_9 + 1));
}
__pyx_L9:;
+0446: if indices:
__pyx_t_2 = (__pyx_v_indices != 0);
if (__pyx_t_2) {
/* … */
goto __pyx_L10;
}
+0447: treenums = <uint32_t *>malloc(allocated * sizeof(int))
__pyx_v_treenums = ((uint32_t *)malloc((__pyx_v_allocated * (sizeof(int)))));
+0448: if treenums is NULL:
__pyx_t_2 = ((__pyx_v_treenums == NULL) != 0);
if (__pyx_t_2) {
/* … */
}
+0449: raise MemoryError
PyErr_NoMemory(); __PYX_ERR(0, 449, __pyx_L1_error)
+0450: if indices == 1:
switch (__pyx_v_indices) {
case 1:
/* … */
break;
+0451: theindices = [clone(uintarray, 0, False) for _ in bitsets]
{ /* enter inner scope */
CYTHON_UNUSED PyObject *__pyx_8genexpr4__pyx_v__ = NULL;
__pyx_t_6 = PyList_New(0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 451, __pyx_L14_error)
__Pyx_GOTREF(__pyx_t_6);
if (unlikely(__pyx_v_bitsets == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 451, __pyx_L14_error)
}
__pyx_t_10 = __pyx_v_bitsets; __Pyx_INCREF(__pyx_t_10); __pyx_t_11 = 0;
for (;;) {
if (__pyx_t_11 >= PyList_GET_SIZE(__pyx_t_10)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_12 = PyList_GET_ITEM(__pyx_t_10, __pyx_t_11); __Pyx_INCREF(__pyx_t_12); __pyx_t_11++; if (unlikely(0 < 0)) __PYX_ERR(0, 451, __pyx_L14_error)
#else
__pyx_t_12 = PySequence_ITEM(__pyx_t_10, __pyx_t_11); __pyx_t_11++; if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 451, __pyx_L14_error)
__Pyx_GOTREF(__pyx_t_12);
#endif
__Pyx_XDECREF_SET(__pyx_8genexpr4__pyx_v__, __pyx_t_12);
__pyx_t_12 = 0;
__pyx_t_12 = ((PyObject *)__pyx_v_8discodop_10_fragments_uintarray);
__Pyx_INCREF(__pyx_t_12);
__pyx_t_13 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_12), 0, 0)); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 451, __pyx_L14_error)
__Pyx_GOTREF(__pyx_t_13);
__Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0;
if (unlikely(__Pyx_ListComp_Append(__pyx_t_6, (PyObject*)__pyx_t_13))) __PYX_ERR(0, 451, __pyx_L14_error)
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
}
__Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
__Pyx_XDECREF(__pyx_8genexpr4__pyx_v__);
goto __pyx_L17_exit_scope;
__pyx_L14_error:;
__Pyx_XDECREF(__pyx_8genexpr4__pyx_v__);
goto __pyx_L1_error;
__pyx_L17_exit_scope:;
} /* exit inner scope */
__Pyx_DECREF_SET(__pyx_v_theindices, ((PyObject*)__pyx_t_6));
__pyx_t_6 = 0;
+0452: elif indices == 2:
case 2:
/* … */
break;
default: break;
}
+0453: theindices = [
{ /* enter inner scope */
CYTHON_UNUSED PyObject *__pyx_8genexpr5__pyx_v__ = NULL;
__pyx_t_6 = PyList_New(0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 453, __pyx_L20_error)
__Pyx_GOTREF(__pyx_t_6);
+0454: (clone(uintarray, 0, False), clone(shortarray, 0, False))
__pyx_t_13 = ((PyObject *)__pyx_v_8discodop_10_fragments_uintarray);
__Pyx_INCREF(__pyx_t_13);
__pyx_t_12 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_13), 0, 0)); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 454, __pyx_L20_error)
__Pyx_GOTREF(__pyx_t_12);
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
__pyx_t_13 = ((PyObject *)__pyx_v_8discodop_10_fragments_shortarray);
__Pyx_INCREF(__pyx_t_13);
__pyx_t_14 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_13), 0, 0)); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 454, __pyx_L20_error)
__Pyx_GOTREF(__pyx_t_14);
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
__pyx_t_13 = PyTuple_New(2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 454, __pyx_L20_error)
__Pyx_GOTREF(__pyx_t_13);
__Pyx_GIVEREF(__pyx_t_12);
PyTuple_SET_ITEM(__pyx_t_13, 0, __pyx_t_12);
__Pyx_GIVEREF(__pyx_t_14);
PyTuple_SET_ITEM(__pyx_t_13, 1, __pyx_t_14);
__pyx_t_12 = 0;
__pyx_t_14 = 0;
if (unlikely(__Pyx_ListComp_Append(__pyx_t_6, (PyObject*)__pyx_t_13))) __PYX_ERR(0, 453, __pyx_L20_error)
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
+0455: for _ in bitsets]
if (unlikely(__pyx_v_bitsets == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 455, __pyx_L20_error)
}
__pyx_t_10 = __pyx_v_bitsets; __Pyx_INCREF(__pyx_t_10); __pyx_t_11 = 0;
for (;;) {
if (__pyx_t_11 >= PyList_GET_SIZE(__pyx_t_10)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_13 = PyList_GET_ITEM(__pyx_t_10, __pyx_t_11); __Pyx_INCREF(__pyx_t_13); __pyx_t_11++; if (unlikely(0 < 0)) __PYX_ERR(0, 455, __pyx_L20_error)
#else
__pyx_t_13 = PySequence_ITEM(__pyx_t_10, __pyx_t_11); __pyx_t_11++; if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 455, __pyx_L20_error)
__Pyx_GOTREF(__pyx_t_13);
#endif
__Pyx_XDECREF_SET(__pyx_8genexpr5__pyx_v__, __pyx_t_13);
__pyx_t_13 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
__Pyx_XDECREF(__pyx_8genexpr5__pyx_v__);
goto __pyx_L23_exit_scope;
__pyx_L20_error:;
__Pyx_XDECREF(__pyx_8genexpr5__pyx_v__);
goto __pyx_L1_error;
__pyx_L23_exit_scope:;
} /* exit inner scope */
__Pyx_DECREF_SET(__pyx_v_theindices, ((PyObject*)__pyx_t_6));
__pyx_t_6 = 0;
+0456: nodenums = <short *>malloc(allocated * sizeof(int))
__pyx_v_nodenums = ((short *)malloc((__pyx_v_allocated * (sizeof(int)))));
+0457: if nodenums is NULL:
__pyx_t_2 = ((__pyx_v_nodenums == NULL) != 0);
if (__pyx_t_2) {
/* … */
}
+0458: raise MemoryError
PyErr_NoMemory(); __PYX_ERR(0, 458, __pyx_L1_error)
0459: else:
+0460: counts = clone(uintarray, len(bitsets), True)
/*else*/ {
__pyx_t_6 = ((PyObject *)__pyx_v_8discodop_10_fragments_uintarray);
__Pyx_INCREF(__pyx_t_6);
if (unlikely(__pyx_v_bitsets == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 460, __pyx_L1_error)
}
__pyx_t_11 = PyList_GET_SIZE(__pyx_v_bitsets); if (unlikely(__pyx_t_11 == -1)) __PYX_ERR(0, 460, __pyx_L1_error)
__pyx_t_10 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_6), __pyx_t_11, 1)); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 460, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_10);
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
__Pyx_DECREF_SET(__pyx_v_counts, ((arrayobject *)__pyx_t_10));
__pyx_t_10 = 0;
+0461: countsp = counts.data.as_uints
__pyx_t_15 = __pyx_v_counts->data.as_uints;
__pyx_v_countsp = __pyx_t_15;
}
__pyx_L10:;
+0462: candidatesarray = clone(uintarray, 0, False)
__pyx_t_10 = ((PyObject *)__pyx_v_8discodop_10_fragments_uintarray); __Pyx_INCREF(__pyx_t_10); __pyx_t_6 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_10), 0, 0)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 462, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; __Pyx_DECREF_SET(__pyx_v_candidatesarray, ((arrayobject *)__pyx_t_6)); __pyx_t_6 = 0;
0463: # compare one bitset to each tree for each unique fragment.
+0464: for n, wrapper in enumerate(bitsets):
__pyx_t_16 = 0; __pyx_t_6 = __pyx_v_bitsets; __Pyx_INCREF(__pyx_t_6); __pyx_t_11 = 0; for (;;) { if (__pyx_t_11 >= PyList_GET_SIZE(__pyx_t_6)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_10 = PyList_GET_ITEM(__pyx_t_6, __pyx_t_11); __Pyx_INCREF(__pyx_t_10); __pyx_t_11++; if (unlikely(0 < 0)) __PYX_ERR(0, 464, __pyx_L1_error) #else __pyx_t_10 = PySequence_ITEM(__pyx_t_6, __pyx_t_11); __pyx_t_11++; if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 464, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); #endif __Pyx_XDECREF_SET(__pyx_v_wrapper, __pyx_t_10); __pyx_t_10 = 0; __pyx_v_n = __pyx_t_16; __pyx_t_16 = (__pyx_t_16 + 1); /* … */ __pyx_L25_continue:; } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+0465: bitset = getpointer(wrapper)
__pyx_v_bitset = __pyx_f_8discodop_10_fragments_getpointer(__pyx_v_wrapper);
+0466: a = &(trees1.trees[getid(bitset, SLOTS)]) # fragment is from this tree
__pyx_v_a = (&(__pyx_v_trees1->trees[__pyx_f_8discodop_10_fragments_getid(__pyx_v_bitset, __pyx_v_SLOTS)]));
+0467: anodes = &trees1.nodes[a.offset]
__pyx_v_anodes = (&(__pyx_v_trees1->nodes[__pyx_v_a->offset]));
+0468: candidates = getcandidates(anodes, bitset, trees2, a.len,
__pyx_t_10 = __pyx_f_8discodop_10_fragments_getcandidates(__pyx_v_anodes, __pyx_v_bitset, __pyx_v_trees2, __pyx_v_a->len, __pyx_v_start_, __pyx_v_end_, __pyx_v_SLOTS); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 468, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_10);
__Pyx_XDECREF_SET(__pyx_v_candidates, __pyx_t_10);
__pyx_t_10 = 0;
0469: start_, end_, SLOTS)
+0470: if candidates is None: # ran across unseen production
__pyx_t_2 = (__pyx_v_candidates == Py_None);
__pyx_t_17 = (__pyx_t_2 != 0);
if (__pyx_t_17) {
/* … */
}
+0471: continue
goto __pyx_L25_continue;
+0472: candidatesarray.extend(candidates)
__pyx_t_13 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_candidatesarray), __pyx_n_s_extend); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 472, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __pyx_t_14 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_13))) { __pyx_t_14 = PyMethod_GET_SELF(__pyx_t_13); if (likely(__pyx_t_14)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_13); __Pyx_INCREF(__pyx_t_14); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_13, function); } } if (!__pyx_t_14) { __pyx_t_10 = __Pyx_PyObject_CallOneArg(__pyx_t_13, __pyx_v_candidates); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 472, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_13)) { PyObject *__pyx_temp[2] = {__pyx_t_14, __pyx_v_candidates}; __pyx_t_10 = __Pyx_PyFunction_FastCall(__pyx_t_13, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 472, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; __Pyx_GOTREF(__pyx_t_10); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_13)) { PyObject *__pyx_temp[2] = {__pyx_t_14, __pyx_v_candidates}; __pyx_t_10 = __Pyx_PyCFunction_FastCall(__pyx_t_13, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 472, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; __Pyx_GOTREF(__pyx_t_10); } else #endif { __pyx_t_12 = PyTuple_New(1+1); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 472, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_12, 0, __pyx_t_14); __pyx_t_14 = NULL; __Pyx_INCREF(__pyx_v_candidates); __Pyx_GIVEREF(__pyx_v_candidates); PyTuple_SET_ITEM(__pyx_t_12, 0+1, __pyx_v_candidates); __pyx_t_10 = __Pyx_PyObject_Call(__pyx_t_13, __pyx_t_12, NULL); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 472, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; } } __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
+0473: numcandidates = len(candidatesarray)
if (unlikely(((PyObject *)__pyx_v_candidatesarray) == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 473, __pyx_L1_error)
}
__pyx_t_18 = Py_SIZE(((PyObject *)__pyx_v_candidatesarray)); if (unlikely(__pyx_t_18 == -1)) __PYX_ERR(0, 473, __pyx_L1_error)
__pyx_v_numcandidates = __pyx_t_18;
+0474: if indices == 1:
switch (__pyx_v_indices) {
case 1:
/* … */
break;
+0475: tmp = theindices[n]
if (unlikely(__pyx_v_theindices == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 475, __pyx_L1_error)
}
if (!(likely(((PyList_GET_ITEM(__pyx_v_theindices, __pyx_v_n)) == Py_None) || likely(__Pyx_TypeTest(PyList_GET_ITEM(__pyx_v_theindices, __pyx_v_n), __pyx_ptype_7cpython_5array_array))))) __PYX_ERR(0, 475, __pyx_L1_error)
__pyx_t_10 = PyList_GET_ITEM(__pyx_v_theindices, __pyx_v_n);
__Pyx_INCREF(__pyx_t_10);
__Pyx_DECREF_SET(__pyx_v_tmp, ((arrayobject *)__pyx_t_10));
__pyx_t_10 = 0;
+0476: elif indices == 2:
case 2:
/* … */
break;
default: break;
}
+0477: tmp, tmp2 = theindices[n]
if (unlikely(__pyx_v_theindices == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 477, __pyx_L1_error)
}
__pyx_t_10 = PyList_GET_ITEM(__pyx_v_theindices, __pyx_v_n);
__Pyx_INCREF(__pyx_t_10);
if ((likely(PyTuple_CheckExact(__pyx_t_10))) || (PyList_CheckExact(__pyx_t_10))) {
PyObject* sequence = __pyx_t_10;
#if !CYTHON_COMPILING_IN_PYPY
Py_ssize_t size = Py_SIZE(sequence);
#else
Py_ssize_t size = PySequence_Size(sequence);
#endif
if (unlikely(size != 2)) {
if (size > 2) __Pyx_RaiseTooManyValuesError(2);
else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
__PYX_ERR(0, 477, __pyx_L1_error)
}
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
if (likely(PyTuple_CheckExact(sequence))) {
__pyx_t_13 = PyTuple_GET_ITEM(sequence, 0);
__pyx_t_12 = PyTuple_GET_ITEM(sequence, 1);
} else {
__pyx_t_13 = PyList_GET_ITEM(sequence, 0);
__pyx_t_12 = PyList_GET_ITEM(sequence, 1);
}
__Pyx_INCREF(__pyx_t_13);
__Pyx_INCREF(__pyx_t_12);
#else
__pyx_t_13 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 477, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
__pyx_t_12 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 477, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_12);
#endif
__Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
} else {
Py_ssize_t index = -1;
__pyx_t_14 = PyObject_GetIter(__pyx_t_10); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 477, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_14);
__Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
__pyx_t_19 = Py_TYPE(__pyx_t_14)->tp_iternext;
index = 0; __pyx_t_13 = __pyx_t_19(__pyx_t_14); if (unlikely(!__pyx_t_13)) goto __pyx_L28_unpacking_failed;
__Pyx_GOTREF(__pyx_t_13);
index = 1; __pyx_t_12 = __pyx_t_19(__pyx_t_14); if (unlikely(!__pyx_t_12)) goto __pyx_L28_unpacking_failed;
__Pyx_GOTREF(__pyx_t_12);
if (__Pyx_IternextUnpackEndCheck(__pyx_t_19(__pyx_t_14), 2) < 0) __PYX_ERR(0, 477, __pyx_L1_error)
__pyx_t_19 = NULL;
__Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
goto __pyx_L29_unpacking_done;
__pyx_L28_unpacking_failed:;
__Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
__pyx_t_19 = NULL;
if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);
__PYX_ERR(0, 477, __pyx_L1_error)
__pyx_L29_unpacking_done:;
}
if (!(likely(((__pyx_t_13) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_13, __pyx_ptype_7cpython_5array_array))))) __PYX_ERR(0, 477, __pyx_L1_error)
if (!(likely(((__pyx_t_12) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_12, __pyx_ptype_7cpython_5array_array))))) __PYX_ERR(0, 477, __pyx_L1_error)
__Pyx_DECREF_SET(__pyx_v_tmp, ((arrayobject *)__pyx_t_13));
__pyx_t_13 = 0;
__Pyx_DECREF_SET(__pyx_v_tmp2, ((arrayobject *)__pyx_t_12));
__pyx_t_12 = 0;
+0478: i = getroot(bitset, SLOTS) # root of fragment in tree 'a'
__pyx_v_i = __pyx_f_8discodop_10_fragments_getroot(__pyx_v_bitset, __pyx_v_SLOTS);
+0479: with nogil:
{
#ifdef WITH_THREAD
PyThreadState *_save;
Py_UNBLOCK_THREADS
__Pyx_FastGIL_Remember();
#endif
/*try:*/ {
/* … */
/*finally:*/ {
/*normal exit:*/{
#ifdef WITH_THREAD
__Pyx_FastGIL_Forget();
Py_BLOCK_THREADS
#endif
goto __pyx_L34;
}
__pyx_L34:;
}
}
+0480: cnt = countbitset(bitset, anodes, i, n,
__pyx_v_cnt = __pyx_f_8discodop_10_fragments_countbitset(__pyx_v_bitset, __pyx_v_anodes, __pyx_v_i, __pyx_v_n, __pyx_v_candidatesarray->data.as_uints, __pyx_v_numcandidates, __pyx_v_indices, __pyx_v_trees2->trees, __pyx_v_trees2->nodes, __pyx_v_maxresults_, (&__pyx_v_nummatches), (&__pyx_v_allocated), __pyx_v_countsp, (&__pyx_v_treenums), (&__pyx_v_nodenums));
}
0481: candidatesarray.data.as_uints,
0482: numcandidates, indices, trees2.trees, trees2.nodes,
0483: maxresults_, &nummatches, &allocated,
0484: countsp, &treenums, &nodenums)
+0485: if cnt == -1:
__pyx_t_17 = ((__pyx_v_cnt == -1L) != 0);
if (__pyx_t_17) {
/* … */
}
+0486: raise MemoryError
PyErr_NoMemory(); __PYX_ERR(0, 486, __pyx_L1_error)
+0487: if indices == 1:
switch (__pyx_v_indices) {
case 1:
/* … */
break;
+0488: extend_buffer(tmp, <char *>treenums, cnt)
__pyx_t_4 = __pyx_f_7cpython_5array_extend_buffer(__pyx_v_tmp, ((char *)__pyx_v_treenums), __pyx_v_cnt); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 488, __pyx_L1_error)
+0489: elif indices == 2:
case 2:
/* … */
break;
default: break;
}
+0490: extend_buffer(tmp, <char *>treenums, cnt)
__pyx_t_4 = __pyx_f_7cpython_5array_extend_buffer(__pyx_v_tmp, ((char *)__pyx_v_treenums), __pyx_v_cnt); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 490, __pyx_L1_error)
+0491: extend_buffer(tmp2, <char *>nodenums, cnt)
__pyx_t_4 = __pyx_f_7cpython_5array_extend_buffer(__pyx_v_tmp2, ((char *)__pyx_v_nodenums), __pyx_v_cnt); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 491, __pyx_L1_error)
+0492: resize(candidatesarray, 0)
__pyx_t_4 = resize(__pyx_v_candidatesarray, 0); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 492, __pyx_L1_error)
+0493: if indices:
__pyx_t_17 = (__pyx_v_indices != 0);
if (__pyx_t_17) {
/* … */
}
+0494: free(treenums)
free(__pyx_v_treenums);
+0495: if indices == 2:
__pyx_t_17 = ((__pyx_v_indices == 2) != 0);
if (__pyx_t_17) {
/* … */
}
+0496: free(nodenums)
free(__pyx_v_nodenums);
+0497: return theindices if indices else counts
__Pyx_XDECREF(__pyx_r); if ((__pyx_v_indices != 0)) { __Pyx_INCREF(__pyx_v_theindices); __pyx_t_6 = __pyx_v_theindices; } else { __Pyx_INCREF(((PyObject *)__pyx_v_counts)); __pyx_t_6 = ((PyObject *)__pyx_v_counts); } __pyx_r = __pyx_t_6; __pyx_t_6 = 0; goto __pyx_L0;
0498:
0499:
+0500: cdef int countbitset(uint64_t *bitset, Node *anodes, int i, int n,
static int __pyx_f_8discodop_10_fragments_countbitset(uint64_t *__pyx_v_bitset, struct __pyx_t_8discodop_10containers_Node *__pyx_v_anodes, int __pyx_v_i, int __pyx_v_n, uint32_t *__pyx_v_candidatesarrayp, int __pyx_v_numcandidates, int __pyx_v_indices, struct __pyx_t_8discodop_10containers_NodeArray *__pyx_v_trees, struct __pyx_t_8discodop_10containers_Node *__pyx_v_nodes, size_t __pyx_v_maxresults, size_t *__pyx_v_nummatches, size_t *__pyx_v_allocated, uint32_t *__pyx_v_countsp, uint32_t **__pyx_v_treenums, short **__pyx_v_nodenums) {
int __pyx_v_cnt;
struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_b;
struct __pyx_t_8discodop_10containers_Node *__pyx_v_bnodes;
int __pyx_v_x;
uint32_t __pyx_v_m;
int16_t __pyx_v_j;
int __pyx_r;
/* … */
/* function exit code */
__pyx_L0:;
return __pyx_r;
}
0501: uint32_t *candidatesarrayp, int numcandidates, int indices,
0502: NodeArray *trees, Node *nodes, size_t maxresults, size_t *nummatches,
0503: size_t *allocated, uint32_t *countsp, uint32_t **treenums,
0504: short **nodenums) nogil:
0505: """Internal function to process a single fragment."""
+0506: cdef int cnt = 0
__pyx_v_cnt = 0;
0507: cdef NodeArray b
0508: cdef Node *bnodes
+0509: for x in range(numcandidates):
__pyx_t_1 = __pyx_v_numcandidates;
for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) {
__pyx_v_x = __pyx_t_2;
+0510: m = candidatesarrayp[x]
__pyx_v_m = (__pyx_v_candidatesarrayp[__pyx_v_x]);
+0511: b = trees[m]
__pyx_v_b = (__pyx_v_trees[__pyx_v_m]);
+0512: bnodes = &nodes[b.offset]
__pyx_v_bnodes = (&(__pyx_v_nodes[__pyx_v_b.offset]));
+0513: for j in range(b.len):
__pyx_t_3 = __pyx_v_b.len;
for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) {
__pyx_v_j = __pyx_t_4;
+0514: if anodes[i].prod == bnodes[j].prod:
__pyx_t_5 = (((__pyx_v_anodes[__pyx_v_i]).prod == (__pyx_v_bnodes[__pyx_v_j]).prod) != 0);
if (__pyx_t_5) {
/* … */
goto __pyx_L7;
}
+0515: if containsbitset(anodes, bnodes, bitset, i, j):
__pyx_t_5 = (__pyx_f_8discodop_10_fragments_containsbitset(__pyx_v_anodes, __pyx_v_bnodes, __pyx_v_bitset, __pyx_v_i, __pyx_v_j) != 0);
if (__pyx_t_5) {
/* … */
}
+0516: if indices == 0:
switch (__pyx_v_indices) {
case 0:
/* … */
break;
+0517: countsp[n] += 1
__pyx_t_6 = __pyx_v_n;
(__pyx_v_countsp[__pyx_t_6]) = ((__pyx_v_countsp[__pyx_t_6]) + 1);
+0518: elif indices == 1:
case 1:
/* … */
break;
+0519: treenums[0][cnt] = m
((__pyx_v_treenums[0])[__pyx_v_cnt]) = __pyx_v_m;
+0520: elif indices == 2:
case 2:
/* … */
break;
default: break;
}
+0521: treenums[0][cnt] = m
((__pyx_v_treenums[0])[__pyx_v_cnt]) = __pyx_v_m;
+0522: nodenums[0][cnt] = j
((__pyx_v_nodenums[0])[__pyx_v_cnt]) = __pyx_v_j;
+0523: nummatches[0] += 1
__pyx_t_7 = 0;
(__pyx_v_nummatches[__pyx_t_7]) = ((__pyx_v_nummatches[__pyx_t_7]) + 1);
+0524: cnt += 1
__pyx_v_cnt = (__pyx_v_cnt + 1);
+0525: if nummatches[0] >= maxresults:
__pyx_t_5 = (((__pyx_v_nummatches[0]) >= __pyx_v_maxresults) != 0);
if (__pyx_t_5) {
/* … */
}
+0526: return cnt
__pyx_r = __pyx_v_cnt;
goto __pyx_L0;
+0527: elif <size_t>cnt >= allocated[0]:
__pyx_t_5 = ((((size_t)__pyx_v_cnt) >= (__pyx_v_allocated[0])) != 0);
if (__pyx_t_5) {
/* … */
}
+0528: allocated[0] = cnt * 2
(__pyx_v_allocated[0]) = (__pyx_v_cnt * 2);
+0529: treenums[0] = <uint32_t *>realloc(
(__pyx_v_treenums[0]) = ((uint32_t *)realloc((__pyx_v_treenums[0]), ((2 * __pyx_v_cnt) * (sizeof(int)))));
0530: treenums[0], 2 * cnt * sizeof(int))
+0531: if treenums[0] is NULL:
__pyx_t_5 = (((__pyx_v_treenums[0]) == NULL) != 0);
if (__pyx_t_5) {
/* … */
}
+0532: return -1
__pyx_r = -1;
goto __pyx_L0;
+0533: if indices == 2:
__pyx_t_5 = ((__pyx_v_indices == 2) != 0);
if (__pyx_t_5) {
/* … */
}
+0534: nodenums[0] = <short *>realloc(
(__pyx_v_nodenums[0]) = ((short *)realloc((__pyx_v_nodenums[0]), ((2 * __pyx_v_cnt) * (sizeof(int)))));
0535: nodenums[0], 2 * cnt * sizeof(int))
+0536: if nodenums[0] is NULL:
__pyx_t_5 = (((__pyx_v_nodenums[0]) == NULL) != 0);
if (__pyx_t_5) {
/* … */
}
+0537: return -1
__pyx_r = -1;
goto __pyx_L0;
+0538: elif anodes[i].prod < bnodes[j].prod:
__pyx_t_5 = (((__pyx_v_anodes[__pyx_v_i]).prod < (__pyx_v_bnodes[__pyx_v_j]).prod) != 0);
if (__pyx_t_5) {
/* … */
}
__pyx_L7:;
}
__pyx_L6_break:;
}
+0539: break
goto __pyx_L6_break;
+0540: return cnt
__pyx_r = __pyx_v_cnt; goto __pyx_L0;
0541:
0542:
+0543: cdef getcandidates(Node *a, uint64_t *bitset, Ctrees trees, short alen,
static PyObject *__pyx_f_8discodop_10_fragments_getcandidates(struct __pyx_t_8discodop_10containers_Node *__pyx_v_a, uint64_t *__pyx_v_bitset, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees, short __pyx_v_alen, int __pyx_v_start, int __pyx_v_end, short __pyx_v_SLOTS) {
uint64_t __pyx_v_cur;
int __pyx_v_i;
int __pyx_v_idx;
PyObject *__pyx_v_tmp = 0;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("getcandidates", 0);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_7);
__Pyx_XDECREF(__pyx_t_8);
__Pyx_AddTraceback("discodop._fragments.getcandidates", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_tmp);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0544: int start, int end, short SLOTS):
0545: """Get candidates from productions in fragment ``bitset`` at ``a[i]``."""
+0546: cdef uint64_t cur = bitset[0]
__pyx_v_cur = (__pyx_v_bitset[0]);
+0547: cdef int i, idx = 0
__pyx_v_idx = 0;
+0548: cdef list tmp = []
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 548, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_tmp = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
+0549: while True:
while (1) {
+0550: i = iteratesetbits(bitset, SLOTS, &cur, &idx)
__pyx_v_i = __pyx_f_8discodop_3bit_iteratesetbits(__pyx_v_bitset, __pyx_v_SLOTS, (&__pyx_v_cur), (&__pyx_v_idx));
+0551: if i == -1 or i >= alen: # FIXME. why is 2nd condition necessary?
__pyx_t_3 = ((__pyx_v_i == -1L) != 0);
if (!__pyx_t_3) {
} else {
__pyx_t_2 = __pyx_t_3;
goto __pyx_L6_bool_binop_done;
}
__pyx_t_3 = ((__pyx_v_i >= __pyx_v_alen) != 0);
__pyx_t_2 = __pyx_t_3;
__pyx_L6_bool_binop_done:;
if (__pyx_t_2) {
/* … */
}
+0552: break
goto __pyx_L4_break;
+0553: tmp.append(a[i].prod)
__pyx_t_1 = __Pyx_PyInt_From_int32_t((__pyx_v_a[__pyx_v_i]).prod); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 553, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_4 = __Pyx_PyList_Append(__pyx_v_tmp, __pyx_t_1); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 553, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
}
__pyx_L4_break:;
+0554: return trees.prodindex.intersection(
__Pyx_XDECREF(__pyx_r); __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_trees->prodindex, __pyx_n_s_intersection); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 554, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); /* … */ __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 554, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_INCREF(__pyx_v_tmp); __Pyx_GIVEREF(__pyx_v_tmp); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_v_tmp); /* … */ __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 554, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_r = __pyx_t_7; __pyx_t_7 = 0; goto __pyx_L0;
+0555: tmp, start=start, stop=end or trees.len)
__pyx_t_6 = PyDict_New(); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 555, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_7 = __Pyx_PyInt_From_int(__pyx_v_start); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 555, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); if (PyDict_SetItem(__pyx_t_6, __pyx_n_s_start, __pyx_t_7) < 0) __PYX_ERR(0, 555, __pyx_L1_error) __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (!__pyx_v_end) { } else { __pyx_t_8 = __Pyx_PyInt_From_int(__pyx_v_end); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 555, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_t_7 = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L8_bool_binop_done; } __pyx_t_8 = __Pyx_PyInt_From_int(__pyx_v_trees->len); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 555, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __pyx_t_7 = __pyx_t_8; __pyx_t_8 = 0; __pyx_L8_bool_binop_done:; if (PyDict_SetItem(__pyx_t_6, __pyx_n_s_stop, __pyx_t_7) < 0) __PYX_ERR(0, 555, __pyx_L1_error) __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
0556:
0557:
+0558: cdef inline int containsbitset(Node *a, Node *b, uint64_t *bitset,
static CYTHON_INLINE int __pyx_f_8discodop_10_fragments_containsbitset(struct __pyx_t_8discodop_10containers_Node *__pyx_v_a, struct __pyx_t_8discodop_10containers_Node *__pyx_v_b, uint64_t *__pyx_v_bitset, short __pyx_v_i, short __pyx_v_j) {
int __pyx_r;
/* … */
/* function exit code */
__pyx_L0:;
return __pyx_r;
}
0559: short i, short j) nogil:
0560: """Test whether the fragment ``bitset`` at ``a[i]`` occurs at ``b[j]``."""
+0561: if a[i].prod != b[j].prod:
__pyx_t_1 = (((__pyx_v_a[__pyx_v_i]).prod != (__pyx_v_b[__pyx_v_j]).prod) != 0);
if (__pyx_t_1) {
/* … */
}
+0562: return 0
__pyx_r = 0;
goto __pyx_L0;
+0563: elif a[i].left < 0:
__pyx_t_1 = (((__pyx_v_a[__pyx_v_i]).left < 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0564: return 1
__pyx_r = 1;
goto __pyx_L0;
+0565: elif TESTBIT(bitset, a[i].left):
__pyx_t_1 = (TESTBIT(__pyx_v_bitset, (__pyx_v_a[__pyx_v_i]).left) != 0);
if (__pyx_t_1) {
/* … */
}
+0566: if not containsbitset(a, b, bitset, a[i].left, b[j].left):
__pyx_t_1 = ((!(__pyx_f_8discodop_10_fragments_containsbitset(__pyx_v_a, __pyx_v_b, __pyx_v_bitset, (__pyx_v_a[__pyx_v_i]).left, (__pyx_v_b[__pyx_v_j]).left) != 0)) != 0);
if (__pyx_t_1) {
/* … */
}
+0567: return 0
__pyx_r = 0;
goto __pyx_L0;
+0568: if a[i].right < 0:
__pyx_t_1 = (((__pyx_v_a[__pyx_v_i]).right < 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0569: return 1
__pyx_r = 1;
goto __pyx_L0;
+0570: elif TESTBIT(bitset, a[i].right):
__pyx_t_1 = (TESTBIT(__pyx_v_bitset, (__pyx_v_a[__pyx_v_i]).right) != 0);
if (__pyx_t_1) {
/* … */
}
+0571: return containsbitset(a, b, bitset, a[i].right, b[j].right)
__pyx_r = __pyx_f_8discodop_10_fragments_containsbitset(__pyx_v_a, __pyx_v_b, __pyx_v_bitset, (__pyx_v_a[__pyx_v_i]).right, (__pyx_v_b[__pyx_v_j]).right);
goto __pyx_L0;
+0572: return 1
__pyx_r = 1; goto __pyx_L0;
0573:
0574:
+0575: cpdef completebitsets(Ctrees trees, Vocabulary vocab,
static PyObject *__pyx_pw_8discodop_10_fragments_7completebitsets(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static PyObject *__pyx_f_8discodop_10_fragments_completebitsets(struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, short __pyx_v_maxnodes, CYTHON_UNUSED int __pyx_skip_dispatch, struct __pyx_opt_args_8discodop_10_fragments_completebitsets *__pyx_optional_args) {
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_AddTraceback("discodop._fragments.completebitsets", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_result);
__Pyx_XDECREF(__pyx_v_bitsets);
__Pyx_XDECREF(__pyx_v_tmp);
__Pyx_XDECREF(__pyx_v_unused_step);
__Pyx_XDECREF(__pyx_v_termindices);
__Pyx_XDECREF(__pyx_v_tree);
__Pyx_XDECREF(__pyx_v_start);
__Pyx_XDECREF(__pyx_v_end);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_7completebitsets(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static char __pyx_doc_8discodop_10_fragments_6completebitsets[] = "completebitsets(Ctrees trees, Vocabulary vocab, short maxnodes, bool disc=False, start=None, end=None, tostring=True)\nGenerate bitsets corresponding to whole trees in the input.\n\n\t:param tostring: when False, do not create list of trees as strings\n\t:returns: a pair of lists with trees as strings and their bitsets,\n\t\trespectively.\n\n\tA tree with a discontinuous substitution site is expected to be binarized\n\twith ``rightmostunary=True``:\n\n\t>>> from discodop.treebank import brackettree\n\t>>> tree, sent = brackettree(u'(S (X 0= 2= 4=))')\n\t>>> print(handledisc(tree))\n\t(S (X 0 (X|<> 2 (X|<> 4))))\n\n\tThese auxiliary nodes will not be part of the returned tree / bitset:\n\n\t>>> tmp = getctrees([(tree, sent)])\n\t>>> print(completebitsets(tmp['trees1'], tmp['vocab'], 2, disc=True)[0][0])\n\t(S (X 0= 2= 4=))\n\t";
static PyObject *__pyx_pw_8discodop_10_fragments_7completebitsets(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees = 0;
struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab = 0;
short __pyx_v_maxnodes;
int __pyx_v_disc;
PyObject *__pyx_v_start = 0;
PyObject *__pyx_v_end = 0;
PyObject *__pyx_v_tostring = 0;
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("completebitsets (wrapper)", 0);
{
static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_trees,&__pyx_n_s_vocab,&__pyx_n_s_maxnodes,&__pyx_n_s_disc,&__pyx_n_s_start,&__pyx_n_s_end,&__pyx_n_s_tostring,0};
PyObject* values[7] = {0,0,0,0,0,0,0};
/* … */
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__pyx_r = NULL;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_8discodop_10_fragments_6completebitsets(CYTHON_UNUSED PyObject *__pyx_self, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, short __pyx_v_maxnodes, int __pyx_v_disc, PyObject *__pyx_v_start, PyObject *__pyx_v_end, PyObject *__pyx_v_tostring) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("completebitsets", 0);
__Pyx_XDECREF(__pyx_r);
__pyx_t_2.__pyx_n = 4;
__pyx_t_2.disc = __pyx_v_disc;
__pyx_t_2.start = __pyx_v_start;
__pyx_t_2.end = __pyx_v_end;
__pyx_t_2.tostring = __pyx_v_tostring;
__pyx_t_1 = __pyx_f_8discodop_10_fragments_completebitsets(__pyx_v_trees, __pyx_v_vocab, __pyx_v_maxnodes, 0, &__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 575, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_r = __pyx_t_1;
__pyx_t_1 = 0;
goto __pyx_L0;
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_AddTraceback("discodop._fragments.completebitsets", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
struct __pyx_opt_args_8discodop_10_fragments_completebitsets {
int __pyx_n;
int disc;
PyObject *start;
PyObject *end;
PyObject *tostring;
};
+0576: short maxnodes, bint disc=False, start=None, end=None, tostring=True):
int __pyx_v_disc = ((int)0); PyObject *__pyx_v_start = ((PyObject *)Py_None); PyObject *__pyx_v_end = ((PyObject *)Py_None); PyObject *__pyx_v_tostring = ((PyObject *)Py_True); PyObject *__pyx_v_result = 0; PyObject *__pyx_v_bitsets = 0; PyObject *__pyx_v_tmp = 0; int __pyx_v_n; int __pyx_v_i; short __pyx_v_SLOTS; uint64_t *__pyx_v_scratch; struct __pyx_t_8discodop_10containers_Node *__pyx_v_nodes; CYTHON_UNUSED PyObject *__pyx_v_unused_step = NULL; PyObject *__pyx_v_termindices = NULL; PyObject *__pyx_v_tree = NULL; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("completebitsets", 0); if (__pyx_optional_args) { if (__pyx_optional_args->__pyx_n > 0) { __pyx_v_disc = __pyx_optional_args->disc; if (__pyx_optional_args->__pyx_n > 1) { __pyx_v_start = __pyx_optional_args->start; if (__pyx_optional_args->__pyx_n > 2) { __pyx_v_end = __pyx_optional_args->end; if (__pyx_optional_args->__pyx_n > 3) { __pyx_v_tostring = __pyx_optional_args->tostring; } } } } } __Pyx_INCREF(__pyx_v_start); __Pyx_INCREF(__pyx_v_end); /* … */ values[4] = ((PyObject *)Py_None); values[5] = ((PyObject *)Py_None); values[6] = ((PyObject *)Py_True); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); CYTHON_FALLTHROUGH; case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); CYTHON_FALLTHROUGH; case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); CYTHON_FALLTHROUGH; case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_trees)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_vocab)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("completebitsets", 0, 3, 7, 1); __PYX_ERR(0, 575, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_maxnodes)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("completebitsets", 0, 3, 7, 2); __PYX_ERR(0, 575, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 3: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_disc); if (value) { values[3] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 4: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_start); if (value) { values[4] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 5: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_end); if (value) { values[5] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 6: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_tostring); if (value) { values[6] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "completebitsets") < 0)) __PYX_ERR(0, 575, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); CYTHON_FALLTHROUGH; case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); CYTHON_FALLTHROUGH; case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); CYTHON_FALLTHROUGH; case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_trees = ((struct __pyx_obj_8discodop_10containers_Ctrees *)values[0]); __pyx_v_vocab = ((struct __pyx_obj_8discodop_10containers_Vocabulary *)values[1]); __pyx_v_maxnodes = __Pyx_PyInt_As_short(values[2]); if (unlikely((__pyx_v_maxnodes == (short)-1) && PyErr_Occurred())) __PYX_ERR(0, 576, __pyx_L3_error) if (values[3]) { __pyx_v_disc = __Pyx_PyObject_IsTrue(values[3]); if (unlikely((__pyx_v_disc == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 576, __pyx_L3_error) } else { __pyx_v_disc = ((int)0); } __pyx_v_start = values[4]; __pyx_v_end = values[5]; __pyx_v_tostring = values[6]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("completebitsets", 0, 3, 7, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 575, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("discodop._fragments.completebitsets", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_trees), __pyx_ptype_8discodop_10containers_Ctrees, 1, "trees", 0))) __PYX_ERR(0, 575, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_vocab), __pyx_ptype_8discodop_10containers_Vocabulary, 1, "vocab", 0))) __PYX_ERR(0, 575, __pyx_L1_error) __pyx_r = __pyx_pf_8discodop_10_fragments_6completebitsets(__pyx_self, __pyx_v_trees, __pyx_v_vocab, __pyx_v_maxnodes, __pyx_v_disc, __pyx_v_start, __pyx_v_end, __pyx_v_tostring);
0577: """Generate bitsets corresponding to whole trees in the input.
0578:
0579: :param tostring: when False, do not create list of trees as strings
0580: :returns: a pair of lists with trees as strings and their bitsets,
0581: respectively.
0582:
0583: A tree with a discontinuous substitution site is expected to be binarized
0584: with ``rightmostunary=True``:
0585:
0586: >>> from discodop.treebank import brackettree
0587: >>> tree, sent = brackettree(u'(S (X 0= 2= 4=))')
0588: >>> print(handledisc(tree))
0589: (S (X 0 (X|<> 2 (X|<> 4))))
0590:
0591: These auxiliary nodes will not be part of the returned tree / bitset:
0592:
0593: >>> tmp = getctrees([(tree, sent)])
0594: >>> print(completebitsets(tmp['trees1'], tmp['vocab'], 2, disc=True)[0][0])
0595: (S (X 0= 2= 4=))
0596: """
0597: cdef:
+0598: list result = [] if tostring else None
__pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_tostring); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 598, __pyx_L1_error) if (__pyx_t_2) { __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 598, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_1 = __pyx_t_3; __pyx_t_3 = 0; } else { __Pyx_INCREF(Py_None); __pyx_t_1 = Py_None; } __pyx_v_result = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
+0599: list bitsets = []
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 599, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_bitsets = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
+0600: list tmp = []
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 600, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_tmp = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
0601: int n, i
+0602: short SLOTS = BITNSLOTS(maxnodes + 1)
__pyx_v_SLOTS = BITNSLOTS((__pyx_v_maxnodes + 1));
+0603: uint64_t *scratch = <uint64_t *>malloc((SLOTS + 2) * sizeof(uint64_t))
__pyx_v_scratch = ((uint64_t *)malloc(((__pyx_v_SLOTS + 2) * (sizeof(uint64_t)))));
0604: Node *nodes
+0605: start, end, unused_step = slice(start, end).indices(trees.len)
__pyx_t_3 = PySlice_New(__pyx_v_start, __pyx_v_end, Py_None); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_indices); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_trees->len); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_5 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_5)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); __Pyx_INCREF(__pyx_t_5); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_4, function); } } if (!__pyx_t_5) { __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_t_3}; __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_t_3}; __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __pyx_t_5 = NULL; __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_3); __pyx_t_3 = 0; __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { PyObject* sequence = __pyx_t_1; #if !CYTHON_COMPILING_IN_PYPY Py_ssize_t size = Py_SIZE(sequence); #else Py_ssize_t size = PySequence_Size(sequence); #endif if (unlikely(size != 3)) { if (size > 3) __Pyx_RaiseTooManyValuesError(3); else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); __PYX_ERR(0, 605, __pyx_L1_error) } #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS if (likely(PyTuple_CheckExact(sequence))) { __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_6 = PyTuple_GET_ITEM(sequence, 1); __pyx_t_3 = PyTuple_GET_ITEM(sequence, 2); } else { __pyx_t_4 = PyList_GET_ITEM(sequence, 0); __pyx_t_6 = PyList_GET_ITEM(sequence, 1); __pyx_t_3 = PyList_GET_ITEM(sequence, 2); } __Pyx_INCREF(__pyx_t_4); __Pyx_INCREF(__pyx_t_6); __Pyx_INCREF(__pyx_t_3); #else __pyx_t_4 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_6 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_3 = PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); #endif __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; } else { Py_ssize_t index = -1; __pyx_t_5 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 605, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_7 = Py_TYPE(__pyx_t_5)->tp_iternext; index = 0; __pyx_t_4 = __pyx_t_7(__pyx_t_5); if (unlikely(!__pyx_t_4)) goto __pyx_L3_unpacking_failed; __Pyx_GOTREF(__pyx_t_4); index = 1; __pyx_t_6 = __pyx_t_7(__pyx_t_5); if (unlikely(!__pyx_t_6)) goto __pyx_L3_unpacking_failed; __Pyx_GOTREF(__pyx_t_6); index = 2; __pyx_t_3 = __pyx_t_7(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L3_unpacking_failed; __Pyx_GOTREF(__pyx_t_3); if (__Pyx_IternextUnpackEndCheck(__pyx_t_7(__pyx_t_5), 3) < 0) __PYX_ERR(0, 605, __pyx_L1_error) __pyx_t_7 = NULL; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; goto __pyx_L4_unpacking_done; __pyx_L3_unpacking_failed:; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_t_7 = NULL; if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); __PYX_ERR(0, 605, __pyx_L1_error) __pyx_L4_unpacking_done:; } __Pyx_DECREF_SET(__pyx_v_start, __pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF_SET(__pyx_v_end, __pyx_t_6); __pyx_t_6 = 0; __pyx_v_unused_step = __pyx_t_3; __pyx_t_3 = 0;
+0606: for n in range(<int>start, <int>end):
__pyx_t_8 = __Pyx_PyInt_As_int(__pyx_v_end); if (unlikely((__pyx_t_8 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 606, __pyx_L1_error) __pyx_t_9 = ((int)__pyx_t_8); __pyx_t_8 = __Pyx_PyInt_As_int(__pyx_v_start); if (unlikely((__pyx_t_8 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 606, __pyx_L1_error) for (__pyx_t_10 = ((int)__pyx_t_8); __pyx_t_10 < __pyx_t_9; __pyx_t_10+=1) { __pyx_v_n = __pyx_t_10;
+0607: memset(<void *>scratch, 0, SLOTS * sizeof(uint64_t))
memset(((void *)__pyx_v_scratch), 0, (__pyx_v_SLOTS * (sizeof(uint64_t))));
+0608: nodes = &trees.nodes[trees.trees[n].offset]
__pyx_v_nodes = (&(__pyx_v_trees->nodes[(__pyx_v_trees->trees[__pyx_v_n]).offset]));
0609: # The set of terminal indices (i.e., minus frontier non-terminals)
+0610: termindices = {termidx(nodes[m].left)
{ /* enter inner scope */
int16_t __pyx_8genexpr6__pyx_v_m;
__pyx_t_1 = PySet_New(NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 610, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
/* … */
__pyx_t_3 = __Pyx_PyInt_From_short(__pyx_f_8discodop_10containers_termidx((__pyx_v_nodes[__pyx_8genexpr6__pyx_v_m]).left)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 610, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
if (unlikely(PySet_Add(__pyx_t_1, (PyObject*)__pyx_t_3))) __PYX_ERR(0, 610, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+0611: for m in range(trees.trees[n].len)
__pyx_t_11 = (__pyx_v_trees->trees[__pyx_v_n]).len;
for (__pyx_t_12 = 0; __pyx_t_12 < __pyx_t_11; __pyx_t_12+=1) {
__pyx_8genexpr6__pyx_v_m = __pyx_t_12;
+0612: if nodes[m].left < 0 and vocab.islexical(nodes[m].prod)}
__pyx_t_13 = (((__pyx_v_nodes[__pyx_8genexpr6__pyx_v_m]).left < 0) != 0);
if (__pyx_t_13) {
} else {
__pyx_t_2 = __pyx_t_13;
goto __pyx_L10_bool_binop_done;
}
__pyx_t_13 = (((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->islexical(__pyx_v_vocab, (__pyx_v_nodes[__pyx_8genexpr6__pyx_v_m]).prod) != 0);
__pyx_t_2 = __pyx_t_13;
__pyx_L10_bool_binop_done:;
if (__pyx_t_2) {
/* … */
}
}
} /* exit inner scope */
__Pyx_XDECREF_SET(__pyx_v_termindices, ((PyObject*)__pyx_t_1));
__pyx_t_1 = 0;
+0613: for i in range(trees.trees[n].len):
__pyx_t_11 = (__pyx_v_trees->trees[__pyx_v_n]).len;
for (__pyx_t_14 = 0; __pyx_t_14 < __pyx_t_11; __pyx_t_14+=1) {
__pyx_v_i = __pyx_t_14;
+0614: if nodes[i].left >= 0 or termidx(nodes[i].left) in termindices:
__pyx_t_13 = (((__pyx_v_nodes[__pyx_v_i]).left >= 0) != 0);
if (!__pyx_t_13) {
} else {
__pyx_t_2 = __pyx_t_13;
goto __pyx_L15_bool_binop_done;
}
__pyx_t_1 = __Pyx_PyInt_From_short(__pyx_f_8discodop_10containers_termidx((__pyx_v_nodes[__pyx_v_i]).left)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 614, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_13 = (__Pyx_PySequence_ContainsTF(__pyx_t_1, __pyx_v_termindices, Py_EQ)); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(0, 614, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
__pyx_t_15 = (__pyx_t_13 != 0);
__pyx_t_2 = __pyx_t_15;
__pyx_L15_bool_binop_done:;
if (__pyx_t_2) {
/* … */
}
}
+0615: SETBIT(scratch, i)
SETBIT(__pyx_v_scratch, __pyx_v_i);
+0616: setrootid(scratch, trees.trees[n].root, n, SLOTS)
__pyx_f_8discodop_10_fragments_setrootid(__pyx_v_scratch, (__pyx_v_trees->trees[__pyx_v_n]).root, __pyx_v_n, __pyx_v_SLOTS);
+0617: if tostring:
__pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_tostring); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 617, __pyx_L1_error)
if (__pyx_t_2) {
/* … */
}
+0618: tmp[:] = []
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 618, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (__Pyx_PyObject_SetSlice(__pyx_v_tmp, __pyx_t_1, 0, 0, NULL, NULL, NULL, 0, 0, 0) < 0) __PYX_ERR(0, 618, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+0619: getsubtree(tmp, nodes, scratch, vocab,
__pyx_t_1 = __pyx_f_8discodop_10_fragments_getsubtree(__pyx_v_tmp, __pyx_v_nodes, __pyx_v_scratch, __pyx_v_vocab, __pyx_v_disc, (__pyx_v_trees->trees[__pyx_v_n]).root); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 619, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
0620: disc, trees.trees[n].root)
+0621: tree = getsent(''.join(tmp)) if disc else ''.join(tmp)
if ((__pyx_v_disc != 0)) {
__pyx_t_3 = PyUnicode_Join(__pyx_kp_u__2, __pyx_v_tmp); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 621, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__pyx_t_6 = __pyx_f_8discodop_10_fragments_getsent(((PyObject*)__pyx_t_3)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 621, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__pyx_t_1 = __pyx_t_6;
__pyx_t_6 = 0;
} else {
__pyx_t_6 = PyUnicode_Join(__pyx_kp_u__2, __pyx_v_tmp); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 621, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__pyx_t_1 = __pyx_t_6;
__pyx_t_6 = 0;
}
__Pyx_XDECREF_SET(__pyx_v_tree, __pyx_t_1);
__pyx_t_1 = 0;
+0622: result.append(tree)
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 622, __pyx_L1_error)
}
__pyx_t_16 = __Pyx_PyList_Append(__pyx_v_result, __pyx_v_tree); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 622, __pyx_L1_error)
+0623: bitsets.append(wrap(scratch, SLOTS))
__pyx_t_1 = __pyx_f_8discodop_10_fragments_wrap(__pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 623, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_16 = __Pyx_PyList_Append(__pyx_v_bitsets, __pyx_t_1); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 623, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
}
+0624: free(scratch)
free(__pyx_v_scratch);
+0625: return result, bitsets
__Pyx_XDECREF(__pyx_r); __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 625, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_result); __Pyx_GIVEREF(__pyx_v_result); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_result); __Pyx_INCREF(__pyx_v_bitsets); __Pyx_GIVEREF(__pyx_v_bitsets); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_bitsets); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0;
0626:
0627:
+0628: cdef twoterminals(NodeArray a, Node *anodes,
static PyObject *__pyx_f_8discodop_10_fragments_twoterminals(struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_a, struct __pyx_t_8discodop_10containers_Node *__pyx_v_anodes, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees2, PyObject *__pyx_v_contentwordprods, PyObject *__pyx_v_lexicalprods) {
int __pyx_v_i;
int __pyx_v_j;
PyObject *__pyx_v_tmp = 0;
PyObject *__pyx_v_candidates = 0;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("twoterminals", 0);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_AddTraceback("discodop._fragments.twoterminals", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_tmp);
__Pyx_XDECREF(__pyx_v_candidates);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0629: Ctrees trees2, set contentwordprods, set lexicalprods):
0630: """Produce tree pairs that share at least two words.
0631:
0632: Specifically, tree pairs sharing one content word and one additional word,
0633: where content words are recognized by a POS tag from the Penn treebank tag
0634: set.
0635:
0636: if trees2 is None, pairs (n, m) are such that n < m."""
0637: cdef int i, j
+0638: cdef object tmp, candidates = RoaringBitmap()
__pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_RoaringBitmap); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 638, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_2, function); } } if (__pyx_t_3) { __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 638, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 638, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_candidates = __pyx_t_1; __pyx_t_1 = 0;
0639: # select candidates from 'trees2' that share productions with tree 'a'
0640: # want to select at least 1 content POS tag, 1 other lexical prod
+0641: for i in range(a.len):
__pyx_t_4 = __pyx_v_a.len;
for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) {
__pyx_v_i = __pyx_t_5;
+0642: if anodes[i].left >= 0 or anodes[i].prod not in contentwordprods:
__pyx_t_7 = (((__pyx_v_anodes[__pyx_v_i]).left >= 0) != 0);
if (!__pyx_t_7) {
} else {
__pyx_t_6 = __pyx_t_7;
goto __pyx_L6_bool_binop_done;
}
__pyx_t_1 = __Pyx_PyInt_From_int32_t((__pyx_v_anodes[__pyx_v_i]).prod); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 642, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_7 = (__Pyx_PySequence_ContainsTF(__pyx_t_1, __pyx_v_contentwordprods, Py_NE)); if (unlikely(__pyx_t_7 < 0)) __PYX_ERR(0, 642, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
__pyx_t_8 = (__pyx_t_7 != 0);
__pyx_t_6 = __pyx_t_8;
__pyx_L6_bool_binop_done:;
if (__pyx_t_6) {
/* … */
}
+0643: continue
goto __pyx_L3_continue;
+0644: tmp = trees2.prodindex[anodes[i].prod]
__pyx_t_1 = __Pyx_GetItemInt(__pyx_v_trees2->prodindex, (__pyx_v_anodes[__pyx_v_i]).prod, int32_t, 1, __Pyx_PyInt_From_int32_t, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 644, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_XDECREF_SET(__pyx_v_tmp, __pyx_t_1); __pyx_t_1 = 0;
+0645: for j in range(a.len):
__pyx_t_9 = __pyx_v_a.len;
for (__pyx_t_10 = 0; __pyx_t_10 < __pyx_t_9; __pyx_t_10+=1) {
__pyx_v_j = __pyx_t_10;
+0646: if (i != j and anodes[j].left < 0 and
__pyx_t_8 = ((__pyx_v_i != __pyx_v_j) != 0);
if (__pyx_t_8) {
} else {
__pyx_t_6 = __pyx_t_8;
goto __pyx_L11_bool_binop_done;
}
__pyx_t_8 = (((__pyx_v_anodes[__pyx_v_j]).left < 0) != 0);
if (__pyx_t_8) {
} else {
__pyx_t_6 = __pyx_t_8;
goto __pyx_L11_bool_binop_done;
}
/* … */
if (__pyx_t_6) {
/* … */
}
}
__pyx_L3_continue:;
}
+0647: anodes[j].prod in lexicalprods):
__pyx_t_1 = __Pyx_PyInt_From_int32_t((__pyx_v_anodes[__pyx_v_j]).prod); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 647, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_8 = (__Pyx_PySequence_ContainsTF(__pyx_t_1, __pyx_v_lexicalprods, Py_EQ)); if (unlikely(__pyx_t_8 < 0)) __PYX_ERR(0, 647, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
__pyx_t_7 = (__pyx_t_8 != 0);
__pyx_t_6 = __pyx_t_7;
__pyx_L11_bool_binop_done:;
+0648: candidates |= tmp & trees2.prodindex[anodes[j].prod]
__pyx_t_1 = __Pyx_GetItemInt(__pyx_v_trees2->prodindex, (__pyx_v_anodes[__pyx_v_j]).prod, int32_t, 1, __Pyx_PyInt_From_int32_t, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 648, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = PyNumber_And(__pyx_v_tmp, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 648, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyNumber_InPlaceOr(__pyx_v_candidates, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 648, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF_SET(__pyx_v_candidates, __pyx_t_1); __pyx_t_1 = 0;
+0649: return candidates
__Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_candidates); __pyx_r = __pyx_v_candidates; goto __pyx_L0;
0650:
0651:
+0652: def allfragments(Ctrees trees, Vocabulary vocab,
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_9allfragments(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static char __pyx_doc_8discodop_10_fragments_8allfragments[] = "allfragments(Ctrees trees, Vocabulary vocab, unsigned int maxdepth, unsigned int maxfrontier=999, bool disc=True, bool indices=False)\nReturn all fragments of trees up to maxdepth.\n\n\t:param maxdepth: maximum depth of fragments; depth 1 gives fragments that\n\t\tare equivalent to a treebank grammar.\n\t:param maxfrontier: maximum number of frontier non-terminals (substitution\n\t\tsites) in fragments; a limit of 0 only gives fragments that bottom out\n\t\tin terminals; 999 is unlimited for practical purposes.\n\t:returns: dictionary fragments with tree strings as keys and integer counts\n\t\tas values.";
static PyMethodDef __pyx_mdef_8discodop_10_fragments_9allfragments = {"allfragments", (PyCFunction)__pyx_pw_8discodop_10_fragments_9allfragments, METH_VARARGS|METH_KEYWORDS, __pyx_doc_8discodop_10_fragments_8allfragments};
static PyObject *__pyx_pw_8discodop_10_fragments_9allfragments(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees = 0;
struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab = 0;
unsigned int __pyx_v_maxdepth;
unsigned int __pyx_v_maxfrontier;
int __pyx_v_disc;
int __pyx_v_indices;
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("allfragments (wrapper)", 0);
{
static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_trees,&__pyx_n_s_vocab,&__pyx_n_s_maxdepth,&__pyx_n_s_maxfrontier,&__pyx_n_s_disc,&__pyx_n_s_indices,0};
PyObject* values[6] = {0,0,0,0,0,0};
if (unlikely(__pyx_kwds)) {
Py_ssize_t kw_args;
const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
switch (pos_args) {
case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5);
CYTHON_FALLTHROUGH;
case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4);
CYTHON_FALLTHROUGH;
case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3);
CYTHON_FALLTHROUGH;
case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
CYTHON_FALLTHROUGH;
case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
CYTHON_FALLTHROUGH;
case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
CYTHON_FALLTHROUGH;
case 0: break;
default: goto __pyx_L5_argtuple_error;
}
kw_args = PyDict_Size(__pyx_kwds);
switch (pos_args) {
case 0:
if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_trees)) != 0)) kw_args--;
else goto __pyx_L5_argtuple_error;
CYTHON_FALLTHROUGH;
case 1:
if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_vocab)) != 0)) kw_args--;
else {
__Pyx_RaiseArgtupleInvalid("allfragments", 0, 3, 6, 1); __PYX_ERR(0, 652, __pyx_L3_error)
}
CYTHON_FALLTHROUGH;
case 2:
if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_maxdepth)) != 0)) kw_args--;
else {
__Pyx_RaiseArgtupleInvalid("allfragments", 0, 3, 6, 2); __PYX_ERR(0, 652, __pyx_L3_error)
}
CYTHON_FALLTHROUGH;
case 3:
if (kw_args > 0) {
PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_maxfrontier);
if (value) { values[3] = value; kw_args--; }
}
CYTHON_FALLTHROUGH;
case 4:
if (kw_args > 0) {
PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_disc);
if (value) { values[4] = value; kw_args--; }
}
CYTHON_FALLTHROUGH;
case 5:
if (kw_args > 0) {
PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_indices);
if (value) { values[5] = value; kw_args--; }
}
}
if (unlikely(kw_args > 0)) {
if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allfragments") < 0)) __PYX_ERR(0, 652, __pyx_L3_error)
}
} else {
switch (PyTuple_GET_SIZE(__pyx_args)) {
case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5);
CYTHON_FALLTHROUGH;
case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4);
CYTHON_FALLTHROUGH;
case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3);
CYTHON_FALLTHROUGH;
case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
break;
default: goto __pyx_L5_argtuple_error;
}
}
__pyx_v_trees = ((struct __pyx_obj_8discodop_10containers_Ctrees *)values[0]);
__pyx_v_vocab = ((struct __pyx_obj_8discodop_10containers_Vocabulary *)values[1]);
__pyx_v_maxdepth = __Pyx_PyInt_As_unsigned_int(values[2]); if (unlikely((__pyx_v_maxdepth == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 653, __pyx_L3_error)
if (values[3]) {
__pyx_v_maxfrontier = __Pyx_PyInt_As_unsigned_int(values[3]); if (unlikely((__pyx_v_maxfrontier == (unsigned int)-1) && PyErr_Occurred())) __PYX_ERR(0, 653, __pyx_L3_error)
} else {
__pyx_v_maxfrontier = ((unsigned int)0x3E7);
}
if (values[4]) {
__pyx_v_disc = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_disc == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 653, __pyx_L3_error)
} else {
/* … */
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__pyx_r = NULL;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_8discodop_10_fragments_8allfragments(CYTHON_UNUSED PyObject *__pyx_self, struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_trees, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, unsigned int __pyx_v_maxdepth, unsigned int __pyx_v_maxfrontier, int __pyx_v_disc, int __pyx_v_indices) {
struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_tree;
struct __pyx_t_8discodop_10containers_Node *__pyx_v_nodes;
int __pyx_v_n;
int __pyx_v_SLOTS;
PyObject *__pyx_v_table = 0;
PyObject *__pyx_v_tmp = 0;
PyObject *__pyx_v_inter = 0;
PyObject *__pyx_v_fragments = 0;
uint64_t *__pyx_v_scratch;
PyObject *__pyx_v_sent = NULL;
Py_ssize_t __pyx_v_i;
PyObject *__pyx_v_frags = NULL;
PyObject *__pyx_v_frag = NULL;
short __pyx_v_depth;
uint32_t __pyx_v_frontiers;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("allfragments", 0);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_7);
__Pyx_XDECREF(__pyx_t_9);
__Pyx_AddTraceback("discodop._fragments.allfragments", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_table);
__Pyx_XDECREF(__pyx_v_tmp);
__Pyx_XDECREF(__pyx_v_inter);
__Pyx_XDECREF(__pyx_v_fragments);
__Pyx_XDECREF(__pyx_v_sent);
__Pyx_XDECREF(__pyx_v_frags);
__Pyx_XDECREF(__pyx_v_frag);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
__pyx_tuple__31 = PyTuple_Pack(21, __pyx_n_s_trees, __pyx_n_s_vocab, __pyx_n_s_maxdepth, __pyx_n_s_maxfrontier, __pyx_n_s_disc, __pyx_n_s_indices, __pyx_n_s_tree, __pyx_n_s_nodes, __pyx_n_s_n, __pyx_n_s_SLOTS, __pyx_n_s_table, __pyx_n_s_tmp, __pyx_n_s_inter, __pyx_n_s_fragments, __pyx_n_s_scratch, __pyx_n_s_sent, __pyx_n_s_i, __pyx_n_s_frags, __pyx_n_s_frag, __pyx_n_s_depth, __pyx_n_s_frontiers); if (unlikely(!__pyx_tuple__31)) __PYX_ERR(0, 652, __pyx_L1_error)
__Pyx_GOTREF(__pyx_tuple__31);
__Pyx_GIVEREF(__pyx_tuple__31);
/* … */
__pyx_t_3 = PyCFunction_NewEx(&__pyx_mdef_8discodop_10_fragments_9allfragments, NULL, __pyx_n_s_discodop__fragments); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 652, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
if (PyDict_SetItem(__pyx_d, __pyx_n_s_allfragments, __pyx_t_3) < 0) __PYX_ERR(0, 652, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__pyx_codeobj__32 = (PyObject*)__Pyx_PyCode_New(6, 0, 21, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__31, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_discodop__fragments_pyx, __pyx_n_s_allfragments, 652, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__32)) __PYX_ERR(0, 652, __pyx_L1_error)
+0653: unsigned int maxdepth, unsigned int maxfrontier=999, bint disc=True,
__pyx_v_disc = ((int)1);
}
if (values[5]) {
__pyx_v_indices = __Pyx_PyObject_IsTrue(values[5]); if (unlikely((__pyx_v_indices == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 654, __pyx_L3_error)
} else {
+0654: bint indices=False):
__pyx_v_indices = ((int)0);
}
}
goto __pyx_L4_argument_unpacking_done;
__pyx_L5_argtuple_error:;
__Pyx_RaiseArgtupleInvalid("allfragments", 0, 3, 6, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 652, __pyx_L3_error)
__pyx_L3_error:;
__Pyx_AddTraceback("discodop._fragments.allfragments", __pyx_clineno, __pyx_lineno, __pyx_filename);
__Pyx_RefNannyFinishContext();
return NULL;
__pyx_L4_argument_unpacking_done:;
if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_trees), __pyx_ptype_8discodop_10containers_Ctrees, 1, "trees", 0))) __PYX_ERR(0, 652, __pyx_L1_error)
if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_vocab), __pyx_ptype_8discodop_10containers_Vocabulary, 1, "vocab", 0))) __PYX_ERR(0, 652, __pyx_L1_error)
__pyx_r = __pyx_pf_8discodop_10_fragments_8allfragments(__pyx_self, __pyx_v_trees, __pyx_v_vocab, __pyx_v_maxdepth, __pyx_v_maxfrontier, __pyx_v_disc, __pyx_v_indices);
0655: """Return all fragments of trees up to maxdepth.
0656:
0657: :param maxdepth: maximum depth of fragments; depth 1 gives fragments that
0658: are equivalent to a treebank grammar.
0659: :param maxfrontier: maximum number of frontier non-terminals (substitution
0660: sites) in fragments; a limit of 0 only gives fragments that bottom out
0661: in terminals; 999 is unlimited for practical purposes.
0662: :returns: dictionary fragments with tree strings as keys and integer counts
0663: as values."""
0664: cdef NodeArray tree
0665: cdef Node *nodes
+0666: cdef int n, SLOTS = BITNSLOTS(trees.maxnodes)
__pyx_v_SLOTS = BITNSLOTS(__pyx_v_trees->maxnodes);
+0667: cdef list table, tmp = []
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 667, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_tmp = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
+0668: cdef set inter = set()
__pyx_t_1 = PySet_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 668, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_inter = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
+0669: cdef dict fragments = {}
__pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 669, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_fragments = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
+0670: cdef uint64_t *scratch = <uint64_t *>malloc((SLOTS + 2) * sizeof(uint64_t))
__pyx_v_scratch = ((uint64_t *)malloc(((__pyx_v_SLOTS + 2) * (sizeof(uint64_t)))));
+0671: if scratch is NULL:
__pyx_t_2 = ((__pyx_v_scratch == NULL) != 0);
if (__pyx_t_2) {
/* … */
}
+0672: raise MemoryError('allocation error')
__pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 672, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __PYX_ERR(0, 672, __pyx_L1_error) /* … */ __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_u_allocation_error); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(0, 672, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__3); __Pyx_GIVEREF(__pyx_tuple__3);
+0673: for n in range(trees.len):
__pyx_t_3 = __pyx_v_trees->len;
for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) {
__pyx_v_n = __pyx_t_4;
+0674: tree = trees.trees[n]
__pyx_v_tree = (__pyx_v_trees->trees[__pyx_v_n]);
+0675: sent = trees.extractsent(n, vocab)
__pyx_t_5 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_trees), __pyx_n_s_extractsent); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 675, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = __Pyx_PyInt_From_int(__pyx_v_n); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 675, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_7 = NULL; __pyx_t_8 = 0; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_5))) { __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_5); if (likely(__pyx_t_7)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); __Pyx_INCREF(__pyx_t_7); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_5, function); __pyx_t_8 = 1; } } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[3] = {__pyx_t_7, __pyx_t_6, ((PyObject *)__pyx_v_vocab)}; __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_8, 2+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 675, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[3] = {__pyx_t_7, __pyx_t_6, ((PyObject *)__pyx_v_vocab)}; __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_8, 2+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 675, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { __pyx_t_9 = PyTuple_New(2+__pyx_t_8); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 675, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); if (__pyx_t_7) { __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_7); __pyx_t_7 = NULL; } __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_9, 0+__pyx_t_8, __pyx_t_6); __Pyx_INCREF(((PyObject *)__pyx_v_vocab)); __Pyx_GIVEREF(((PyObject *)__pyx_v_vocab)); PyTuple_SET_ITEM(__pyx_t_9, 1+__pyx_t_8, ((PyObject *)__pyx_v_vocab)); __pyx_t_6 = 0; __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 675, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_XDECREF_SET(__pyx_v_sent, __pyx_t_1); __pyx_t_1 = 0;
+0676: nodes = &trees.nodes[tree.offset]
__pyx_v_nodes = (&(__pyx_v_trees->nodes[__pyx_v_tree.offset]));
0677: # for each node, list of bitsets wrapped in bytes
+0678: table = [[] for _ in range(tree.len)]
{ /* enter inner scope */
CYTHON_UNUSED int16_t __pyx_8genexpr7__pyx_v__;
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 678, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_10 = __pyx_v_tree.len;
for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_10; __pyx_t_11+=1) {
__pyx_8genexpr7__pyx_v__ = __pyx_t_11;
__pyx_t_5 = PyList_New(0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 678, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(0, 678, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
}
} /* exit inner scope */
__Pyx_XDECREF_SET(__pyx_v_table, ((PyObject*)__pyx_t_1));
__pyx_t_1 = 0;
+0679: traverse(tree, nodes, tree.root, n, maxdepth, maxfrontier, table,
__pyx_t_1 = __pyx_f_8discodop_10_fragments_traverse(__pyx_v_tree, __pyx_v_nodes, __pyx_v_tree.root, __pyx_v_n, __pyx_v_maxdepth, __pyx_v_maxfrontier, __pyx_v_table, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 679, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
0680: scratch, SLOTS)
0681: # collect subtrees at each node in a single set
+0682: for i, frags in enumerate(table):
__pyx_t_12 = 0;
__pyx_t_1 = __pyx_v_table; __Pyx_INCREF(__pyx_t_1); __pyx_t_13 = 0;
for (;;) {
if (__pyx_t_13 >= PyList_GET_SIZE(__pyx_t_1)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_5 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_13); __Pyx_INCREF(__pyx_t_5); __pyx_t_13++; if (unlikely(0 < 0)) __PYX_ERR(0, 682, __pyx_L1_error)
#else
__pyx_t_5 = PySequence_ITEM(__pyx_t_1, __pyx_t_13); __pyx_t_13++; if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 682, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
#endif
__Pyx_XDECREF_SET(__pyx_v_frags, __pyx_t_5);
__pyx_t_5 = 0;
__pyx_v_i = __pyx_t_12;
__pyx_t_12 = (__pyx_t_12 + 1);
/* … */
}
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+0683: for frag in frags:
if (likely(PyList_CheckExact(__pyx_v_frags)) || PyTuple_CheckExact(__pyx_v_frags)) { __pyx_t_5 = __pyx_v_frags; __Pyx_INCREF(__pyx_t_5); __pyx_t_14 = 0; __pyx_t_15 = NULL; } else { __pyx_t_14 = -1; __pyx_t_5 = PyObject_GetIter(__pyx_v_frags); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 683, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_15 = Py_TYPE(__pyx_t_5)->tp_iternext; if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 683, __pyx_L1_error) } for (;;) { if (likely(!__pyx_t_15)) { if (likely(PyList_CheckExact(__pyx_t_5))) { if (__pyx_t_14 >= PyList_GET_SIZE(__pyx_t_5)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_9 = PyList_GET_ITEM(__pyx_t_5, __pyx_t_14); __Pyx_INCREF(__pyx_t_9); __pyx_t_14++; if (unlikely(0 < 0)) __PYX_ERR(0, 683, __pyx_L1_error) #else __pyx_t_9 = PySequence_ITEM(__pyx_t_5, __pyx_t_14); __pyx_t_14++; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 683, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); #endif } else { if (__pyx_t_14 >= PyTuple_GET_SIZE(__pyx_t_5)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_9 = PyTuple_GET_ITEM(__pyx_t_5, __pyx_t_14); __Pyx_INCREF(__pyx_t_9); __pyx_t_14++; if (unlikely(0 < 0)) __PYX_ERR(0, 683, __pyx_L1_error) #else __pyx_t_9 = PySequence_ITEM(__pyx_t_5, __pyx_t_14); __pyx_t_14++; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 683, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); #endif } } else { __pyx_t_9 = __pyx_t_15(__pyx_t_5); if (unlikely(!__pyx_t_9)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 683, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_9); } __Pyx_XDECREF_SET(__pyx_v_frag, __pyx_t_9); __pyx_t_9 = 0; /* … */ } __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
0684: # in the table, the root & id attributes are used to store
0685: # the depth and frontiers of fragments; replace these
+0686: depth = getroot(getpointer(frag), SLOTS)
__pyx_v_depth = __pyx_f_8discodop_10_fragments_getroot(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_frag), __pyx_v_SLOTS);
+0687: frontiers = getid(getpointer(frag), SLOTS)
__pyx_v_frontiers = __pyx_f_8discodop_10_fragments_getid(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_frag), __pyx_v_SLOTS);
+0688: assert <size_t>depth <= maxdepth
#ifndef CYTHON_WITHOUT_ASSERTIONS
if (unlikely(!Py_OptimizeFlag)) {
if (unlikely(!((((size_t)__pyx_v_depth) <= __pyx_v_maxdepth) != 0))) {
PyErr_SetNone(PyExc_AssertionError);
__PYX_ERR(0, 688, __pyx_L1_error)
}
}
#endif
+0689: assert frontiers <= maxfrontier
#ifndef CYTHON_WITHOUT_ASSERTIONS
if (unlikely(!Py_OptimizeFlag)) {
if (unlikely(!((__pyx_v_frontiers <= __pyx_v_maxfrontier) != 0))) {
PyErr_SetNone(PyExc_AssertionError);
__PYX_ERR(0, 689, __pyx_L1_error)
}
}
#endif
+0690: memcpy(scratch, getpointer(frag), SLOTS * sizeof(uint64_t))
memcpy(__pyx_v_scratch, __pyx_f_8discodop_10_fragments_getpointer(__pyx_v_frag), (__pyx_v_SLOTS * (sizeof(uint64_t))));
+0691: setrootid(scratch, i, n, SLOTS)
__pyx_f_8discodop_10_fragments_setrootid(__pyx_v_scratch, __pyx_v_i, __pyx_v_n, __pyx_v_SLOTS);
+0692: inter.add(wrap(scratch, SLOTS))
__pyx_t_9 = __pyx_f_8discodop_10_fragments_wrap(__pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 692, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_9);
__pyx_t_16 = PySet_Add(__pyx_v_inter, __pyx_t_9); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 692, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+0693: collectfragments(fragments, inter, nodes, sent, vocab,
if (!(likely(PyList_CheckExact(__pyx_v_sent))||((__pyx_v_sent) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "list", Py_TYPE(__pyx_v_sent)->tp_name), 0))) __PYX_ERR(0, 693, __pyx_L1_error) /* … */ __pyx_t_1 = __pyx_f_8discodop_10_fragments_collectfragments(__pyx_v_fragments, __pyx_v_inter, __pyx_v_nodes, ((PyObject*)__pyx_v_sent), __pyx_v_vocab, __pyx_v_disc, (!(__pyx_v_indices != 0)), __pyx_v_indices, __pyx_v_tmp, __pyx_v_SLOTS); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 693, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
0694: disc, not indices, indices, tmp, SLOTS)
+0695: del table[:]
if (__Pyx_PyObject_DelSlice(__pyx_v_table, 0, 0, NULL, NULL, NULL, 0, 0, 0) < 0) __PYX_ERR(0, 695, __pyx_L1_error)
}
+0696: free(scratch)
free(__pyx_v_scratch);
+0697: return fragments
__Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_fragments); __pyx_r = __pyx_v_fragments; goto __pyx_L0;
0698:
0699:
+0700: cdef traverse(NodeArray tree, Node *nodes, int i, int n, int maxdepth,
static PyObject *__pyx_f_8discodop_10_fragments_traverse(struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_tree, struct __pyx_t_8discodop_10containers_Node *__pyx_v_nodes, int __pyx_v_i, int __pyx_v_n, int __pyx_v_maxdepth, int __pyx_v_maxfrontier, PyObject *__pyx_v_table, uint64_t *__pyx_v_scratch, int __pyx_v_SLOTS) {
PyObject *__pyx_v_lfrag = 0;
PyObject *__pyx_v_rfrag = 0;
int __pyx_v_ldepth;
int __pyx_v_rdepth;
int __pyx_v_lfrontiers;
int __pyx_v_rfrontiers;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("traverse", 0);
/* … */
/* function exit code */
__pyx_r = Py_None; __Pyx_INCREF(Py_None);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_10);
__Pyx_AddTraceback("discodop._fragments.traverse", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_lfrag);
__Pyx_XDECREF(__pyx_v_rfrag);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0701: int maxfrontier, list table, uint64_t *scratch, int SLOTS):
0702: """Collect all fragments of a tree up to maxdepth, maxfrontier."""
0703: cdef bytes lfrag, rfrag
0704: cdef int ldepth, rdepth, lfrontiers, rfrontiers
0705: # First collect fragments of children
+0706: if nodes[i].left >= 0:
__pyx_t_1 = (((__pyx_v_nodes[__pyx_v_i]).left >= 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0707: traverse(tree, nodes, nodes[i].left, n, maxdepth, maxfrontier,
__pyx_t_2 = __pyx_f_8discodop_10_fragments_traverse(__pyx_v_tree, __pyx_v_nodes, (__pyx_v_nodes[__pyx_v_i]).left, __pyx_v_n, __pyx_v_maxdepth, __pyx_v_maxfrontier, __pyx_v_table, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 707, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
0708: table, scratch, SLOTS)
+0709: if nodes[i].right >= 0:
__pyx_t_1 = (((__pyx_v_nodes[__pyx_v_i]).right >= 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0710: traverse(tree, nodes, nodes[i].right, n, maxdepth, maxfrontier,
__pyx_t_2 = __pyx_f_8discodop_10_fragments_traverse(__pyx_v_tree, __pyx_v_nodes, (__pyx_v_nodes[__pyx_v_i]).right, __pyx_v_n, __pyx_v_maxdepth, __pyx_v_maxfrontier, __pyx_v_table, __pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 710, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
0711: table, scratch, SLOTS)
0712: # Production at current idx is a depth 1 fragment; 0, 1, or 2 frontiers
+0713: if nodes[i].left < 0:
__pyx_t_1 = (((__pyx_v_nodes[__pyx_v_i]).left < 0) != 0);
if (__pyx_t_1) {
/* … */
goto __pyx_L5;
}
+0714: lfrontiers = 0
__pyx_v_lfrontiers = 0;
+0715: elif nodes[i].right < 0:
__pyx_t_1 = (((__pyx_v_nodes[__pyx_v_i]).right < 0) != 0);
if (__pyx_t_1) {
/* … */
goto __pyx_L5;
}
+0716: lfrontiers = 1
__pyx_v_lfrontiers = 1;
0717: else:
+0718: lfrontiers = 2
/*else*/ {
__pyx_v_lfrontiers = 2;
}
__pyx_L5:;
+0719: if lfrontiers <= maxfrontier:
__pyx_t_1 = ((__pyx_v_lfrontiers <= __pyx_v_maxfrontier) != 0);
if (__pyx_t_1) {
/* … */
}
+0720: memset(<void *>scratch, 0, SLOTS * sizeof(uint64_t))
memset(((void *)__pyx_v_scratch), 0, (__pyx_v_SLOTS * (sizeof(uint64_t))));
+0721: SETBIT(scratch, i)
SETBIT(__pyx_v_scratch, __pyx_v_i);
+0722: setrootid(scratch, 1, lfrontiers, SLOTS)
__pyx_f_8discodop_10_fragments_setrootid(__pyx_v_scratch, 1, __pyx_v_lfrontiers, __pyx_v_SLOTS);
+0723: table[i].append(wrap(scratch, SLOTS))
if (unlikely(__pyx_v_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 723, __pyx_L1_error)
}
__pyx_t_2 = __pyx_f_8discodop_10_fragments_wrap(__pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 723, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_3 = __Pyx_PyObject_Append(PyList_GET_ITEM(__pyx_v_table, __pyx_v_i), __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(0, 723, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
0724: # Then combine with fragments of children to form prunes of this subtree
+0725: if nodes[i].left >= 0:
__pyx_t_1 = (((__pyx_v_nodes[__pyx_v_i]).left >= 0) != 0);
if (__pyx_t_1) {
/* … */
}
0726: # unary, or right node as frontier with all lfrags
+0727: for lfrag in table[nodes[i].left]:
if (unlikely(__pyx_v_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 727, __pyx_L1_error)
}
if (likely(PyList_CheckExact(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).left))) || PyTuple_CheckExact(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).left))) {
__pyx_t_2 = PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).left); __Pyx_INCREF(__pyx_t_2); __pyx_t_4 = 0;
__pyx_t_5 = NULL;
} else {
__pyx_t_4 = -1; __pyx_t_2 = PyObject_GetIter(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).left)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 727, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_5 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 727, __pyx_L1_error)
}
for (;;) {
if (likely(!__pyx_t_5)) {
if (likely(PyList_CheckExact(__pyx_t_2))) {
if (__pyx_t_4 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_6 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_4); __Pyx_INCREF(__pyx_t_6); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 727, __pyx_L1_error)
#else
__pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 727, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
#endif
} else {
if (__pyx_t_4 >= PyTuple_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_4); __Pyx_INCREF(__pyx_t_6); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 727, __pyx_L1_error)
#else
__pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 727, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
#endif
}
} else {
__pyx_t_6 = __pyx_t_5(__pyx_t_2);
if (unlikely(!__pyx_t_6)) {
PyObject* exc_type = PyErr_Occurred();
if (exc_type) {
if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
else __PYX_ERR(0, 727, __pyx_L1_error)
}
break;
}
__Pyx_GOTREF(__pyx_t_6);
}
if (!(likely(PyBytes_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_6)->tp_name), 0))) __PYX_ERR(0, 727, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_lfrag, ((PyObject*)__pyx_t_6));
__pyx_t_6 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0728: ldepth = getroot(getpointer(lfrag), SLOTS)
__pyx_v_ldepth = __pyx_f_8discodop_10_fragments_getroot(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_lfrag), __pyx_v_SLOTS);
+0729: lfrontiers = getid(getpointer(lfrag), SLOTS)
__pyx_v_lfrontiers = __pyx_f_8discodop_10_fragments_getid(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_lfrag), __pyx_v_SLOTS);
+0730: if (ldepth + 1 <= maxdepth
__pyx_t_7 = (((__pyx_v_ldepth + 1) <= __pyx_v_maxdepth) != 0);
if (__pyx_t_7) {
} else {
__pyx_t_1 = __pyx_t_7;
goto __pyx_L11_bool_binop_done;
}
/* … */
if (__pyx_t_1) {
/* … */
}
+0731: and lfrontiers + 1 <= maxfrontier):
__pyx_t_7 = (((__pyx_v_lfrontiers + 1) <= __pyx_v_maxfrontier) != 0);
__pyx_t_1 = __pyx_t_7;
__pyx_L11_bool_binop_done:;
0732: # bitset with current idx + left bitset
+0733: memset(<void *>scratch, 0, SLOTS * sizeof(uint64_t))
memset(((void *)__pyx_v_scratch), 0, (__pyx_v_SLOTS * (sizeof(uint64_t))));
+0734: SETBIT(scratch, i)
SETBIT(__pyx_v_scratch, __pyx_v_i);
+0735: setunioninplace(scratch, getpointer(lfrag), SLOTS)
__pyx_f_8discodop_3bit_setunioninplace(__pyx_v_scratch, __pyx_f_8discodop_10_fragments_getpointer(__pyx_v_lfrag), __pyx_v_SLOTS);
+0736: setrootid(scratch, ldepth + 1, lfrontiers + 1, SLOTS)
__pyx_f_8discodop_10_fragments_setrootid(__pyx_v_scratch, (__pyx_v_ldepth + 1), (__pyx_v_lfrontiers + 1), __pyx_v_SLOTS);
+0737: table[i].append(wrap(scratch, SLOTS))
if (unlikely(__pyx_v_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 737, __pyx_L1_error)
}
__pyx_t_6 = __pyx_f_8discodop_10_fragments_wrap(__pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 737, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__pyx_t_3 = __Pyx_PyObject_Append(PyList_GET_ITEM(__pyx_v_table, __pyx_v_i), __pyx_t_6); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(0, 737, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+0738: if nodes[i].right >= 0: # binary node
__pyx_t_1 = (((__pyx_v_nodes[__pyx_v_i]).right >= 0) != 0);
if (__pyx_t_1) {
/* … */
}
0739: # left node as frontier with all rfrags
+0740: for rfrag in table[nodes[i].right]:
if (unlikely(__pyx_v_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 740, __pyx_L1_error)
}
if (likely(PyList_CheckExact(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).right))) || PyTuple_CheckExact(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).right))) {
__pyx_t_2 = PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).right); __Pyx_INCREF(__pyx_t_2); __pyx_t_4 = 0;
__pyx_t_5 = NULL;
} else {
__pyx_t_4 = -1; __pyx_t_2 = PyObject_GetIter(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).right)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 740, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_5 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 740, __pyx_L1_error)
}
for (;;) {
if (likely(!__pyx_t_5)) {
if (likely(PyList_CheckExact(__pyx_t_2))) {
if (__pyx_t_4 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_6 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_4); __Pyx_INCREF(__pyx_t_6); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 740, __pyx_L1_error)
#else
__pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 740, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
#endif
} else {
if (__pyx_t_4 >= PyTuple_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_4); __Pyx_INCREF(__pyx_t_6); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 740, __pyx_L1_error)
#else
__pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 740, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
#endif
}
} else {
__pyx_t_6 = __pyx_t_5(__pyx_t_2);
if (unlikely(!__pyx_t_6)) {
PyObject* exc_type = PyErr_Occurred();
if (exc_type) {
if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
else __PYX_ERR(0, 740, __pyx_L1_error)
}
break;
}
__Pyx_GOTREF(__pyx_t_6);
}
if (!(likely(PyBytes_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_6)->tp_name), 0))) __PYX_ERR(0, 740, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_rfrag, ((PyObject*)__pyx_t_6));
__pyx_t_6 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0741: rdepth = getroot(getpointer(rfrag), SLOTS)
__pyx_v_rdepth = __pyx_f_8discodop_10_fragments_getroot(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_rfrag), __pyx_v_SLOTS);
+0742: rfrontiers = getid(getpointer(rfrag), SLOTS)
__pyx_v_rfrontiers = __pyx_f_8discodop_10_fragments_getid(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_rfrag), __pyx_v_SLOTS);
+0743: if (rdepth + 1 <= maxdepth
__pyx_t_7 = (((__pyx_v_rdepth + 1) <= __pyx_v_maxdepth) != 0);
if (__pyx_t_7) {
} else {
__pyx_t_1 = __pyx_t_7;
goto __pyx_L17_bool_binop_done;
}
/* … */
if (__pyx_t_1) {
/* … */
}
+0744: and rfrontiers + 1 <= maxfrontier):
__pyx_t_7 = (((__pyx_v_rfrontiers + 1) <= __pyx_v_maxfrontier) != 0);
__pyx_t_1 = __pyx_t_7;
__pyx_L17_bool_binop_done:;
0745: # bitset with current idx + right bitset
+0746: memset(<void *>scratch, 0, SLOTS * sizeof(uint64_t))
memset(((void *)__pyx_v_scratch), 0, (__pyx_v_SLOTS * (sizeof(uint64_t))));
+0747: SETBIT(scratch, i)
SETBIT(__pyx_v_scratch, __pyx_v_i);
+0748: setunioninplace(scratch, getpointer(rfrag), SLOTS)
__pyx_f_8discodop_3bit_setunioninplace(__pyx_v_scratch, __pyx_f_8discodop_10_fragments_getpointer(__pyx_v_rfrag), __pyx_v_SLOTS);
+0749: setrootid(scratch, rdepth + 1, rfrontiers + 1, SLOTS)
__pyx_f_8discodop_10_fragments_setrootid(__pyx_v_scratch, (__pyx_v_rdepth + 1), (__pyx_v_rfrontiers + 1), __pyx_v_SLOTS);
+0750: table[i].append(wrap(scratch, SLOTS))
if (unlikely(__pyx_v_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 750, __pyx_L1_error)
}
__pyx_t_6 = __pyx_f_8discodop_10_fragments_wrap(__pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 750, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__pyx_t_3 = __Pyx_PyObject_Append(PyList_GET_ITEM(__pyx_v_table, __pyx_v_i), __pyx_t_6); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(0, 750, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
0751: # cartesian product of left x right fragments
+0752: for lfrag in table[nodes[i].left]:
if (unlikely(__pyx_v_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 752, __pyx_L1_error)
}
if (likely(PyList_CheckExact(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).left))) || PyTuple_CheckExact(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).left))) {
__pyx_t_2 = PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).left); __Pyx_INCREF(__pyx_t_2); __pyx_t_4 = 0;
__pyx_t_5 = NULL;
} else {
__pyx_t_4 = -1; __pyx_t_2 = PyObject_GetIter(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).left)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 752, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_5 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 752, __pyx_L1_error)
}
for (;;) {
if (likely(!__pyx_t_5)) {
if (likely(PyList_CheckExact(__pyx_t_2))) {
if (__pyx_t_4 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_6 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_4); __Pyx_INCREF(__pyx_t_6); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 752, __pyx_L1_error)
#else
__pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 752, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
#endif
} else {
if (__pyx_t_4 >= PyTuple_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_4); __Pyx_INCREF(__pyx_t_6); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 752, __pyx_L1_error)
#else
__pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 752, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
#endif
}
} else {
__pyx_t_6 = __pyx_t_5(__pyx_t_2);
if (unlikely(!__pyx_t_6)) {
PyObject* exc_type = PyErr_Occurred();
if (exc_type) {
if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
else __PYX_ERR(0, 752, __pyx_L1_error)
}
break;
}
__Pyx_GOTREF(__pyx_t_6);
}
if (!(likely(PyBytes_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_6)->tp_name), 0))) __PYX_ERR(0, 752, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_lfrag, ((PyObject*)__pyx_t_6));
__pyx_t_6 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0753: ldepth = getroot(getpointer(lfrag), SLOTS)
__pyx_v_ldepth = __pyx_f_8discodop_10_fragments_getroot(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_lfrag), __pyx_v_SLOTS);
+0754: lfrontiers = getid(getpointer(lfrag), SLOTS)
__pyx_v_lfrontiers = __pyx_f_8discodop_10_fragments_getid(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_lfrag), __pyx_v_SLOTS);
+0755: for rfrag in table[nodes[i].right]:
if (unlikely(__pyx_v_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 755, __pyx_L1_error)
}
if (likely(PyList_CheckExact(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).right))) || PyTuple_CheckExact(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).right))) {
__pyx_t_6 = PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).right); __Pyx_INCREF(__pyx_t_6); __pyx_t_8 = 0;
__pyx_t_9 = NULL;
} else {
__pyx_t_8 = -1; __pyx_t_6 = PyObject_GetIter(PyList_GET_ITEM(__pyx_v_table, (__pyx_v_nodes[__pyx_v_i]).right)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 755, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__pyx_t_9 = Py_TYPE(__pyx_t_6)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 755, __pyx_L1_error)
}
for (;;) {
if (likely(!__pyx_t_9)) {
if (likely(PyList_CheckExact(__pyx_t_6))) {
if (__pyx_t_8 >= PyList_GET_SIZE(__pyx_t_6)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_10 = PyList_GET_ITEM(__pyx_t_6, __pyx_t_8); __Pyx_INCREF(__pyx_t_10); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 755, __pyx_L1_error)
#else
__pyx_t_10 = PySequence_ITEM(__pyx_t_6, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 755, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_10);
#endif
} else {
if (__pyx_t_8 >= PyTuple_GET_SIZE(__pyx_t_6)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_10 = PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_8); __Pyx_INCREF(__pyx_t_10); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 755, __pyx_L1_error)
#else
__pyx_t_10 = PySequence_ITEM(__pyx_t_6, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 755, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_10);
#endif
}
} else {
__pyx_t_10 = __pyx_t_9(__pyx_t_6);
if (unlikely(!__pyx_t_10)) {
PyObject* exc_type = PyErr_Occurred();
if (exc_type) {
if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
else __PYX_ERR(0, 755, __pyx_L1_error)
}
break;
}
__Pyx_GOTREF(__pyx_t_10);
}
if (!(likely(PyBytes_CheckExact(__pyx_t_10))||((__pyx_t_10) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_10)->tp_name), 0))) __PYX_ERR(0, 755, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_rfrag, ((PyObject*)__pyx_t_10));
__pyx_t_10 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+0756: rdepth = getroot(getpointer(rfrag), SLOTS)
__pyx_v_rdepth = __pyx_f_8discodop_10_fragments_getroot(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_rfrag), __pyx_v_SLOTS);
+0757: rfrontiers = getid(getpointer(rfrag), SLOTS)
__pyx_v_rfrontiers = __pyx_f_8discodop_10_fragments_getid(__pyx_f_8discodop_10_fragments_getpointer(__pyx_v_rfrag), __pyx_v_SLOTS);
+0758: if (max(ldepth, rdepth) + 1 <= maxdepth
__pyx_t_11 = __pyx_v_rdepth;
__pyx_t_12 = __pyx_v_ldepth;
if (((__pyx_t_11 > __pyx_t_12) != 0)) {
__pyx_t_13 = __pyx_t_11;
} else {
__pyx_t_13 = __pyx_t_12;
}
/* … */
__pyx_t_7 = (((__pyx_t_13 + 1) <= __pyx_v_maxdepth) != 0);
if (__pyx_t_7) {
} else {
__pyx_t_1 = __pyx_t_7;
goto __pyx_L24_bool_binop_done;
}
/* … */
if (__pyx_t_1) {
/* … */
}
+0759: and lfrontiers + rfrontiers <= maxfrontier):
__pyx_t_7 = (((__pyx_v_lfrontiers + __pyx_v_rfrontiers) <= __pyx_v_maxfrontier) != 0);
__pyx_t_1 = __pyx_t_7;
__pyx_L24_bool_binop_done:;
0760: # bitset with current idx + pairs of bitsets in l, r
+0761: memset(<void *>scratch, 0, SLOTS * sizeof(uint64_t))
memset(((void *)__pyx_v_scratch), 0, (__pyx_v_SLOTS * (sizeof(uint64_t))));
+0762: SETBIT(scratch, i)
SETBIT(__pyx_v_scratch, __pyx_v_i);
+0763: setunioninplace(scratch, getpointer(lfrag), SLOTS)
__pyx_f_8discodop_3bit_setunioninplace(__pyx_v_scratch, __pyx_f_8discodop_10_fragments_getpointer(__pyx_v_lfrag), __pyx_v_SLOTS);
+0764: setunioninplace(scratch, getpointer(rfrag), SLOTS)
__pyx_f_8discodop_3bit_setunioninplace(__pyx_v_scratch, __pyx_f_8discodop_10_fragments_getpointer(__pyx_v_rfrag), __pyx_v_SLOTS);
+0765: setrootid(scratch, max(ldepth, rdepth) + 1,
__pyx_t_13 = __pyx_v_rdepth;
__pyx_t_11 = __pyx_v_ldepth;
if (((__pyx_t_13 > __pyx_t_11) != 0)) {
__pyx_t_12 = __pyx_t_13;
} else {
__pyx_t_12 = __pyx_t_11;
}
/* … */
__pyx_f_8discodop_10_fragments_setrootid(__pyx_v_scratch, (__pyx_t_12 + 1), (__pyx_v_lfrontiers + __pyx_v_rfrontiers), __pyx_v_SLOTS);
0766: lfrontiers + rfrontiers, SLOTS)
+0767: table[i].append(wrap(scratch, SLOTS))
if (unlikely(__pyx_v_table == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 767, __pyx_L1_error)
}
__pyx_t_10 = __pyx_f_8discodop_10_fragments_wrap(__pyx_v_scratch, __pyx_v_SLOTS); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 767, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_10);
__pyx_t_3 = __Pyx_PyObject_Append(PyList_GET_ITEM(__pyx_v_table, __pyx_v_i), __pyx_t_10); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(0, 767, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
0768:
0769:
+0770: cdef inline getsubtree(list result, Node *tree, uint64_t *bitset,
static CYTHON_INLINE PyObject *__pyx_f_8discodop_10_fragments_getsubtree(PyObject *__pyx_v_result, struct __pyx_t_8discodop_10containers_Node *__pyx_v_tree, uint64_t *__pyx_v_bitset, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, int __pyx_v_disc, int __pyx_v_i) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("getsubtree", 0);
/* … */
/* function exit code */
__pyx_r = Py_None; __Pyx_INCREF(Py_None);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_AddTraceback("discodop._fragments.getsubtree", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0771: Vocabulary vocab, bint disc, int i):
0772: """Get string of tree fragment denoted by bitset; indices as terminals.
0773:
0774: :param result: provide an empty list for the initial call.
0775: :param disc: pass True to get a tree with indices as leaves
0776: (discontinuous trees); otherwise the result will be a
0777: continuous tree with words as leaves."""
+0778: result.append('(')
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 778, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyList_Append(__pyx_v_result, __pyx_kp_u__4); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 778, __pyx_L1_error)
+0779: result.append(vocab.getlabel(tree[i].prod))
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 779, __pyx_L1_error)
}
__pyx_t_2 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getlabel(__pyx_v_vocab, (__pyx_v_tree[__pyx_v_i]).prod); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 779, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_1 = __Pyx_PyList_Append(__pyx_v_result, __pyx_t_2); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 779, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0780: result.append(' ')
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 780, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyList_Append(__pyx_v_result, __pyx_kp_u__5); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 780, __pyx_L1_error)
+0781: if TESTBIT(bitset, i):
__pyx_t_3 = (TESTBIT(__pyx_v_bitset, __pyx_v_i) != 0);
if (__pyx_t_3) {
/* … */
goto __pyx_L3;
}
+0782: if tree[i].left >= 0:
__pyx_t_3 = (((__pyx_v_tree[__pyx_v_i]).left >= 0) != 0);
if (__pyx_t_3) {
/* … */
goto __pyx_L4;
}
+0783: getsubtree(result, tree, bitset, vocab, disc, tree[i].left)
__pyx_t_2 = __pyx_f_8discodop_10_fragments_getsubtree(__pyx_v_result, __pyx_v_tree, __pyx_v_bitset, __pyx_v_vocab, __pyx_v_disc, (__pyx_v_tree[__pyx_v_i]).left); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 783, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0784: if tree[i].right >= 0:
__pyx_t_3 = (((__pyx_v_tree[__pyx_v_i]).right >= 0) != 0);
if (__pyx_t_3) {
/* … */
}
+0785: result.append(' ')
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 785, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyList_Append(__pyx_v_result, __pyx_kp_u__5); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 785, __pyx_L1_error)
+0786: getsubtree(result, tree, bitset, vocab, disc, tree[i].right)
__pyx_t_2 = __pyx_f_8discodop_10_fragments_getsubtree(__pyx_v_result, __pyx_v_tree, __pyx_v_bitset, __pyx_v_vocab, __pyx_v_disc, (__pyx_v_tree[__pyx_v_i]).right); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 786, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0787: elif disc:
__pyx_t_3 = (__pyx_v_disc != 0);
if (__pyx_t_3) {
/* … */
goto __pyx_L4;
}
+0788: if vocab.islexical(tree[i].prod):
__pyx_t_3 = (((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->islexical(__pyx_v_vocab, (__pyx_v_tree[__pyx_v_i]).prod) != 0);
if (__pyx_t_3) {
/* … */
goto __pyx_L6;
}
+0789: result.append('%d=%s' % (
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 789, __pyx_L1_error)
}
/* … */
__pyx_t_4 = PyUnicode_Format(__pyx_kp_u_d_s, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 789, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_4);
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
__pyx_t_1 = __Pyx_PyList_Append(__pyx_v_result, __pyx_t_4); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 789, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+0790: termidx(tree[i].left), vocab.getword(tree[i].prod)))
__pyx_t_2 = __Pyx_PyInt_From_short(__pyx_f_8discodop_10containers_termidx((__pyx_v_tree[__pyx_v_i]).left)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 790, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_4 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getword(__pyx_v_vocab, (__pyx_v_tree[__pyx_v_i]).prod); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 790, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 790, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_2); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4); __pyx_t_2 = 0; __pyx_t_4 = 0;
0791: else:
+0792: result.append('%d:%d' % (
/*else*/ {
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 792, __pyx_L1_error)
}
/* … */
__pyx_t_5 = PyUnicode_Format(__pyx_kp_u_d_d, __pyx_t_2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 792, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__pyx_t_1 = __Pyx_PyList_Append(__pyx_v_result, __pyx_t_5); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 792, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
}
__pyx_L6:;
+0793: termidx(tree[i].left), termidx(tree[i].left)))
__pyx_t_4 = __Pyx_PyInt_From_short(__pyx_f_8discodop_10containers_termidx((__pyx_v_tree[__pyx_v_i]).left)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 793, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __Pyx_PyInt_From_short(__pyx_f_8discodop_10containers_termidx((__pyx_v_tree[__pyx_v_i]).left)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 793, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 793, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_5); __pyx_t_4 = 0; __pyx_t_5 = 0;
0794: else:
+0795: result.append(vocab.getword(tree[i].prod) or '')
/*else*/ {
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 795, __pyx_L1_error)
}
__pyx_t_2 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getword(__pyx_v_vocab, (__pyx_v_tree[__pyx_v_i]).prod); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 795, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 795, __pyx_L1_error)
if (!__pyx_t_3) {
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
} else {
__Pyx_INCREF(__pyx_t_2);
__pyx_t_5 = __pyx_t_2;
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
goto __pyx_L7_bool_binop_done;
}
__Pyx_INCREF(__pyx_kp_u__2);
__pyx_t_5 = __pyx_kp_u__2;
__pyx_L7_bool_binop_done:;
__pyx_t_1 = __Pyx_PyList_Append(__pyx_v_result, __pyx_t_5); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 795, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
}
__pyx_L4:;
+0796: elif disc: # node not in bitset, frontier non-terminal
__pyx_t_3 = (__pyx_v_disc != 0);
if (__pyx_t_3) {
/* … */
}
__pyx_L3:;
+0797: result.append(yieldranges(sorted(getyield(tree, i))))
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 797, __pyx_L1_error)
}
__pyx_t_2 = __pyx_f_8discodop_10_fragments_getyield(__pyx_v_tree, __pyx_v_i); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 797, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_4 = PySequence_List(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 797, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_4);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__pyx_t_5 = ((PyObject*)__pyx_t_4);
__pyx_t_4 = 0;
__pyx_t_1 = PyList_Sort(__pyx_t_5); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 797, __pyx_L1_error)
__pyx_t_4 = __pyx_f_8discodop_10containers_yieldranges(((PyObject*)__pyx_t_5)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 797, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_4);
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
__pyx_t_1 = __Pyx_PyList_Append(__pyx_v_result, __pyx_t_4); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 797, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+0798: result.append(')')
if (unlikely(__pyx_v_result == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 798, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyList_Append(__pyx_v_result, __pyx_kp_u__6); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 798, __pyx_L1_error)
0799:
0800:
+0801: cdef inline list getyield(Node *tree, int i):
static CYTHON_INLINE PyObject *__pyx_f_8discodop_10_fragments_getyield(struct __pyx_t_8discodop_10containers_Node *__pyx_v_tree, int __pyx_v_i) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("getyield", 0);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_AddTraceback("discodop._fragments.getyield", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0802: """Recursively collect indices of terminals under a node."""
+0803: if tree[i].left < 0: # terminal / substitution site
__pyx_t_1 = (((__pyx_v_tree[__pyx_v_i]).left < 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0804: if tree[i].right >= 0: # substitution site with more indices
__pyx_t_1 = (((__pyx_v_tree[__pyx_v_i]).right >= 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0805: return [termidx(tree[i].left)] + getyield(tree, tree[i].right)
__Pyx_XDECREF(__pyx_r); __pyx_t_2 = __Pyx_PyInt_From_short(__pyx_f_8discodop_10containers_termidx((__pyx_v_tree[__pyx_v_i]).left)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 805, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = PyList_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 805, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_2); PyList_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __pyx_f_8discodop_10_fragments_getyield(__pyx_v_tree, (__pyx_v_tree[__pyx_v_i]).right); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 805, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_4 = PyNumber_Add(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 805, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; goto __pyx_L0;
+0806: return [termidx(tree[i].left)]
__Pyx_XDECREF(__pyx_r); __pyx_t_4 = __Pyx_PyInt_From_short(__pyx_f_8discodop_10containers_termidx((__pyx_v_tree[__pyx_v_i]).left)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 806, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 806, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_4); PyList_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); __pyx_t_4 = 0; __pyx_r = ((PyObject*)__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0;
+0807: elif tree[i].right < 0: # unary node
__pyx_t_1 = (((__pyx_v_tree[__pyx_v_i]).right < 0) != 0);
if (__pyx_t_1) {
/* … */
}
+0808: return getyield(tree, tree[i].left)
__Pyx_XDECREF(__pyx_r); __pyx_t_2 = __pyx_f_8discodop_10_fragments_getyield(__pyx_v_tree, (__pyx_v_tree[__pyx_v_i]).left); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 808, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = ((PyObject*)__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0;
0809: # binary node
+0810: return getyield(tree, tree[i].left) + getyield(tree, tree[i].right)
__Pyx_XDECREF(__pyx_r); __pyx_t_2 = __pyx_f_8discodop_10_fragments_getyield(__pyx_v_tree, (__pyx_v_tree[__pyx_v_i]).left); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 810, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_4 = __pyx_f_8discodop_10_fragments_getyield(__pyx_v_tree, (__pyx_v_tree[__pyx_v_i]).right); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 810, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyNumber_Add(__pyx_t_2, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 810, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_r = ((PyObject*)__pyx_t_3); __pyx_t_3 = 0; goto __pyx_L0;
0811:
0812:
+0813: def pygetsent(str frag):
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_11pygetsent(PyObject *__pyx_self, PyObject *__pyx_v_frag); /*proto*/
static char __pyx_doc_8discodop_10_fragments_10pygetsent[] = "pygetsent(unicode frag)\nWrapper of ``getsent()`` to make doctests possible.\n\n\t>>> print(pygetsent(u'(S (NP 2=man) (VP 4=walks))'))\n\t(S (NP 0=man) (VP 2=walks))\n\t>>> print(pygetsent(u'(VP (VB 0=Wake) (PRT 3=up))'))\n\t(VP (VB 0=Wake) (PRT 2=up))\n\t>>> print(pygetsent(u'(S (NP 2:2 4:4) (VP 1:1 3:3))'))\n\t(S (NP 1= 3=) (VP 0= 2=))\n\t>>> print(pygetsent(u'(ROOT (S 0:2) ($. 3=.))'))\n\t(ROOT (S 0=) ($. 1=.))\n\t>>> print(pygetsent(u'(ROOT (S 0=Foo) ($. 3=.))'))\n\t(ROOT (S 0=Foo) ($. 2=.))\n\t>>> print(pygetsent(\n\t... u'(S|<VP>_2 (VP_3 0:1 3:3 16:16) (VAFIN 2=wird))'))\n\t(S|<VP>_2 (VP_3 0= 2= 4=) (VAFIN 1=wird))\n\t";
static PyMethodDef __pyx_mdef_8discodop_10_fragments_11pygetsent = {"pygetsent", (PyCFunction)__pyx_pw_8discodop_10_fragments_11pygetsent, METH_O, __pyx_doc_8discodop_10_fragments_10pygetsent};
static PyObject *__pyx_pw_8discodop_10_fragments_11pygetsent(PyObject *__pyx_self, PyObject *__pyx_v_frag) {
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("pygetsent (wrapper)", 0);
if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_frag), (&PyUnicode_Type), 1, "frag", 1))) __PYX_ERR(0, 813, __pyx_L1_error)
__pyx_r = __pyx_pf_8discodop_10_fragments_10pygetsent(__pyx_self, ((PyObject*)__pyx_v_frag));
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__pyx_r = NULL;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_8discodop_10_fragments_10pygetsent(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_frag) {
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("pygetsent", 0);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_4);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_7);
__Pyx_XDECREF(__pyx_t_8);
__Pyx_AddTraceback("discodop._fragments.pygetsent", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
__pyx_tuple__33 = PyTuple_Pack(1, __pyx_n_s_frag); if (unlikely(!__pyx_tuple__33)) __PYX_ERR(0, 813, __pyx_L1_error)
__Pyx_GOTREF(__pyx_tuple__33);
__Pyx_GIVEREF(__pyx_tuple__33);
/* … */
__pyx_t_3 = PyCFunction_NewEx(&__pyx_mdef_8discodop_10_fragments_11pygetsent, NULL, __pyx_n_s_discodop__fragments); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 813, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
if (PyDict_SetItem(__pyx_d, __pyx_n_s_pygetsent, __pyx_t_3) < 0) __PYX_ERR(0, 813, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__pyx_codeobj__34 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__33, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_discodop__fragments_pyx, __pyx_n_s_pygetsent, 813, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__34)) __PYX_ERR(0, 813, __pyx_L1_error)
0814: """Wrapper of ``getsent()`` to make doctests possible.
0815:
0816: >>> print(pygetsent(u'(S (NP 2=man) (VP 4=walks))'))
0817: (S (NP 0=man) (VP 2=walks))
0818: >>> print(pygetsent(u'(VP (VB 0=Wake) (PRT 3=up))'))
0819: (VP (VB 0=Wake) (PRT 2=up))
0820: >>> print(pygetsent(u'(S (NP 2:2 4:4) (VP 1:1 3:3))'))
0821: (S (NP 1= 3=) (VP 0= 2=))
0822: >>> print(pygetsent(u'(ROOT (S 0:2) ($. 3=.))'))
0823: (ROOT (S 0=) ($. 1=.))
0824: >>> print(pygetsent(u'(ROOT (S 0=Foo) ($. 3=.))'))
0825: (ROOT (S 0=Foo) ($. 2=.))
0826: >>> print(pygetsent(
0827: ... u'(S|<VP>_2 (VP_3 0:1 3:3 16:16) (VAFIN 2=wird))'))
0828: (S|<VP>_2 (VP_3 0= 2= 4=) (VAFIN 1=wird))
0829: """
+0830: try:
{
/*try:*/ {
/* … */
}
__pyx_L3_error:;
__Pyx_PyThreadState_assign
__Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
/* … */
__Pyx_PyThreadState_assign
__Pyx_XGIVEREF(__pyx_t_1);
__Pyx_XGIVEREF(__pyx_t_2);
__Pyx_XGIVEREF(__pyx_t_3);
__Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3);
goto __pyx_L1_error;
__pyx_L7_try_return:;
__Pyx_PyThreadState_assign
__Pyx_XGIVEREF(__pyx_t_1);
__Pyx_XGIVEREF(__pyx_t_2);
__Pyx_XGIVEREF(__pyx_t_3);
__Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3);
goto __pyx_L0;
}
+0831: return getsent(frag)
__Pyx_XDECREF(__pyx_r); __pyx_t_4 = __pyx_f_8discodop_10_fragments_getsent(__pyx_v_frag); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 831, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_4); __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L7_try_return;
+0832: except:
/*except:*/ {
__Pyx_AddTraceback("discodop._fragments.pygetsent", __pyx_clineno, __pyx_lineno, __pyx_filename);
if (__Pyx_GetException(&__pyx_t_4, &__pyx_t_5, &__pyx_t_6) < 0) __PYX_ERR(0, 832, __pyx_L5_except_error)
__Pyx_GOTREF(__pyx_t_4);
__Pyx_GOTREF(__pyx_t_5);
__Pyx_GOTREF(__pyx_t_6);
+0833: print(frag)
__pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 833, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_INCREF(__pyx_v_frag); __Pyx_GIVEREF(__pyx_v_frag); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_frag); __pyx_t_8 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_7, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 833, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+0834: raise
__Pyx_GIVEREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_5); __Pyx_XGIVEREF(__pyx_t_6); __Pyx_ErrRestoreWithState(__pyx_t_4, __pyx_t_5, __pyx_t_6); __pyx_t_4 = 0; __pyx_t_5 = 0; __pyx_t_6 = 0; __PYX_ERR(0, 834, __pyx_L5_except_error) } __pyx_L5_except_error:;
0835:
0836:
+0837: cdef getsent(str frag):
static PyObject *__pyx_f_8discodop_10_fragments_getsent(PyObject *__pyx_v_frag) {
int __pyx_v_n;
int __pyx_v_m;
int __pyx_v_maxl;
PyObject *__pyx_v_leafmap = 0;
PyObject *__pyx_v_spans = 0;
PyObject *__pyx_v_leaves = 0;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("getsent", 0);
__Pyx_INCREF(__pyx_v_frag);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_XDECREF(__pyx_t_8);
__Pyx_AddTraceback("discodop._fragments.getsent", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_leafmap);
__Pyx_XDECREF(__pyx_v_spans);
__Pyx_XDECREF(__pyx_v_leaves);
__Pyx_XDECREF(__pyx_v_frag);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0838: """Renumber indices in fragment and select terminals it contains.
0839:
0840: Returns a transformed copy of fragment. Renumber terminals and substitution
0841: sites in fragment such that the first index is 0 and any gaps have a width
0842: of 1. Expects a tree as string where frontier nodes are marked with
0843: closed intervals ``n:m``, while terminals are denoted with ``n=word``."""
+0844: cdef int n, m = 0, maxl
__pyx_v_m = 0;
+0845: cdef dict leafmap = {}
__pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 845, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_leafmap = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
+0846: cdef dict spans = {int(start): int(end) + 1
{ /* enter inner scope */
PyObject *__pyx_8genexpr8__pyx_v_start = NULL;
PyObject *__pyx_8genexpr8__pyx_v_end = NULL;
__pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 846, __pyx_L5_error)
__Pyx_GOTREF(__pyx_t_1);
/* … */
__pyx_t_2 = __Pyx_PyNumber_Int(__pyx_8genexpr8__pyx_v_start); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 846, __pyx_L5_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_3 = __Pyx_PyNumber_Int(__pyx_8genexpr8__pyx_v_end); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 846, __pyx_L5_error)
__Pyx_GOTREF(__pyx_t_3);
__pyx_t_5 = __Pyx_PyInt_AddObjC(__pyx_t_3, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 846, __pyx_L5_error)
__Pyx_GOTREF(__pyx_t_5);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
if (unlikely(PyDict_SetItem(__pyx_t_1, (PyObject*)__pyx_t_2, (PyObject*)__pyx_t_5))) __PYX_ERR(0, 846, __pyx_L5_error)
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+0847: for start, end in FRONTIERRE.findall(frag)}
__pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_FRONTIERRE); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_findall); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_4))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_4, function); } } if (!__pyx_t_3) { __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_frag); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_2); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_frag}; __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_frag}; __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; __Pyx_INCREF(__pyx_v_frag); __Pyx_GIVEREF(__pyx_v_frag); PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_v_frag); __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (likely(PyList_CheckExact(__pyx_t_2)) || PyTuple_CheckExact(__pyx_t_2)) { __pyx_t_4 = __pyx_t_2; __Pyx_INCREF(__pyx_t_4); __pyx_t_6 = 0; __pyx_t_7 = NULL; } else { __pyx_t_6 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_7 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 847, __pyx_L5_error) } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; for (;;) { if (likely(!__pyx_t_7)) { if (likely(PyList_CheckExact(__pyx_t_4))) { if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_2 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_6); __Pyx_INCREF(__pyx_t_2); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 847, __pyx_L5_error) #else __pyx_t_2 = PySequence_ITEM(__pyx_t_4, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_2); #endif } else { if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_6); __Pyx_INCREF(__pyx_t_2); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 847, __pyx_L5_error) #else __pyx_t_2 = PySequence_ITEM(__pyx_t_4, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_2); #endif } } else { __pyx_t_2 = __pyx_t_7(__pyx_t_4); if (unlikely(!__pyx_t_2)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 847, __pyx_L5_error) } break; } __Pyx_GOTREF(__pyx_t_2); } if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { PyObject* sequence = __pyx_t_2; #if !CYTHON_COMPILING_IN_PYPY Py_ssize_t size = Py_SIZE(sequence); #else Py_ssize_t size = PySequence_Size(sequence); #endif if (unlikely(size != 2)) { if (size > 2) __Pyx_RaiseTooManyValuesError(2); else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); __PYX_ERR(0, 847, __pyx_L5_error) } #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS if (likely(PyTuple_CheckExact(sequence))) { __pyx_t_5 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); } else { __pyx_t_5 = PyList_GET_ITEM(sequence, 0); __pyx_t_3 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_5); __Pyx_INCREF(__pyx_t_3); #else __pyx_t_5 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_3 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_3); #endif __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { Py_ssize_t index = -1; __pyx_t_8 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 847, __pyx_L5_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_9 = Py_TYPE(__pyx_t_8)->tp_iternext; index = 0; __pyx_t_5 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_5)) goto __pyx_L8_unpacking_failed; __Pyx_GOTREF(__pyx_t_5); index = 1; __pyx_t_3 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_3)) goto __pyx_L8_unpacking_failed; __Pyx_GOTREF(__pyx_t_3); if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_8), 2) < 0) __PYX_ERR(0, 847, __pyx_L5_error) __pyx_t_9 = NULL; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L9_unpacking_done; __pyx_L8_unpacking_failed:; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __pyx_t_9 = NULL; if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); __PYX_ERR(0, 847, __pyx_L5_error) __pyx_L9_unpacking_done:; } __Pyx_XDECREF_SET(__pyx_8genexpr8__pyx_v_start, __pyx_t_5); __pyx_t_5 = 0; __Pyx_XDECREF_SET(__pyx_8genexpr8__pyx_v_end, __pyx_t_3); __pyx_t_3 = 0; /* … */ } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_XDECREF(__pyx_8genexpr8__pyx_v_start); __Pyx_XDECREF(__pyx_8genexpr8__pyx_v_end); goto __pyx_L10_exit_scope; __pyx_L5_error:; __Pyx_XDECREF(__pyx_8genexpr8__pyx_v_start); __Pyx_XDECREF(__pyx_8genexpr8__pyx_v_end); goto __pyx_L1_error; __pyx_L10_exit_scope:; } /* exit inner scope */ __pyx_v_spans = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0;
+0848: cdef list leaves = [int(a) for a in TERMINDICESRE.findall(frag)]
{ /* enter inner scope */
PyObject *__pyx_8genexpr9__pyx_v_a = NULL;
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_TERMINDICESRE); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_5);
__pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_findall); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
__pyx_t_5 = NULL;
if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) {
__pyx_t_5 = PyMethod_GET_SELF(__pyx_t_2);
if (likely(__pyx_t_5)) {
PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2);
__Pyx_INCREF(__pyx_t_5);
__Pyx_INCREF(function);
__Pyx_DECREF_SET(__pyx_t_2, function);
}
}
if (!__pyx_t_5) {
__pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_frag); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_4);
} else {
#if CYTHON_FAST_PYCALL
if (PyFunction_Check(__pyx_t_2)) {
PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_v_frag};
__pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
__Pyx_GOTREF(__pyx_t_4);
} else
#endif
#if CYTHON_FAST_PYCCALL
if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) {
PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_v_frag};
__pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
__Pyx_GOTREF(__pyx_t_4);
} else
#endif
{
__pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_5); __pyx_t_5 = NULL;
__Pyx_INCREF(__pyx_v_frag);
__Pyx_GIVEREF(__pyx_v_frag);
PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_v_frag);
__pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_4);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
}
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
if (likely(PyList_CheckExact(__pyx_t_4)) || PyTuple_CheckExact(__pyx_t_4)) {
__pyx_t_2 = __pyx_t_4; __Pyx_INCREF(__pyx_t_2); __pyx_t_6 = 0;
__pyx_t_7 = NULL;
} else {
__pyx_t_6 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_7 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 848, __pyx_L13_error)
}
__Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
for (;;) {
if (likely(!__pyx_t_7)) {
if (likely(PyList_CheckExact(__pyx_t_2))) {
if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_4 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_6); __Pyx_INCREF(__pyx_t_4); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 848, __pyx_L13_error)
#else
__pyx_t_4 = PySequence_ITEM(__pyx_t_2, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_4);
#endif
} else {
if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_4 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_6); __Pyx_INCREF(__pyx_t_4); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 848, __pyx_L13_error)
#else
__pyx_t_4 = PySequence_ITEM(__pyx_t_2, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_4);
#endif
}
} else {
__pyx_t_4 = __pyx_t_7(__pyx_t_2);
if (unlikely(!__pyx_t_4)) {
PyObject* exc_type = PyErr_Occurred();
if (exc_type) {
if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
else __PYX_ERR(0, 848, __pyx_L13_error)
}
break;
}
__Pyx_GOTREF(__pyx_t_4);
}
__Pyx_XDECREF_SET(__pyx_8genexpr9__pyx_v_a, __pyx_t_4);
__pyx_t_4 = 0;
__pyx_t_4 = __Pyx_PyNumber_Int(__pyx_8genexpr9__pyx_v_a); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_GOTREF(__pyx_t_4);
if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) __PYX_ERR(0, 848, __pyx_L13_error)
__Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__Pyx_XDECREF(__pyx_8genexpr9__pyx_v_a);
goto __pyx_L16_exit_scope;
__pyx_L13_error:;
__Pyx_XDECREF(__pyx_8genexpr9__pyx_v_a);
goto __pyx_L1_error;
__pyx_L16_exit_scope:;
} /* exit inner scope */
__pyx_v_leaves = ((PyObject*)__pyx_t_1);
__pyx_t_1 = 0;
+0849: for n in leaves:
__pyx_t_1 = __pyx_v_leaves; __Pyx_INCREF(__pyx_t_1); __pyx_t_6 = 0; for (;;) { if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_1)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_6); __Pyx_INCREF(__pyx_t_2); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 849, __pyx_L1_error) #else __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 849, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #endif __pyx_t_10 = __Pyx_PyInt_As_int(__pyx_t_2); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 849, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_n = __pyx_t_10; /* … */ } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+0850: spans[n] = n + 1
__pyx_t_2 = __Pyx_PyInt_From_long((__pyx_v_n + 1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 850, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_4 = __Pyx_PyInt_From_int(__pyx_v_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 850, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); if (unlikely(PyDict_SetItem(__pyx_v_spans, __pyx_t_4, __pyx_t_2) < 0)) __PYX_ERR(0, 850, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0851: maxl = max(spans)
__pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 851, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_spans); __Pyx_GIVEREF(__pyx_v_spans); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_spans); __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_max, __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 851, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_10 = __Pyx_PyInt_As_int(__pyx_t_2); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 851, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_maxl = __pyx_t_10;
+0852: for n in sorted(spans):
__pyx_t_1 = PySequence_List(__pyx_v_spans); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 852, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; __pyx_t_11 = PyList_Sort(__pyx_t_2); if (unlikely(__pyx_t_11 == -1)) __PYX_ERR(0, 852, __pyx_L1_error) if (unlikely(__pyx_t_2 == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); __PYX_ERR(0, 852, __pyx_L1_error) } __pyx_t_1 = __pyx_t_2; __Pyx_INCREF(__pyx_t_1); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; for (;;) { if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_1)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_6); __Pyx_INCREF(__pyx_t_2); __pyx_t_6++; if (unlikely(0 < 0)) __PYX_ERR(0, 852, __pyx_L1_error) #else __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 852, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #endif __pyx_t_10 = __Pyx_PyInt_As_int(__pyx_t_2); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 852, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_n = __pyx_t_10; /* … */ } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+0853: leafmap[n] = ' %d=' % m
__pyx_t_2 = __Pyx_PyInt_From_int(__pyx_v_m); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 853, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_4 = PyUnicode_Format(__pyx_kp_u_d, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 853, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_v_n); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 853, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (unlikely(PyDict_SetItem(__pyx_v_leafmap, __pyx_t_2, __pyx_t_4) < 0)) __PYX_ERR(0, 853, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+0854: m += 1
__pyx_v_m = (__pyx_v_m + 1);
+0855: if spans[n] not in spans and n != maxl: # a gap
__pyx_t_4 = __Pyx_PyInt_From_int(__pyx_v_n); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 855, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_2 = __Pyx_PyDict_GetItem(__pyx_v_spans, __pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 855, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_13 = (__Pyx_PyDict_ContainsTF(__pyx_t_2, __pyx_v_spans, Py_NE)); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(0, 855, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { } else { __pyx_t_12 = __pyx_t_14; goto __pyx_L22_bool_binop_done; } __pyx_t_14 = ((__pyx_v_n != __pyx_v_maxl) != 0); __pyx_t_12 = __pyx_t_14; __pyx_L22_bool_binop_done:; if (__pyx_t_12) { /* … */ }
+0856: m += 1
__pyx_v_m = (__pyx_v_m + 1);
+0857: frag = FRONTIERORTERMRE.sub(repl(leafmap), frag)
__pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_FRONTIERORTERMRE); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_sub); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_repl); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_5 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_5)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); __Pyx_INCREF(__pyx_t_5); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_3, function); } } if (!__pyx_t_5) { __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_v_leafmap); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_v_leafmap}; __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_v_leafmap}; __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_5); __pyx_t_5 = NULL; __Pyx_INCREF(__pyx_v_leafmap); __Pyx_GIVEREF(__pyx_v_leafmap); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_leafmap); __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } } __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; __pyx_t_10 = 0; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_4))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_4, function); __pyx_t_10 = 1; } } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[3] = {__pyx_t_3, __pyx_t_2, __pyx_v_frag}; __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[3] = {__pyx_t_3, __pyx_t_2, __pyx_v_frag}; __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { __pyx_t_8 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_3); __pyx_t_3 = NULL; } __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_8, 0+__pyx_t_10, __pyx_t_2); __Pyx_INCREF(__pyx_v_frag); __Pyx_GIVEREF(__pyx_v_frag); PyTuple_SET_ITEM(__pyx_t_8, 1+__pyx_t_10, __pyx_v_frag); __pyx_t_2 = 0; __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "unicode", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 857, __pyx_L1_error) __Pyx_DECREF_SET(__pyx_v_frag, ((PyObject*)__pyx_t_1)); __pyx_t_1 = 0;
+0858: return frag
__Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_frag); __pyx_r = __pyx_v_frag; goto __pyx_L0;
0859:
0860:
+0861: cdef dumpmatrix(uint64_t *matrix, NodeArray a, NodeArray b, Node *anodes,
static PyObject *__pyx_f_8discodop_10_fragments_dumpmatrix(uint64_t *__pyx_v_matrix, struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_a, struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_b, struct __pyx_t_8discodop_10containers_Node *__pyx_v_anodes, struct __pyx_t_8discodop_10containers_Node *__pyx_v_bnodes, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, uint64_t *__pyx_v_scratch, short __pyx_v_SLOTS) {
PyObject *__pyx_v_m = NULL;
PyObject *__pyx_v_n = NULL;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("dumpmatrix", 0);
/* … */
/* function exit code */
__pyx_r = Py_None; __Pyx_INCREF(Py_None);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_7);
__Pyx_XDECREF(__pyx_t_12);
__Pyx_AddTraceback("discodop._fragments.dumpmatrix", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_m);
__Pyx_XDECREF(__pyx_v_n);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0862: Node *bnodes, Vocabulary vocab,
0863: uint64_t *scratch, short SLOTS):
0864: """Dump a table of the common productions of a tree pair."""
+0865: dumptree(a, anodes, [], vocab, scratch) # FIXME get sent
__pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 865, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_f_8discodop_10_fragments_dumptree(__pyx_v_a, __pyx_v_anodes, ((PyObject*)__pyx_t_1), __pyx_v_vocab, __pyx_v_scratch); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 865, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0866: dumptree(b, bnodes, [], vocab, scratch)
__pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 866, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = __pyx_f_8discodop_10_fragments_dumptree(__pyx_v_b, __pyx_v_bnodes, ((PyObject*)__pyx_t_2), __pyx_v_vocab, __pyx_v_scratch); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 866, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+0867: print('\t'.join([''] + ['%2d' % x for x in range(b.len)
__pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 867, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_kp_u__2); __Pyx_GIVEREF(__pyx_kp_u__2); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u__2); { /* enter inner scope */ int16_t __pyx_9genexpr10__pyx_v_x; __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 867, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __pyx_v_b.len; for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { __pyx_9genexpr10__pyx_v_x = __pyx_t_4; /* … */ __pyx_t_6 = __Pyx_PyInt_From_int16_t(__pyx_9genexpr10__pyx_v_x); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 867, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_7 = PyUnicode_Format(__pyx_kp_u_2d, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 867, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_7))) __PYX_ERR(0, 867, __pyx_L1_error) __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; /* … */ __pyx_t_7 = PyNumber_Add(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 867, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = PyUnicode_Join(__pyx_kp_u__7, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 867, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 867, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 867, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0868: if bnodes[x].prod != -1]))
__pyx_t_5 = (((__pyx_v_bnodes[__pyx_9genexpr10__pyx_v_x]).prod != -1L) != 0);
if (__pyx_t_5) {
/* … */
}
}
} /* exit inner scope */
+0869: for m in range(b.len):
__pyx_t_2 = __Pyx_PyInt_From_int16_t(__pyx_v_b.len); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 869, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 869, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 869, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (likely(PyList_CheckExact(__pyx_t_2)) || PyTuple_CheckExact(__pyx_t_2)) { __pyx_t_7 = __pyx_t_2; __Pyx_INCREF(__pyx_t_7); __pyx_t_8 = 0; __pyx_t_9 = NULL; } else { __pyx_t_8 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 869, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_9 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 869, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; for (;;) { if (likely(!__pyx_t_9)) { if (likely(PyList_CheckExact(__pyx_t_7))) { if (__pyx_t_8 >= PyList_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_2 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 869, __pyx_L1_error) #else __pyx_t_2 = PySequence_ITEM(__pyx_t_7, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 869, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #endif } else { if (__pyx_t_8 >= PyTuple_GET_SIZE(__pyx_t_7)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 869, __pyx_L1_error) #else __pyx_t_2 = PySequence_ITEM(__pyx_t_7, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 869, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #endif } } else { __pyx_t_2 = __pyx_t_9(__pyx_t_7); if (unlikely(!__pyx_t_2)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 869, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_2); } __Pyx_XDECREF_SET(__pyx_v_m, __pyx_t_2); __pyx_t_2 = 0; /* … */ } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+0870: print('\t', vocab.getlabel((<Node>bnodes[m]).prod)[:3], end='')
__pyx_t_10 = __Pyx_PyIndex_AsSsize_t(__pyx_v_m); if (unlikely((__pyx_t_10 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 870, __pyx_L1_error) __pyx_t_2 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getlabel(__pyx_v_vocab, (__pyx_v_bnodes[__pyx_t_10]).prod); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 870, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (unlikely(__pyx_t_2 == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); __PYX_ERR(0, 870, __pyx_L1_error) } __pyx_t_1 = __Pyx_PyUnicode_Substring(__pyx_t_2, 0, 3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 870, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 870, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_kp_u__7); __Pyx_GIVEREF(__pyx_kp_u__7); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_kp_u__7); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 870, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_end, __pyx_kp_u__2) < 0) __PYX_ERR(0, 870, __pyx_L1_error) __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_2, __pyx_t_1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 870, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+0871: for n in range(a.len):
__pyx_t_7 = __Pyx_PyInt_From_int16_t(__pyx_v_a.len); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 871, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 871, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_6, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 871, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; if (likely(PyList_CheckExact(__pyx_t_7)) || PyTuple_CheckExact(__pyx_t_7)) { __pyx_t_6 = __pyx_t_7; __Pyx_INCREF(__pyx_t_6); __pyx_t_8 = 0; __pyx_t_9 = NULL; } else { __pyx_t_8 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 871, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_9 = Py_TYPE(__pyx_t_6)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 871, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; for (;;) { if (likely(!__pyx_t_9)) { if (likely(PyList_CheckExact(__pyx_t_6))) { if (__pyx_t_8 >= PyList_GET_SIZE(__pyx_t_6)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_7 = PyList_GET_ITEM(__pyx_t_6, __pyx_t_8); __Pyx_INCREF(__pyx_t_7); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 871, __pyx_L1_error) #else __pyx_t_7 = PySequence_ITEM(__pyx_t_6, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 871, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); #endif } else { if (__pyx_t_8 >= PyTuple_GET_SIZE(__pyx_t_6)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_8); __Pyx_INCREF(__pyx_t_7); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 871, __pyx_L1_error) #else __pyx_t_7 = PySequence_ITEM(__pyx_t_6, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 871, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); #endif } } else { __pyx_t_7 = __pyx_t_9(__pyx_t_6); if (unlikely(!__pyx_t_7)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 871, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_7); } __Pyx_XDECREF_SET(__pyx_v_n, __pyx_t_7); __pyx_t_7 = 0; /* … */ } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+0872: print('\n%2d' % n, vocab.getlabel((<Node>anodes[n]).prod)[:3], end='')
__pyx_t_7 = PyUnicode_Format(__pyx_kp_u_2d_2, __pyx_v_n); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 872, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_10 = __Pyx_PyIndex_AsSsize_t(__pyx_v_n); if (unlikely((__pyx_t_10 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 872, __pyx_L1_error) __pyx_t_1 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getlabel(__pyx_v_vocab, (__pyx_v_anodes[__pyx_t_10]).prod); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 872, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (unlikely(__pyx_t_1 == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); __PYX_ERR(0, 872, __pyx_L1_error) } __pyx_t_2 = __Pyx_PyUnicode_Substring(__pyx_t_1, 0, 3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 872, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 872, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_7); __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_2); __pyx_t_7 = 0; __pyx_t_2 = 0; __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 872, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_end, __pyx_kp_u__2) < 0) __PYX_ERR(0, 872, __pyx_L1_error) __pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 872, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+0873: for m in range(b.len):
__pyx_t_7 = __Pyx_PyInt_From_int16_t(__pyx_v_b.len); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 873, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 873, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_GIVEREF(__pyx_t_7);
PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_7);
__pyx_t_7 = 0;
__pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_2, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 873, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
if (likely(PyList_CheckExact(__pyx_t_7)) || PyTuple_CheckExact(__pyx_t_7)) {
__pyx_t_2 = __pyx_t_7; __Pyx_INCREF(__pyx_t_2); __pyx_t_10 = 0;
__pyx_t_11 = NULL;
} else {
__pyx_t_10 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 873, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_11 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 873, __pyx_L1_error)
}
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
for (;;) {
if (likely(!__pyx_t_11)) {
if (likely(PyList_CheckExact(__pyx_t_2))) {
if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_7 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_10); __Pyx_INCREF(__pyx_t_7); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 873, __pyx_L1_error)
#else
__pyx_t_7 = PySequence_ITEM(__pyx_t_2, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 873, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
#endif
} else {
if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_10); __Pyx_INCREF(__pyx_t_7); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 873, __pyx_L1_error)
#else
__pyx_t_7 = PySequence_ITEM(__pyx_t_2, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 873, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
#endif
}
} else {
__pyx_t_7 = __pyx_t_11(__pyx_t_2);
if (unlikely(!__pyx_t_7)) {
PyObject* exc_type = PyErr_Occurred();
if (exc_type) {
if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
else __PYX_ERR(0, 873, __pyx_L1_error)
}
break;
}
__Pyx_GOTREF(__pyx_t_7);
}
__Pyx_XDECREF_SET(__pyx_v_m, __pyx_t_7);
__pyx_t_7 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0874: print('\t', end='')
__pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 874, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_end, __pyx_kp_u__2) < 0) __PYX_ERR(0, 874, __pyx_L1_error) __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_tuple__8, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 874, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* … */ __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_u__7); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(0, 874, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__8); __Pyx_GIVEREF(__pyx_tuple__8);
+0875: print('1' if TESTBIT(&matrix[m * SLOTS], n) else ' ', end='')
__pyx_t_7 = __Pyx_PyInt_From_short(__pyx_v_SLOTS); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 875, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_12 = PyNumber_Multiply(__pyx_v_m, __pyx_t_7); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 875, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_12); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 875, __pyx_L1_error) __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; __pyx_t_14 = __Pyx_PyInt_As_int(__pyx_v_n); if (unlikely((__pyx_t_14 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 875, __pyx_L1_error) if ((TESTBIT((&(__pyx_v_matrix[__pyx_t_13])), __pyx_t_14) != 0)) { __Pyx_INCREF(__pyx_kp_u_1); __pyx_t_1 = __pyx_kp_u_1; } else { __Pyx_INCREF(__pyx_kp_u__5); __pyx_t_1 = __pyx_kp_u__5; } __pyx_t_12 = PyTuple_New(1); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 875, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_12, 0, __pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 875, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_end, __pyx_kp_u__2) < 0) __PYX_ERR(0, 875, __pyx_L1_error) __pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_12, __pyx_t_1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 875, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+0876: print('\ncommon productions:', end=' ')
__pyx_t_6 = PyDict_New(); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 876, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (PyDict_SetItem(__pyx_t_6, __pyx_n_s_end, __pyx_kp_u__5) < 0) __PYX_ERR(0, 876, __pyx_L1_error) __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_tuple__9, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 876, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* … */ __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_u_common_productions); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(0, 876, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__9); __Pyx_GIVEREF(__pyx_tuple__9);
+0877: print(len({anodes[n].prod for n in range(a.len)} &
{ /* enter inner scope */
int16_t __pyx_9genexpr11__pyx_v_n;
__pyx_t_2 = PySet_New(NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 877, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__pyx_t_3 = __pyx_v_a.len;
for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) {
__pyx_9genexpr11__pyx_v_n = __pyx_t_4;
__pyx_t_6 = __Pyx_PyInt_From_int32_t((__pyx_v_anodes[__pyx_9genexpr11__pyx_v_n]).prod); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 877, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
if (unlikely(PySet_Add(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(0, 877, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
}
} /* exit inner scope */
{ /* enter inner scope */
int16_t __pyx_9genexpr12__pyx_v_n;
/* … */
__pyx_t_7 = PyNumber_And(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 877, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
__pyx_t_8 = PySet_GET_SIZE(__pyx_t_7); if (unlikely(__pyx_t_8 == -1)) __PYX_ERR(0, 877, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
__pyx_t_7 = PyInt_FromSsize_t(__pyx_t_8); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 877, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 877, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__Pyx_GIVEREF(__pyx_t_7);
PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_7);
__pyx_t_7 = 0;
__pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_6, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 877, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+0878: {bnodes[n].prod for n in range(b.len)}))
__pyx_t_6 = PySet_New(NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 878, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_3 = __pyx_v_b.len; for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { __pyx_9genexpr12__pyx_v_n = __pyx_t_4; __pyx_t_7 = __Pyx_PyInt_From_int32_t((__pyx_v_bnodes[__pyx_9genexpr12__pyx_v_n]).prod); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 878, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); if (unlikely(PySet_Add(__pyx_t_6, (PyObject*)__pyx_t_7))) __PYX_ERR(0, 878, __pyx_L1_error) __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } } /* exit inner scope */
+0879: print('found:', end=' ')
__pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 879, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_end, __pyx_kp_u__5) < 0) __PYX_ERR(0, 879, __pyx_L1_error) __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_tuple__10, __pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 879, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; /* … */ __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_u_found); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(0, 879, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__10); __Pyx_GIVEREF(__pyx_tuple__10);
+0880: print('horz', sum([abitcount(&matrix[n * SLOTS], SLOTS) > 0
{ /* enter inner scope */
int16_t __pyx_9genexpr13__pyx_v_n;
__pyx_t_6 = PyList_New(0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 880, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
/* … */
__pyx_t_7 = __Pyx_PyBool_FromLong((__pyx_f_8discodop_3bit_abitcount((&(__pyx_v_matrix[(__pyx_9genexpr13__pyx_v_n * __pyx_v_SLOTS)])), __pyx_v_SLOTS) > 0)); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 880, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
if (unlikely(__Pyx_ListComp_Append(__pyx_t_6, (PyObject*)__pyx_t_7))) __PYX_ERR(0, 880, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
}
} /* exit inner scope */
__pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 880, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__Pyx_GIVEREF(__pyx_t_6);
PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_6);
__pyx_t_6 = 0;
__pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_sum, __pyx_t_7, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 880, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
/* … */
__pyx_t_1 = PyTuple_New(6); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 880, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__Pyx_INCREF(__pyx_n_u_horz);
__Pyx_GIVEREF(__pyx_n_u_horz);
PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_n_u_horz);
__Pyx_GIVEREF(__pyx_t_6);
PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_6);
__Pyx_INCREF(__pyx_n_u_vert);
__Pyx_GIVEREF(__pyx_n_u_vert);
PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_n_u_vert);
__Pyx_GIVEREF(__pyx_t_7);
PyTuple_SET_ITEM(__pyx_t_1, 3, __pyx_t_7);
__Pyx_INCREF(__pyx_n_u_both);
__Pyx_GIVEREF(__pyx_n_u_both);
PyTuple_SET_ITEM(__pyx_t_1, 4, __pyx_n_u_both);
__Pyx_GIVEREF(__pyx_t_2);
PyTuple_SET_ITEM(__pyx_t_1, 5, __pyx_t_2);
__pyx_t_6 = 0;
__pyx_t_7 = 0;
__pyx_t_2 = 0;
__pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 880, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0881: for n in range(b.len)]),
__pyx_t_3 = __pyx_v_b.len;
for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) {
__pyx_9genexpr13__pyx_v_n = __pyx_t_4;
+0882: 'vert', sum([any([TESTBIT(&matrix[n * SLOTS], m)
{ /* enter inner scope */
int16_t __pyx_9genexpr14__pyx_v_m;
__pyx_t_7 = PyList_New(0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 882, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
/* … */
{ /* enter inner scope */
int16_t __pyx_9genexpr15__pyx_9genexpr14__pyx_v_n;
__pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 882, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
/* … */
__pyx_t_1 = __Pyx_PyInt_From_uint64_t(TESTBIT((&(__pyx_v_matrix[(__pyx_9genexpr15__pyx_9genexpr14__pyx_v_n * __pyx_v_SLOTS)])), __pyx_9genexpr14__pyx_v_m)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 882, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_1))) __PYX_ERR(0, 882, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
}
} /* exit inner scope */
__pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 882, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__Pyx_GIVEREF(__pyx_t_2);
PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_2);
__pyx_t_2 = 0;
__pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_any, __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 882, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
if (unlikely(__Pyx_ListComp_Append(__pyx_t_7, (PyObject*)__pyx_t_2))) __PYX_ERR(0, 882, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
}
} /* exit inner scope */
__pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 882, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_2);
__Pyx_GIVEREF(__pyx_t_7);
PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_7);
__pyx_t_7 = 0;
__pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_sum, __pyx_t_2, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 882, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+0883: for n in range(b.len)]) for m in range(a.len)]),
__pyx_t_3 = __pyx_v_a.len;
for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) {
__pyx_9genexpr14__pyx_v_m = __pyx_t_4;
/* … */
__pyx_t_15 = __pyx_v_b.len;
for (__pyx_t_16 = 0; __pyx_t_16 < __pyx_t_15; __pyx_t_16+=1) {
__pyx_9genexpr15__pyx_9genexpr14__pyx_v_n = __pyx_t_16;
+0884: 'both', abitcount(matrix, b.len * SLOTS))
__pyx_t_2 = __Pyx_PyInt_From_int(__pyx_f_8discodop_3bit_abitcount(__pyx_v_matrix, (__pyx_v_b.len * __pyx_v_SLOTS))); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 884, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2);
0885:
0886:
+0887: cdef dumptree(NodeArray a, Node *anodes, list asent, Vocabulary vocab,
static PyObject *__pyx_f_8discodop_10_fragments_dumptree(struct __pyx_t_8discodop_10containers_NodeArray __pyx_v_a, struct __pyx_t_8discodop_10containers_Node *__pyx_v_anodes, PyObject *__pyx_v_asent, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, uint64_t *__pyx_v_scratch) {
int16_t __pyx_v_n;
PyObject *__pyx_v_tmp = NULL;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("dumptree", 0);
/* … */
/* function exit code */
__pyx_r = Py_None; __Pyx_INCREF(Py_None);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_7);
__Pyx_XDECREF(__pyx_t_8);
__Pyx_AddTraceback("discodop._fragments.dumptree", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_tmp);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
0888: uint64_t *scratch):
0889: """Print debug information of a given tree."""
+0890: for n in range(a.len):
__pyx_t_1 = __pyx_v_a.len;
for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) {
__pyx_v_n = __pyx_t_2;
+0891: print('idx=%2d\tleft=%2d\tright=%2d\tprod=%2d\tlabel=%s' % (n,
__pyx_t_3 = __Pyx_PyInt_From_int16_t(__pyx_v_n); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 891, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
/* … */
__pyx_t_8 = PyTuple_New(5); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 891, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_8);
__Pyx_GIVEREF(__pyx_t_3);
PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_3);
__Pyx_GIVEREF(__pyx_t_4);
PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_4);
__Pyx_GIVEREF(__pyx_t_5);
PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_5);
__Pyx_GIVEREF(__pyx_t_6);
PyTuple_SET_ITEM(__pyx_t_8, 3, __pyx_t_6);
__Pyx_GIVEREF(__pyx_t_7);
PyTuple_SET_ITEM(__pyx_t_8, 4, __pyx_t_7);
__pyx_t_3 = 0;
__pyx_t_4 = 0;
__pyx_t_5 = 0;
__pyx_t_6 = 0;
__pyx_t_7 = 0;
__pyx_t_7 = PyUnicode_Format(__pyx_kp_u_idx_2d_left_2d_right_2d_prod_2d, __pyx_t_8); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 891, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
__pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 891, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_8);
__Pyx_GIVEREF(__pyx_t_7);
PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7);
__pyx_t_7 = 0;
/* … */
__pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_8, __pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 891, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+0892: termidx(anodes[n].left) if anodes[n].left < 0
if ((((__pyx_v_anodes[__pyx_v_n]).left < 0) != 0)) {
__pyx_t_5 = __Pyx_PyInt_From_short(__pyx_f_8discodop_10containers_termidx((__pyx_v_anodes[__pyx_v_n]).left)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 892, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
__pyx_t_4 = __pyx_t_5;
__pyx_t_5 = 0;
} else {
+0893: else anodes[n].left,
__pyx_t_5 = __Pyx_PyInt_From_int16_t((__pyx_v_anodes[__pyx_v_n]).left); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 893, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
__pyx_t_4 = __pyx_t_5;
__pyx_t_5 = 0;
}
+0894: anodes[n].right, anodes[n].prod,
__pyx_t_5 = __Pyx_PyInt_From_int16_t((__pyx_v_anodes[__pyx_v_n]).right); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 894, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
__pyx_t_6 = __Pyx_PyInt_From_int32_t((__pyx_v_anodes[__pyx_v_n]).prod); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 894, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
+0895: vocab.getlabel(anodes[n].prod)),
__pyx_t_7 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getlabel(__pyx_v_vocab, (__pyx_v_anodes[__pyx_v_n]).prod); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 895, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
+0896: end='')
__pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 896, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_end, __pyx_kp_u__2) < 0) __PYX_ERR(0, 896, __pyx_L1_error)
+0897: if anodes[n].left < 0:
__pyx_t_9 = (((__pyx_v_anodes[__pyx_v_n]).left < 0) != 0);
if (__pyx_t_9) {
/* … */
goto __pyx_L5;
}
+0898: if vocab.islexical(anodes[n].prod):
__pyx_t_9 = (((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->islexical(__pyx_v_vocab, (__pyx_v_anodes[__pyx_v_n]).prod) != 0);
if (__pyx_t_9) {
/* … */
goto __pyx_L6;
}
+0899: print('\t%s=%s' % ('terminal', vocab.getword(anodes[n].prod)))
__pyx_t_6 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getword(__pyx_v_vocab, (__pyx_v_anodes[__pyx_v_n]).prod); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 899, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 899, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__Pyx_INCREF(__pyx_n_u_terminal);
__Pyx_GIVEREF(__pyx_n_u_terminal);
PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_n_u_terminal);
__Pyx_GIVEREF(__pyx_t_6);
PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6);
__pyx_t_6 = 0;
__pyx_t_6 = PyUnicode_Format(__pyx_kp_u_s_s, __pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 899, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
__pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 899, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_7);
__Pyx_GIVEREF(__pyx_t_6);
PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_6);
__pyx_t_6 = 0;
__pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_7, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 899, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
0900: else:
+0901: print('\tfrontier non-terminal')
/*else*/ {
__pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_tuple__11, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 901, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
}
__pyx_L6:;
/* … */
__pyx_tuple__11 = PyTuple_Pack(1, __pyx_kp_u_frontier_non_terminal); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(0, 901, __pyx_L1_error)
__Pyx_GOTREF(__pyx_tuple__11);
__Pyx_GIVEREF(__pyx_tuple__11);
0902: else:
+0903: print()
/*else*/ {
__pyx_t_6 = __Pyx_PyObject_CallNoArg(__pyx_builtin_print); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 903, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
}
__pyx_L5:;
}
+0904: tmp = []
__pyx_t_6 = PyList_New(0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 904, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_v_tmp = ((PyObject*)__pyx_t_6); __pyx_t_6 = 0;
+0905: memset(<void *>scratch, 255, BITNSLOTS(a.len) * sizeof(uint64_t))
memset(((void *)__pyx_v_scratch), 0xFF, (BITNSLOTS(__pyx_v_a.len) * (sizeof(uint64_t))));
+0906: getsubtree(tmp, anodes, scratch, vocab, True, a.root)
__pyx_t_6 = __pyx_f_8discodop_10_fragments_getsubtree(__pyx_v_tmp, __pyx_v_anodes, __pyx_v_scratch, __pyx_v_vocab, 1, __pyx_v_a.root); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 906, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+0907: print(''.join(tmp), '\n', asent, '\n')
__pyx_t_6 = PyUnicode_Join(__pyx_kp_u__2, __pyx_v_tmp); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 907, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_7 = PyTuple_New(4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 907, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_6); __Pyx_INCREF(__pyx_kp_u__12); __Pyx_GIVEREF(__pyx_kp_u__12); PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_kp_u__12); __Pyx_INCREF(__pyx_v_asent); __Pyx_GIVEREF(__pyx_v_asent); PyTuple_SET_ITEM(__pyx_t_7, 2, __pyx_v_asent); __Pyx_INCREF(__pyx_kp_u__12); __Pyx_GIVEREF(__pyx_kp_u__12); PyTuple_SET_ITEM(__pyx_t_7, 3, __pyx_kp_u__12); __pyx_t_6 = 0; __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_7, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 907, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
0908:
0909:
+0910: def repl(d):
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_13repl(PyObject *__pyx_self, PyObject *__pyx_v_d); /*proto*/
static char __pyx_doc_8discodop_10_fragments_12repl[] = "repl(d)\nA function for use with re.sub that looks up numeric IDs in a dict.\n\t";
static PyMethodDef __pyx_mdef_8discodop_10_fragments_13repl = {"repl", (PyCFunction)__pyx_pw_8discodop_10_fragments_13repl, METH_O, __pyx_doc_8discodop_10_fragments_12repl};
static PyObject *__pyx_pw_8discodop_10_fragments_13repl(PyObject *__pyx_self, PyObject *__pyx_v_d) {
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("repl (wrapper)", 0);
__pyx_r = __pyx_pf_8discodop_10_fragments_12repl(__pyx_self, ((PyObject *)__pyx_v_d));
/* function exit code */
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
static PyObject *__pyx_pf_8discodop_10_fragments_12repl(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_d) {
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct__repl *__pyx_cur_scope;
PyObject *__pyx_v_f = 0;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("repl", 0);
__pyx_cur_scope = (struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct__repl *)__pyx_tp_new_8discodop_10_fragments___pyx_scope_struct__repl(__pyx_ptype_8discodop_10_fragments___pyx_scope_struct__repl, __pyx_empty_tuple, NULL);
if (unlikely(!__pyx_cur_scope)) {
__pyx_cur_scope = ((struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct__repl *)Py_None);
__Pyx_INCREF(Py_None);
__PYX_ERR(0, 910, __pyx_L1_error)
} else {
__Pyx_GOTREF(__pyx_cur_scope);
}
__pyx_cur_scope->__pyx_v_d = __pyx_v_d;
__Pyx_INCREF(__pyx_cur_scope->__pyx_v_d);
__Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_d);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_AddTraceback("discodop._fragments.repl", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_f);
__Pyx_DECREF(((PyObject *)__pyx_cur_scope));
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_gb_8discodop_10_fragments_8readtree_2generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
/* … */
__pyx_tuple__35 = PyTuple_Pack(3, __pyx_n_s_d_2, __pyx_n_s_f, __pyx_n_s_f); if (unlikely(!__pyx_tuple__35)) __PYX_ERR(0, 910, __pyx_L1_error)
__Pyx_GOTREF(__pyx_tuple__35);
__Pyx_GIVEREF(__pyx_tuple__35);
/* … */
__pyx_t_3 = PyCFunction_NewEx(&__pyx_mdef_8discodop_10_fragments_13repl, NULL, __pyx_n_s_discodop__fragments); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 910, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
if (PyDict_SetItem(__pyx_d, __pyx_n_s_repl, __pyx_t_3) < 0) __PYX_ERR(0, 910, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__pyx_codeobj__36 = (PyObject*)__Pyx_PyCode_New(1, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__35, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_discodop__fragments_pyx, __pyx_n_s_repl, 910, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__36)) __PYX_ERR(0, 910, __pyx_L1_error)
/* … */
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct__repl {
PyObject_HEAD
PyObject *__pyx_v_d;
};
0911: """A function for use with re.sub that looks up numeric IDs in a dict.
0912: """
+0913: def f(x):
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_4repl_1f(PyObject *__pyx_self, PyObject *__pyx_v_x); /*proto*/
static PyMethodDef __pyx_mdef_8discodop_10_fragments_4repl_1f = {"f", (PyCFunction)__pyx_pw_8discodop_10_fragments_4repl_1f, METH_O, 0};
static PyObject *__pyx_pw_8discodop_10_fragments_4repl_1f(PyObject *__pyx_self, PyObject *__pyx_v_x) {
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("f (wrapper)", 0);
__pyx_r = __pyx_pf_8discodop_10_fragments_4repl_f(__pyx_self, ((PyObject *)__pyx_v_x));
/* function exit code */
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_8discodop_10_fragments_4repl_f(PyObject *__pyx_self, PyObject *__pyx_v_x) {
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct__repl *__pyx_cur_scope;
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct__repl *__pyx_outer_scope;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("f", 0);
__pyx_outer_scope = (struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct__repl *) __Pyx_CyFunction_GetClosure(__pyx_self);
__pyx_cur_scope = __pyx_outer_scope;
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_AddTraceback("discodop._fragments.repl.f", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
__pyx_tuple__14 = PyTuple_Pack(1, __pyx_n_s_x); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(0, 913, __pyx_L1_error)
__Pyx_GOTREF(__pyx_tuple__14);
__Pyx_GIVEREF(__pyx_tuple__14);
/* … */
__pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_8discodop_10_fragments_4repl_1f, 0, __pyx_n_s_repl_locals_f, ((PyObject*)__pyx_cur_scope), __pyx_n_s_discodop__fragments, __pyx_d, ((PyObject *)__pyx_codeobj__15)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 913, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_v_f = __pyx_t_1;
__pyx_t_1 = 0;
__pyx_codeobj__15 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_discodop__fragments_pyx, __pyx_n_s_f, 913, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__15)) __PYX_ERR(0, 913, __pyx_L1_error)
+0914: return d[int(x.group(1))]
__Pyx_XDECREF(__pyx_r); if (unlikely(!__pyx_cur_scope->__pyx_v_d)) { __Pyx_RaiseClosureNameError("d"); __PYX_ERR(0, 914, __pyx_L1_error) } __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_x, __pyx_n_s_group); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 914, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__13, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 914, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = __Pyx_PyNumber_Int(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 914, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = PyObject_GetItem(__pyx_cur_scope->__pyx_v_d, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 914, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; /* … */ __pyx_tuple__13 = PyTuple_Pack(1, __pyx_int_1); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(0, 914, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__13); __Pyx_GIVEREF(__pyx_tuple__13);
+0915: return f
__Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_f); __pyx_r = __pyx_v_f; goto __pyx_L0;
0916:
0917:
+0918: cdef int readtree(
static int __pyx_f_8discodop_10_fragments_readtree(PyObject *__pyx_v_line, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, struct __pyx_t_8discodop_10containers_Node *__pyx_v_nodes, arrayobject *__pyx_v_stack, arrayobject *__pyx_v_labels, PyObject *__pyx_v_children) {
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree *__pyx_cur_scope;
short __pyx_v_idx;
short __pyx_v_n;
short __pyx_v_y;
short __pyx_v_cur;
short __pyx_v_parent;
short __pyx_v_lenline;
short __pyx_v_lensent;
int __pyx_v_label;
int __pyx_v_binlabel;
int __pyx_v_binlabel2;
PyObject *__pyx_v_terminal = 0;
PyObject *__pyx_v_prod = 0;
PyObject *__pyx_v_binchildren = 0;
Rule __pyx_v_rule;
short __pyx_v_startlabel;
short __pyx_v_start;
int __pyx_r;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("readtree", 0);
__pyx_cur_scope = (struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree *)__pyx_tp_new_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree(__pyx_ptype_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree, __pyx_empty_tuple, NULL);
if (unlikely(!__pyx_cur_scope)) {
__pyx_cur_scope = ((struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree *)Py_None);
__Pyx_INCREF(Py_None);
__PYX_ERR(0, 918, __pyx_L1_error)
} else {
__Pyx_GOTREF(__pyx_cur_scope);
}
__pyx_cur_scope->__pyx_v_vocab = __pyx_v_vocab;
__Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_vocab);
__Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_vocab);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_9);
__Pyx_XDECREF(__pyx_t_13);
__Pyx_XDECREF(__pyx_t_14);
__Pyx_XDECREF(__pyx_t_15);
__Pyx_XDECREF(__pyx_t_23);
__Pyx_AddTraceback("discodop._fragments.readtree", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = -9;
__pyx_L0:;
__Pyx_XDECREF(__pyx_v_terminal);
__Pyx_XDECREF(__pyx_v_prod);
__Pyx_XDECREF(__pyx_v_binchildren);
__Pyx_DECREF(((PyObject *)__pyx_cur_scope));
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree {
PyObject_HEAD
PyObject *__pyx_v_binlabels;
short __pyx_v_m;
struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab;
};
0919: str line, Vocabulary vocab, Node *nodes, array stack,
0920: array labels, list children) except -9:
0921: """Parse tree in bracket format into pre-allocated array of Node structs.
0922:
0923: Tree will be binarized on the fly, equivalent to
0924: ```binarize(tree, dot=True)```. Whitespace is not significant.
0925: Terminals must have preterminals, without siblings; e.g., ``(X x y)``
0926: will raise an error.
0927:
0928: :param line: a complete bracketed tree.
0929: :param vocab: collects productions, labels, and words.
0930: :parem nodes: primary result.
0931: :param stack: provide an empty ``array.array('h')``.
0932: :param labels: provide empty ``array.array('i')``.
0933: :param children: provide empty list.
0934: """
0935: # :param disc: whether to expect discbracket format.
+0936: cdef short idx = 0, n = 0, m, y, cur, parent = -1
__pyx_v_idx = 0; __pyx_v_n = 0; __pyx_v_parent = -1;
+0937: cdef short lenline = len(line), lensent = 0
if (unlikely(__pyx_v_line == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 937, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyUnicode_GET_LENGTH(__pyx_v_line); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 937, __pyx_L1_error)
__pyx_v_lenline = __pyx_t_1;
__pyx_v_lensent = 0;
0938: cdef int label, binlabel, binlabel2
+0939: cdef str terminal = None
__Pyx_INCREF(Py_None);
__pyx_v_terminal = ((PyObject*)Py_None);
+0940: cdef bytes prod = None
__Pyx_INCREF(Py_None);
__pyx_v_prod = ((PyObject*)Py_None);
0941: cdef list binchildren, binlabels
0942: cdef Rule rule
+0943: while n < lenline:
while (1) {
__pyx_t_2 = ((__pyx_v_n < __pyx_v_lenline) != 0);
if (!__pyx_t_2) break;
+0944: while n < lenline and line[n] in ' \t\n':
while (1) {
__pyx_t_3 = ((__pyx_v_n < __pyx_v_lenline) != 0);
if (__pyx_t_3) {
} else {
__pyx_t_2 = __pyx_t_3;
goto __pyx_L7_bool_binop_done;
}
__pyx_t_4 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_v_n, short, 1, __Pyx_PyInt_From_short, 0, 0, 0); if (unlikely(__pyx_t_4 == (Py_UCS4)-1)) __PYX_ERR(0, 944, __pyx_L1_error)
switch (__pyx_t_4) {
case 9:
case 10:
case 32:
__pyx_t_3 = 1;
break;
default:
__pyx_t_3 = 0;
break;
}
__pyx_t_5 = (__pyx_t_3 != 0);
__pyx_t_2 = __pyx_t_5;
__pyx_L7_bool_binop_done:;
if (!__pyx_t_2) break;
+0945: n += 1
__pyx_v_n = (__pyx_v_n + 1);
}
+0946: if n >= lenline:
__pyx_t_2 = ((__pyx_v_n >= __pyx_v_lenline) != 0);
if (__pyx_t_2) {
/* … */
}
+0947: break
goto __pyx_L4_break;
+0948: elif line[n] == '(':
__pyx_t_6 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_v_n, short, 1, __Pyx_PyInt_From_short, 0, 0, 0); if (unlikely(__pyx_t_6 == (Py_UCS4)-1)) __PYX_ERR(0, 948, __pyx_L1_error)
__pyx_t_2 = ((__pyx_t_6 == 40) != 0);
if (__pyx_t_2) {
/* … */
goto __pyx_L9;
}
+0949: n += 1
__pyx_v_n = (__pyx_v_n + 1);
+0950: while n < lenline and line[n] in ' \t\n':
while (1) {
__pyx_t_5 = ((__pyx_v_n < __pyx_v_lenline) != 0);
if (__pyx_t_5) {
} else {
__pyx_t_2 = __pyx_t_5;
goto __pyx_L12_bool_binop_done;
}
__pyx_t_6 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_v_n, short, 1, __Pyx_PyInt_From_short, 0, 0, 0); if (unlikely(__pyx_t_6 == (Py_UCS4)-1)) __PYX_ERR(0, 950, __pyx_L1_error)
switch (__pyx_t_6) {
case 9:
case 10:
case 32:
__pyx_t_5 = 1;
break;
default:
__pyx_t_5 = 0;
break;
}
__pyx_t_3 = (__pyx_t_5 != 0);
__pyx_t_2 = __pyx_t_3;
__pyx_L12_bool_binop_done:;
if (!__pyx_t_2) break;
+0951: n += 1
__pyx_v_n = (__pyx_v_n + 1);
}
+0952: if n >= lenline:
__pyx_t_2 = ((__pyx_v_n >= __pyx_v_lenline) != 0);
if (__pyx_t_2) {
/* … */
}
+0953: return -1 # unexpected end of string after opening paren
__pyx_r = -1;
goto __pyx_L0;
+0954: startlabel = n
__pyx_v_startlabel = __pyx_v_n;
+0955: while n + 1 < lenline and line[n + 1] not in ' \t\n()':
while (1) {
__pyx_t_3 = (((__pyx_v_n + 1) < __pyx_v_lenline) != 0);
if (__pyx_t_3) {
} else {
__pyx_t_2 = __pyx_t_3;
goto __pyx_L17_bool_binop_done;
}
__pyx_t_7 = (__pyx_v_n + 1);
__pyx_t_8 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_t_7, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(__pyx_t_8 == (Py_UCS4)-1)) __PYX_ERR(0, 955, __pyx_L1_error)
switch (__pyx_t_8) {
case 9:
case 10:
case 32:
case 40:
case 41:
__pyx_t_3 = 0;
break;
default:
__pyx_t_3 = 1;
break;
}
__pyx_t_5 = (__pyx_t_3 != 0);
__pyx_t_2 = __pyx_t_5;
__pyx_L17_bool_binop_done:;
if (!__pyx_t_2) break;
+0956: n += 1
__pyx_v_n = (__pyx_v_n + 1);
}
+0957: if n + 1>= lenline:
__pyx_t_2 = (((__pyx_v_n + 1) >= __pyx_v_lenline) != 0);
if (__pyx_t_2) {
/* … */
}
+0958: return -2 # unexpected end of string after label
__pyx_r = -2;
goto __pyx_L0;
+0959: label = vocab._getlabelid(line[startlabel:n + 1])
if (unlikely(__pyx_v_line == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 959, __pyx_L1_error)
}
__pyx_t_9 = __Pyx_PyUnicode_Substring(__pyx_v_line, __pyx_v_startlabel, (__pyx_v_n + 1)); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 959, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_9);
__pyx_t_10 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->_getlabelid(__pyx_cur_scope->__pyx_v_vocab, ((PyObject*)__pyx_t_9)); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 959, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
__pyx_v_label = __pyx_t_10;
+0960: parent = stack.data.as_shorts[len(stack) - 1] if stack else -1
__pyx_t_2 = __Pyx_PyObject_IsTrue(((PyObject *)__pyx_v_stack)); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 960, __pyx_L1_error) if (__pyx_t_2) { if (unlikely(((PyObject *)__pyx_v_stack) == Py_None)) { PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); __PYX_ERR(0, 960, __pyx_L1_error) } __pyx_t_1 = Py_SIZE(((PyObject *)__pyx_v_stack)); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 960, __pyx_L1_error) __pyx_t_11 = (__pyx_v_stack->data.as_shorts[(__pyx_t_1 - 1)]); } else { __pyx_t_11 = -1; } __pyx_v_parent = __pyx_t_11;
+0961: stack.append(idx)
__pyx_t_9 = __Pyx_PyInt_From_short(__pyx_v_idx); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 961, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); __pyx_t_12 = __Pyx_PyObject_Append(((PyObject *)__pyx_v_stack), __pyx_t_9); if (unlikely(__pyx_t_12 == -1)) __PYX_ERR(0, 961, __pyx_L1_error) __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+0962: labels.append(label)
__pyx_t_9 = __Pyx_PyInt_From_int(__pyx_v_label); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 962, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); __pyx_t_12 = __Pyx_PyObject_Append(((PyObject *)__pyx_v_labels), __pyx_t_9); if (unlikely(__pyx_t_12 == -1)) __PYX_ERR(0, 962, __pyx_L1_error) __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+0963: nodes[idx].left = nodes[idx].right = nodes[idx].prod = -1
(__pyx_v_nodes[__pyx_v_idx]).left = -1;
(__pyx_v_nodes[__pyx_v_idx]).right = -1;
(__pyx_v_nodes[__pyx_v_idx]).prod = -1;
+0964: idx += 1
__pyx_v_idx = (__pyx_v_idx + 1);
+0965: if parent >= 0 and nodes[parent].right >= 0:
__pyx_t_5 = ((__pyx_v_parent >= 0) != 0);
if (__pyx_t_5) {
} else {
__pyx_t_2 = __pyx_t_5;
goto __pyx_L21_bool_binop_done;
}
__pyx_t_5 = (((__pyx_v_nodes[__pyx_v_parent]).right >= 0) != 0);
__pyx_t_2 = __pyx_t_5;
__pyx_L21_bool_binop_done:;
if (__pyx_t_2) {
/* … */
goto __pyx_L20;
}
+0966: children.append([nodes[parent].left, nodes[parent].right,
if (unlikely(__pyx_v_children == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "append");
__PYX_ERR(0, 966, __pyx_L1_error)
}
__pyx_t_9 = __Pyx_PyInt_From_int16_t((__pyx_v_nodes[__pyx_v_parent]).left); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 966, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_9);
__pyx_t_13 = __Pyx_PyInt_From_int16_t((__pyx_v_nodes[__pyx_v_parent]).right); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 966, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
/* … */
__pyx_t_15 = PyList_New(3); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 966, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
__Pyx_GIVEREF(__pyx_t_9);
PyList_SET_ITEM(__pyx_t_15, 0, __pyx_t_9);
__Pyx_GIVEREF(__pyx_t_13);
PyList_SET_ITEM(__pyx_t_15, 1, __pyx_t_13);
__Pyx_GIVEREF(__pyx_t_14);
PyList_SET_ITEM(__pyx_t_15, 2, __pyx_t_14);
__pyx_t_9 = 0;
__pyx_t_13 = 0;
__pyx_t_14 = 0;
__pyx_t_12 = __Pyx_PyList_Append(__pyx_v_children, __pyx_t_15); if (unlikely(__pyx_t_12 == -1)) __PYX_ERR(0, 966, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
+0967: stack.data.as_shorts[len(stack) - 1]])
if (unlikely(((PyObject *)__pyx_v_stack) == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 967, __pyx_L1_error)
}
__pyx_t_1 = Py_SIZE(((PyObject *)__pyx_v_stack)); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 967, __pyx_L1_error)
__pyx_t_14 = __Pyx_PyInt_From_short((__pyx_v_stack->data.as_shorts[(__pyx_t_1 - 1)])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 967, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_14);
+0968: nodes[parent].right = -2
(__pyx_v_nodes[__pyx_v_parent]).right = -2;
+0969: elif parent >= 0 and nodes[parent].right == -2:
__pyx_t_5 = ((__pyx_v_parent >= 0) != 0);
if (__pyx_t_5) {
} else {
__pyx_t_2 = __pyx_t_5;
goto __pyx_L23_bool_binop_done;
}
__pyx_t_5 = (((__pyx_v_nodes[__pyx_v_parent]).right == -2L) != 0);
__pyx_t_2 = __pyx_t_5;
__pyx_L23_bool_binop_done:;
if (__pyx_t_2) {
/* … */
}
__pyx_L20:;
+0970: children[len(children) - 1].append(
if (unlikely(__pyx_v_children == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 970, __pyx_L1_error)
}
if (unlikely(__pyx_v_children == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 970, __pyx_L1_error)
}
__pyx_t_1 = PyList_GET_SIZE(__pyx_v_children); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 970, __pyx_L1_error)
__pyx_t_16 = (__pyx_t_1 - 1);
/* … */
__pyx_t_12 = __Pyx_PyObject_Append(PyList_GET_ITEM(__pyx_v_children, __pyx_t_16), __pyx_t_15); if (unlikely(__pyx_t_12 == -1)) __PYX_ERR(0, 970, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
+0971: stack.data.as_shorts[len(stack) - 1])
if (unlikely(((PyObject *)__pyx_v_stack) == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 971, __pyx_L1_error)
}
__pyx_t_1 = Py_SIZE(((PyObject *)__pyx_v_stack)); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 971, __pyx_L1_error)
__pyx_t_15 = __Pyx_PyInt_From_short((__pyx_v_stack->data.as_shorts[(__pyx_t_1 - 1)])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 971, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
+0972: elif line[n] == ')':
__pyx_t_17 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_v_n, short, 1, __Pyx_PyInt_From_short, 0, 0, 0); if (unlikely(__pyx_t_17 == (Py_UCS4)-1)) __PYX_ERR(0, 972, __pyx_L1_error)
__pyx_t_2 = ((__pyx_t_17 == 41) != 0);
if (__pyx_t_2) {
/* … */
goto __pyx_L9;
}
+0973: prod = terminal = None
__Pyx_INCREF(Py_None); __Pyx_DECREF_SET(__pyx_v_prod, ((PyObject*)Py_None)); __Pyx_INCREF(Py_None); __Pyx_DECREF_SET(__pyx_v_terminal, ((PyObject*)Py_None));
+0974: cur = stack.pop()
__pyx_t_15 = __Pyx_PyObject_Pop(((PyObject *)__pyx_v_stack)); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 974, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); __pyx_t_11 = __Pyx_PyInt_As_short(__pyx_t_15); if (unlikely((__pyx_t_11 == (short)-1) && PyErr_Occurred())) __PYX_ERR(0, 974, __pyx_L1_error) __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; __pyx_v_cur = __pyx_t_11;
+0975: if stack:
__pyx_t_2 = __Pyx_PyObject_IsTrue(((PyObject *)__pyx_v_stack)); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 975, __pyx_L1_error)
if (__pyx_t_2) {
/* … */
goto __pyx_L25;
}
+0976: parent = stack.data.as_shorts[len(stack) - 1]
if (unlikely(((PyObject *)__pyx_v_stack) == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 976, __pyx_L1_error)
}
__pyx_t_16 = Py_SIZE(((PyObject *)__pyx_v_stack)); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 976, __pyx_L1_error)
__pyx_v_parent = (__pyx_v_stack->data.as_shorts[(__pyx_t_16 - 1)]);
0977: # lookahead for next paren
+0978: while n + 1 < lenline and line[n + 1] in ' \t\n':
while (1) {
__pyx_t_5 = (((__pyx_v_n + 1) < __pyx_v_lenline) != 0);
if (__pyx_t_5) {
} else {
__pyx_t_2 = __pyx_t_5;
goto __pyx_L28_bool_binop_done;
}
__pyx_t_7 = (__pyx_v_n + 1);
__pyx_t_17 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_t_7, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(__pyx_t_17 == (Py_UCS4)-1)) __PYX_ERR(0, 978, __pyx_L1_error)
switch (__pyx_t_17) {
case 9:
case 10:
case 32:
__pyx_t_5 = 1;
break;
default:
__pyx_t_5 = 0;
break;
}
__pyx_t_3 = (__pyx_t_5 != 0);
__pyx_t_2 = __pyx_t_3;
__pyx_L28_bool_binop_done:;
if (!__pyx_t_2) break;
+0979: n += 1
__pyx_v_n = (__pyx_v_n + 1);
}
+0980: if n + 1 >= lenline:
__pyx_t_2 = (((__pyx_v_n + 1) >= __pyx_v_lenline) != 0);
if (__pyx_t_2) {
/* … */
}
+0981: return -3 # unexpected end of string after closing paren
__pyx_r = -3;
goto __pyx_L0;
+0982: if nodes[cur].prod == -1: # frontier non-terminal?
__pyx_t_2 = (((__pyx_v_nodes[__pyx_v_cur]).prod == -1L) != 0);
if (__pyx_t_2) {
/* … */
}
+0983: nodes[cur].left = termidx(lensent)
(__pyx_v_nodes[__pyx_v_cur]).left = __pyx_f_8discodop_10containers_termidx(__pyx_v_lensent);
+0984: lensent += 1
__pyx_v_lensent = (__pyx_v_lensent + 1);
+0985: rule.lhs = labels.data.as_ints[cur]
__pyx_v_rule.lhs = (__pyx_v_labels->data.as_ints[__pyx_v_cur]);
+0986: rule.rhs1 = rule.rhs2 = rule.args = rule.lengths = 0
__pyx_v_rule.rhs1 = 0;
__pyx_v_rule.rhs2 = 0;
__pyx_v_rule.args = 0;
__pyx_v_rule.lengths = 0;
+0987: prod = (<char *>&rule)[:sizeof(Rule)]
__pyx_t_15 = __Pyx_PyBytes_FromStringAndSize(((char *)(&__pyx_v_rule)) + 0, (sizeof(Rule)) - 0); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 987, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); __Pyx_DECREF_SET(__pyx_v_prod, ((PyObject*)__pyx_t_15)); __pyx_t_15 = 0;
+0988: nodes[cur].prod = vocab._getprodid(prod)
__pyx_t_10 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->_getprodid(__pyx_cur_scope->__pyx_v_vocab, __pyx_v_prod); if (unlikely(__pyx_t_10 == -2)) __PYX_ERR(0, 988, __pyx_L1_error)
(__pyx_v_nodes[__pyx_v_cur]).prod = __pyx_t_10;
+0989: if nodes[parent].left == -1:
__pyx_t_2 = (((__pyx_v_nodes[__pyx_v_parent]).left == -1L) != 0);
if (__pyx_t_2) {
/* … */
goto __pyx_L32;
}
+0990: nodes[parent].left = cur
(__pyx_v_nodes[__pyx_v_parent]).left = __pyx_v_cur;
+0991: if line[n + 1] == ')': # first and only child, unary node
__pyx_t_7 = (__pyx_v_n + 1);
__pyx_t_18 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_t_7, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(__pyx_t_18 == (Py_UCS4)-1)) __PYX_ERR(0, 991, __pyx_L1_error)
__pyx_t_2 = ((__pyx_t_18 == 41) != 0);
if (__pyx_t_2) {
/* … */
}
+0992: rule.lhs = labels.data.as_ints[parent]
__pyx_v_rule.lhs = (__pyx_v_labels->data.as_ints[__pyx_v_parent]);
+0993: rule.rhs1 = labels.data.as_ints[cur]
__pyx_v_rule.rhs1 = (__pyx_v_labels->data.as_ints[__pyx_v_cur]);
+0994: rule.rhs2 = rule.args = 0
__pyx_v_rule.rhs2 = 0;
__pyx_v_rule.args = 0;
+0995: rule.lengths = 1
__pyx_v_rule.lengths = 1;
+0996: prod = (<char *>&rule)[:sizeof(Rule)]
__pyx_t_15 = __Pyx_PyBytes_FromStringAndSize(((char *)(&__pyx_v_rule)) + 0, (sizeof(Rule)) - 0); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 996, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); __Pyx_DECREF_SET(__pyx_v_prod, ((PyObject*)__pyx_t_15)); __pyx_t_15 = 0;
+0997: elif nodes[parent].right == -1:
__pyx_t_2 = (((__pyx_v_nodes[__pyx_v_parent]).right == -1L) != 0);
if (__pyx_t_2) {
/* … */
goto __pyx_L32;
}
+0998: nodes[parent].right = cur
(__pyx_v_nodes[__pyx_v_parent]).right = __pyx_v_cur;
+0999: if line[n + 1] == ')': # 2nd and last child, binary node
__pyx_t_7 = (__pyx_v_n + 1);
__pyx_t_18 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_t_7, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(__pyx_t_18 == (Py_UCS4)-1)) __PYX_ERR(0, 999, __pyx_L1_error)
__pyx_t_2 = ((__pyx_t_18 == 41) != 0);
if (__pyx_t_2) {
/* … */
}
+1000: rule.lhs = labels.data.as_ints[parent]
__pyx_v_rule.lhs = (__pyx_v_labels->data.as_ints[__pyx_v_parent]);
+1001: rule.rhs1 = labels.data.as_ints[parent + 1]
__pyx_v_rule.rhs1 = (__pyx_v_labels->data.as_ints[(__pyx_v_parent + 1)]);
+1002: rule.rhs2 = labels.data.as_ints[cur]
__pyx_v_rule.rhs2 = (__pyx_v_labels->data.as_ints[__pyx_v_cur]);
+1003: rule.args = rule.lengths = 0b10
__pyx_v_rule.args = 2;
__pyx_v_rule.lengths = 2;
+1004: prod = (<char *>&rule)[:sizeof(Rule)]
__pyx_t_15 = __Pyx_PyBytes_FromStringAndSize(((char *)(&__pyx_v_rule)) + 0, (sizeof(Rule)) - 0); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1004, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); __Pyx_DECREF_SET(__pyx_v_prod, ((PyObject*)__pyx_t_15)); __pyx_t_15 = 0;
+1005: elif line[n + 1] == ')': # last of > 2 children
__pyx_t_7 = (__pyx_v_n + 1);
__pyx_t_18 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_t_7, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(__pyx_t_18 == (Py_UCS4)-1)) __PYX_ERR(0, 1005, __pyx_L1_error)
__pyx_t_2 = ((__pyx_t_18 == 41) != 0);
if (__pyx_t_2) {
/* … */
}
__pyx_L32:;
1006: # collect labels
+1007: binchildren = children.pop()
if (unlikely(__pyx_v_children == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "pop");
__PYX_ERR(0, 1007, __pyx_L1_error)
}
__pyx_t_15 = __Pyx_PyList_Pop(__pyx_v_children); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1007, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
if (!(likely(PyList_CheckExact(__pyx_t_15))||((__pyx_t_15) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "list", Py_TYPE(__pyx_t_15)->tp_name), 0))) __PYX_ERR(0, 1007, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_binchildren, ((PyObject*)__pyx_t_15));
__pyx_t_15 = 0;
+1008: binlabels = [labels.data.as_ints[m] for m in binchildren]
{ /* enter inner scope */
short __pyx_9genexpr16__pyx_v_m;
__pyx_t_15 = PyList_New(0); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1008, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
if (unlikely(__pyx_v_binchildren == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
__PYX_ERR(0, 1008, __pyx_L1_error)
}
__pyx_t_14 = __pyx_v_binchildren; __Pyx_INCREF(__pyx_t_14); __pyx_t_16 = 0;
for (;;) {
if (__pyx_t_16 >= PyList_GET_SIZE(__pyx_t_14)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_13 = PyList_GET_ITEM(__pyx_t_14, __pyx_t_16); __Pyx_INCREF(__pyx_t_13); __pyx_t_16++; if (unlikely(0 < 0)) __PYX_ERR(0, 1008, __pyx_L1_error)
#else
__pyx_t_13 = PySequence_ITEM(__pyx_t_14, __pyx_t_16); __pyx_t_16++; if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1008, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
#endif
__pyx_t_11 = __Pyx_PyInt_As_short(__pyx_t_13); if (unlikely((__pyx_t_11 == (short)-1) && PyErr_Occurred())) __PYX_ERR(0, 1008, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
__pyx_9genexpr16__pyx_v_m = __pyx_t_11;
__pyx_t_13 = __Pyx_PyInt_From_int((__pyx_v_labels->data.as_ints[__pyx_9genexpr16__pyx_v_m])); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1008, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
if (unlikely(__Pyx_ListComp_Append(__pyx_t_15, (PyObject*)__pyx_t_13))) __PYX_ERR(0, 1008, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
}
__Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
} /* exit inner scope */
__Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_binlabels);
__Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_binlabels, ((PyObject*)__pyx_t_15));
__Pyx_GIVEREF(__pyx_t_15);
__pyx_t_15 = 0;
+1009: binlabel = vocab._getlabelid('%s|<%s.%s>' % (
__pyx_t_13 = PyUnicode_Format(__pyx_kp_u_s_s_s, __pyx_t_9); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1009, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __pyx_t_10 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->_getlabelid(__pyx_cur_scope->__pyx_v_vocab, ((PyObject*)__pyx_t_13)); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 1009, __pyx_L1_error) __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; __pyx_v_binlabel = __pyx_t_10;
+1010: vocab.idtolabel(labels.data.as_ints[parent]),
__pyx_t_15 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->idtolabel(__pyx_cur_scope->__pyx_v_vocab, (__pyx_v_labels->data.as_ints[__pyx_v_parent])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1010, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
/* … */
__pyx_t_9 = PyTuple_New(3); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 1010, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_9);
__Pyx_GIVEREF(__pyx_t_15);
PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_15);
__Pyx_GIVEREF(__pyx_t_14);
PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_14);
__Pyx_GIVEREF(__pyx_t_13);
PyTuple_SET_ITEM(__pyx_t_9, 2, __pyx_t_13);
__pyx_t_15 = 0;
__pyx_t_14 = 0;
__pyx_t_13 = 0;
+1011: vocab.idtolabel(binlabels[0]),
__pyx_t_19 = __Pyx_PyInt_As_uint32_t(PyList_GET_ITEM(__pyx_cur_scope->__pyx_v_binlabels, 0)); if (unlikely((__pyx_t_19 == ((uint32_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 1011, __pyx_L1_error) __pyx_t_14 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->idtolabel(__pyx_cur_scope->__pyx_v_vocab, __pyx_t_19); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 1011, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_14);
+1012: ','.join(vocab.idtolabel(x)
static PyObject *__pyx_pf_8discodop_10_fragments_8readtree_genexpr(PyObject *__pyx_self) {
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_2_genexpr *__pyx_cur_scope;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("genexpr", 0);
__pyx_cur_scope = (struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_2_genexpr *)__pyx_tp_new_8discodop_10_fragments___pyx_scope_struct_2_genexpr(__pyx_ptype_8discodop_10_fragments___pyx_scope_struct_2_genexpr, __pyx_empty_tuple, NULL);
if (unlikely(!__pyx_cur_scope)) {
__pyx_cur_scope = ((struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_2_genexpr *)Py_None);
__Pyx_INCREF(Py_None);
__PYX_ERR(0, 1012, __pyx_L1_error)
} else {
__Pyx_GOTREF(__pyx_cur_scope);
}
__pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree *) __pyx_self;
__Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_outer_scope));
__Pyx_GIVEREF(__pyx_cur_scope->__pyx_outer_scope);
{
__pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_8discodop_10_fragments_8readtree_2generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_genexpr, __pyx_n_s_readtree_locals_genexpr, __pyx_n_s_discodop__fragments); if (unlikely(!gen)) __PYX_ERR(0, 1012, __pyx_L1_error)
__Pyx_DECREF(__pyx_cur_scope);
__Pyx_RefNannyFinishContext();
return (PyObject *) gen;
}
/* function exit code */
__pyx_L1_error:;
__Pyx_AddTraceback("discodop._fragments.readtree.genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__Pyx_DECREF(((PyObject *)__pyx_cur_scope));
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_gb_8discodop_10_fragments_8readtree_2generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
{
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("None", 0);
__pyx_L3_first_run:;
if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 1012, __pyx_L1_error)
__pyx_r = PyList_New(0); if (unlikely(!__pyx_r)) __PYX_ERR(0, 1012, __pyx_L1_error)
__Pyx_GOTREF(__pyx_r);
/* … */
if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_vocab)) { __Pyx_RaiseClosureNameError("vocab"); __PYX_ERR(0, 1012, __pyx_L1_error) }
__pyx_t_4 = __Pyx_PyInt_As_uint32_t(__pyx_cur_scope->__pyx_v_x); if (unlikely((__pyx_t_4 == ((uint32_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 1012, __pyx_L1_error)
__pyx_t_1 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_outer_scope->__pyx_v_vocab->__pyx_vtab)->idtolabel(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_vocab, __pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1012, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
if (unlikely(__Pyx_ListComp_Append(__pyx_r, (PyObject*)__pyx_t_1))) __PYX_ERR(0, 1012, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
/* … */
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_r); __pyx_r = 0;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_AddTraceback("genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__pyx_generator->resume_label = -1;
__Pyx_Coroutine_clear((PyObject*)__pyx_generator);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_gb_8discodop_10_fragments_8readtree_5generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
/* … */
__pyx_t_13 = __pyx_pf_8discodop_10_fragments_8readtree_genexpr(((PyObject*)__pyx_cur_scope)); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1012, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
__pyx_t_9 = __Pyx_Generator_Next(__pyx_t_13); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 1012, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_9);
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
__pyx_t_13 = PyUnicode_Join(__pyx_kp_u__16, __pyx_t_9); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1012, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
__Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
/* … */
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_2_genexpr {
PyObject_HEAD
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree *__pyx_outer_scope;
PyObject *__pyx_v_x;
};
+1013: for x in binlabels[1:])))
if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_binlabels)) { __Pyx_RaiseClosureNameError("binlabels"); __PYX_ERR(0, 1013, __pyx_L1_error) }
if (unlikely(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_binlabels == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 1013, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyList_GetSlice(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_binlabels, 1, PY_SSIZE_T_MAX); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1013, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_2 = __pyx_t_1; __Pyx_INCREF(__pyx_t_2); __pyx_t_3 = 0;
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
for (;;) {
if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_1 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_1); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(0, 1013, __pyx_L1_error)
#else
__pyx_t_1 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1013, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
#endif
__Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_x);
__Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_x, __pyx_t_1);
__Pyx_GIVEREF(__pyx_t_1);
__pyx_t_1 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope);
+1014: nodes[parent].right = idx
(__pyx_v_nodes[__pyx_v_parent]).right = __pyx_v_idx;
+1015: rule.lhs = labels.data.as_ints[parent]
__pyx_v_rule.lhs = (__pyx_v_labels->data.as_ints[__pyx_v_parent]);
+1016: rule.rhs1 = labels.data.as_ints[parent + 1]
__pyx_v_rule.rhs1 = (__pyx_v_labels->data.as_ints[(__pyx_v_parent + 1)]);
+1017: rule.rhs2 = binlabel
__pyx_v_rule.rhs2 = __pyx_v_binlabel;
+1018: rule.args = rule.lengths = 0b10
__pyx_v_rule.args = 2;
__pyx_v_rule.lengths = 2;
+1019: prod = (<char *>&rule)[:sizeof(Rule)]
__pyx_t_13 = __Pyx_PyBytes_FromStringAndSize(((char *)(&__pyx_v_rule)) + 0, (sizeof(Rule)) - 0); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1019, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_DECREF_SET(__pyx_v_prod, ((PyObject*)__pyx_t_13)); __pyx_t_13 = 0;
1020: # add intermediate nodes
+1021: for m, y in enumerate(
__pyx_t_11 = 1;
/* … */
__pyx_t_9 = __pyx_t_13; __Pyx_INCREF(__pyx_t_9); __pyx_t_16 = 0;
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
for (;;) {
if (__pyx_t_16 >= PyList_GET_SIZE(__pyx_t_9)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_13 = PyList_GET_ITEM(__pyx_t_9, __pyx_t_16); __Pyx_INCREF(__pyx_t_13); __pyx_t_16++; if (unlikely(0 < 0)) __PYX_ERR(0, 1021, __pyx_L1_error)
#else
__pyx_t_13 = PySequence_ITEM(__pyx_t_9, __pyx_t_16); __pyx_t_16++; if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1021, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
#endif
__pyx_t_20 = __Pyx_PyInt_As_short(__pyx_t_13); if (unlikely((__pyx_t_20 == (short)-1) && PyErr_Occurred())) __PYX_ERR(0, 1021, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
__pyx_v_y = __pyx_t_20;
__pyx_cur_scope->__pyx_v_m = __pyx_t_11;
__pyx_t_11 = (__pyx_t_11 + 1);
/* … */
}
__Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+1022: binchildren[1:len(binchildren) - 1], 1):
if (unlikely(__pyx_v_binchildren == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 1022, __pyx_L1_error)
}
if (unlikely(__pyx_v_binchildren == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 1022, __pyx_L1_error)
}
__pyx_t_16 = PyList_GET_SIZE(__pyx_v_binchildren); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 1022, __pyx_L1_error)
__pyx_t_13 = __Pyx_PyList_GetSlice(__pyx_v_binchildren, 1, (__pyx_t_16 - 1)); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1022, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
+1023: nodes[idx].left = y
(__pyx_v_nodes[__pyx_v_idx]).left = __pyx_v_y;
+1024: if m == len(binlabels) - 2:
__pyx_t_13 = __pyx_cur_scope->__pyx_v_binlabels;
__Pyx_INCREF(__pyx_t_13);
if (unlikely(__pyx_t_13 == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 1024, __pyx_L1_error)
}
__pyx_t_1 = PyList_GET_SIZE(__pyx_t_13); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 1024, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
__pyx_t_2 = ((__pyx_cur_scope->__pyx_v_m == (__pyx_t_1 - 2)) != 0);
if (__pyx_t_2) {
/* … */
goto __pyx_L39;
}
+1025: nodes[idx].right = binchildren[m + 1]
if (unlikely(__pyx_v_binchildren == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 1025, __pyx_L1_error)
}
__pyx_t_7 = (__pyx_cur_scope->__pyx_v_m + 1);
__pyx_t_21 = __Pyx_PyInt_As_int16_t(PyList_GET_ITEM(__pyx_v_binchildren, __pyx_t_7)); if (unlikely((__pyx_t_21 == ((int16_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 1025, __pyx_L1_error)
(__pyx_v_nodes[__pyx_v_idx]).right = __pyx_t_21;
+1026: binlabel2 = binlabels[len(binlabels) - 1]
__pyx_t_13 = __pyx_cur_scope->__pyx_v_binlabels;
__Pyx_INCREF(__pyx_t_13);
if (unlikely(__pyx_t_13 == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 1026, __pyx_L1_error)
}
__pyx_t_1 = PyList_GET_SIZE(__pyx_t_13); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 1026, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
__pyx_t_22 = (__pyx_t_1 - 1);
__pyx_t_10 = __Pyx_PyInt_As_int(PyList_GET_ITEM(__pyx_cur_scope->__pyx_v_binlabels, __pyx_t_22)); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 1026, __pyx_L1_error)
__pyx_v_binlabel2 = __pyx_t_10;
1027: else:
+1028: nodes[idx].right = idx + 1
/*else*/ {
(__pyx_v_nodes[__pyx_v_idx]).right = (__pyx_v_idx + 1);
+1029: binlabel2 = vocab._getlabelid('%s|<%s.%s>' % (
__pyx_t_15 = PyUnicode_Format(__pyx_kp_u_s_s_s, __pyx_t_23); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1029, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); __Pyx_DECREF(__pyx_t_23); __pyx_t_23 = 0; __pyx_t_10 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->_getlabelid(__pyx_cur_scope->__pyx_v_vocab, ((PyObject*)__pyx_t_15)); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 1029, __pyx_L1_error) __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; __pyx_v_binlabel2 = __pyx_t_10; } __pyx_L39:;
+1030: vocab.idtolabel(
__pyx_t_13 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->idtolabel(__pyx_cur_scope->__pyx_v_vocab, (__pyx_v_labels->data.as_ints[__pyx_v_parent])); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1030, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
/* … */
__pyx_t_23 = PyTuple_New(3); if (unlikely(!__pyx_t_23)) __PYX_ERR(0, 1030, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_23);
__Pyx_GIVEREF(__pyx_t_13);
PyTuple_SET_ITEM(__pyx_t_23, 0, __pyx_t_13);
__Pyx_GIVEREF(__pyx_t_14);
PyTuple_SET_ITEM(__pyx_t_23, 1, __pyx_t_14);
__Pyx_GIVEREF(__pyx_t_15);
PyTuple_SET_ITEM(__pyx_t_23, 2, __pyx_t_15);
__pyx_t_13 = 0;
__pyx_t_14 = 0;
__pyx_t_15 = 0;
1031: labels.data.as_ints[parent]),
+1032: ','.join(vocab.idtolabel(x)
static PyObject *__pyx_pf_8discodop_10_fragments_8readtree_3genexpr(PyObject *__pyx_self) {
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_3_genexpr *__pyx_cur_scope;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("genexpr", 0);
__pyx_cur_scope = (struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_3_genexpr *)__pyx_tp_new_8discodop_10_fragments___pyx_scope_struct_3_genexpr(__pyx_ptype_8discodop_10_fragments___pyx_scope_struct_3_genexpr, __pyx_empty_tuple, NULL);
if (unlikely(!__pyx_cur_scope)) {
__pyx_cur_scope = ((struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_3_genexpr *)Py_None);
__Pyx_INCREF(Py_None);
__PYX_ERR(0, 1032, __pyx_L1_error)
} else {
__Pyx_GOTREF(__pyx_cur_scope);
}
__pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree *) __pyx_self;
__Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_outer_scope));
__Pyx_GIVEREF(__pyx_cur_scope->__pyx_outer_scope);
{
__pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_8discodop_10_fragments_8readtree_5generator1, (PyObject *) __pyx_cur_scope, __pyx_n_s_genexpr, __pyx_n_s_readtree_locals_genexpr, __pyx_n_s_discodop__fragments); if (unlikely(!gen)) __PYX_ERR(0, 1032, __pyx_L1_error)
__Pyx_DECREF(__pyx_cur_scope);
__Pyx_RefNannyFinishContext();
return (PyObject *) gen;
}
/* function exit code */
__pyx_L1_error:;
__Pyx_AddTraceback("discodop._fragments.readtree.genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__Pyx_DECREF(((PyObject *)__pyx_cur_scope));
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_gb_8discodop_10_fragments_8readtree_5generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
{
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("None", 0);
__pyx_L3_first_run:;
if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 1032, __pyx_L1_error)
__pyx_r = PyList_New(0); if (unlikely(!__pyx_r)) __PYX_ERR(0, 1032, __pyx_L1_error)
__Pyx_GOTREF(__pyx_r);
/* … */
if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_vocab)) { __Pyx_RaiseClosureNameError("vocab"); __PYX_ERR(0, 1032, __pyx_L1_error) }
__pyx_t_4 = __Pyx_PyInt_As_uint32_t(__pyx_cur_scope->__pyx_v_x); if (unlikely((__pyx_t_4 == ((uint32_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 1032, __pyx_L1_error)
__pyx_t_1 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_outer_scope->__pyx_v_vocab->__pyx_vtab)->idtolabel(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_vocab, __pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1032, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
if (unlikely(__Pyx_ListComp_Append(__pyx_r, (PyObject*)__pyx_t_1))) __PYX_ERR(0, 1032, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
/* … */
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_r); __pyx_r = 0;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_AddTraceback("genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__pyx_generator->resume_label = -1;
__Pyx_Coroutine_clear((PyObject*)__pyx_generator);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_gb_8discodop_10_fragments_8readtree_8generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
/* … */
__pyx_t_14 = __pyx_pf_8discodop_10_fragments_8readtree_3genexpr(((PyObject*)__pyx_cur_scope)); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 1032, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_14);
__pyx_t_15 = __Pyx_Generator_Next(__pyx_t_14); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1032, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
__Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
__pyx_t_14 = PyUnicode_Join(__pyx_kp_u__16, __pyx_t_15); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 1032, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_14);
__Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
/* … */
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_3_genexpr {
PyObject_HEAD
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree *__pyx_outer_scope;
PyObject *__pyx_v_x;
};
+1033: for x in binlabels[:m + 1]),
if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_binlabels)) { __Pyx_RaiseClosureNameError("binlabels"); __PYX_ERR(0, 1033, __pyx_L1_error) }
if (unlikely(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_binlabels == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 1033, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyList_GetSlice(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_binlabels, 0, (__pyx_cur_scope->__pyx_outer_scope->__pyx_v_m + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1033, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_2 = __pyx_t_1; __Pyx_INCREF(__pyx_t_2); __pyx_t_3 = 0;
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
for (;;) {
if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_1 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_1); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(0, 1033, __pyx_L1_error)
#else
__pyx_t_1 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1033, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
#endif
__Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_x);
__Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_x, __pyx_t_1);
__Pyx_GIVEREF(__pyx_t_1);
__pyx_t_1 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope);
+1034: ','.join(vocab.idtolabel(x)
static PyObject *__pyx_pf_8discodop_10_fragments_8readtree_6genexpr(PyObject *__pyx_self) {
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_4_genexpr *__pyx_cur_scope;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("genexpr", 0);
__pyx_cur_scope = (struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_4_genexpr *)__pyx_tp_new_8discodop_10_fragments___pyx_scope_struct_4_genexpr(__pyx_ptype_8discodop_10_fragments___pyx_scope_struct_4_genexpr, __pyx_empty_tuple, NULL);
if (unlikely(!__pyx_cur_scope)) {
__pyx_cur_scope = ((struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_4_genexpr *)Py_None);
__Pyx_INCREF(Py_None);
__PYX_ERR(0, 1034, __pyx_L1_error)
} else {
__Pyx_GOTREF(__pyx_cur_scope);
}
__pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree *) __pyx_self;
__Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_outer_scope));
__Pyx_GIVEREF(__pyx_cur_scope->__pyx_outer_scope);
{
__pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_8discodop_10_fragments_8readtree_8generator2, (PyObject *) __pyx_cur_scope, __pyx_n_s_genexpr, __pyx_n_s_readtree_locals_genexpr, __pyx_n_s_discodop__fragments); if (unlikely(!gen)) __PYX_ERR(0, 1034, __pyx_L1_error)
__Pyx_DECREF(__pyx_cur_scope);
__Pyx_RefNannyFinishContext();
return (PyObject *) gen;
}
/* function exit code */
__pyx_L1_error:;
__Pyx_AddTraceback("discodop._fragments.readtree.genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__Pyx_DECREF(((PyObject *)__pyx_cur_scope));
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_gb_8discodop_10_fragments_8readtree_8generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
{
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("None", 0);
__pyx_L3_first_run:;
if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 1034, __pyx_L1_error)
__pyx_r = PyList_New(0); if (unlikely(!__pyx_r)) __PYX_ERR(0, 1034, __pyx_L1_error)
__Pyx_GOTREF(__pyx_r);
/* … */
if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_vocab)) { __Pyx_RaiseClosureNameError("vocab"); __PYX_ERR(0, 1034, __pyx_L1_error) }
__pyx_t_4 = __Pyx_PyInt_As_uint32_t(__pyx_cur_scope->__pyx_v_x); if (unlikely((__pyx_t_4 == ((uint32_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 1034, __pyx_L1_error)
__pyx_t_1 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_outer_scope->__pyx_v_vocab->__pyx_vtab)->idtolabel(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_vocab, __pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1034, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
if (unlikely(__Pyx_ListComp_Append(__pyx_r, (PyObject*)__pyx_t_1))) __PYX_ERR(0, 1034, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
/* … */
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_r); __pyx_r = 0;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_AddTraceback("genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__pyx_generator->resume_label = -1;
__Pyx_Coroutine_clear((PyObject*)__pyx_generator);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
__pyx_t_15 = __pyx_pf_8discodop_10_fragments_8readtree_6genexpr(((PyObject*)__pyx_cur_scope)); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1034, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
__pyx_t_23 = __Pyx_Generator_Next(__pyx_t_15); if (unlikely(!__pyx_t_23)) __PYX_ERR(0, 1034, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_23);
__Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
__pyx_t_15 = PyUnicode_Join(__pyx_kp_u__16, __pyx_t_23); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1034, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_15);
__Pyx_DECREF(__pyx_t_23); __pyx_t_23 = 0;
/* … */
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_4_genexpr {
PyObject_HEAD
struct __pyx_obj_8discodop_10_fragments___pyx_scope_struct_1___pyx_f_8discodop_10_fragments_readtree *__pyx_outer_scope;
PyObject *__pyx_v_x;
};
+1035: for x in binlabels[m + 1:])))
if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_binlabels)) { __Pyx_RaiseClosureNameError("binlabels"); __PYX_ERR(0, 1035, __pyx_L1_error) }
if (unlikely(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_binlabels == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 1035, __pyx_L1_error)
}
__pyx_t_1 = __Pyx_PyList_GetSlice(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_binlabels, (__pyx_cur_scope->__pyx_outer_scope->__pyx_v_m + 1), PY_SSIZE_T_MAX); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1035, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__pyx_t_2 = __pyx_t_1; __Pyx_INCREF(__pyx_t_2); __pyx_t_3 = 0;
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
for (;;) {
if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_2)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_1 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_1); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(0, 1035, __pyx_L1_error)
#else
__pyx_t_1 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1035, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
#endif
__Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_x);
__Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_x, __pyx_t_1);
__Pyx_GIVEREF(__pyx_t_1);
__pyx_t_1 = 0;
/* … */
}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope);
+1036: nodes[parent].prod = vocab._getprodid(prod)
__pyx_t_10 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->_getprodid(__pyx_cur_scope->__pyx_v_vocab, __pyx_v_prod); if (unlikely(__pyx_t_10 == -2)) __PYX_ERR(0, 1036, __pyx_L1_error)
(__pyx_v_nodes[__pyx_v_parent]).prod = __pyx_t_10;
+1037: parent = idx
__pyx_v_parent = __pyx_v_idx;
+1038: label, binlabel = binlabel, binlabel2
__pyx_t_10 = __pyx_v_binlabel;
__pyx_t_24 = __pyx_v_binlabel2;
__pyx_v_label = __pyx_t_10;
__pyx_v_binlabel = __pyx_t_24;
+1039: rule.lhs, rule.rhs1, rule.rhs2 = (
__pyx_v_rule.lhs = __pyx_t_24;
__pyx_v_rule.rhs1 = __pyx_t_10;
__pyx_v_rule.rhs2 = __pyx_t_25;
+1040: label, labels.data.as_ints[y], binlabel)
__pyx_t_24 = __pyx_v_label;
__pyx_t_10 = (__pyx_v_labels->data.as_ints[__pyx_v_y]);
__pyx_t_25 = __pyx_v_binlabel;
+1041: rule.args = rule.lengths = 0b10
__pyx_v_rule.args = 2;
__pyx_v_rule.lengths = 2;
+1042: prod = (<char *>&rule)[:sizeof(Rule)]
__pyx_t_15 = __Pyx_PyBytes_FromStringAndSize(((char *)(&__pyx_v_rule)) + 0, (sizeof(Rule)) - 0); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1042, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); __Pyx_DECREF_SET(__pyx_v_prod, ((PyObject*)__pyx_t_15)); __pyx_t_15 = 0;
+1043: idx += 1
__pyx_v_idx = (__pyx_v_idx + 1);
+1044: labels.append(label)
__pyx_t_15 = __Pyx_PyInt_From_int(__pyx_v_label); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1044, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_15); __pyx_t_12 = __Pyx_PyObject_Append(((PyObject *)__pyx_v_labels), __pyx_t_15); if (unlikely(__pyx_t_12 == -1)) __PYX_ERR(0, 1044, __pyx_L1_error) __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
1045: # else: # there is no else.
1046: else: # end of stack, should be end of tree
+1047: n += 1
/*else*/ {
__pyx_v_n = (__pyx_v_n + 1);
+1048: while n < lenline and line[n] in ' \t\n':
while (1) {
__pyx_t_3 = ((__pyx_v_n < __pyx_v_lenline) != 0);
if (__pyx_t_3) {
} else {
__pyx_t_2 = __pyx_t_3;
goto __pyx_L42_bool_binop_done;
}
__pyx_t_18 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_v_n, short, 1, __Pyx_PyInt_From_short, 0, 0, 0); if (unlikely(__pyx_t_18 == (Py_UCS4)-1)) __PYX_ERR(0, 1048, __pyx_L1_error)
switch (__pyx_t_18) {
case 9:
case 10:
case 32:
__pyx_t_3 = 1;
break;
default:
__pyx_t_3 = 0;
break;
}
__pyx_t_5 = (__pyx_t_3 != 0);
__pyx_t_2 = __pyx_t_5;
__pyx_L42_bool_binop_done:;
if (!__pyx_t_2) break;
+1049: n += 1
__pyx_v_n = (__pyx_v_n + 1);
}
+1050: if n < lenline:
__pyx_t_2 = ((__pyx_v_n < __pyx_v_lenline) != 0);
if (__pyx_t_2) {
/* … */
}
}
__pyx_L25:;
+1051: return -4 # unexpected data after end of tree
__pyx_r = -4;
goto __pyx_L0;
1052: else: # terminal
+1053: start = n
/*else*/ {
__pyx_v_start = __pyx_v_n;
+1054: while n < lenline and line[n] not in ') \t\n':
while (1) {
__pyx_t_5 = ((__pyx_v_n < __pyx_v_lenline) != 0);
if (__pyx_t_5) {
} else {
__pyx_t_2 = __pyx_t_5;
goto __pyx_L47_bool_binop_done;
}
__pyx_t_26 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_v_n, short, 1, __Pyx_PyInt_From_short, 0, 0, 0); if (unlikely(__pyx_t_26 == (Py_UCS4)-1)) __PYX_ERR(0, 1054, __pyx_L1_error)
switch (__pyx_t_26) {
case 9:
case 10:
case 32:
case 41:
__pyx_t_5 = 0;
break;
default:
__pyx_t_5 = 1;
break;
}
__pyx_t_3 = (__pyx_t_5 != 0);
__pyx_t_2 = __pyx_t_3;
__pyx_L47_bool_binop_done:;
if (!__pyx_t_2) break;
+1055: n += 1
__pyx_v_n = (__pyx_v_n + 1);
}
+1056: terminal = line[start:n]
if (unlikely(__pyx_v_line == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 1056, __pyx_L1_error)
}
__pyx_t_9 = __Pyx_PyUnicode_Substring(__pyx_v_line, __pyx_v_start, __pyx_v_n); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 1056, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_9);
__Pyx_DECREF_SET(__pyx_v_terminal, ((PyObject*)__pyx_t_9));
__pyx_t_9 = 0;
+1057: while n < lenline and line[n] in ' \t\n':
while (1) {
__pyx_t_3 = ((__pyx_v_n < __pyx_v_lenline) != 0);
if (__pyx_t_3) {
} else {
__pyx_t_2 = __pyx_t_3;
goto __pyx_L51_bool_binop_done;
}
__pyx_t_27 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_v_n, short, 1, __Pyx_PyInt_From_short, 0, 0, 0); if (unlikely(__pyx_t_27 == (Py_UCS4)-1)) __PYX_ERR(0, 1057, __pyx_L1_error)
switch (__pyx_t_27) {
case 9:
case 10:
case 32:
__pyx_t_3 = 1;
break;
default:
__pyx_t_3 = 0;
break;
}
__pyx_t_5 = (__pyx_t_3 != 0);
__pyx_t_2 = __pyx_t_5;
__pyx_L51_bool_binop_done:;
if (!__pyx_t_2) break;
+1058: n += 1
__pyx_v_n = (__pyx_v_n + 1);
}
+1059: if line[n] != ')':
__pyx_t_28 = __Pyx_GetItemInt_Unicode(__pyx_v_line, __pyx_v_n, short, 1, __Pyx_PyInt_From_short, 0, 0, 0); if (unlikely(__pyx_t_28 == (Py_UCS4)-1)) __PYX_ERR(0, 1059, __pyx_L1_error)
__pyx_t_2 = ((__pyx_t_28 != 41) != 0);
if (__pyx_t_2) {
/* … */
}
+1060: return -5 # unexpected data after terminal
__pyx_r = -5;
goto __pyx_L0;
+1061: n -= 1
__pyx_v_n = (__pyx_v_n - 1);
+1062: parent = stack.data.as_shorts[len(stack) - 1]
if (unlikely(((PyObject *)__pyx_v_stack) == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 1062, __pyx_L1_error)
}
__pyx_t_16 = Py_SIZE(((PyObject *)__pyx_v_stack)); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 1062, __pyx_L1_error)
__pyx_v_parent = (__pyx_v_stack->data.as_shorts[(__pyx_t_16 - 1)]);
1063: # if disc:
1064: # m = terminal.find('=')
1065: # if m <= 0:
1066: # return -6 # missing terminal index
1067: # m, terminal = int(terminal[:m]), terminal[m + 1:]
1068: # nodes[parent].left = termidx(m)
1069: # else:
1070: # nodes[parent].left = termidx(lensent)
+1071: nodes[parent].left = termidx(lensent)
(__pyx_v_nodes[__pyx_v_parent]).left = __pyx_f_8discodop_10containers_termidx(__pyx_v_lensent);
+1072: lensent += 1
__pyx_v_lensent = (__pyx_v_lensent + 1);
+1073: rule.lhs = labels.data.as_ints[parent]
__pyx_v_rule.lhs = (__pyx_v_labels->data.as_ints[__pyx_v_parent]);
+1074: rule.rhs1 = rule.rhs2 = rule.lengths = 0
__pyx_v_rule.rhs1 = 0;
__pyx_v_rule.rhs2 = 0;
__pyx_v_rule.lengths = 0;
+1075: rule.args = vocab._getlabelid(terminal)
__pyx_t_25 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->_getlabelid(__pyx_cur_scope->__pyx_v_vocab, __pyx_v_terminal); if (unlikely(__pyx_t_25 == -1)) __PYX_ERR(0, 1075, __pyx_L1_error)
__pyx_v_rule.args = __pyx_t_25;
+1076: prod = (<char *>&rule)[:sizeof(Rule)]
__pyx_t_9 = __Pyx_PyBytes_FromStringAndSize(((char *)(&__pyx_v_rule)) + 0, (sizeof(Rule)) - 0); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 1076, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); __Pyx_DECREF_SET(__pyx_v_prod, ((PyObject*)__pyx_t_9)); __pyx_t_9 = 0; } __pyx_L9:;
+1077: if prod is None:
__pyx_t_2 = (__pyx_v_prod == ((PyObject*)Py_None));
__pyx_t_5 = (__pyx_t_2 != 0);
if (__pyx_t_5) {
goto __pyx_L54;
}
1078: pass
1079: else:
+1080: nodes[parent].prod = vocab._getprodid(prod)
/*else*/ {
__pyx_t_25 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_cur_scope->__pyx_v_vocab->__pyx_vtab)->_getprodid(__pyx_cur_scope->__pyx_v_vocab, __pyx_v_prod); if (unlikely(__pyx_t_25 == -2)) __PYX_ERR(0, 1080, __pyx_L1_error)
(__pyx_v_nodes[__pyx_v_parent]).prod = __pyx_t_25;
}
__pyx_L54:;
+1081: n += 1
__pyx_v_n = (__pyx_v_n + 1); } __pyx_L4_break:;
1082:
+1083: if len(stack) != 0 or len(children) != 0:
if (unlikely(((PyObject *)__pyx_v_stack) == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 1083, __pyx_L1_error)
}
__pyx_t_16 = Py_SIZE(((PyObject *)__pyx_v_stack)); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 1083, __pyx_L1_error)
__pyx_t_2 = ((__pyx_t_16 != 0) != 0);
if (!__pyx_t_2) {
} else {
__pyx_t_5 = __pyx_t_2;
goto __pyx_L56_bool_binop_done;
}
if (unlikely(__pyx_v_children == Py_None)) {
PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()");
__PYX_ERR(0, 1083, __pyx_L1_error)
}
__pyx_t_16 = PyList_GET_SIZE(__pyx_v_children); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 1083, __pyx_L1_error)
__pyx_t_2 = ((__pyx_t_16 != 0) != 0);
__pyx_t_5 = __pyx_t_2;
__pyx_L56_bool_binop_done:;
if (__pyx_t_5) {
/* … */
}
+1084: return -7 # stack / children not empty after end of string
__pyx_r = -7;
goto __pyx_L0;
+1085: return idx
__pyx_r = __pyx_v_idx; goto __pyx_L0;
1086:
1087:
+1088: cdef inline copynodes(tree, list prodsintree, Node *result, int *idx):
static CYTHON_INLINE PyObject *__pyx_f_8discodop_10_fragments_copynodes(PyObject *__pyx_v_tree, PyObject *__pyx_v_prodsintree, struct __pyx_t_8discodop_10containers_Node *__pyx_v_result, int *__pyx_v_idx) {
size_t __pyx_v_n;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("copynodes", 0);
/* … */
/* function exit code */
__pyx_r = Py_None; __Pyx_INCREF(Py_None);
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_AddTraceback("discodop._fragments.copynodes", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
1089: """Convert a binarized Tree object to an array of Node structs."""
+1090: cdef size_t n = idx[0]
__pyx_v_n = (__pyx_v_idx[0]);
+1091: if not isinstance(tree, Tree):
__pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_Tree); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1091, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = PyObject_IsInstance(__pyx_v_tree, __pyx_t_1); if (unlikely(__pyx_t_2 == -1)) __PYX_ERR(0, 1091, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_3 = ((!(__pyx_t_2 != 0)) != 0); if (__pyx_t_3) { /* … */ }
+1092: raise ValueError('Expected Tree node, got %s\n%r' % (type(tree), tree))
__pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1092, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(((PyObject *)Py_TYPE(__pyx_v_tree))); __Pyx_GIVEREF(((PyObject *)Py_TYPE(__pyx_v_tree))); PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(__pyx_v_tree))); __Pyx_INCREF(__pyx_v_tree); __Pyx_GIVEREF(__pyx_v_tree); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_tree); __pyx_t_4 = PyUnicode_Format(__pyx_kp_u_Expected_Tree_node_got_s_r, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1092, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1092, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1092, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(0, 1092, __pyx_L1_error)
+1093: elif not 1 <= len(tree) <= 2:
__pyx_t_5 = PyObject_Length(__pyx_v_tree); if (unlikely(__pyx_t_5 == -1)) __PYX_ERR(0, 1093, __pyx_L1_error)
__pyx_t_3 = (1 <= __pyx_t_5);
if (__pyx_t_3) {
__pyx_t_3 = (__pyx_t_5 <= 2);
}
__pyx_t_2 = ((!(__pyx_t_3 != 0)) != 0);
if (__pyx_t_2) {
/* … */
}
+1094: raise ValueError('trees must be non-empty and binarized\n%s' % tree)
__pyx_t_4 = PyUnicode_Format(__pyx_kp_u_trees_must_be_non_empty_and_bina, __pyx_v_tree); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1094, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1094, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1094, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(0, 1094, __pyx_L1_error)
+1095: result[n].prod = prodsintree[n]
if (unlikely(__pyx_v_prodsintree == Py_None)) {
PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
__PYX_ERR(0, 1095, __pyx_L1_error)
}
__pyx_t_6 = __Pyx_PyInt_As_int32_t(PyList_GET_ITEM(__pyx_v_prodsintree, __pyx_v_n)); if (unlikely((__pyx_t_6 == ((int32_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 1095, __pyx_L1_error)
(__pyx_v_result[__pyx_v_n]).prod = __pyx_t_6;
+1096: idx[0] += 1
__pyx_t_7 = 0; (__pyx_v_idx[__pyx_t_7]) = ((__pyx_v_idx[__pyx_t_7]) + 1);
+1097: if isinstance(tree[0], int): # a terminal index
__pyx_t_4 = __Pyx_GetItemInt(__pyx_v_tree, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1097, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_2 = PyInt_Check(__pyx_t_4); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_3 = (__pyx_t_2 != 0); if (__pyx_t_3) { /* … */ goto __pyx_L4; }
+1098: result[n].left = -tree[0] - 1
__pyx_t_4 = __Pyx_GetItemInt(__pyx_v_tree, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1098, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_1 = PyNumber_Negative(__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1098, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = __Pyx_PyInt_SubtractObjC(__pyx_t_1, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1098, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_8 = __Pyx_PyInt_As_int16_t(__pyx_t_4); if (unlikely((__pyx_t_8 == ((int16_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 1098, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; (__pyx_v_result[__pyx_v_n]).left = __pyx_t_8;
1099: else: # non-terminal
+1100: result[n].left = idx[0]
/*else*/ {
(__pyx_v_result[__pyx_v_n]).left = (__pyx_v_idx[0]);
+1101: copynodes(tree[0], prodsintree, result, idx)
__pyx_t_4 = __Pyx_GetItemInt(__pyx_v_tree, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_1 = __pyx_f_8discodop_10_fragments_copynodes(__pyx_t_4, __pyx_v_prodsintree, __pyx_v_result, __pyx_v_idx); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1101, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; } __pyx_L4:;
+1102: if len(tree) == 1: # unary node
__pyx_t_5 = PyObject_Length(__pyx_v_tree); if (unlikely(__pyx_t_5 == -1)) __PYX_ERR(0, 1102, __pyx_L1_error)
__pyx_t_3 = ((__pyx_t_5 == 1) != 0);
if (__pyx_t_3) {
/* … */
goto __pyx_L5;
}
+1103: result[n].right = -1
(__pyx_v_result[__pyx_v_n]).right = -1;
+1104: elif isinstance(tree[1], int): # a terminal index
__pyx_t_1 = __Pyx_GetItemInt(__pyx_v_tree, 1, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1104, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = PyInt_Check(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_2 = (__pyx_t_3 != 0); if (__pyx_t_2) { /* … */ }
+1105: raise ValueError('right child can only be non-terminal.')
__pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__17, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1105, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __PYX_ERR(0, 1105, __pyx_L1_error) /* … */ __pyx_tuple__17 = PyTuple_Pack(1, __pyx_kp_u_right_child_can_only_be_non_term); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(0, 1105, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__17); __Pyx_GIVEREF(__pyx_tuple__17);
1106: else: # binary node
+1107: result[n].right = idx[0]
/*else*/ {
(__pyx_v_result[__pyx_v_n]).right = (__pyx_v_idx[0]);
+1108: copynodes(tree[1], prodsintree, result, idx)
__pyx_t_1 = __Pyx_GetItemInt(__pyx_v_tree, 1, long, 1, __Pyx_PyInt_From_long, 0, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1108, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_4 = __pyx_f_8discodop_10_fragments_copynodes(__pyx_t_1, __pyx_v_prodsintree, __pyx_v_result, __pyx_v_idx); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1108, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } __pyx_L5:;
1109:
1110:
+1111: def getctrees(items1, items2=None, Vocabulary vocab=None, bint index=True):
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_15getctrees(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static char __pyx_doc_8discodop_10_fragments_14getctrees[] = "getctrees(items1, items2=None, Vocabulary vocab=None, bool index=True)\nConvert binarized Tree objects to Ctrees object.\n\n\t:param items1: an iterable with tuples of the form ``(tree, sent)``.\n\t:param items2: optionally, a second iterable of trees.\n\t:param update: if False, do not update ``vocab``; unseen productions are\n\t\tassigned a sentinel that never matches.\n\t:param index: whether to create production index of trees.\n\t:returns: dictionary with same keys as arguments, where trees1 and\n\t\ttrees2 are Ctrees objects for disc. binary trees and sentences.";
static PyMethodDef __pyx_mdef_8discodop_10_fragments_15getctrees = {"getctrees", (PyCFunction)__pyx_pw_8discodop_10_fragments_15getctrees, METH_VARARGS|METH_KEYWORDS, __pyx_doc_8discodop_10_fragments_14getctrees};
static PyObject *__pyx_pw_8discodop_10_fragments_15getctrees(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
PyObject *__pyx_v_items1 = 0;
PyObject *__pyx_v_items2 = 0;
struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab = 0;
int __pyx_v_index;
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("getctrees (wrapper)", 0);
{
static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_items1,&__pyx_n_s_items2,&__pyx_n_s_vocab,&__pyx_n_s_index,0};
PyObject* values[4] = {0,0,0,0};
values[1] = ((PyObject *)Py_None);
values[2] = (PyObject *)((struct __pyx_obj_8discodop_10containers_Vocabulary *)Py_None);
if (unlikely(__pyx_kwds)) {
Py_ssize_t kw_args;
const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
switch (pos_args) {
case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3);
CYTHON_FALLTHROUGH;
case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
CYTHON_FALLTHROUGH;
case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
CYTHON_FALLTHROUGH;
case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
CYTHON_FALLTHROUGH;
case 0: break;
default: goto __pyx_L5_argtuple_error;
}
kw_args = PyDict_Size(__pyx_kwds);
switch (pos_args) {
case 0:
if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_items1)) != 0)) kw_args--;
else goto __pyx_L5_argtuple_error;
CYTHON_FALLTHROUGH;
case 1:
if (kw_args > 0) {
PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_items2);
if (value) { values[1] = value; kw_args--; }
}
CYTHON_FALLTHROUGH;
case 2:
if (kw_args > 0) {
PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_vocab);
if (value) { values[2] = value; kw_args--; }
}
CYTHON_FALLTHROUGH;
case 3:
if (kw_args > 0) {
PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_index);
if (value) { values[3] = value; kw_args--; }
}
}
if (unlikely(kw_args > 0)) {
if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "getctrees") < 0)) __PYX_ERR(0, 1111, __pyx_L3_error)
}
} else {
switch (PyTuple_GET_SIZE(__pyx_args)) {
case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3);
CYTHON_FALLTHROUGH;
case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
CYTHON_FALLTHROUGH;
case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
CYTHON_FALLTHROUGH;
case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
break;
default: goto __pyx_L5_argtuple_error;
}
}
__pyx_v_items1 = values[0];
__pyx_v_items2 = values[1];
__pyx_v_vocab = ((struct __pyx_obj_8discodop_10containers_Vocabulary *)values[2]);
if (values[3]) {
__pyx_v_index = __Pyx_PyObject_IsTrue(values[3]); if (unlikely((__pyx_v_index == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 1111, __pyx_L3_error)
} else {
__pyx_v_index = ((int)1);
}
}
goto __pyx_L4_argument_unpacking_done;
__pyx_L5_argtuple_error:;
__Pyx_RaiseArgtupleInvalid("getctrees", 0, 1, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 1111, __pyx_L3_error)
__pyx_L3_error:;
__Pyx_AddTraceback("discodop._fragments.getctrees", __pyx_clineno, __pyx_lineno, __pyx_filename);
__Pyx_RefNannyFinishContext();
return NULL;
__pyx_L4_argument_unpacking_done:;
if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_vocab), __pyx_ptype_8discodop_10containers_Vocabulary, 1, "vocab", 0))) __PYX_ERR(0, 1111, __pyx_L1_error)
__pyx_r = __pyx_pf_8discodop_10_fragments_14getctrees(__pyx_self, __pyx_v_items1, __pyx_v_items2, __pyx_v_vocab, __pyx_v_index);
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__pyx_r = NULL;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_8discodop_10_fragments_14getctrees(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_items1, PyObject *__pyx_v_items2, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, int __pyx_v_index) {
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_ctrees = 0;
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_ctrees1 = 0;
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_ctrees2 = 0;
struct __pyx_t_8discodop_10containers_Node *__pyx_v_scratch;
int __pyx_v_cnt;
PyObject *__pyx_v_maxnodes = NULL;
PyObject *__pyx_v_m = NULL;
PyObject *__pyx_v_items = NULL;
PyObject *__pyx_v_tree = NULL;
PyObject *__pyx_v_sent = NULL;
PyObject *__pyx_v_prodsintree = NULL;
PyObject *__pyx_v_r = NULL;
PyObject *__pyx_v_yf = NULL;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("getctrees", 0);
__Pyx_INCREF((PyObject *)__pyx_v_vocab);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_10);
__Pyx_XDECREF(__pyx_t_11);
__Pyx_XDECREF(__pyx_t_12);
__Pyx_XDECREF(__pyx_t_13);
__Pyx_XDECREF(__pyx_t_17);
__Pyx_AddTraceback("discodop._fragments.getctrees", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XDECREF((PyObject *)__pyx_v_ctrees);
__Pyx_XDECREF((PyObject *)__pyx_v_ctrees1);
__Pyx_XDECREF((PyObject *)__pyx_v_ctrees2);
__Pyx_XDECREF(__pyx_v_maxnodes);
__Pyx_XDECREF(__pyx_v_m);
__Pyx_XDECREF(__pyx_v_items);
__Pyx_XDECREF(__pyx_v_tree);
__Pyx_XDECREF(__pyx_v_sent);
__Pyx_XDECREF(__pyx_v_prodsintree);
__Pyx_XDECREF(__pyx_v_r);
__Pyx_XDECREF(__pyx_v_yf);
__Pyx_XDECREF((PyObject *)__pyx_v_vocab);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
__pyx_tuple__37 = PyTuple_Pack(17, __pyx_n_s_items1, __pyx_n_s_items2, __pyx_n_s_vocab, __pyx_n_s_index, __pyx_n_s_ctrees, __pyx_n_s_ctrees1, __pyx_n_s_ctrees2, __pyx_n_s_scratch, __pyx_n_s_cnt, __pyx_n_s_maxnodes, __pyx_n_s_m, __pyx_n_s_items, __pyx_n_s_tree, __pyx_n_s_sent, __pyx_n_s_prodsintree, __pyx_n_s_r, __pyx_n_s_yf); if (unlikely(!__pyx_tuple__37)) __PYX_ERR(0, 1111, __pyx_L1_error)
__Pyx_GOTREF(__pyx_tuple__37);
__Pyx_GIVEREF(__pyx_tuple__37);
/* … */
__pyx_t_3 = PyCFunction_NewEx(&__pyx_mdef_8discodop_10_fragments_15getctrees, NULL, __pyx_n_s_discodop__fragments); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1111, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
if (PyDict_SetItem(__pyx_d, __pyx_n_s_getctrees, __pyx_t_3) < 0) __PYX_ERR(0, 1111, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__pyx_codeobj__38 = (PyObject*)__Pyx_PyCode_New(4, 0, 17, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__37, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_discodop__fragments_pyx, __pyx_n_s_getctrees, 1111, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__38)) __PYX_ERR(0, 1111, __pyx_L1_error)
1112: """Convert binarized Tree objects to Ctrees object.
1113:
1114: :param items1: an iterable with tuples of the form ``(tree, sent)``.
1115: :param items2: optionally, a second iterable of trees.
1116: :param update: if False, do not update ``vocab``; unseen productions are
1117: assigned a sentinel that never matches.
1118: :param index: whether to create production index of trees.
1119: :returns: dictionary with same keys as arguments, where trees1 and
1120: trees2 are Ctrees objects for disc. binary trees and sentences."""
+1121: cdef Ctrees ctrees, ctrees1, ctrees2 = None
__Pyx_INCREF(Py_None);
__pyx_v_ctrees2 = ((struct __pyx_obj_8discodop_10containers_Ctrees *)Py_None);
1122: cdef Node *scratch
1123: cdef int cnt
+1124: maxnodes = 512
__Pyx_INCREF(__pyx_int_512);
__pyx_v_maxnodes = __pyx_int_512;
+1125: scratch = <Node *>malloc(maxnodes * sizeof(Node))
__pyx_t_1 = __Pyx_PyInt_FromSize_t((sizeof(struct __pyx_t_8discodop_10containers_Node))); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1125, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = PyNumber_Multiply(__pyx_v_maxnodes, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1125, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_3 = __Pyx_PyInt_As_size_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 1125, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_scratch = ((struct __pyx_t_8discodop_10containers_Node *)malloc(__pyx_t_3));
+1126: if scratch is NULL:
__pyx_t_4 = ((__pyx_v_scratch == NULL) != 0);
if (__pyx_t_4) {
/* … */
}
+1127: raise MemoryError('allocation error')
__pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__18, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1127, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __PYX_ERR(0, 1127, __pyx_L1_error) /* … */ __pyx_tuple__18 = PyTuple_Pack(1, __pyx_kp_u_allocation_error); if (unlikely(!__pyx_tuple__18)) __PYX_ERR(0, 1127, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__18); __Pyx_GIVEREF(__pyx_tuple__18);
+1128: if vocab is None:
__pyx_t_4 = (((PyObject *)__pyx_v_vocab) == Py_None);
__pyx_t_5 = (__pyx_t_4 != 0);
if (__pyx_t_5) {
/* … */
}
+1129: vocab = Vocabulary()
__pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_8discodop_10containers_Vocabulary), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1129, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF_SET(__pyx_v_vocab, ((struct __pyx_obj_8discodop_10containers_Vocabulary *)__pyx_t_2)); __pyx_t_2 = 0;
+1130: ctrees = ctrees1 = Ctrees()
__pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_8discodop_10containers_Ctrees), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1130, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_2); __pyx_v_ctrees = ((struct __pyx_obj_8discodop_10containers_Ctrees *)__pyx_t_2); __Pyx_INCREF(__pyx_t_2); __pyx_v_ctrees1 = ((struct __pyx_obj_8discodop_10containers_Ctrees *)__pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+1131: ctrees.alloc(512, 512 * 512)
__pyx_t_2 = ((struct __pyx_vtabstruct_8discodop_10containers_Ctrees *)__pyx_v_ctrees->__pyx_vtab)->alloc(__pyx_v_ctrees, 0x200, 0x40000, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1131, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+1132: for m, items in enumerate((items1, items2)):
__Pyx_INCREF(__pyx_int_0); __pyx_t_2 = __pyx_int_0; __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1132, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_items1); __Pyx_GIVEREF(__pyx_v_items1); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_items1); __Pyx_INCREF(__pyx_v_items2); __Pyx_GIVEREF(__pyx_v_items2); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_items2); __pyx_t_6 = __pyx_t_1; __Pyx_INCREF(__pyx_t_6); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; for (;;) { if (__pyx_t_7 >= 2) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_7); __Pyx_INCREF(__pyx_t_1); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(0, 1132, __pyx_L1_error) #else __pyx_t_1 = PySequence_ITEM(__pyx_t_6, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1132, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); #endif __Pyx_XDECREF_SET(__pyx_v_items, __pyx_t_1); __pyx_t_1 = 0; __Pyx_INCREF(__pyx_t_2); __Pyx_XDECREF_SET(__pyx_v_m, __pyx_t_2); __pyx_t_1 = __Pyx_PyInt_AddObjC(__pyx_t_2, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1132, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = __pyx_t_1; __pyx_t_1 = 0; /* … */ } __pyx_L6_break:; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+1133: if items is None:
__pyx_t_5 = (__pyx_v_items == Py_None);
__pyx_t_4 = (__pyx_t_5 != 0);
if (__pyx_t_4) {
/* … */
}
+1134: break
goto __pyx_L6_break;
+1135: elif m == 1:
__pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_v_m, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1135, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 1135, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_4) { /* … */ }
+1136: ctrees = ctrees2 = Ctrees()
__pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_8discodop_10containers_Ctrees), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_t_1); __Pyx_DECREF_SET(__pyx_v_ctrees, ((struct __pyx_obj_8discodop_10containers_Ctrees *)__pyx_t_1)); __Pyx_INCREF(__pyx_t_1); __Pyx_DECREF_SET(__pyx_v_ctrees2, ((struct __pyx_obj_8discodop_10containers_Ctrees *)__pyx_t_1)); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+1137: ctrees.alloc(512, 512 * 512)
__pyx_t_1 = ((struct __pyx_vtabstruct_8discodop_10containers_Ctrees *)__pyx_v_ctrees->__pyx_vtab)->alloc(__pyx_v_ctrees, 0x200, 0x40000, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1137, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_1);
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+1138: for tree, sent in items:
if (likely(PyList_CheckExact(__pyx_v_items)) || PyTuple_CheckExact(__pyx_v_items)) { __pyx_t_1 = __pyx_v_items; __Pyx_INCREF(__pyx_t_1); __pyx_t_8 = 0; __pyx_t_9 = NULL; } else { __pyx_t_8 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_v_items); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_9 = Py_TYPE(__pyx_t_1)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 1138, __pyx_L1_error) } for (;;) { if (likely(!__pyx_t_9)) { if (likely(PyList_CheckExact(__pyx_t_1))) { if (__pyx_t_8 >= PyList_GET_SIZE(__pyx_t_1)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_10 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_8); __Pyx_INCREF(__pyx_t_10); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 1138, __pyx_L1_error) #else __pyx_t_10 = PySequence_ITEM(__pyx_t_1, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); #endif } else { if (__pyx_t_8 >= PyTuple_GET_SIZE(__pyx_t_1)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_10 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_8); __Pyx_INCREF(__pyx_t_10); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 1138, __pyx_L1_error) #else __pyx_t_10 = PySequence_ITEM(__pyx_t_1, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); #endif } } else { __pyx_t_10 = __pyx_t_9(__pyx_t_1); if (unlikely(!__pyx_t_10)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 1138, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_10); } if ((likely(PyTuple_CheckExact(__pyx_t_10))) || (PyList_CheckExact(__pyx_t_10))) { PyObject* sequence = __pyx_t_10; #if !CYTHON_COMPILING_IN_PYPY Py_ssize_t size = Py_SIZE(sequence); #else Py_ssize_t size = PySequence_Size(sequence); #endif if (unlikely(size != 2)) { if (size > 2) __Pyx_RaiseTooManyValuesError(2); else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); __PYX_ERR(0, 1138, __pyx_L1_error) } #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS if (likely(PyTuple_CheckExact(sequence))) { __pyx_t_11 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_12 = PyTuple_GET_ITEM(sequence, 1); } else { __pyx_t_11 = PyList_GET_ITEM(sequence, 0); __pyx_t_12 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_11); __Pyx_INCREF(__pyx_t_12); #else __pyx_t_11 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 1138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); __pyx_t_12 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 1138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_12); #endif __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; } else { Py_ssize_t index = -1; __pyx_t_13 = PyObject_GetIter(__pyx_t_10); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; __pyx_t_14 = Py_TYPE(__pyx_t_13)->tp_iternext; index = 0; __pyx_t_11 = __pyx_t_14(__pyx_t_13); if (unlikely(!__pyx_t_11)) goto __pyx_L10_unpacking_failed; __Pyx_GOTREF(__pyx_t_11); index = 1; __pyx_t_12 = __pyx_t_14(__pyx_t_13); if (unlikely(!__pyx_t_12)) goto __pyx_L10_unpacking_failed; __Pyx_GOTREF(__pyx_t_12); if (__Pyx_IternextUnpackEndCheck(__pyx_t_14(__pyx_t_13), 2) < 0) __PYX_ERR(0, 1138, __pyx_L1_error) __pyx_t_14 = NULL; __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; goto __pyx_L11_unpacking_done; __pyx_L10_unpacking_failed:; __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; __pyx_t_14 = NULL; if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); __PYX_ERR(0, 1138, __pyx_L1_error) __pyx_L11_unpacking_done:; } __Pyx_XDECREF_SET(__pyx_v_tree, __pyx_t_11); __pyx_t_11 = 0; __Pyx_XDECREF_SET(__pyx_v_sent, __pyx_t_12); __pyx_t_12 = 0; /* … */ } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+1139: cnt = 0
__pyx_v_cnt = 0;
+1140: prodsintree = []
__pyx_t_10 = PyList_New(0); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_XDECREF_SET(__pyx_v_prodsintree, ((PyObject*)__pyx_t_10)); __pyx_t_10 = 0;
+1141: for r, yf in lcfrsproductions(tree, sent, frontiers=True):
__pyx_t_10 = __Pyx_GetModuleGlobalName(__pyx_n_s_lcfrsproductions); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __pyx_t_12 = PyTuple_New(2); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_INCREF(__pyx_v_tree); __Pyx_GIVEREF(__pyx_v_tree); PyTuple_SET_ITEM(__pyx_t_12, 0, __pyx_v_tree); __Pyx_INCREF(__pyx_v_sent); __Pyx_GIVEREF(__pyx_v_sent); PyTuple_SET_ITEM(__pyx_t_12, 1, __pyx_v_sent); __pyx_t_11 = PyDict_New(); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); if (PyDict_SetItem(__pyx_t_11, __pyx_n_s_frontiers, Py_True) < 0) __PYX_ERR(0, 1141, __pyx_L1_error) __pyx_t_13 = __Pyx_PyObject_Call(__pyx_t_10, __pyx_t_12, __pyx_t_11); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; if (likely(PyList_CheckExact(__pyx_t_13)) || PyTuple_CheckExact(__pyx_t_13)) { __pyx_t_11 = __pyx_t_13; __Pyx_INCREF(__pyx_t_11); __pyx_t_15 = 0; __pyx_t_16 = NULL; } else { __pyx_t_15 = -1; __pyx_t_11 = PyObject_GetIter(__pyx_t_13); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); __pyx_t_16 = Py_TYPE(__pyx_t_11)->tp_iternext; if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 1141, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; for (;;) { if (likely(!__pyx_t_16)) { if (likely(PyList_CheckExact(__pyx_t_11))) { if (__pyx_t_15 >= PyList_GET_SIZE(__pyx_t_11)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_13 = PyList_GET_ITEM(__pyx_t_11, __pyx_t_15); __Pyx_INCREF(__pyx_t_13); __pyx_t_15++; if (unlikely(0 < 0)) __PYX_ERR(0, 1141, __pyx_L1_error) #else __pyx_t_13 = PySequence_ITEM(__pyx_t_11, __pyx_t_15); __pyx_t_15++; if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); #endif } else { if (__pyx_t_15 >= PyTuple_GET_SIZE(__pyx_t_11)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_13 = PyTuple_GET_ITEM(__pyx_t_11, __pyx_t_15); __Pyx_INCREF(__pyx_t_13); __pyx_t_15++; if (unlikely(0 < 0)) __PYX_ERR(0, 1141, __pyx_L1_error) #else __pyx_t_13 = PySequence_ITEM(__pyx_t_11, __pyx_t_15); __pyx_t_15++; if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); #endif } } else { __pyx_t_13 = __pyx_t_16(__pyx_t_11); if (unlikely(!__pyx_t_13)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 1141, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_13); } if ((likely(PyTuple_CheckExact(__pyx_t_13))) || (PyList_CheckExact(__pyx_t_13))) { PyObject* sequence = __pyx_t_13; #if !CYTHON_COMPILING_IN_PYPY Py_ssize_t size = Py_SIZE(sequence); #else Py_ssize_t size = PySequence_Size(sequence); #endif if (unlikely(size != 2)) { if (size > 2) __Pyx_RaiseTooManyValuesError(2); else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); __PYX_ERR(0, 1141, __pyx_L1_error) } #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS if (likely(PyTuple_CheckExact(sequence))) { __pyx_t_12 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_10 = PyTuple_GET_ITEM(sequence, 1); } else { __pyx_t_12 = PyList_GET_ITEM(sequence, 0); __pyx_t_10 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_12); __Pyx_INCREF(__pyx_t_10); #else __pyx_t_12 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_12); __pyx_t_10 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); #endif __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; } else { Py_ssize_t index = -1; __pyx_t_17 = PyObject_GetIter(__pyx_t_13); if (unlikely(!__pyx_t_17)) __PYX_ERR(0, 1141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_17); __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; __pyx_t_14 = Py_TYPE(__pyx_t_17)->tp_iternext; index = 0; __pyx_t_12 = __pyx_t_14(__pyx_t_17); if (unlikely(!__pyx_t_12)) goto __pyx_L14_unpacking_failed; __Pyx_GOTREF(__pyx_t_12); index = 1; __pyx_t_10 = __pyx_t_14(__pyx_t_17); if (unlikely(!__pyx_t_10)) goto __pyx_L14_unpacking_failed; __Pyx_GOTREF(__pyx_t_10); if (__Pyx_IternextUnpackEndCheck(__pyx_t_14(__pyx_t_17), 2) < 0) __PYX_ERR(0, 1141, __pyx_L1_error) __pyx_t_14 = NULL; __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; goto __pyx_L15_unpacking_done; __pyx_L14_unpacking_failed:; __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; __pyx_t_14 = NULL; if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); __PYX_ERR(0, 1141, __pyx_L1_error) __pyx_L15_unpacking_done:; } __Pyx_XDECREF_SET(__pyx_v_r, __pyx_t_12); __pyx_t_12 = 0; __Pyx_XDECREF_SET(__pyx_v_yf, __pyx_t_10); __pyx_t_10 = 0; /* … */ } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
+1142: prodsintree.append(vocab.getprod(r, yf))
if (!(likely(PyTuple_CheckExact(__pyx_v_r))||((__pyx_v_r) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "tuple", Py_TYPE(__pyx_v_r)->tp_name), 0))) __PYX_ERR(0, 1142, __pyx_L1_error) if (!(likely(PyTuple_CheckExact(__pyx_v_yf))||((__pyx_v_yf) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "tuple", Py_TYPE(__pyx_v_yf)->tp_name), 0))) __PYX_ERR(0, 1142, __pyx_L1_error) __pyx_t_18 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getprod(__pyx_v_vocab, ((PyObject*)__pyx_v_r), ((PyObject*)__pyx_v_yf)); if (unlikely(__pyx_t_18 == -2)) __PYX_ERR(0, 1142, __pyx_L1_error) __pyx_t_13 = __Pyx_PyInt_From_int(__pyx_t_18); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1142, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __pyx_t_19 = __Pyx_PyList_Append(__pyx_v_prodsintree, __pyx_t_13); if (unlikely(__pyx_t_19 == -1)) __PYX_ERR(0, 1142, __pyx_L1_error) __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
+1143: cnt += 1
__pyx_v_cnt = (__pyx_v_cnt + 1);
+1144: if cnt > maxnodes:
__pyx_t_11 = __Pyx_PyInt_From_int(__pyx_v_cnt); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 1144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); __pyx_t_13 = PyObject_RichCompare(__pyx_t_11, __pyx_v_maxnodes, Py_GT); __Pyx_XGOTREF(__pyx_t_13); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1144, __pyx_L1_error) __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_13); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 1144, __pyx_L1_error) __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; if (__pyx_t_4) { /* … */ }
+1145: maxnodes = cnt
__pyx_t_13 = __Pyx_PyInt_From_int(__pyx_v_cnt); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_DECREF_SET(__pyx_v_maxnodes, __pyx_t_13); __pyx_t_13 = 0;
+1146: scratch = <Node *>realloc(scratch, maxnodes * sizeof(Node))
__pyx_t_13 = __Pyx_PyInt_FromSize_t((sizeof(struct __pyx_t_8discodop_10containers_Node))); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __pyx_t_11 = PyNumber_Multiply(__pyx_v_maxnodes, __pyx_t_13); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 1146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; __pyx_t_3 = __Pyx_PyInt_As_size_t(__pyx_t_11); if (unlikely((__pyx_t_3 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 1146, __pyx_L1_error) __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __pyx_v_scratch = ((struct __pyx_t_8discodop_10containers_Node *)realloc(__pyx_v_scratch, __pyx_t_3));
+1147: if scratch is NULL:
__pyx_t_4 = ((__pyx_v_scratch == NULL) != 0);
if (__pyx_t_4) {
/* … */
}
+1148: raise MemoryError('allocation error')
__pyx_t_11 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__19, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 1148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_Raise(__pyx_t_11, 0, 0, 0); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __PYX_ERR(0, 1148, __pyx_L1_error) /* … */ __pyx_tuple__19 = PyTuple_Pack(1, __pyx_kp_u_allocation_error); if (unlikely(!__pyx_tuple__19)) __PYX_ERR(0, 1148, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__19); __Pyx_GIVEREF(__pyx_tuple__19);
+1149: cnt = 0
__pyx_v_cnt = 0;
+1150: copynodes(tree, prodsintree, scratch, &cnt)
__pyx_t_11 = __pyx_f_8discodop_10_fragments_copynodes(__pyx_v_tree, __pyx_v_prodsintree, __pyx_v_scratch, (&__pyx_v_cnt)); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 1150, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_11);
__Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
+1151: ctrees.addnodes(scratch, cnt, 0)
__pyx_t_11 = ((struct __pyx_vtabstruct_8discodop_10containers_Ctrees *)__pyx_v_ctrees->__pyx_vtab)->addnodes(__pyx_v_ctrees, __pyx_v_scratch, __pyx_v_cnt, 0); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 1151, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_11);
__Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
+1152: if index:
__pyx_t_4 = (__pyx_v_index != 0);
if (__pyx_t_4) {
/* … */
}
+1153: ctrees.indextrees(vocab)
__pyx_t_11 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_ctrees), __pyx_n_s_indextrees); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 1153, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); __pyx_t_13 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_13 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_13)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); __Pyx_INCREF(__pyx_t_13); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_11, function); } } if (!__pyx_t_13) { __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_11, ((PyObject *)__pyx_v_vocab)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1153, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_11)) { PyObject *__pyx_temp[2] = {__pyx_t_13, ((PyObject *)__pyx_v_vocab)}; __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1153, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_13); __pyx_t_13 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { PyObject *__pyx_temp[2] = {__pyx_t_13, ((PyObject *)__pyx_v_vocab)}; __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1153, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_13); __pyx_t_13 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { __pyx_t_10 = PyTuple_New(1+1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1153, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_GIVEREF(__pyx_t_13); PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_t_13); __pyx_t_13 = NULL; __Pyx_INCREF(((PyObject *)__pyx_v_vocab)); __Pyx_GIVEREF(((PyObject *)__pyx_v_vocab)); PyTuple_SET_ITEM(__pyx_t_10, 0+1, ((PyObject *)__pyx_v_vocab)); __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_10, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1153, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+1154: free(scratch)
free(__pyx_v_scratch);
+1155: return dict(trees1=ctrees1, trees2=ctrees2, vocab=vocab)
__Pyx_XDECREF(__pyx_r); __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1155, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_trees1, ((PyObject *)__pyx_v_ctrees1)) < 0) __PYX_ERR(0, 1155, __pyx_L1_error) if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_trees2, ((PyObject *)__pyx_v_ctrees2)) < 0) __PYX_ERR(0, 1155, __pyx_L1_error) if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_vocab, ((PyObject *)__pyx_v_vocab)) < 0) __PYX_ERR(0, 1155, __pyx_L1_error) __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0;
1156:
1157:
+1158: def readtreebank(treebankfile, Vocabulary vocab,
/* Python wrapper */
static PyObject *__pyx_pw_8discodop_10_fragments_17readtreebank(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static char __pyx_doc_8discodop_10_fragments_16readtreebank[] = "readtreebank(treebankfile, Vocabulary vocab, fmt=u'bracket', limit=None, encoding=u'utf8')\nRead a treebank from a given filename.\n\n\t``vocab`` should be re-used when reading multiple treebanks.\n\n\t:returns: tuple of Ctrees object and list of sentences.";
static PyMethodDef __pyx_mdef_8discodop_10_fragments_17readtreebank = {"readtreebank", (PyCFunction)__pyx_pw_8discodop_10_fragments_17readtreebank, METH_VARARGS|METH_KEYWORDS, __pyx_doc_8discodop_10_fragments_16readtreebank};
static PyObject *__pyx_pw_8discodop_10_fragments_17readtreebank(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
PyObject *__pyx_v_treebankfile = 0;
struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab = 0;
PyObject *__pyx_v_fmt = 0;
PyObject *__pyx_v_limit = 0;
PyObject *__pyx_v_encoding = 0;
PyObject *__pyx_r = 0;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("readtreebank (wrapper)", 0);
{
static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_treebankfile,&__pyx_n_s_vocab,&__pyx_n_s_fmt,&__pyx_n_s_limit,&__pyx_n_s_encoding,0};
PyObject* values[5] = {0,0,0,0,0};
values[2] = ((PyObject *)__pyx_n_u_bracket);
/* … */
/* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__pyx_r = NULL;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
static PyObject *__pyx_pf_8discodop_10_fragments_16readtreebank(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_treebankfile, struct __pyx_obj_8discodop_10containers_Vocabulary *__pyx_v_vocab, PyObject *__pyx_v_fmt, PyObject *__pyx_v_limit, PyObject *__pyx_v_encoding) {
struct __pyx_obj_8discodop_10containers_Ctrees *__pyx_v_ctrees = 0;
struct __pyx_t_8discodop_10containers_Node *__pyx_v_scratch;
short __pyx_v_maxnodes;
short __pyx_v_binfactor;
int __pyx_v_cnt;
int __pyx_v_n;
PyObject *__pyx_v_line = 0;
arrayobject *__pyx_v_stack = 0;
arrayobject *__pyx_v_labels = 0;
PyObject *__pyx_v_children = 0;
PyObject *__pyx_v_READERS = NULL;
PyObject *__pyx_v_corpus = NULL;
CYTHON_UNUSED PyObject *__pyx_v__ = NULL;
PyObject *__pyx_v_item = NULL;
PyObject *__pyx_v_tree = NULL;
PyObject *__pyx_v_prodsintree = NULL;
PyObject *__pyx_v_r = NULL;
PyObject *__pyx_v_yf = NULL;
PyObject *__pyx_v_data = NULL;
PyObject *__pyx_r = NULL;
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("readtreebank", 0);
/* … */
/* function exit code */
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_4);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_XDECREF(__pyx_t_10);
__Pyx_XDECREF(__pyx_t_12);
__Pyx_XDECREF(__pyx_t_13);
__Pyx_AddTraceback("discodop._fragments.readtreebank", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = NULL;
__pyx_L0:;
__Pyx_XDECREF((PyObject *)__pyx_v_ctrees);
__Pyx_XDECREF(__pyx_v_line);
__Pyx_XDECREF((PyObject *)__pyx_v_stack);
__Pyx_XDECREF((PyObject *)__pyx_v_labels);
__Pyx_XDECREF(__pyx_v_children);
__Pyx_XDECREF(__pyx_v_READERS);
__Pyx_XDECREF(__pyx_v_corpus);
__Pyx_XDECREF(__pyx_v__);
__Pyx_XDECREF(__pyx_v_item);
__Pyx_XDECREF(__pyx_v_tree);
__Pyx_XDECREF(__pyx_v_prodsintree);
__Pyx_XDECREF(__pyx_v_r);
__Pyx_XDECREF(__pyx_v_yf);
__Pyx_XDECREF(__pyx_v_data);
__Pyx_XGIVEREF(__pyx_r);
__Pyx_RefNannyFinishContext();
return __pyx_r;
}
/* … */
__pyx_tuple__40 = PyTuple_Pack(24, __pyx_n_s_treebankfile, __pyx_n_s_vocab, __pyx_n_s_fmt, __pyx_n_s_limit, __pyx_n_s_encoding, __pyx_n_s_ctrees, __pyx_n_s_scratch, __pyx_n_s_maxnodes, __pyx_n_s_binfactor, __pyx_n_s_cnt, __pyx_n_s_n, __pyx_n_s_line, __pyx_n_s_stack, __pyx_n_s_labels, __pyx_n_s_children, __pyx_n_s_READERS, __pyx_n_s_corpus, __pyx_n_s__39, __pyx_n_s_item, __pyx_n_s_tree, __pyx_n_s_prodsintree, __pyx_n_s_r, __pyx_n_s_yf, __pyx_n_s_data); if (unlikely(!__pyx_tuple__40)) __PYX_ERR(0, 1158, __pyx_L1_error)
__Pyx_GOTREF(__pyx_tuple__40);
__Pyx_GIVEREF(__pyx_tuple__40);
/* … */
__pyx_t_3 = PyCFunction_NewEx(&__pyx_mdef_8discodop_10_fragments_17readtreebank, NULL, __pyx_n_s_discodop__fragments); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1158, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
if (PyDict_SetItem(__pyx_d, __pyx_n_s_readtreebank, __pyx_t_3) < 0) __PYX_ERR(0, 1158, __pyx_L1_error)
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+1159: fmt='bracket', limit=None, encoding='utf8'):
values[3] = ((PyObject *)Py_None);
values[4] = ((PyObject *)__pyx_n_u_utf8);
if (unlikely(__pyx_kwds)) {
Py_ssize_t kw_args;
const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
switch (pos_args) {
case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4);
CYTHON_FALLTHROUGH;
case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3);
CYTHON_FALLTHROUGH;
case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
CYTHON_FALLTHROUGH;
case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
CYTHON_FALLTHROUGH;
case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
CYTHON_FALLTHROUGH;
case 0: break;
default: goto __pyx_L5_argtuple_error;
}
kw_args = PyDict_Size(__pyx_kwds);
switch (pos_args) {
case 0:
if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_treebankfile)) != 0)) kw_args--;
else goto __pyx_L5_argtuple_error;
CYTHON_FALLTHROUGH;
case 1:
if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_vocab)) != 0)) kw_args--;
else {
__Pyx_RaiseArgtupleInvalid("readtreebank", 0, 2, 5, 1); __PYX_ERR(0, 1158, __pyx_L3_error)
}
CYTHON_FALLTHROUGH;
case 2:
if (kw_args > 0) {
PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_fmt);
if (value) { values[2] = value; kw_args--; }
}
CYTHON_FALLTHROUGH;
case 3:
if (kw_args > 0) {
PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_limit);
if (value) { values[3] = value; kw_args--; }
}
CYTHON_FALLTHROUGH;
case 4:
if (kw_args > 0) {
PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_encoding);
if (value) { values[4] = value; kw_args--; }
}
}
if (unlikely(kw_args > 0)) {
if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "readtreebank") < 0)) __PYX_ERR(0, 1158, __pyx_L3_error)
}
} else {
switch (PyTuple_GET_SIZE(__pyx_args)) {
case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4);
CYTHON_FALLTHROUGH;
case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3);
CYTHON_FALLTHROUGH;
case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
CYTHON_FALLTHROUGH;
case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
break;
default: goto __pyx_L5_argtuple_error;
}
}
__pyx_v_treebankfile = values[0];
__pyx_v_vocab = ((struct __pyx_obj_8discodop_10containers_Vocabulary *)values[1]);
__pyx_v_fmt = values[2];
__pyx_v_limit = values[3];
__pyx_v_encoding = values[4];
}
goto __pyx_L4_argument_unpacking_done;
__pyx_L5_argtuple_error:;
__Pyx_RaiseArgtupleInvalid("readtreebank", 0, 2, 5, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 1158, __pyx_L3_error)
__pyx_L3_error:;
__Pyx_AddTraceback("discodop._fragments.readtreebank", __pyx_clineno, __pyx_lineno, __pyx_filename);
__Pyx_RefNannyFinishContext();
return NULL;
__pyx_L4_argument_unpacking_done:;
if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_vocab), __pyx_ptype_8discodop_10containers_Vocabulary, 1, "vocab", 0))) __PYX_ERR(0, 1158, __pyx_L1_error)
__pyx_r = __pyx_pf_8discodop_10_fragments_16readtreebank(__pyx_self, __pyx_v_treebankfile, __pyx_v_vocab, __pyx_v_fmt, __pyx_v_limit, __pyx_v_encoding);
1160: """Read a treebank from a given filename.
1161:
1162: ``vocab`` should be re-used when reading multiple treebanks.
1163:
1164: :returns: tuple of Ctrees object and list of sentences."""
1165: cdef Ctrees ctrees
1166: cdef Node *scratch
+1167: cdef short maxnodes = 512
__pyx_v_maxnodes = 0x200;
+1168: cdef short binfactor = 2 # conservative estimate to accommodate binarization
__pyx_v_binfactor = 2;
1169: cdef int cnt, n
1170: cdef str line
1171: cdef array stack, labels
1172: cdef list children
+1173: if treebankfile is None:
__pyx_t_1 = (__pyx_v_treebankfile == Py_None);
__pyx_t_2 = (__pyx_t_1 != 0);
if (__pyx_t_2) {
/* … */
}
+1174: return None
__Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; goto __pyx_L0;
+1175: ctrees = Ctrees()
__pyx_t_3 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_8discodop_10containers_Ctrees), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1175, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_v_ctrees = ((struct __pyx_obj_8discodop_10containers_Ctrees *)__pyx_t_3); __pyx_t_3 = 0;
+1176: ctrees.alloc(512, 512 * 512) # dummy values, array will be realloc'd
__pyx_t_3 = ((struct __pyx_vtabstruct_8discodop_10containers_Ctrees *)__pyx_v_ctrees->__pyx_vtab)->alloc(__pyx_v_ctrees, 0x200, 0x40000, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1176, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+1177: scratch = <Node *>malloc(maxnodes * binfactor * sizeof(Node))
__pyx_v_scratch = ((struct __pyx_t_8discodop_10containers_Node *)malloc(((__pyx_v_maxnodes * __pyx_v_binfactor) * (sizeof(struct __pyx_t_8discodop_10containers_Node)))));
+1178: if scratch is NULL:
__pyx_t_2 = ((__pyx_v_scratch == NULL) != 0);
if (__pyx_t_2) {
/* … */
}
+1179: raise MemoryError('allocation error')
__pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__20, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1179, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(0, 1179, __pyx_L1_error) /* … */ __pyx_tuple__20 = PyTuple_Pack(1, __pyx_kp_u_allocation_error); if (unlikely(!__pyx_tuple__20)) __PYX_ERR(0, 1179, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__20); __Pyx_GIVEREF(__pyx_tuple__20);
+1180: if fmt != 'bracket':
__pyx_t_2 = (__Pyx_PyUnicode_Equals(__pyx_v_fmt, __pyx_n_u_bracket, Py_NE)); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 1180, __pyx_L1_error)
if (__pyx_t_2) {
/* … */
goto __pyx_L5;
}
+1181: from .treebank import READERS
__pyx_t_3 = PyList_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_n_s_READERS); __Pyx_GIVEREF(__pyx_n_s_READERS); PyList_SET_ITEM(__pyx_t_3, 0, __pyx_n_s_READERS); __pyx_t_4 = __Pyx_Import(__pyx_n_s_treebank, __pyx_t_3, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_ImportFrom(__pyx_t_4, __pyx_n_s_READERS); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_t_3); __pyx_v_READERS = __pyx_t_3; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+1182: corpus = READERS[fmt](treebankfile, encoding=encoding)
__pyx_t_4 = PyObject_GetItem(__pyx_v_READERS, __pyx_v_fmt); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1182, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1182, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_v_treebankfile); __Pyx_GIVEREF(__pyx_v_treebankfile); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_treebankfile); __pyx_t_5 = PyDict_New(); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1182, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_encoding, __pyx_v_encoding) < 0) __PYX_ERR(0, 1182, __pyx_L1_error) __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_3, __pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1182, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_v_corpus = __pyx_t_6; __pyx_t_6 = 0;
+1183: for _, item in corpus.itertrees(0, limit):
__pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_corpus, __pyx_n_s_itertrees); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_3 = NULL; __pyx_t_7 = 0; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_5))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_5); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_5, function); __pyx_t_7 = 1; } } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[3] = {__pyx_t_3, __pyx_int_0, __pyx_v_limit}; __pyx_t_6 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_7, 2+__pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_6); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[3] = {__pyx_t_3, __pyx_int_0, __pyx_v_limit}; __pyx_t_6 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_7, 2+__pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_6); } else #endif { __pyx_t_4 = PyTuple_New(2+__pyx_t_7); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __pyx_t_3 = NULL; } __Pyx_INCREF(__pyx_int_0); __Pyx_GIVEREF(__pyx_int_0); PyTuple_SET_ITEM(__pyx_t_4, 0+__pyx_t_7, __pyx_int_0); __Pyx_INCREF(__pyx_v_limit); __Pyx_GIVEREF(__pyx_v_limit); PyTuple_SET_ITEM(__pyx_t_4, 1+__pyx_t_7, __pyx_v_limit); __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_4, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (likely(PyList_CheckExact(__pyx_t_6)) || PyTuple_CheckExact(__pyx_t_6)) { __pyx_t_5 = __pyx_t_6; __Pyx_INCREF(__pyx_t_5); __pyx_t_8 = 0; __pyx_t_9 = NULL; } else { __pyx_t_8 = -1; __pyx_t_5 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_9 = Py_TYPE(__pyx_t_5)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 1183, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; for (;;) { if (likely(!__pyx_t_9)) { if (likely(PyList_CheckExact(__pyx_t_5))) { if (__pyx_t_8 >= PyList_GET_SIZE(__pyx_t_5)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_6 = PyList_GET_ITEM(__pyx_t_5, __pyx_t_8); __Pyx_INCREF(__pyx_t_6); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 1183, __pyx_L1_error) #else __pyx_t_6 = PySequence_ITEM(__pyx_t_5, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); #endif } else { if (__pyx_t_8 >= PyTuple_GET_SIZE(__pyx_t_5)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_5, __pyx_t_8); __Pyx_INCREF(__pyx_t_6); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 1183, __pyx_L1_error) #else __pyx_t_6 = PySequence_ITEM(__pyx_t_5, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); #endif } } else { __pyx_t_6 = __pyx_t_9(__pyx_t_5); if (unlikely(!__pyx_t_6)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 1183, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_6); } if ((likely(PyTuple_CheckExact(__pyx_t_6))) || (PyList_CheckExact(__pyx_t_6))) { PyObject* sequence = __pyx_t_6; #if !CYTHON_COMPILING_IN_PYPY Py_ssize_t size = Py_SIZE(sequence); #else Py_ssize_t size = PySequence_Size(sequence); #endif if (unlikely(size != 2)) { if (size > 2) __Pyx_RaiseTooManyValuesError(2); else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); __PYX_ERR(0, 1183, __pyx_L1_error) } #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS if (likely(PyTuple_CheckExact(sequence))) { __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); } else { __pyx_t_4 = PyList_GET_ITEM(sequence, 0); __pyx_t_3 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_4); __Pyx_INCREF(__pyx_t_3); #else __pyx_t_4 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); #endif __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { Py_ssize_t index = -1; __pyx_t_10 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1183, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_t_11 = Py_TYPE(__pyx_t_10)->tp_iternext; index = 0; __pyx_t_4 = __pyx_t_11(__pyx_t_10); if (unlikely(!__pyx_t_4)) goto __pyx_L8_unpacking_failed; __Pyx_GOTREF(__pyx_t_4); index = 1; __pyx_t_3 = __pyx_t_11(__pyx_t_10); if (unlikely(!__pyx_t_3)) goto __pyx_L8_unpacking_failed; __Pyx_GOTREF(__pyx_t_3); if (__Pyx_IternextUnpackEndCheck(__pyx_t_11(__pyx_t_10), 2) < 0) __PYX_ERR(0, 1183, __pyx_L1_error) __pyx_t_11 = NULL; __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; goto __pyx_L9_unpacking_done; __pyx_L8_unpacking_failed:; __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; __pyx_t_11 = NULL; if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); __PYX_ERR(0, 1183, __pyx_L1_error) __pyx_L9_unpacking_done:; } __Pyx_XDECREF_SET(__pyx_v__, __pyx_t_4); __pyx_t_4 = 0; __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_3); __pyx_t_3 = 0; /* … */ } __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+1184: tree = binarize(handledisc(item.tree), dot=True)
__pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_binarize); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_handledisc); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_item, __pyx_n_s_tree); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __pyx_t_12 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_4))) { __pyx_t_12 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_12)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); __Pyx_INCREF(__pyx_t_12); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_4, function); } } if (!__pyx_t_12) { __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_10); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; __Pyx_GOTREF(__pyx_t_3); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_12, __pyx_t_10}; __pyx_t_3 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_12, __pyx_t_10}; __pyx_t_3 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; } else #endif { __pyx_t_13 = PyTuple_New(1+1); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_GIVEREF(__pyx_t_12); PyTuple_SET_ITEM(__pyx_t_13, 0, __pyx_t_12); __pyx_t_12 = NULL; __Pyx_GIVEREF(__pyx_t_10); PyTuple_SET_ITEM(__pyx_t_13, 0+1, __pyx_t_10); __pyx_t_10 = 0; __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_13, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; } } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = PyDict_New(); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (PyDict_SetItem(__pyx_t_3, __pyx_n_s_dot, Py_True) < 0) __PYX_ERR(0, 1184, __pyx_L1_error) __pyx_t_13 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_XDECREF_SET(__pyx_v_tree, __pyx_t_13); __pyx_t_13 = 0;
+1185: prodsintree = []
__pyx_t_13 = PyList_New(0); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1185, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_XDECREF_SET(__pyx_v_prodsintree, ((PyObject*)__pyx_t_13)); __pyx_t_13 = 0;
+1186: cnt = 0
__pyx_v_cnt = 0;
+1187: for r, yf in lcfrsproductions(tree, item.sent, frontiers=True):
__pyx_t_13 = __Pyx_GetModuleGlobalName(__pyx_n_s_lcfrsproductions); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_item, __pyx_n_s_sent); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(__pyx_v_tree); __Pyx_GIVEREF(__pyx_v_tree); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_tree); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = PyDict_New(); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (PyDict_SetItem(__pyx_t_3, __pyx_n_s_frontiers, Py_True) < 0) __PYX_ERR(0, 1187, __pyx_L1_error) __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_13, __pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (likely(PyList_CheckExact(__pyx_t_6)) || PyTuple_CheckExact(__pyx_t_6)) { __pyx_t_3 = __pyx_t_6; __Pyx_INCREF(__pyx_t_3); __pyx_t_14 = 0; __pyx_t_15 = NULL; } else { __pyx_t_14 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_15 = Py_TYPE(__pyx_t_3)->tp_iternext; if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 1187, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; for (;;) { if (likely(!__pyx_t_15)) { if (likely(PyList_CheckExact(__pyx_t_3))) { if (__pyx_t_14 >= PyList_GET_SIZE(__pyx_t_3)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_6 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_14); __Pyx_INCREF(__pyx_t_6); __pyx_t_14++; if (unlikely(0 < 0)) __PYX_ERR(0, 1187, __pyx_L1_error) #else __pyx_t_6 = PySequence_ITEM(__pyx_t_3, __pyx_t_14); __pyx_t_14++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); #endif } else { if (__pyx_t_14 >= PyTuple_GET_SIZE(__pyx_t_3)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_14); __Pyx_INCREF(__pyx_t_6); __pyx_t_14++; if (unlikely(0 < 0)) __PYX_ERR(0, 1187, __pyx_L1_error) #else __pyx_t_6 = PySequence_ITEM(__pyx_t_3, __pyx_t_14); __pyx_t_14++; if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); #endif } } else { __pyx_t_6 = __pyx_t_15(__pyx_t_3); if (unlikely(!__pyx_t_6)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); else __PYX_ERR(0, 1187, __pyx_L1_error) } break; } __Pyx_GOTREF(__pyx_t_6); } if ((likely(PyTuple_CheckExact(__pyx_t_6))) || (PyList_CheckExact(__pyx_t_6))) { PyObject* sequence = __pyx_t_6; #if !CYTHON_COMPILING_IN_PYPY Py_ssize_t size = Py_SIZE(sequence); #else Py_ssize_t size = PySequence_Size(sequence); #endif if (unlikely(size != 2)) { if (size > 2) __Pyx_RaiseTooManyValuesError(2); else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); __PYX_ERR(0, 1187, __pyx_L1_error) } #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS if (likely(PyTuple_CheckExact(sequence))) { __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_13 = PyTuple_GET_ITEM(sequence, 1); } else { __pyx_t_4 = PyList_GET_ITEM(sequence, 0); __pyx_t_13 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_4); __Pyx_INCREF(__pyx_t_13); #else __pyx_t_4 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_13 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); #endif __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { Py_ssize_t index = -1; __pyx_t_10 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1187, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_t_11 = Py_TYPE(__pyx_t_10)->tp_iternext; index = 0; __pyx_t_4 = __pyx_t_11(__pyx_t_10); if (unlikely(!__pyx_t_4)) goto __pyx_L12_unpacking_failed; __Pyx_GOTREF(__pyx_t_4); index = 1; __pyx_t_13 = __pyx_t_11(__pyx_t_10); if (unlikely(!__pyx_t_13)) goto __pyx_L12_unpacking_failed; __Pyx_GOTREF(__pyx_t_13); if (__Pyx_IternextUnpackEndCheck(__pyx_t_11(__pyx_t_10), 2) < 0) __PYX_ERR(0, 1187, __pyx_L1_error) __pyx_t_11 = NULL; __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; goto __pyx_L13_unpacking_done; __pyx_L12_unpacking_failed:; __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; __pyx_t_11 = NULL; if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); __PYX_ERR(0, 1187, __pyx_L1_error) __pyx_L13_unpacking_done:; } __Pyx_XDECREF_SET(__pyx_v_r, __pyx_t_4); __pyx_t_4 = 0; __Pyx_XDECREF_SET(__pyx_v_yf, __pyx_t_13); __pyx_t_13 = 0; /* … */ } __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+1188: prodsintree.append(vocab.getprod(r, yf))
if (!(likely(PyTuple_CheckExact(__pyx_v_r))||((__pyx_v_r) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "tuple", Py_TYPE(__pyx_v_r)->tp_name), 0))) __PYX_ERR(0, 1188, __pyx_L1_error) if (!(likely(PyTuple_CheckExact(__pyx_v_yf))||((__pyx_v_yf) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "tuple", Py_TYPE(__pyx_v_yf)->tp_name), 0))) __PYX_ERR(0, 1188, __pyx_L1_error) __pyx_t_7 = ((struct __pyx_vtabstruct_8discodop_10containers_Vocabulary *)__pyx_v_vocab->__pyx_vtab)->getprod(__pyx_v_vocab, ((PyObject*)__pyx_v_r), ((PyObject*)__pyx_v_yf)); if (unlikely(__pyx_t_7 == -2)) __PYX_ERR(0, 1188, __pyx_L1_error) __pyx_t_6 = __Pyx_PyInt_From_int(__pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_16 = __Pyx_PyList_Append(__pyx_v_prodsintree, __pyx_t_6); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 1188, __pyx_L1_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+1189: cnt += 1
__pyx_v_cnt = (__pyx_v_cnt + 1);
+1190: if cnt > maxnodes:
__pyx_t_2 = ((__pyx_v_cnt > __pyx_v_maxnodes) != 0);
if (__pyx_t_2) {
/* … */
}
+1191: maxnodes = cnt
__pyx_v_maxnodes = __pyx_v_cnt;
+1192: scratch = <Node *>realloc(scratch,
__pyx_v_scratch = ((struct __pyx_t_8discodop_10containers_Node *)realloc(__pyx_v_scratch, ((__pyx_v_maxnodes * __pyx_v_binfactor) * (sizeof(struct __pyx_t_8discodop_10containers_Node)))));
1193: maxnodes * binfactor * sizeof(Node))
+1194: if scratch is NULL:
__pyx_t_2 = ((__pyx_v_scratch == NULL) != 0);
if (__pyx_t_2) {
/* … */
}
+1195: raise MemoryError('allocation error')
__pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__21, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1195, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(0, 1195, __pyx_L1_error) /* … */ __pyx_tuple__21 = PyTuple_Pack(1, __pyx_kp_u_allocation_error); if (unlikely(!__pyx_tuple__21)) __PYX_ERR(0, 1195, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__21); __Pyx_GIVEREF(__pyx_tuple__21);
+1196: cnt = 0
__pyx_v_cnt = 0;
+1197: copynodes(tree, prodsintree, scratch, &cnt)
__pyx_t_3 = __pyx_f_8discodop_10_fragments_copynodes(__pyx_v_tree, __pyx_v_prodsintree, __pyx_v_scratch, (&__pyx_v_cnt)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1197, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+1198: ctrees.addnodes(scratch, cnt, 0)
__pyx_t_3 = ((struct __pyx_vtabstruct_8discodop_10containers_Ctrees *)__pyx_v_ctrees->__pyx_vtab)->addnodes(__pyx_v_ctrees, __pyx_v_scratch, __pyx_v_cnt, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1198, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
1199: else: # do incremental reading of bracket trees
+1200: stack = clone(shortarray, 0, False)
/*else*/ {
__pyx_t_5 = ((PyObject *)__pyx_v_8discodop_10_fragments_shortarray);
__Pyx_INCREF(__pyx_t_5);
__pyx_t_3 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_5), 0, 0)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1200, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
__pyx_v_stack = ((arrayobject *)__pyx_t_3);
__pyx_t_3 = 0;
+1201: labels = clone(intarray, 0, False)
__pyx_t_3 = ((PyObject *)__pyx_v_8discodop_10_fragments_intarray);
__Pyx_INCREF(__pyx_t_3);
__pyx_t_5 = ((PyObject *)__pyx_f_7cpython_5array_clone(((arrayobject *)__pyx_t_3), 0, 0)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1201, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
__Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
__pyx_v_labels = ((arrayobject *)__pyx_t_5);
__pyx_t_5 = 0;
+1202: children = [] # list of lists
__pyx_t_5 = PyList_New(0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1202, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_v_children = ((PyObject*)__pyx_t_5); __pyx_t_5 = 0;
+1203: data = openread(treebankfile, encoding=encoding)
__pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_openread); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1203, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1203, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_v_treebankfile); __Pyx_GIVEREF(__pyx_v_treebankfile); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_treebankfile); __pyx_t_6 = PyDict_New(); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1203, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (PyDict_SetItem(__pyx_t_6, __pyx_n_s_encoding, __pyx_v_encoding) < 0) __PYX_ERR(0, 1203, __pyx_L1_error) __pyx_t_13 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_3, __pyx_t_6); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1203, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_v_data = __pyx_t_13; __pyx_t_13 = 0;
+1204: for n, line in enumerate(islice(data, limit), 1):
__pyx_t_7 = 1;
__pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_islice); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1204, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__pyx_t_3 = NULL;
__pyx_t_17 = 0;
if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) {
__pyx_t_3 = PyMethod_GET_SELF(__pyx_t_6);
if (likely(__pyx_t_3)) {
PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6);
__Pyx_INCREF(__pyx_t_3);
__Pyx_INCREF(function);
__Pyx_DECREF_SET(__pyx_t_6, function);
__pyx_t_17 = 1;
}
}
#if CYTHON_FAST_PYCALL
if (PyFunction_Check(__pyx_t_6)) {
PyObject *__pyx_temp[3] = {__pyx_t_3, __pyx_v_data, __pyx_v_limit};
__pyx_t_13 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-__pyx_t_17, 2+__pyx_t_17); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1204, __pyx_L1_error)
__Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
__Pyx_GOTREF(__pyx_t_13);
} else
#endif
#if CYTHON_FAST_PYCCALL
if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) {
PyObject *__pyx_temp[3] = {__pyx_t_3, __pyx_v_data, __pyx_v_limit};
__pyx_t_13 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-__pyx_t_17, 2+__pyx_t_17); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1204, __pyx_L1_error)
__Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
__Pyx_GOTREF(__pyx_t_13);
} else
#endif
{
__pyx_t_5 = PyTuple_New(2+__pyx_t_17); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1204, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_5);
if (__pyx_t_3) {
__Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL;
}
__Pyx_INCREF(__pyx_v_data);
__Pyx_GIVEREF(__pyx_v_data);
PyTuple_SET_ITEM(__pyx_t_5, 0+__pyx_t_17, __pyx_v_data);
__Pyx_INCREF(__pyx_v_limit);
__Pyx_GIVEREF(__pyx_v_limit);
PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_17, __pyx_v_limit);
__pyx_t_13 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_5, NULL); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1204, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
__Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
}
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
if (likely(PyList_CheckExact(__pyx_t_13)) || PyTuple_CheckExact(__pyx_t_13)) {
__pyx_t_6 = __pyx_t_13; __Pyx_INCREF(__pyx_t_6); __pyx_t_8 = 0;
__pyx_t_9 = NULL;
} else {
__pyx_t_8 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_t_13); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1204, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_6);
__pyx_t_9 = Py_TYPE(__pyx_t_6)->tp_iternext; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 1204, __pyx_L1_error)
}
__Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
for (;;) {
if (likely(!__pyx_t_9)) {
if (likely(PyList_CheckExact(__pyx_t_6))) {
if (__pyx_t_8 >= PyList_GET_SIZE(__pyx_t_6)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_13 = PyList_GET_ITEM(__pyx_t_6, __pyx_t_8); __Pyx_INCREF(__pyx_t_13); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 1204, __pyx_L1_error)
#else
__pyx_t_13 = PySequence_ITEM(__pyx_t_6, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1204, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
#endif
} else {
if (__pyx_t_8 >= PyTuple_GET_SIZE(__pyx_t_6)) break;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
__pyx_t_13 = PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_8); __Pyx_INCREF(__pyx_t_13); __pyx_t_8++; if (unlikely(0 < 0)) __PYX_ERR(0, 1204, __pyx_L1_error)
#else
__pyx_t_13 = PySequence_ITEM(__pyx_t_6, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1204, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_13);
#endif
}
} else {
__pyx_t_13 = __pyx_t_9(__pyx_t_6);
if (unlikely(!__pyx_t_13)) {
PyObject* exc_type = PyErr_Occurred();
if (exc_type) {
if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
else __PYX_ERR(0, 1204, __pyx_L1_error)
}
break;
}
__Pyx_GOTREF(__pyx_t_13);
}
if (!(likely(PyUnicode_CheckExact(__pyx_t_13))||((__pyx_t_13) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "unicode", Py_TYPE(__pyx_t_13)->tp_name), 0))) __PYX_ERR(0, 1204, __pyx_L1_error)
__Pyx_XDECREF_SET(__pyx_v_line, ((PyObject*)__pyx_t_13));
__pyx_t_13 = 0;
__pyx_v_n = __pyx_t_7;
__pyx_t_7 = (__pyx_t_7 + 1);
/* … */
}
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+1205: cnt = line.count('(')
if (unlikely(__pyx_v_line == Py_None)) {
PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", "count");
__PYX_ERR(0, 1205, __pyx_L1_error)
}
__pyx_t_14 = PyUnicode_Count(__pyx_v_line, __pyx_kp_u__4, 0, PY_SSIZE_T_MAX); if (unlikely(__pyx_t_14 == -1)) __PYX_ERR(0, 1205, __pyx_L1_error)
__pyx_v_cnt = __pyx_t_14;
+1206: if cnt > maxnodes:
__pyx_t_2 = ((__pyx_v_cnt > __pyx_v_maxnodes) != 0);
if (__pyx_t_2) {
/* … */
}
+1207: maxnodes = 2 * cnt
__pyx_v_maxnodes = (2 * __pyx_v_cnt);
+1208: scratch = <Node *>realloc(scratch,
__pyx_v_scratch = ((struct __pyx_t_8discodop_10containers_Node *)realloc(__pyx_v_scratch, ((__pyx_v_maxnodes * __pyx_v_binfactor) * (sizeof(struct __pyx_t_8discodop_10containers_Node)))));
1209: maxnodes * binfactor * sizeof(Node))
+1210: if scratch is NULL:
__pyx_t_2 = ((__pyx_v_scratch == NULL) != 0);
if (__pyx_t_2) {
/* … */
}
+1211: raise MemoryError('allocation error')
__pyx_t_13 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__22, NULL); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1211, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __Pyx_Raise(__pyx_t_13, 0, 0, 0); __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; __PYX_ERR(0, 1211, __pyx_L1_error) /* … */ __pyx_tuple__22 = PyTuple_Pack(1, __pyx_kp_u_allocation_error); if (unlikely(!__pyx_tuple__22)) __PYX_ERR(0, 1211, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__22); __Pyx_GIVEREF(__pyx_tuple__22);
+1212: cnt = readtree(line, vocab, scratch, stack, labels, children)
__pyx_t_17 = __pyx_f_8discodop_10_fragments_readtree(__pyx_v_line, __pyx_v_vocab, __pyx_v_scratch, __pyx_v_stack, __pyx_v_labels, __pyx_v_children); if (unlikely(__pyx_t_17 == -9)) __PYX_ERR(0, 1212, __pyx_L1_error)
__pyx_v_cnt = __pyx_t_17;
+1213: if cnt <= 0:
__pyx_t_2 = ((__pyx_v_cnt <= 0) != 0);
if (__pyx_t_2) {
/* … */
}
+1214: raise ValueError('error %d in line %d: %s\n%s' % (
__pyx_t_4 = PyUnicode_Format(__pyx_kp_u_error_d_in_line_d_s_s, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1214, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1214, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1214, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(0, 1214, __pyx_L1_error)
+1215: cnt, n, TREEPARSEMSG[-cnt], line))
__pyx_t_13 = __Pyx_PyInt_From_int(__pyx_v_cnt); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 1215, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_13); __pyx_t_5 = __Pyx_PyInt_From_int(__pyx_v_n); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1215, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_TREEPARSEMSG); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1215, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_17 = (-__pyx_v_cnt); __pyx_t_4 = __Pyx_GetItemInt(__pyx_t_3, __pyx_t_17, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1215, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = PyTuple_New(4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1215, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_13); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_13); __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_5); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_t_4); __Pyx_INCREF(__pyx_v_line); __Pyx_GIVEREF(__pyx_v_line); PyTuple_SET_ITEM(__pyx_t_3, 3, __pyx_v_line); __pyx_t_13 = 0; __pyx_t_5 = 0; __pyx_t_4 = 0;
+1216: ctrees.addnodes(scratch, cnt, 0)
__pyx_t_4 = ((struct __pyx_vtabstruct_8discodop_10containers_Ctrees *)__pyx_v_ctrees->__pyx_vtab)->addnodes(__pyx_v_ctrees, __pyx_v_scratch, __pyx_v_cnt, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1216, __pyx_L1_error)
__Pyx_GOTREF(__pyx_t_4);
__Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+1217: del labels[:], children[:]
if (__Pyx_PyObject_DelSlice(((PyObject *)__pyx_v_labels), 0, 0, NULL, NULL, &__pyx_slice__23, 0, 0, 0) < 0) __PYX_ERR(0, 1217, __pyx_L1_error) if (__Pyx_PyObject_DelSlice(__pyx_v_children, 0, 0, NULL, NULL, NULL, 0, 0, 0) < 0) __PYX_ERR(0, 1217, __pyx_L1_error) /* … */ __pyx_slice__23 = PySlice_New(Py_None, Py_None, Py_None); if (unlikely(!__pyx_slice__23)) __PYX_ERR(0, 1217, __pyx_L1_error) __Pyx_GOTREF(__pyx_slice__23); __Pyx_GIVEREF(__pyx_slice__23);
+1218: if not ctrees.len:
__pyx_t_2 = ((!(__pyx_v_ctrees->len != 0)) != 0);
if (__pyx_t_2) {
/* … */
}
}
__pyx_L5:;
+1219: raise ValueError('%r appears to be empty' % treebankfile)
__pyx_t_6 = PyUnicode_Format(__pyx_kp_u_r_appears_to_be_empty, __pyx_v_treebankfile); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1219, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1219, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_6); __pyx_t_6 = 0; __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_4, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1219, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_Raise(__pyx_t_6, 0, 0, 0); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __PYX_ERR(0, 1219, __pyx_L1_error)
+1220: free(scratch)
free(__pyx_v_scratch);
+1221: return ctrees
__Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_ctrees)); __pyx_r = ((PyObject *)__pyx_v_ctrees); goto __pyx_L0;
1222:
1223:
+1224: __all__ = ['extractfragments', 'exactcounts', 'completebitsets',
__pyx_t_3 = PyList_New(9); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1224, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_n_u_extractfragments); __Pyx_GIVEREF(__pyx_n_u_extractfragments); PyList_SET_ITEM(__pyx_t_3, 0, __pyx_n_u_extractfragments); __Pyx_INCREF(__pyx_n_u_exactcounts); __Pyx_GIVEREF(__pyx_n_u_exactcounts); PyList_SET_ITEM(__pyx_t_3, 1, __pyx_n_u_exactcounts); __Pyx_INCREF(__pyx_n_u_completebitsets); __Pyx_GIVEREF(__pyx_n_u_completebitsets); PyList_SET_ITEM(__pyx_t_3, 2, __pyx_n_u_completebitsets); __Pyx_INCREF(__pyx_n_u_allfragments); __Pyx_GIVEREF(__pyx_n_u_allfragments); PyList_SET_ITEM(__pyx_t_3, 3, __pyx_n_u_allfragments); __Pyx_INCREF(__pyx_n_u_repl); __Pyx_GIVEREF(__pyx_n_u_repl); PyList_SET_ITEM(__pyx_t_3, 4, __pyx_n_u_repl); __Pyx_INCREF(__pyx_n_u_pygetsent); __Pyx_GIVEREF(__pyx_n_u_pygetsent); PyList_SET_ITEM(__pyx_t_3, 5, __pyx_n_u_pygetsent); __Pyx_INCREF(__pyx_n_u_getctrees); __Pyx_GIVEREF(__pyx_n_u_getctrees); PyList_SET_ITEM(__pyx_t_3, 6, __pyx_n_u_getctrees); __Pyx_INCREF(__pyx_n_u_readtreebank); __Pyx_GIVEREF(__pyx_n_u_readtreebank); PyList_SET_ITEM(__pyx_t_3, 7, __pyx_n_u_readtreebank); __Pyx_INCREF(__pyx_n_u_exactcountsslice); __Pyx_GIVEREF(__pyx_n_u_exactcountsslice); PyList_SET_ITEM(__pyx_t_3, 8, __pyx_n_u_exactcountsslice); if (PyDict_SetItem(__pyx_d, __pyx_n_s_all, __pyx_t_3) < 0) __PYX_ERR(0, 1224, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
1225: 'allfragments', 'repl', 'pygetsent', 'getctrees',
1226: 'readtreebank', 'exactcountsslice']