1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
"""
Utility functions for the loading and conversion of CoNLL-format files.
"""
import os
import io
FIELD_NUM = 10
ID = 'id'
TEXT = 'text'
LEMMA = 'lemma'
UPOS = 'upos'
XPOS = 'xpos'
FEATS = 'feats'
HEAD = 'head'
DEPREL = 'deprel'
DEPS = 'deps'
MISC = 'misc'
FIELD_TO_IDX = {ID: 0, TEXT: 1, LEMMA: 2, UPOS: 3, XPOS: 4, FEATS: 5, HEAD: 6, DEPREL: 7, DEPS: 8, MISC: 9}
class CoNLL:
@staticmethod
def load_conll(f, ignore_gapping=True):
""" Load the file or string into the CoNLL-U format data.
Input: file or string reader, where the data is in CoNLL-U format.
Output: a list of list of list for each token in each sentence in the data, where the innermost list represents
all fields of a token.
"""
# f is open() or io.StringIO()
doc, sent = [], []
for line in f:
line = line.strip()
if len(line) == 0:
if len(sent) > 0:
doc.append(sent)
sent = []
else:
if line.startswith('#'): # skip comment line
continue
array = line.split('\t')
if ignore_gapping and '.' in array[0]:
continue
assert len(array) == FIELD_NUM, \
f"Cannot parse CoNLL line: expecting {FIELD_NUM} fields, {len(array)} found."
sent += [array]
if len(sent) > 0:
doc.append(sent)
return doc
@staticmethod
def convert_conll(doc_conll):
""" Convert the CoNLL-U format input data to a dictionary format output data.
Input: list of token fields loaded from the CoNLL-U format data, where the outmost list represents a list of sentences, and the inside list represents all fields of a token.
Output: a list of list of dictionaries for each token in each sentence in the document.
"""
doc_dict = []
for sent_conll in doc_conll:
sent_dict = []
for token_conll in sent_conll:
token_dict = CoNLL.convert_conll_token(token_conll)
sent_dict.append(token_dict)
doc_dict.append(sent_dict)
return doc_dict
@staticmethod
def convert_conll_token(token_conll):
""" Convert the CoNLL-U format input token to the dictionary format output token.
Input: a list of all CoNLL-U fields for the token.
Output: a dictionary that maps from field name to value.
"""
token_dict = {}
for field in FIELD_TO_IDX:
value = token_conll[FIELD_TO_IDX[field]]
if value != '_':
if field == HEAD:
token_dict[field] = int(value)
elif field == ID:
token_dict[field] = tuple(int(x) for x in value.split('-'))
else:
token_dict[field] = value
# special case if text is '_'
if token_conll[FIELD_TO_IDX[TEXT]] == '_':
token_dict[TEXT] = token_conll[FIELD_TO_IDX[TEXT]]
token_dict[LEMMA] = token_conll[FIELD_TO_IDX[LEMMA]]
return token_dict
@staticmethod
def conll2dict(input_file=None, input_str=None, ignore_gapping=True):
""" Load the CoNLL-U format data from file or string into lists of dictionaries.
"""
assert any([input_file, input_str]) and not all([input_file, input_str]), 'either input input file or input string'
if input_str:
infile = io.StringIO(input_str)
else:
infile = open(input_file)
doc_conll = CoNLL.load_conll(infile, ignore_gapping)
doc_dict = CoNLL.convert_conll(doc_conll)
return doc_dict
@staticmethod
def convert_dict(doc_dict):
""" Convert the dictionary format input data to the CoNLL-U format output data. This is the reverse function of
`convert_conll`.
Input: dictionary format data, which is a list of list of dictionaries for each token in each sentence in the data.
Output: CoNLL-U format data, which is a list of list of list for each token in each sentence in the data.
"""
doc_conll = []
for sent_dict in doc_dict:
sent_conll = []
for token_dict in sent_dict:
token_conll = CoNLL.convert_token_dict(token_dict)
sent_conll.append(token_conll)
doc_conll.append(sent_conll)
return doc_conll
@staticmethod
def convert_token_dict(token_dict):
""" Convert the dictionary format input token to the CoNLL-U format output token. This is the reverse function of
`convert_conll_token`.
Input: dictionary format token, which is a dictionaries for the token.
Output: CoNLL-U format token, which is a list for the token.
"""
token_conll = ['_' for i in range(FIELD_NUM)]
for key in token_dict:
if key == ID:
token_conll[FIELD_TO_IDX[key]] = '-'.join([str(x) for x in token_dict[key]]) if isinstance(token_dict[key], tuple) else str(token_dict[key])
elif key in FIELD_TO_IDX:
token_conll[FIELD_TO_IDX[key]] = str(token_dict[key])
# when a word (not mwt token) without head is found, we insert dummy head as required by the UD eval script
if '-' not in token_conll[FIELD_TO_IDX[ID]] and HEAD not in token_dict:
token_conll[FIELD_TO_IDX[HEAD]] = str(int(token_dict[ID] if isinstance(token_dict[ID], int) else token_dict[ID][0]) - 1) # evaluation script requires head: int
return token_conll
@staticmethod
def conll_as_string(doc):
""" Dump the loaded CoNLL-U format list data to string. """
return_string = ""
for sent in doc:
for ln in sent:
return_string += ("\t".join(ln)+"\n")
return_string += "\n"
return return_string
@staticmethod
def dict2conll(doc_dict, filename):
""" Convert the dictionary format input data to the CoNLL-U format output data and write to a file.
"""
doc_conll = CoNLL.convert_dict(doc_dict)
conll_string = CoNLL.conll_as_string(doc_conll)
with open(filename, 'w') as outfile:
outfile.write(conll_string)
return
|
