instant-segment/instant-segment-py/test/test.py

24 lines
740 B
Python

import instant_segment, os, sys
DATA_DIR = os.path.join(os.path.dirname(__file__), '../../data/')
def unigrams():
for ln in open(os.path.join(DATA_DIR, 'unigrams.txt')):
parts = ln.split('\t', 1)
yield (parts[0], float(parts[1].strip()))
def bigrams():
for ln in open(os.path.join(DATA_DIR, 'bigrams.txt')):
word_split = ln.split(' ', 1)
score_split = word_split[1].split('\t', 1)
yield ((word_split[0], score_split[0]), float(score_split[1].strip()))
def main():
segmenter = instant_segment.Segmenter(unigrams(), bigrams())
search = instant_segment.Search()
segmenter.segment('thisisatest', search)
print([word for word in search])
if __name__ == '__main__':
main()