instant-labs
/
instant-segment
mirror of https://github.com/instant-labs/instant-segment.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add new data files

This commit is contained in:
Dirkjan Ochtman 2021-06-02 01:18:36 -07:00
parent fcf24c7543
commit fee2adb995
3 changed files with 354784 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

262144
data/en-bigrams.txt Normal file
View File

File diff suppressed because it is too large Load Diff

92638
data/en-unigrams.txt Normal file
View File

File diff suppressed because it is too large Load Diff

4
instant-segment/src/test_data.rs
View File

@ -16,7 +16,7 @@ pub fn segmenter() -> Segmenter {
let dir = PathBuf::from(format!("{}/../data", env!("CARGO_MANIFEST_DIR"))); let dir = PathBuf::from(format!("{}/../data", env!("CARGO_MANIFEST_DIR")));
let mut ln = String::new(); let mut ln = String::new();
let uni_file = dir.join("unigrams.txt"); let uni_file = dir.join("en-unigrams.txt");
let mut reader = BufReader::new(File::open(&uni_file).unwrap()); let mut reader = BufReader::new(File::open(&uni_file).unwrap());
let mut i = 0; let mut i = 0;
let mut unigrams = HashMap::default(); let mut unigrams = HashMap::default();
@ -33,7 +33,7 @@ pub fn segmenter() -> Segmenter {
ln.clear(); ln.clear();
} }
let bi_file = dir.join("bigrams.txt"); let bi_file = dir.join("en-bigrams.txt");
let mut reader = BufReader::new(File::open(&bi_file).unwrap()); let mut reader = BufReader::new(File::open(&bi_file).unwrap());
let mut i = 0; let mut i = 0;
let mut bigrams = HashMap::default(); let mut bigrams = HashMap::default();