#!/bin/sh

export LANG=ja_JP.UTF-8

cat corpus.*.txt > corpus
#cat corpus.kanjikai.txt > corpus
#cat corpus.kanjikai.txt corpus.ryomou.txt > corpus
#cat corpus.misc.txt > corpus

#cat corpus \
cat corpus.*.txt \
| grep -v EOS | sed 's/	/,0,0,0,/' | sort | uniq > misc.corpus.csv
