#!/usr/bin/env python
# * mode: python-mode *

import deepcut
import json
import sys

for line in sys.stdin:
    line = line.strip()
    print(json.dumps(deepcut.tokenize(line)))

#!/usr/bin/env python
# * mode: python-mode *

from pythainlp.tokenize import word_tokenize

import json
import sys

for line in sys.stdin:
    line = line.strip()
    print(json.dumps(word_tokenize(line ,engine='newmm')))

#!/usr/bin/env python
# * mode: python-mode *

from wordcut import Wordcut
import json
import sys

wordcut = Wordcut.bigthai()

for line in sys.stdin:
    line = line.strip()
    print(json.dumps(wordcut.tokenize(line)))

$ newmm < file1.txt > file1.ndjson

$ for x in *.txt; do newmm < $x > $x.ndjson; done

#!/usr/bin/env ruby
# * mode: ruby-mode *

require "json"
require "csv"

h = Hash.new(0)
while gets
  JSON.parse($_.chomp)
    .each{|w| h[w.chomp] += 1}
end
h.to_a.each{|row| puts CSV::generate_line(row)}

cat file1.txt | newmm | wcnt