DataBaser.Net: Hive Python

파이썬 프로그램을 짠다.

import sys
import string
from pykospacing import spacing

for line in sys.stdin:
    line = line.strip()
    pc_id, msg = line.split("\t")
    print ("\t".join([pc_id, spacing(msg)]))

hdfs에 넣는다.

hadoop fs -put -f /home/hdfs/py/chat_spacing.py /user/hive/udfs

transform 함수를 써서 사용한다.

set hive.execution.engine=mr;
add file hdfs:///user/hive/udfs/chat_spacing.py;

select transform(msg) using 'python3.4 chat_spacing.py' as (pc_id bigint, msg string)
from (
    select concat(cast(id as string), "\t", sss) msg
    from sample
) t;

[edit]

pyhive #

from pyhive import hive
conn = hive.Connection(host="192.168.0.100", port=10000, username="hdfs")
 
cursor = conn.cursor()
cursor.execute("SHOW TABLES")
for table in cursor.fetchall():
    print(table)

export http_proxy='http://xxx.com'
export https_proxy='https://xxx.com'

pip3.4 install pip3.4 install tensorflow --trusted-host pypi.python.org --index-url=https://pypi.python.org/simple/