如何在python代码中使用sqoop命令进行增量导入

hvvq6cgz  于 2021-06-02  发布在  Hadoop
关注(0)|答案(1)|浏览(334)

我想从用户位置历史进行增量导入,在增量导入之后,我想保存用户位置更新中的最后一个id,以便将来可以自动进行导入。


# !/usr/bin/python

import subprocess
import time
import subprocess
import MySQLdb
import datetime
import sys
import pytz
import os
from subprocess import call

def get_mysql_cursor():
        conn_1 = MySQLdb.connect(user='db', passwd='bazookadb', host='10.216.204.20', db='bazooka')
        conn_2 = MySQLdb.connect(user='db', passwd='bazookadb', host='10.216.204.7', db='bazooka')
        #print conn_1,conn_2
        return conn_1.cursor(),conn_2.cursor()

def get_records():
        cur_1,cur_2 = get_mysql_cursor()
        cur_1.execute("select updated from user_location_updated")
        cur_2.execute("select max(moving_date) from user_location_history")
        return cur_1.fetchone(),cur_2.fetchone()

def update_records(update_date):
        cur_1,cur_2 = get_mysql_cursor()
        print update_date
        query = "update user_location_updated set updated = '"+str(update_date)+"' where id='1' "
        print query
        result = cur_1.execute(query)
        print result

result = get_records()
update_result = update_records(result[1][0])
print result[0][0]
print result[1][0]

sqoopcom = "sqoop import --connect jdbc:mysql://10.216.204.7:3306/bazooka --username db --password bazookadb --fields-terminated-by , --escaped-by \\ --enclosed-by '\"' --table user_location_history  -m 1  --hive-delims-replacement ' ' --as-textfile --incremental append --check-column moving_date --last-value 2016-08-04 19:00:36 --target-dir hdfs://example:9000/user/bigdata/sqoopip --verbose"

# os.system(sqoopcom)

exec (sqoopcom)

----but this code is giving error
wwtsj6pe

wwtsj6pe1#

包裹 --last-value 用单引号括起来。
使用 --last-value '2016-08-04 19:00:36'

相关问题