我正在创建一个胶水作业，需要处理来自s3路径的4tb的每日数据量- s3://<path>/<year>/<month>/<day>/<hour>/ . 因此，我创建了一个循环，通过每小时一次的文件夹（每个155gb）将数据读入spark df，对某些类别进行过滤，并将其作为按过滤类别划分的Parquet文件写回s3( s3://<path>/category=<category>/year=<year>/month=<month>/day=<day>/hour=<hour>/ ). 我使用60个g2.x工作节点，每个节点都有8个vcpu、32 gb内存、128 gb磁盘。s3写操作非常慢，需要10个多小时才能完成运行。除了增加节点数量之外，还有没有其他方法可以加速/优化s3写操作？

def s3_load_job(input_list):

    hour, year, month, day = input_list
    logger.info(f"hour in s3 func {hour}")

    # get data from s3
    s3_path = f"s3://<path>/{year}/{month}/{day}/{hour}/"
    logger.info(f"print s3 path {s3_path}")

    #user defined library function that return spark df
    df = get_df_from_s3(glueContext, s3_path)

    df = df.withColumn('category', F.lower(F.col('category')))

    df_rep = df.where(F.col('category').isin({ "A", "B", "C","D"}))

    #write to s3
    datasink4 = DynamicFrame.fromDF(df_rep, glueContext, "datasink4")

    glueContext.write_dynamic_frame.from_options(frame = datasink4,
                                                             connection_type = "s3",
                                                             connection_options = 
                                                             {"path":"s3://<path>/"
                                           ,"partitionKeys"["category","year","month","day","hour"]}
                                                             ,format = "glueparquet" )

def main():

    year = '2020'
    month = '08'
    day = '01'
    hours = ["%.2d" % i for i in range(24)]

    input_list = [[hour, year, month, day] for hour in hours]
    logger.info(f"input_list {input_list}")

    for i in input_list:
        s3_load_job(i)

    job.commit()

if __name__ == "__main__":
    main()