您可以使用write方法将pysark DataFrame以csv格式写入hdfs。在Spark 2.0+中，您可以直接使用csv数据源。

from pyspark.sql.types import IntegerType, StringType, StructField, StructType
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

data = [
    {
        "row_nr": 1,
        "payment": "p1",
        "ref1": "c1",
        "ref2": "c1a",
        "original_ref": "c1",
    },
    {
        "row_nr": 2,
        "payment": "p1",
        "ref1": "c1a",
        "ref2": "c1b",
        "original_ref": None,
    }
]

schema = StructType(
    [
        StructField("row_nr", IntegerType()),
        StructField("payment", StringType()),
        StructField("ref1", StringType()),
        StructField("ref2", StringType()),
        StructField("original_ref", StringType()),
    ]
)

df = spark.createDataFrame(data=data, schema=schema)
df.write.csv(path="/my_path/my_file.csv", mode="overwrite")