您可以加入并分组 DEPT_ID 以及 DEPT_NAME 将员工详细信息列表收集到结构中。使用 to_json 要获取json字符串：

from pyspark.sql import functions as F

df_c = df_a.join(df_b, ["DEPT_ID"]).groupBy("DEPT_ID", "DEPT_NAME").agg(
    F.to_json(
        F.struct(
            F.lit("normal").alias("_status"),
            F.collect_list(
                F.struct(
                    F.col("EMP_ID"),
                    F.col("EMP_NAME"),
                    F.col("EMP_SALARY")
                )
            ).alias("EmpDetails")
        )
    ).alias("Json_Data")
)

df_c.show(truncate=False)

# +-------+---------+-----------------------------------------------------------------------------------------------------------------------------------------+

# |DEPT_ID|DEPT_NAME|Json_Data                                                                                                                                |

# +-------+---------+-----------------------------------------------------------------------------------------------------------------------------------------+

# |10     |Finance  |{"_status":"normal","EmpDetails":[{"EMP_ID":101,"EMP_NAME":"AAAA","EMP_SALARY":1000},{"EMP_ID":103,"EMP_NAME":"CCCC","EMP_SALARY":1500}]}|

# |20     |Marketing|{"_status":"normal","EmpDetails":[{"EMP_ID":102,"EMP_NAME":"BBBB","EMP_SALARY":2000},{"EMP_ID":104,"EMP_NAME":"DDDD","EMP_SALARY":3000}]}|

# +-------+---------+-----------------------------------------------------------------------------------------------------------------------------------------+