我有一个解决方案，我加入df2两次，第一次是获得不支持，第二次是获得支持。完成后，很容易按管理器分组并收集值列表
你可以试试这样的方法：

import pyspark.sql.functions as F

df = [
    {"Manager": "Abhinav", "Department": "Medical", "isHospRelated": "t"},
    {"Manager": "Abhinav", "Department": "Surgery", "isHospRelated": "t"},
    {"Manager": "Rajesh", "Department": "Payments", "isHospRelated": "t"},
    {"Manager": "Sonu", "Department": "Onboarding", "isHospRelated": "t"},
    {"Manager": "Sonu", "Department": "Surgery", "isHospRelated": "t"},
]

df2 = [
    {"OrgSupported": "Medical", "OrgNonSupported": "Payments"},
    {"OrgSupported": "Surgery", "OrgNonSupported": "Onboarding"},
]

df = spark.createDataFrame(df)
df2 = spark.createDataFrame(df2)

dfWithNonSupported = df.join(
    df2.drop("OrgNonSupported"), df.Department == df2.OrgSupported, "left"
)
dfWithSupportedAndNonSupported = dfWithNonSupported.join(
    df2.drop("OrgSupported"),
    dfWithNonSupported.Department == df2.OrgNonSupported,
    "left",
)

dfWithSupportedAndNonSupported.groupBy("Manager").agg(
    F.collect_list("Department").alias("Department"),
    F.collect_list("OrgSupported").alias("OrgSupported"),
    F.collect_list("OrgNonSupported").alias("OrgNonSupported"),
    F.first("isHospRelated").alias("isHospRelated")
).show()

输出：

+-------+--------------------+------------------+---------------+-------------+
|Manager|          Department|      OrgSupported|OrgNonSupported|isHospRelated|
+-------+--------------------+------------------+---------------+-------------+
|Abhinav|  [Medical, Surgery]|[Medical, Surgery]|             []|            t|
| Rajesh|          [Payments]|                []|     [Payments]|            t|
|   Sonu|[Onboarding, Surg...|         [Surgery]|   [Onboarding]|            t|
+-------+--------------------+------------------+---------------+-------------+