df2=spark.read.option("header","True").option("inferSchema","True").csv("df1.csv")
df1=spark.read.option("header","True").option("inferSchema","True").csv("df2.csv")

df2 = df2.withColumnRenamed("ID",'df2_ID').withColumnRenamed("Total_Count",'df2_Total_Count')

final_df = df1.join(df2,(df1.ID ==  df2.df2_ID) & (df1.Total_Count ==  df2.df2_Total_Count),"left")

from pyspark.sql.functions import when
for i in ('A','B','C','D'):
  final_df = final_df.withColumn(i, when(final_df[i] == 0, final_df["Final_{}".format(i)]).otherwise(final_df[i]))

cols = df2.columns
final_df = final_df.drop(*cols)

df = df1.join(df2.select('Final_A', 'Final_B', 'Final_C', 'Final_D'), 'ID'], 'left')
df =df.withColumn('A', coalesce(df['Final_A'],df['A'])).\
       withColumn('B', coalesce(df['Final_B'],df['B'])).\
       withColumn('C', coalesce(df['Final_C'],df['C'])).\
       withColumn('D', coalesce(df['Final_D'],df['D']))

df1 = df.select('ID', 'Total_Count','A', 'B', 'C', 'D')

df1.show()