My current code produces a new column with the expected outcome, however, I would like to do this iteratively since I have to execute the same logic 3 times.

With one dependencies
1. (key, value) mappings have to produce a value in order for new columns to be added. For example,
dict notDefined_conformedAttributeCode does not have a value, thus my code fails there.

What would be a more efficient way of replicating this logic?

#--AttributeName
#Dictionary of corresponding conform column name 
notDefined_conformedAttributeNames =dict(Entity.agg(collect_set(array('DomainName', 'TargetAttributeForName')).alias('m')).first().m)
print(notDefined_conformedAttributeNames)
#{'RequestPriority': 'ConformedRequestPriorityName', 'RequestActivityGroup': 'ConformedRequestActivityGroupName', 'RequestStatus': 'ConformedRequestStatusName', 'RequestType': 'ConformedRequestTypeName'}

#Dictionary of corresponding column to search values on 
notDefined_matchedAttributeNames =dict(filterDomainLookup.agg(collect_set(array('DomainName', 'PrimaryLookupAttributeName')).alias('m')).first().m)
print(notDefined_matchedAttributeNames)
#{}

dataset_notDefinedName = dataset
for i in notDefined_conformedAttributeNames.values():
  dataset_notDefinedName = dataset_notDefinedName.withColumn(i,lit(notDefined_item['name']))

#display(dataset_notDefinedName.select("ConformedRequestPriorityName", "ConformedRequestActivityGroupName", "ConformedRequestStatusName", "ConformedRequestTypeName"))


#--AttributeId
#Dictionary of corresponding conform column name 
notDefined_conformedAttributeId =dict(Entity.agg(collect_set(array('DomainName', 'TargetAttributeForId')).alias('m')).first().m)
print(notDefined_conformedAttributeId)
#{'RequestActivityGroup': 'ConformedRequestActivityGroupId', 'RequestPriority': 'ConformedRequestPriorityId', 'RequestType': 'ConformedRequestTypeId', 'RequestStatus': 'ConformedRequestTypeId'}

dataset_notDefinedId = dataset_notDefinedName
for i in notDefined_conformedAttributeId.values():
  dataset_notDefinedId = dataset_notDefinedId.withColumn(i,lit(notDefined_id))

display(dataset_notDefinedId.select("ConformedRequestPriorityName", "ConformedRequestActivityGroupName", "ConformedRequestStatusName", "ConformedRequestTypeName", "ConformedRequestActivityGroupId","ConformedRequestPriorityId","ConformedRequestTypeId","ConformedRequestTypeId" ))

+----------------------------+---------------------------------+--------------------------+------------------------+-------------------------------+--------------------------+----------------------+----------------------+
|ConformedRequestPriorityName|ConformedRequestActivityGroupName|ConformedRequestStatusName|ConformedRequestTypeName|ConformedRequestActivityGroupId|ConformedRequestPriorityId|ConformedRequestTypeId|ConformedRequestTypeId|
+----------------------------+---------------------------------+--------------------------+------------------------+-------------------------------+--------------------------+----------------------+----------------------+
|                 Not Defined|                      Not Defined|               Not Defined|             Not Defined|                              4|                         4|                     4|                     4|
|                 Not Defined|                      Not Defined|               Not Defined|             Not Defined|                              4|                         4|                     4|                     4|
|                 Not Defined|                      Not Defined|               Not Defined|             Not Defined|                              4|                         4|                     4|                     4|
|                 Not Defined|                      Not Defined|               Not Defined|             Not Defined|                              4|                         4|                     4|                     4|
|                 Not Defined|                      Not Defined|               Not Defined|             Not Defined|                              4|                         4|                     4|                     4|
|                 Not Defined|                      Not Defined|               Not Defined|             Not Defined|                              4|                         4|                     4|                     4|
|                 Not Defined|                      Not Defined|               Not Defined|             Not Defined|                              4|                         4|                     4|                     4|

——–

#--AttributeCode
#Dictionary of corresponding conform column name 
notDefined_conformedAttributeCode =dict(Entity.agg(collect_set(array('DomainName', 'TargetAttributeForCode')).alias('m')).first().m)
print(notDefined_conformedAttributeCode)
#{'RequestStatus': None, 'RequestType': None, 'RequestActivityGroup': None, 'RequestPriority': None}

dataset_notDefinedCode = dataset_notDefinedId
for i in notDefined_conformedAttributeCode.values():
   dataset_notDefinedFinal = dataset_notDefinedCode.withColumn(i,lit(notDefined_item['code']))

Error java.lang.NullPointerException since my notDefined_conformedAttributeCode.values() are None