My current code produces a new column with the expected outcome, however, I would like to do this iteratively since I have to execute the same logic 3 times.
With one dependencies
1. (key, value) mappings have to produce a value in order for new columns to be added. For example,
dict notDefined_conformedAttributeCode
does not have a value, thus my code fails there.
What would be a more efficient way of replicating this logic?
#--AttributeName
#Dictionary of corresponding conform column name
notDefined_conformedAttributeNames =dict(Entity.agg(collect_set(array('DomainName', 'TargetAttributeForName')).alias('m')).first().m)
print(notDefined_conformedAttributeNames)
#{'RequestPriority': 'ConformedRequestPriorityName', 'RequestActivityGroup': 'ConformedRequestActivityGroupName', 'RequestStatus': 'ConformedRequestStatusName', 'RequestType': 'ConformedRequestTypeName'}
#Dictionary of corresponding column to search values on
notDefined_matchedAttributeNames =dict(filterDomainLookup.agg(collect_set(array('DomainName', 'PrimaryLookupAttributeName')).alias('m')).first().m)
print(notDefined_matchedAttributeNames)
#{}
dataset_notDefinedName = dataset
for i in notDefined_conformedAttributeNames.values():
dataset_notDefinedName = dataset_notDefinedName.withColumn(i,lit(notDefined_item['name']))
#display(dataset_notDefinedName.select("ConformedRequestPriorityName", "ConformedRequestActivityGroupName", "ConformedRequestStatusName", "ConformedRequestTypeName"))
#--AttributeId
#Dictionary of corresponding conform column name
notDefined_conformedAttributeId =dict(Entity.agg(collect_set(array('DomainName', 'TargetAttributeForId')).alias('m')).first().m)
print(notDefined_conformedAttributeId)
#{'RequestActivityGroup': 'ConformedRequestActivityGroupId', 'RequestPriority': 'ConformedRequestPriorityId', 'RequestType': 'ConformedRequestTypeId', 'RequestStatus': 'ConformedRequestTypeId'}
dataset_notDefinedId = dataset_notDefinedName
for i in notDefined_conformedAttributeId.values():
dataset_notDefinedId = dataset_notDefinedId.withColumn(i,lit(notDefined_id))
display(dataset_notDefinedId.select("ConformedRequestPriorityName", "ConformedRequestActivityGroupName", "ConformedRequestStatusName", "ConformedRequestTypeName", "ConformedRequestActivityGroupId","ConformedRequestPriorityId","ConformedRequestTypeId","ConformedRequestTypeId" ))
+----------------------------+---------------------------------+--------------------------+------------------------+-------------------------------+--------------------------+----------------------+----------------------+
|ConformedRequestPriorityName|ConformedRequestActivityGroupName|ConformedRequestStatusName|ConformedRequestTypeName|ConformedRequestActivityGroupId|ConformedRequestPriorityId|ConformedRequestTypeId|ConformedRequestTypeId|
+----------------------------+---------------------------------+--------------------------+------------------------+-------------------------------+--------------------------+----------------------+----------------------+
| Not Defined| Not Defined| Not Defined| Not Defined| 4| 4| 4| 4|
| Not Defined| Not Defined| Not Defined| Not Defined| 4| 4| 4| 4|
| Not Defined| Not Defined| Not Defined| Not Defined| 4| 4| 4| 4|
| Not Defined| Not Defined| Not Defined| Not Defined| 4| 4| 4| 4|
| Not Defined| Not Defined| Not Defined| Not Defined| 4| 4| 4| 4|
| Not Defined| Not Defined| Not Defined| Not Defined| 4| 4| 4| 4|
| Not Defined| Not Defined| Not Defined| Not Defined| 4| 4| 4| 4|
——–
#--AttributeCode
#Dictionary of corresponding conform column name
notDefined_conformedAttributeCode =dict(Entity.agg(collect_set(array('DomainName', 'TargetAttributeForCode')).alias('m')).first().m)
print(notDefined_conformedAttributeCode)
#{'RequestStatus': None, 'RequestType': None, 'RequestActivityGroup': None, 'RequestPriority': None}
dataset_notDefinedCode = dataset_notDefinedId
for i in notDefined_conformedAttributeCode.values():
dataset_notDefinedFinal = dataset_notDefinedCode.withColumn(i,lit(notDefined_item['code']))
Error java.lang.NullPointerException
since my notDefined_conformedAttributeCode.values()
are None
Go to Source
Author: jgtrz