PySpark - Aggregation and Join

Create DataFrame from Dictionary Array

from pyspark.sql import Row
valuesA = [('Pirate',1),('Monkey',2),('Ninja',3),('Spaghetti',4)]
rdd1 = sc.parallelize(valuesA)
rddA = rdd1.map(lambda x: Row(name = x[0], id = x[1]))
TableA = sqlContext.createDataFrame(rddA)
# TableA1 = spark.createDataFrame(valuesA, ['name', 'id'])

valuesB = [('Rutabaga',1),('Pirate',2),('Ninja',3),('Darth Vader',4)]
rdd2 = sc.parallelize(valuesB)
rddB = rdd2.map(lambda x: Row(name = x[0], id = x[1]))
TableB = sqlContext.createDataFrame(rddB)
# TableB1 = spark.createDataFrame(valuesB, ['name', 'id'])

TableA.show()
TableB.show()

Alias Table

ta = TableA.alias('ta')
tb = TableB.alias('tb')

ta.show()
tb.show()

Inner Join between two Tables

inner_join = ta.join(tb, ta.name == tb.name)
inner_join.show()

Left Join between two Tables

left_join = ta.join(tb, on = "name", how = "left")
left_join.show()

left_join2 = ta.join(tb, ta.name == tb.name, how='left') # Could also use 'left_outer'
left_join2.show()