15 Matching Annotations
  1. Feb 2023
    1. import org.apache.spark.sql._ import org.apache.spark.sql.types._ val rows1 = Seq( Row("1", Row("a", "b"), "8.00", Seq(Row("1","2"), Row("12","22"))), Row("2", Row("c", "d"), "9.00", Seq(Row("3","4"), Row("33","44"))) ) val rows1Rdd = spark.sparkContext.parallelize(rows1, 4) val schema1 = StructType( Seq( StructField("id", StringType, true), StructField("s1", StructType( Seq( StructField("x", StringType, true), StructField("y", StringType, true) ) ), true), StructField("d", StringType, true), StructField("s2", ArrayType(StructType( Seq( StructField("u", StringType, true), StructField("v", StringType, true) ) )), true) ) ) val df1 = spark.createDataFrame(rows1Rdd, schema1)

      create a dataframe schema

    1. df = df.withColumn( "person", struct( $"person.*", struct( lit("value_1").as("person_field_1"), lit("value_2").as("person_field_2"), ).as("nested_column_within_person") ) )

      Example code for adding a complex structure into a nested column

  2. Aug 2021
  3. Oct 2019
  4. Sep 2019
  5. May 2019
  6. Sep 2017
  7. Aug 2017
  8. Jun 2017
  9. May 2017
  10. Apr 2017