Skip to content

Instantly share code, notes, and snippets.

@ebuildy

ebuildy/flatten.java

Last active December 22, 2024 03:01
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ebuildy/3de0e2855498e5358e4eed1a4f72ea48 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ebuildy/3de0e2855498e5358e4eed1a4f72ea48 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. ebuildy revised this gist Apr 11, 2017. No changes.
  2. ebuildy created this gist Apr 11, 2017.
    53 changes: 53 additions & 0 deletions flatten.java
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,53 @@
    class Toto
    {
    public void Main()
    {
    final DataFrame source = GetDataFrame();
    final String querySelectSQL = flattenSchema(source.schema(), null);

    source.registerTempTable("source");

    final DataFrame flattenData = sqlContext.sql("SELECT " + querySelectSQL + " FROM source")
    }

    /**
    * Generate SQL to select columns as flat.
    */
    public String flattenSchema(StructType schema, String prefix)
    {
    final StringBuilder selectSQLQuery = new StringBuilder();

    for (StructField field : schema.fields())
    {
    final String fieldName = field.name();

    if (fieldName.startsWith("@"))
    {
    continue;
    }

    String colName = prefix == null ? fieldName : (prefix + "." + fieldName);
    String colNameTarget = colName.replace(".", "_");

    if (field.dataType().getClass().equals(StructType.class))
    {
    selectSQLQuery.append(flattenSchema((StructType) field.dataType(), colName));
    }
    else
    {
    selectSQLQuery.append(colName);
    selectSQLQuery.append(" as ");
    selectSQLQuery.append(colNameTarget);
    }

    selectSQLQuery.append(",");
    }

    if (selectSQLQuery.length() > 0)
    {
    selectSQLQuery.deleteCharAt(selectSQLQuery.length() - 1);
    }

    return selectSQLQuery.toString();
    }
    }