如何将json对象映射到apache spark中的java对象

Question

我有以下 json 对象，我想将此 json 映射到 java 对象中，请帮助我。

{
  "address": {
    "building": "1007",
    "coord": [
      -73.856077,
      40.848447
    ],
    "street": "Morris Park Ave",
    "zipcode": "10462"
  },
  "borough": "Bronx",
  "cuisine": "Bakery",
  "grades": [
    {
      "date": {
        "$date": 1393804800000
      },
      "grade": "A",
      "score": 2
    },
    {
      "date": {
        "$date": 1378857600000
      },
      "grade": "A",
      "score": 6
    },
    {
      "date": {
        "$date": 1358985600000
      },
      "grade": "A",
      "score": 10
    },
    {
      "date": {
        "$date": 1322006400000
      },
      "grade": "A",
      "score": 9
    },
    {
      "date": {
        "$date": 1299715200000
      },
      "grade": "B",
      "score": 14
    }
  ],
  "name": "Morris Park Bake Shop",
  "restaurant_id": "30075445"
}

当我使用 scala printschema 方法时，它显示如下，请帮助我将此模式映射到 java 对象。

root
 |-- address: struct (nullable = true)
 |    |-- building: string (nullable = true)
 |    |-- coord: array (nullable = true)
 |    |    |-- element: double (containsNull = true)
 |    |-- street: string (nullable = true)
 |    |-- zipcode: string (nullable = true)
 |-- borough: string (nullable = true)
 |-- cuisine: string (nullable = true)
 |-- grades: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- date: struct (nullable = true)
 |    |    |    |-- $date: long (nullable = true)
 |    |    |-- grade: string (nullable = true)
 |    |    |-- score: long (nullable = true)
 |-- name: string (nullable = true)
 |-- restaurant_id: string (nullable = true)

Answer 1

如果你想将 json 字符串转换为 java 你可以使用 Gson..

Gson g = new Gson(); yourvenillabean p = g.fromJson(jsonString, yourvanillabean.class)

在此处查看完整示例

阅读更多here

---

因为你添加了火花标签如果我是对的，如果你想创建一个数据框，下面是火花处理 json 的方式......

您可以将此 json 转换为 DataFrame，如下所示

package examples;


import org.apache.spark.SparkConf;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SparkSession;

import java.util.Arrays;
import java.util.List;

public class JsonDF {

    SparkSession sparkSession;

    public static void main(String[] args) {


 SparkConf conf = new SparkConf();
        conf.setMaster("local[2]");


        SparkSession spark = SparkSession
                .builder()
                .appName("Dataset-json")
                .master("local[4]")
                .getOrCreate();
        System.out.println("Session created");
        String str = "{\n" +
                "  \"address\": {\n" +
                "    \"building\": \"1007\",\n" +
                "    \"coord\": [\n" +
                "      -73.856077,\n" +
                "      40.848447\n" +
                "    ],\n" +
                "    \"street\": \"Morris Park Ave\",\n" +
                "    \"zipcode\": \"10462\"\n" +
                "  },\n" +
                "  \"borough\": \"Bronx\",\n" +
                "  \"cuisine\": \"Bakery\",\n" +
                "  \"grades\": [\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1393804800000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 2\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1378857600000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 6\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1358985600000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 10\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1322006400000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 9\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1299715200000\n" +
                "      },\n" +
                "      \"grade\": \"B\",\n" +
                "      \"score\": 14\n" +
                "    }\n" +
                "  ],\n" +
                "  \"name\": \"Morris Park Bake Shop\",\n" +
                "  \"restaurant_id\": \"30075445\"\n" +
                "}";
        List<String> data = Arrays.asList(str);

        Dataset<String> ds = spark.createDataset(data, Encoders.STRING());
        spark.read().json(ds).show(false);


    }
}

结果：

+-------------------------------------------------------+-------+-------+-------------------------------------------------------------------------------------------------------------------------------+---------------------+-------------+
|address                                                |borough|cuisine|grades                                                                                                                         |name                 |restaurant_id|
+-------------------------------------------------------+-------+-------+-------------------------------------------------------------------------------------------------------------------------------+---------------------+-------------+
|[1007, [-73.856077, 40.848447], Morris Park Ave, 10462]|Bronx  |Bakery |[[[1393804800000], A, 2], [[1378857600000], A, 6], [[1358985600000], A, 10], [[1322006400000], A, 9], [[1299715200000], B, 14]]|Morris Park Bake Shop|30075445     |
+-------------------------------------------------------+-------+-------+-------------------------------------------------------------------------------------------------------------------------------+---------------------+-------------+

更新：

由于您想将 json 转换为 java 类，您可以使用 this 来处理您的 json 字符串...

转换后可以得到这样的类：

public class Application {
  Address AddressObject;
  private String borough;
  private String cuisine;
  ArrayList<Object> grades = new ArrayList<Object>();
  private String name;
  private String restaurant_id;


 // Getter Methods 

  public Address getAddress() {
    return AddressObject;
  }

  public String getBorough() {
    return borough;
  }

  public String getCuisine() {
    return cuisine;
  }

  public String getName() {
    return name;
  }

  public String getRestaurant_id() {
    return restaurant_id;
  }

 // Setter Methods 

  public void setAddress( Address addressObject ) {
    this.AddressObject = addressObject;
  }

  public void setBorough( String borough ) {
    this.borough = borough;
  }

  public void setCuisine( String cuisine ) {
    this.cuisine = cuisine;
  }

  public void setName( String name ) {
    this.name = name;
  }

  public void setRestaurant_id( String restaurant_id ) {
    this.restaurant_id = restaurant_id;
  }
}
public class Address {
  private String building;
  ArrayList<Object> coord = new ArrayList<Object>();
  private String street;
  private String zipcode;


 // Getter Methods 

  public String getBuilding() {
    return building;
  }

  public String getStreet() {
    return street;
  }

  public String getZipcode() {
    return zipcode;
  }

 // Setter Methods 

  public void setBuilding( String building ) {
    this.building = building;
  }

  public void setStreet( String street ) {
    this.street = street;
  }

  public void setZipcode( String zipcode ) {
    this.zipcode = zipcode;
  }
}

这与你得到的火花模式兼容..

root
 |-- address: struct (nullable = true)
 |    |-- building: string (nullable = true)
 |    |-- coord: array (nullable = true)
 |    |    |-- element: double (containsNull = true)
 |    |-- street: string (nullable = true)
 |    |-- zipcode: string (nullable = true)
 |-- borough: string (nullable = true)
 |-- cuisine: string (nullable = true)
 |-- grades: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- date: struct (nullable = true)
 |    |    |    |-- $date: long (nullable = true)
 |    |    |-- grade: string (nullable = true)
 |    |    |-- score: long (nullable = true)
 |-- name: string (nullable = true)
 |-- restaurant_id: string (nullable = true)

这里是How to create a Dataframe in spark that contains complex Java Beans的例子

Answer 2

我想目的是从 json String 创建 Java 对象。它会很长，但很容易实现。 该解决方案基于 Jackson API。ObjectMapper 是用于数据绑定的主要 api。

第 1 步： 如果您使用 pom.xml，请在 maven 依赖项中包含 jackson，否则从 maven 存储库https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core下载相应的 jar 文件

<dependencies>
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>2.10.1</version>
    </dependency>
  </dependencies>

第 2 步：创建 POJO 类。这些对象将从 JSON 创建。

我们需要下面的 POJO，并将采用下面的结构，

         AddressDetails  
           Address
           Grades  
             Grade   
               Date

我将添加 POJO 自下而上的代码。

日期：

package org.personal.TestProject.jsontoobject;

public class Date {
    private float $date;

    // Getter Methods
    public float get$date() {
        return $date;
    }

    // Setter Methods
    public void set$date(float $date) {
        this.$date = $date;
    }
}

年级：

package org.personal.TestProject.jsontoobject;

    public class Grade {

        Date date = new Date();
        private String grade;
        private float score;

        // Getter Methods
        public float getScore() {
            return score;
        }
        public String getGrade() {
            return grade;
        }
        public Date getDate() {
            return date;
        }

        // Setter Methods
        public void setScore(float score) {
            this.score = score;
        }
        public void setGrade(String grade) {
            this.grade = grade;
        }

        public void setDate(Date date) {
            this.date = date;
        }
    }

成绩：

package org.personal.TestProject.jsontoobject;

import java.util.ArrayList;
import java.util.List;
import org.personal.TestProject.jsontoobject.Grade;

public class Grades {

    private List<Grade> grades = new ArrayList<Grade>();

    public List<Grade> getGrades() {
        return grades;
    }

    public void setGrades(List<Grade> grades) {
        this.grades = grades;
    }
}

地址：

package org.personal.TestProject.jsontoobject;
import java.util.ArrayList;
import java.util.List;

public class Address {
    private String building;
    List<Float> coord = new ArrayList<Float>();
    private String street;
    private String zipcode;


    // Getter Methods

    public String getBuilding() {
        return building;
    }
    public List<Float> getCoords() {
        return coord;
    }

    public String getStreet() {
        return street;
    }
    public String getZipcode() {
        return zipcode;
    }

    // Setter Methods
    public void setBuilding(String building) {
        this.building = building;
    }
    public void setCoord(List<Float> coord) {
        this.coord = coord;
    }
    public void setStreet(String street) {
        this.street = street;
    }
    public void setZipcode(String zipcode) {
        this.zipcode = zipcode;
    }
}

详细地址：

package org.personal.TestProject.jsontoobject;

import java.util.ArrayList;
import java.util.List;
import org.personal.TestProject.jsontoobject.Grade;
import org.personal.TestProject.jsontoobject.Address;

public class AddressDetails {

    Address address = new Address();
    private String borough;
    private String cuisine;
    List<Grade> grades = new ArrayList<Grade>();
    private String name;
    private String restaurant_id;

    //Setters
    public List<Grade> getGrades() {
        return grades;
    }
    public Address getAddress() {
        return address;
    }
    public String getBorough() {
        return borough;
    }
    public String getCuisine() {
        return cuisine;
    }
    public String getName() {
        return name;
    }
    public String getRestaurant_id() {
        return restaurant_id;
    }

    //Getters
    public void setGrades(List<Grade> grades) {
        this.grades = grades;
    }
    public void setAddress(Address address) {
        this.address = address;
    }
    public void setBorough(String borough) {
        this.borough = borough;
    }  
    public void setCuisine(String cuisine) {
        this.cuisine = cuisine;
    }    
    public void setName(String name) {
        this.name = name;
    }   
    public void setRestaurant_id(String restaurant_id) {
        this.restaurant_id = restaurant_id;
    }
}

第 3 步：我们使用 ObjectMapper 将 json（文件或字符串）转换为 Object。现在您可以根据需要使用 getter 访问对象中的字段。

package org.personal.TestProject.jsontoobject;

        import java.io.File;
        import java.util.ArrayList;
        import java.util.List;

        import com.fasterxml.jackson.databind.ObjectMapper;
        import org.personal.TestProject.jsontoobject.AddressDetails;

public class ReadJsonFile {

    public static void main(String args[]){

        ObjectMapper mapper = new ObjectMapper();

        String json = "{\n" +
                "  \"address\": {\n" +
                "    \"building\": \"1007\",\n" +
                "    \"coord\": [\n" +
                "      -73.856077,\n" +
                "      40.848447\n" +
                "    ],\n" +
                "    \"street\": \"Morris Park Ave\",\n" +
                "    \"zipcode\": \"10462\"\n" +
                "  },\n" +
                "  \"borough\": \"Bronx\",\n" +
                "  \"cuisine\": \"Bakery\",\n" +
                "  \"grades\": [\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1393804800000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 2\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1378857600000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 6\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1358985600000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 10\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1322006400000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 9\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1299715200000\n" +
                "      },\n" +
                "      \"grade\": \"B\",\n" +
                "      \"score\": 14\n" +
                "    }\n" +
                "  ],\n" +
                "  \"name\": \"Morris Park Bake Shop\",\n" +
                "  \"restaurant_id\": \"30075445\"\n" +
                "}";

        AddressDetails value = null;
        try {
            value = mapper.readValue(json, AddressDetails.class);
             // For converting a json file to object use below code
            //value = mapper.readValue(new File("result.json"), AddressDetails.class);
            // Use getter methods to Access any fields on the object
            System.out.println("Restaurant ID " +value.getRestaurant_id());
        } catch (Exception e) {
            e.printStackTrace();
        }     
    }

}