dataflow

Convert Java object to BigQuery TableRow


I am exploring Google Cloud Dataflow.

I was wondering if automatic conversion between java object or JSON to TableRow can be done.

Just like we can automatically parse JSON to POJO class.

I could not find relevant information. Hope not to duplicate question.

Will be grateful for any info!

Greetings


Solution

  • I've looking for examples for the same with no luck. I created a POJO class that almost match the schema of the bigquery table and matches the structure of the JSON objects that are the input for the pipeline. Finally, when I have to convert those objects to TableRow, for the nested and repeated values I made something like below, and the conversion was made by the API

        TableRow row = new TableRow()
                .set("items", c.element().getItems())
                .set("orderDate", c.element().getOrderDate())
                .set("orderNumber", c.element().getOrderNumber());
    

    Where Item class is part of the Order object :

    @JsonProperty("items")
    private List<Item> items = null;
    

    This is the code for Item class:

    import java.io.Serializable;
    import java.util.HashMap;
    import java.util.List;
    import java.util.Map;
    import com.fasterxml.jackson.annotation.JsonAnyGetter;
    import com.fasterxml.jackson.annotation.JsonAnySetter;
    import com.fasterxml.jackson.annotation.JsonIgnore;
    import com.fasterxml.jackson.annotation.JsonInclude;
    import com.fasterxml.jackson.annotation.JsonProperty;
    import com.fasterxml.jackson.annotation.JsonPropertyOrder;
    
    @JsonInclude(JsonInclude.Include.NON_NULL)
    @JsonPropertyOrder({
        "id",
        "code",
        "detail",
        "name",
        "shortName",
        "description",
        "sku",
        "quantity",
        "category",
        "products"
    })
    public class Item implements Serializable
    {
    
        @JsonProperty("id")
        private Integer id;
        @JsonProperty("code")
        private String code;
        @JsonProperty("detail")
        private String detail;
        @JsonProperty("name")
        private String name;
        @JsonProperty("shortName")
        private String shortName;
        @JsonProperty("description")
        private String description;
        @JsonProperty("sku")
        private String sku;
        @JsonProperty("quantity")
        private Integer quantity;
        @JsonProperty("category")
        private Category category;
        @JsonProperty("products")
        private List<Product> products = null;
        @JsonIgnore
        private Map<String, Object> additionalProperties = new HashMap<String, Object>();
        private final static long serialVersionUID = -5644586446669059821L;
    
        @JsonProperty("id")
        public Integer getId() {
            return id;
        }
    
        @JsonProperty("id")
        public void setId(Integer id) {
            this.id = id;
        }
    
        @JsonProperty("code")
        public String getCode() {
            return code;
        }
    
        @JsonProperty("code")
        public void setCode(String code) {
            this.code = code;
        }
    
        @JsonProperty("detail")
        public String getDetail() {
            return detail;
        }
    
        @JsonProperty("detail")
        public void setDetail(String detail) {
            this.detail = detail;
        }
    
        @JsonProperty("name")
        public String getName() {
            return name;
        }
    
        @JsonProperty("name")
        public void setName(String name) {
            this.name = name;
        }
    
        @JsonProperty("shortName")
        public String getShortName() {
            return shortName;
        }
    
        @JsonProperty("shortName")
        public void setShortName(String shortName) {
            this.shortName = shortName;
        }
    
        @JsonProperty("description")
        public String getDescription() {
            return description;
        }
    
        @JsonProperty("description")
        public void setDescription(String description) {
            this.description = description;
        }
    
        @JsonProperty("sku")
        public String getSku() {
            return sku;
        }
    
        @JsonProperty("sku")
        public void setSku(String sku) {
            this.sku = sku;
        }
    
        @JsonProperty("quantity")
        public Integer getQuantity() {
            return quantity;
        }
    
        @JsonProperty("quantity")
        public void setQuantity(Integer quantity) {
            this.quantity = quantity;
        }
    
        @JsonProperty("category")
        public Category getCategory() {
            return category;
        }
    
        @JsonProperty("category")
        public void setCategory(Category category) {
            this.category = category;
        }
    
        @JsonProperty("products")
        public List<Product> getProducts() {
            return products;
        }
    
    @JsonProperty("products")
    public void setProducts(List<Product> products) {
        this.products = products;
    }
    
    @JsonAnyGetter
    public Map<String, Object> getAdditionalProperties() {
        return this.additionalProperties;
    }
    
    @JsonAnySetter
    public void setAdditionalProperty(String name, Object value) {
        this.additionalProperties.put(name, value);
    }
    }
    

    And this is the schema of the BigQuery table in regards Items, where Item is a RECORD and REPEATED field and also contain a nested RECORD and REPEATED field: products. See the screenshot of the schema

    Item schema fields in BQ