«Serialization»

C2AB 53 65 72 69 61 6C 69 7A 61 74 69 6F 6E C2BB

Miki Tebeka

Selection Criteria

General Health

The Lindy Effect

The Lindy effect is a theory that the future life expectancy of some non-perishable things like a technology or an idea is proportional to their current age, so that every additional period of survival implies a longer remaining life expectancy. Where the Lindy effect applies, mortality rate decreases with time.

Language Support

Types

i := complex(0, 1)
_, err := json.Marshal(i)
fmt.Println(err) // json: unsupported type: complex128

data, _ := json.Marshal(time.Now())
var out interface{}
json.Unmarshal(data, &out)
fmt.Printf("%T\n", out) // string

n := 20
data, _ = json.Marshal(n)
json.Unmarshal(data, &out)
fmt.Printf("%T\n", out) // float64

Schema

DATE         SNOW  TMAX  TMIN  PGTM
2000-01-01   0     100    11   1337
2000-01-02   0     156    61   2313
2000-01-03   0     178   106    320
2000-01-04   0     156    78   1819
2000-01-05   0      83   -17    843
2000-07-16   8     312   245    937

Performance

"The energy needed to refute benchmarks is order of magnitude bigger than to run them." - Brendan Gregg

Security

Streaming

Standard Library

$ go mod tidy

Our Software Dependency Problem

Formats

gob

t1 := Trade{
	Time:   time.Now(),
	Symbol: "BRK-A",
	Volume: 1,
	Price:  321_801.07,
	IsBuy:  true,
}

var network bytes.Buffer
gob.NewEncoder(&network).Encode(t1)
var t2 Trade
gob.NewDecoder(&network).Decode(&t2)
fmt.Printf("%+v\n", t2) // {Time:2019-11-25 12:41:04.41768293 +0200 IST Symbol:BRK-A Volume:1 Price:321801.07 IsBuy:true}

JSON

{
  "time": "2019-11-25T12:27:16.6051444+02:00",
  "symbol": "BRK-A",
  "volume": 1,
  "price": 321801.07,
  "buy": true
}

YAML

time: 2019-11-25T12:52:28.129106987+02:00
symbol: "BRK-A"
volume: 1
price: 321801.07
buy: true

TOML

time = 2019-11-25T10:59:25Z
symbol = "BRK-A"
volume = 1
price = 321801.07
buy = true

XML

<?xml version="1.0"?>
<trade>
  <time>2019-11-25T13:01:04.592473082+02:00</time>
  <symbol>BRK-A</symbol>
  <price>321801.07</price>
  <volume>1</volume>
  <buy>true</buy>
</trade>

CSV

time,symbol,volume,price,buy
2019-11-25T13:01:04.592473082+02:00,BRK-A,1,321801.07,true

msgpack

BSON

protobuf

syntax = "proto3";
package pb;
import "google/protobuf/timestamp.proto";


message Trade {
    google.protobuf.Timestamp time = 1;
    string symbol = 2;
    int64 volume = 3;
    double price = 4;
    bool buy = 5;
}

flatbuffers

capnproto

SQL

CREATE TABLE trades (
    time TIMESTAMP,
    symbol VARCHAR(32),
    volume INTEGER,
    price FLOAT,
    buy BOOLEAN
);

parquet

orc

HDF5

Arrow

...

Invent Your Own

JSON Tips

Struct Tags

type User struct {
	Login  string   `json:"login"`
	Name   string   `json:"name,omitempty"`
	Groups []string `json:"groups"`
	Token  string   `json:"-"`
}
// ...
u := User{
	Login:  "bugs",
	Groups: []string{"wheel", "network", "lp", "daffy"},
	Token:  "duck season",
}
json.NewEncoder(os.Stdout).Encode(u)
// {"login":"bugs","groups":["wheel","network","lp","daffy"]}

protobuf generated

type Trade struct {
	Time                 *timestamp.Timestamp `protobuf:"bytes,1,opt,name=time,proto3" json:"time,omitempty"`
	Symbol               string               `protobuf:"bytes,2,opt,name=symbol,proto3" json:"symbol,omitempty"`
	Volume               int64                `protobuf:"varint,3,opt,name=volume,proto3" json:"volume,omitempty"`
	Price                float64              `protobuf:"fixed64,4,opt,name=price,proto3" json:"price,omitempty"`
	Buy                  bool                 `protobuf:"varint,5,opt,name=buy,proto3" json:"buy,omitempty"`
	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
	XXX_unrecognized     []byte               `json:"-"`
	XXX_sizecache        int32                `json:"-"`
}
			

Streaming

type Payment struct {
	Name   string
	Amount float64
}

var data = `{"name": "bugs", "amount": 12.3}{"name": "daffy", "amount": 13.7}`

	total := 0.0
	r := strings.NewReader(data)
	dec := json.NewDecoder(r)
loop:
	for {
		var p Payment
		err := dec.Decode(&p)
		switch err {
		case nil:
			total += p.Amount
		case io.EOF:
			break loop
		default:
			log.Fatal(err)
		}
	}

	fmt.Println("total:", total)

json.Marshaller

type List struct {
	Value string
	Next  *List
}

func (l *List) MarshalJSON() ([]byte, error) {
	var values []string
	for ; l != nil; l = l.Next {
		values = append(values, l.Value)
	}
	return json.Marshal(values)
}

cart := NewList("onions")
cart.Append("steak")
cart.Append("potatos")

out, _ := json.Marshal(cart)
fmt.Println(string(out)) // ["onions","steak","potatos"]
			

json.Unmarshaller

func (l *List) UnmarshalJSON(data []byte) error {
	var values []string
	if err := json.Unmarshal(data, &values); err != nil {
		return err
	}

	if len(values) == 0 {
		return fmt.Errorf("empty list")
	}

	l.Value = values[0]
	for _, v := range values[1:] {
		l.Append(v)
	}
	return nil
}

var user struct {
	Name string
	Cart *List
}

data := []byte(`{
	"name": "Wile E. Coyote",
	"cart": ["canon", "brick", "gun powder"]
}`)
json.Unmarshal(data, &user)
fmt.Printf("%s\n", user.Cart) // canon → brick → gun powder

Missing Data

type Job struct {
	User   string
	Action string
	Count  int
}

func main() {
	data := []byte(`{
		"user": "saitama",
		"action": "punch"
	}`)
	var job Job
	json.Unmarshal(data, &job)
	fmt.Printf("%+v\n", job)
	// {User:saitama Action:punch Count:0}
}

Solution - Pointers

type Job struct {
	User   string
	Action string
	JCount *int `json:"count"`
}

func (j *Job) Count() int {
	if j.JCount == nil {
		return 1
	}
	return *j.JCount
}

func main() {
	data := []byte(`{
		"user": "saitama",
		"action": "punch"
	}`)

	var job Job
	json.Unmarshal(data, &job)
	fmt.Println(job.Count()) // 1
}

%#v

fmt.Printf("%#v\n", t)

/*
	main.Trade{
		Time:time.Time{wall:0xbf6f0bebf6db79e5, ext:64591, loc:(*time.Location)(0x570b60)},
		Symbol:"BRK-A",
		Volume:1,
		Price:321801.07,
		IsBuy:true
	}
*/

Go

go.353solutions.com/config

package config

// Config holds configuration fields
type Config struct {
	MaxWorkers  int
	Port        int
	AuthServers []string
}

config.go

import "go.353solutions.com/config"

var Config = config.Config{
	Port: 9091,
	AuthServers : []string{"auth1.local", "auth2.local"},
}

func init() {
	if runtime.GOOS == "windows" {
		Config.MaxWorkers = 100
	} else {
		Config.MaxWorkers = 1000
	}
}

app.go

plug, _ := plugin.Open("config.so")
v, _ := plug.Lookup("Config")
cfg := v.(*config.Config)
fmt.Printf("%+v\n", cfg)
// &{MaxWorkers:1000 Port:9091 AuthServers:[auth1.local auth2.local]}