jsonpb: add option to ignore unknown fields in a message I added a new Unmarshaler type for symmetry with Marshaler.
diff --git a/jsonpb/jsonpb.go b/jsonpb/jsonpb.go index 6308548..e40d1d3 100644 --- a/jsonpb/jsonpb.go +++ b/jsonpb/jsonpb.go
@@ -510,41 +510,63 @@ return out.err } +// Unmarshaler is a configurable object for converting from a JSON +// representation to a protocol buffer object. +type Unmarshaler struct { + // Whether to allow messages to contain unknown fields, as opposed to + // failing to unmarshal. + AllowUnknownFields bool +} + // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. // This function is lenient and will decode any options permutations of the // related Marshaler. -func UnmarshalNext(dec *json.Decoder, pb proto.Message) error { +func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error { inputValue := json.RawMessage{} if err := dec.Decode(&inputValue); err != nil { return err } - return unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil) + return u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil) +} + +// Unmarshal unmarshals a JSON object stream into a protocol +// buffer. This function is lenient and will decode any options +// permutations of the related Marshaler. +func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error { + dec := json.NewDecoder(r) + return u.UnmarshalNext(dec, pb) +} + +// UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. +// This function is lenient and will decode any options permutations of the +// related Marshaler. +func UnmarshalNext(dec *json.Decoder, pb proto.Message) error { + return new(Unmarshaler).UnmarshalNext(dec, pb) } // Unmarshal unmarshals a JSON object stream into a protocol // buffer. This function is lenient and will decode any options // permutations of the related Marshaler. func Unmarshal(r io.Reader, pb proto.Message) error { - dec := json.NewDecoder(r) - return UnmarshalNext(dec, pb) + return new(Unmarshaler).Unmarshal(r, pb) } // UnmarshalString will populate the fields of a protocol buffer based // on a JSON string. This function is lenient and will decode any options // permutations of the related Marshaler. func UnmarshalString(str string, pb proto.Message) error { - return Unmarshal(strings.NewReader(str), pb) + return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb) } // unmarshalValue converts/copies a value into the target. // prop may be nil. -func unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error { +func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error { targetType := target.Type() // Allocate memory for pointer fields. if targetType.Kind() == reflect.Ptr { target.Set(reflect.New(targetType.Elem())) - return unmarshalValue(target.Elem(), inputValue, prop) + return u.unmarshalValue(target.Elem(), inputValue, prop) } // Handle well-known types. @@ -559,7 +581,7 @@ // as the wrapped primitive type, except that null is allowed." // encoding/json will turn JSON `null` into Go `nil`, // so we don't have to do any extra work. - return unmarshalValue(target.Field(0), inputValue, prop) + return u.unmarshalValue(target.Field(0), inputValue, prop) case "Any": return fmt.Errorf("unmarshaling Any not supported yet") case "Duration": @@ -657,7 +679,7 @@ continue } - if err := unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil { + if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil { return err } } @@ -670,12 +692,12 @@ } nv := reflect.New(oop.Type.Elem()) target.Field(oop.Field).Set(nv) - if err := unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil { + if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil { return err } } } - if len(jsonFields) > 0 { + if !u.AllowUnknownFields && len(jsonFields) > 0 { // Pick any field to be the scapegoat. var f string for fname := range jsonFields { @@ -696,7 +718,7 @@ len := len(slc) target.Set(reflect.MakeSlice(targetType, len, len)) for i := 0; i < len; i++ { - if err := unmarshalValue(target.Index(i), slc[i], prop); err != nil { + if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil { return err } } @@ -725,14 +747,14 @@ k = reflect.ValueOf(ks) } else { k = reflect.New(targetType.Key()).Elem() - if err := unmarshalValue(k, json.RawMessage(ks), keyprop); err != nil { + if err := u.unmarshalValue(k, json.RawMessage(ks), keyprop); err != nil { return err } } // Unmarshal map value. v := reflect.New(targetType.Elem()).Elem() - if err := unmarshalValue(v, raw, valprop); err != nil { + if err := u.unmarshalValue(v, raw, valprop); err != nil { return err } target.SetMapIndex(k, v)
diff --git a/jsonpb/jsonpb_test.go b/jsonpb/jsonpb_test.go index 659dbed..8cbc8ec 100644 --- a/jsonpb/jsonpb_test.go +++ b/jsonpb/jsonpb_test.go
@@ -36,6 +36,7 @@ "encoding/json" "io" "reflect" + "strings" "testing" "github.com/golang/protobuf/proto" @@ -410,66 +411,69 @@ } var unmarshalingTests = []struct { - desc string - json string - pb proto.Message + desc string + unmarshaler Unmarshaler + json string + pb proto.Message }{ - {"simple flat object", simpleObjectJSON, simpleObject}, - {"simple pretty object", simpleObjectPrettyJSON, simpleObject}, - {"repeated fields flat object", repeatsObjectJSON, repeatsObject}, - {"repeated fields pretty object", repeatsObjectPrettyJSON, repeatsObject}, - {"nested message/enum flat object", complexObjectJSON, complexObject}, - {"nested message/enum pretty object", complexObjectPrettyJSON, complexObject}, - {"enum-string object", `{"color":"BLUE"}`, &pb.Widget{Color: pb.Widget_BLUE.Enum()}}, - {"enum-value object", "{\n \"color\": 2\n}", &pb.Widget{Color: pb.Widget_BLUE.Enum()}}, - {"proto3 enum string", `{"hilarity":"PUNS"}`, &proto3pb.Message{Hilarity: proto3pb.Message_PUNS}}, - {"proto3 enum value", `{"hilarity":1}`, &proto3pb.Message{Hilarity: proto3pb.Message_PUNS}}, + {"simple flat object", Unmarshaler{}, simpleObjectJSON, simpleObject}, + {"simple pretty object", Unmarshaler{}, simpleObjectPrettyJSON, simpleObject}, + {"repeated fields flat object", Unmarshaler{}, repeatsObjectJSON, repeatsObject}, + {"repeated fields pretty object", Unmarshaler{}, repeatsObjectPrettyJSON, repeatsObject}, + {"nested message/enum flat object", Unmarshaler{}, complexObjectJSON, complexObject}, + {"nested message/enum pretty object", Unmarshaler{}, complexObjectPrettyJSON, complexObject}, + {"enum-string object", Unmarshaler{}, `{"color":"BLUE"}`, &pb.Widget{Color: pb.Widget_BLUE.Enum()}}, + {"enum-value object", Unmarshaler{}, "{\n \"color\": 2\n}", &pb.Widget{Color: pb.Widget_BLUE.Enum()}}, + {"unknown field with allowed option", Unmarshaler{AllowUnknownFields: true}, `{"unknown": "foo"}`, new(pb.Simple)}, + {"proto3 enum string", Unmarshaler{}, `{"hilarity":"PUNS"}`, &proto3pb.Message{Hilarity: proto3pb.Message_PUNS}}, + {"proto3 enum value", Unmarshaler{}, `{"hilarity":1}`, &proto3pb.Message{Hilarity: proto3pb.Message_PUNS}}, {"unknown enum value object", + Unmarshaler{}, "{\n \"color\": 1000,\n \"r_color\": [\n \"RED\"\n ]\n}", &pb.Widget{Color: pb.Widget_Color(1000).Enum(), RColor: []pb.Widget_Color{pb.Widget_RED}}}, - {"repeated proto3 enum", `{"rFunny":["PUNS","SLAPSTICK"]}`, + {"repeated proto3 enum", Unmarshaler{}, `{"rFunny":["PUNS","SLAPSTICK"]}`, &proto3pb.Message{RFunny: []proto3pb.Message_Humour{ proto3pb.Message_PUNS, proto3pb.Message_SLAPSTICK, }}}, - {"repeated proto3 enum as int", `{"rFunny":[1,2]}`, + {"repeated proto3 enum as int", Unmarshaler{}, `{"rFunny":[1,2]}`, &proto3pb.Message{RFunny: []proto3pb.Message_Humour{ proto3pb.Message_PUNS, proto3pb.Message_SLAPSTICK, }}}, - {"repeated proto3 enum as mix of strings and ints", `{"rFunny":["PUNS",2]}`, + {"repeated proto3 enum as mix of strings and ints", Unmarshaler{}, `{"rFunny":["PUNS",2]}`, &proto3pb.Message{RFunny: []proto3pb.Message_Humour{ proto3pb.Message_PUNS, proto3pb.Message_SLAPSTICK, }}}, - {"unquoted int64 object", `{"oInt64":-314}`, &pb.Simple{OInt64: proto.Int64(-314)}}, - {"unquoted uint64 object", `{"oUint64":123}`, &pb.Simple{OUint64: proto.Uint64(123)}}, - {"map<int64, int32>", `{"nummy":{"1":2,"3":4}}`, &pb.Mappy{Nummy: map[int64]int32{1: 2, 3: 4}}}, - {"map<string, string>", `{"strry":{"\"one\"":"two","three":"four"}}`, &pb.Mappy{Strry: map[string]string{`"one"`: "two", "three": "four"}}}, - {"map<int32, Object>", `{"objjy":{"1":{"dub":1}}}`, &pb.Mappy{Objjy: map[int32]*pb.Simple3{1: &pb.Simple3{Dub: 1}}}}, + {"unquoted int64 object", Unmarshaler{}, `{"oInt64":-314}`, &pb.Simple{OInt64: proto.Int64(-314)}}, + {"unquoted uint64 object", Unmarshaler{}, `{"oUint64":123}`, &pb.Simple{OUint64: proto.Uint64(123)}}, + {"map<int64, int32>", Unmarshaler{}, `{"nummy":{"1":2,"3":4}}`, &pb.Mappy{Nummy: map[int64]int32{1: 2, 3: 4}}}, + {"map<string, string>", Unmarshaler{}, `{"strry":{"\"one\"":"two","three":"four"}}`, &pb.Mappy{Strry: map[string]string{`"one"`: "two", "three": "four"}}}, + {"map<int32, Object>", Unmarshaler{}, `{"objjy":{"1":{"dub":1}}}`, &pb.Mappy{Objjy: map[int32]*pb.Simple3{1: &pb.Simple3{Dub: 1}}}}, // TODO: This is broken. - //{"map<string, enum>", `{"enumy":{"XIV":"ROMAN"}`, &pb.Mappy{Enumy: map[string]pb.Numeral{"XIV": pb.Numeral_ROMAN}}}, - {"map<string, enum as int>", `{"enumy":{"XIV":2}}`, &pb.Mappy{Enumy: map[string]pb.Numeral{"XIV": pb.Numeral_ROMAN}}}, - {"oneof", `{"salary":31000}`, &pb.MsgWithOneof{Union: &pb.MsgWithOneof_Salary{31000}}}, - {"oneof spec name", `{"country":"Australia"}`, &pb.MsgWithOneof{Union: &pb.MsgWithOneof_Country{"Australia"}}}, - {"oneof orig_name", `{"Country":"Australia"}`, &pb.MsgWithOneof{Union: &pb.MsgWithOneof_Country{"Australia"}}}, - {"orig_name input", `{"o_bool":true}`, &pb.Simple{OBool: proto.Bool(true)}}, - {"camelName input", `{"oBool":true}`, &pb.Simple{OBool: proto.Bool(true)}}, + //{"map<string, enum>", Unmarshaler{}, `{"enumy":{"XIV":"ROMAN"}`, &pb.Mappy{Enumy: map[string]pb.Numeral{"XIV": pb.Numeral_ROMAN}}}, + {"map<string, enum as int>", Unmarshaler{}, `{"enumy":{"XIV":2}}`, &pb.Mappy{Enumy: map[string]pb.Numeral{"XIV": pb.Numeral_ROMAN}}}, + {"oneof", Unmarshaler{}, `{"salary":31000}`, &pb.MsgWithOneof{Union: &pb.MsgWithOneof_Salary{31000}}}, + {"oneof spec name", Unmarshaler{}, `{"country":"Australia"}`, &pb.MsgWithOneof{Union: &pb.MsgWithOneof_Country{"Australia"}}}, + {"oneof orig_name", Unmarshaler{}, `{"Country":"Australia"}`, &pb.MsgWithOneof{Union: &pb.MsgWithOneof_Country{"Australia"}}}, + {"orig_name input", Unmarshaler{}, `{"o_bool":true}`, &pb.Simple{OBool: proto.Bool(true)}}, + {"camelName input", Unmarshaler{}, `{"oBool":true}`, &pb.Simple{OBool: proto.Bool(true)}}, - {"Duration", `{"dur":"3.000s"}`, &pb.KnownTypes{Dur: &durpb.Duration{Seconds: 3}}}, - {"Timestamp", `{"ts":"2014-05-13T16:53:20.021Z"}`, &pb.KnownTypes{Ts: &tspb.Timestamp{Seconds: 14e8, Nanos: 21e6}}}, + {"Duration", Unmarshaler{}, `{"dur":"3.000s"}`, &pb.KnownTypes{Dur: &durpb.Duration{Seconds: 3}}}, + {"Timestamp", Unmarshaler{}, `{"ts":"2014-05-13T16:53:20.021Z"}`, &pb.KnownTypes{Ts: &tspb.Timestamp{Seconds: 14e8, Nanos: 21e6}}}, - {"DoubleValue", `{"dbl":1.2}`, &pb.KnownTypes{Dbl: &wpb.DoubleValue{Value: 1.2}}}, - {"FloatValue", `{"flt":1.2}`, &pb.KnownTypes{Flt: &wpb.FloatValue{Value: 1.2}}}, - {"Int64Value", `{"i64":"-3"}`, &pb.KnownTypes{I64: &wpb.Int64Value{Value: -3}}}, - {"UInt64Value", `{"u64":"3"}`, &pb.KnownTypes{U64: &wpb.UInt64Value{Value: 3}}}, - {"Int32Value", `{"i32":-4}`, &pb.KnownTypes{I32: &wpb.Int32Value{Value: -4}}}, - {"UInt32Value", `{"u32":4}`, &pb.KnownTypes{U32: &wpb.UInt32Value{Value: 4}}}, - {"BoolValue", `{"bool":true}`, &pb.KnownTypes{Bool: &wpb.BoolValue{Value: true}}}, - {"StringValue", `{"str":"plush"}`, &pb.KnownTypes{Str: &wpb.StringValue{Value: "plush"}}}, - {"BytesValue", `{"bytes":"d293"}`, &pb.KnownTypes{Bytes: &wpb.BytesValue{Value: []byte("wow")}}}, + {"DoubleValue", Unmarshaler{}, `{"dbl":1.2}`, &pb.KnownTypes{Dbl: &wpb.DoubleValue{Value: 1.2}}}, + {"FloatValue", Unmarshaler{}, `{"flt":1.2}`, &pb.KnownTypes{Flt: &wpb.FloatValue{Value: 1.2}}}, + {"Int64Value", Unmarshaler{}, `{"i64":"-3"}`, &pb.KnownTypes{I64: &wpb.Int64Value{Value: -3}}}, + {"UInt64Value", Unmarshaler{}, `{"u64":"3"}`, &pb.KnownTypes{U64: &wpb.UInt64Value{Value: 3}}}, + {"Int32Value", Unmarshaler{}, `{"i32":-4}`, &pb.KnownTypes{I32: &wpb.Int32Value{Value: -4}}}, + {"UInt32Value", Unmarshaler{}, `{"u32":4}`, &pb.KnownTypes{U32: &wpb.UInt32Value{Value: 4}}}, + {"BoolValue", Unmarshaler{}, `{"bool":true}`, &pb.KnownTypes{Bool: &wpb.BoolValue{Value: true}}}, + {"StringValue", Unmarshaler{}, `{"str":"plush"}`, &pb.KnownTypes{Str: &wpb.StringValue{Value: "plush"}}}, + {"BytesValue", Unmarshaler{}, `{"bytes":"d293"}`, &pb.KnownTypes{Bytes: &wpb.BytesValue{Value: []byte("wow")}}}, // `null` is also a permissible value. Let's just test one. - {"null DoubleValue", `{"dbl":null}`, &pb.KnownTypes{Dbl: &wpb.DoubleValue{}}}, + {"null DoubleValue", Unmarshaler{}, `{"dbl":null}`, &pb.KnownTypes{Dbl: &wpb.DoubleValue{}}}, } func TestUnmarshaling(t *testing.T) { @@ -477,7 +481,7 @@ // Make a new instance of the type of our expected object. p := reflect.New(reflect.TypeOf(tt.pb).Elem()).Interface().(proto.Message) - err := UnmarshalString(tt.json, p) + err := tt.unmarshaler.Unmarshal(strings.NewReader(tt.json), p) if err != nil { t.Errorf("%s: %v", tt.desc, err) continue @@ -507,7 +511,7 @@ // Make a new instance of the type of our expected object. p := reflect.New(reflect.TypeOf(tt.pb).Elem()).Interface().(proto.Message) - err := UnmarshalNext(dec, p) + err := tt.unmarshaler.UnmarshalNext(dec, p) if err != nil { t.Errorf("%s: %v", tt.desc, err) continue @@ -522,7 +526,7 @@ } p := &pb.Simple{} - err := UnmarshalNext(dec, p) + err := new(Unmarshaler).UnmarshalNext(dec, p) if err != io.EOF { t.Errorf("eof: got %v, expected io.EOF", err) } @@ -535,6 +539,7 @@ }{ {"a value", "666", new(pb.Simple)}, {"gibberish", "{adskja123;l23=-=", new(pb.Simple)}, + {"unknown field", `{"unknown": "foo"}`, new(pb.Simple)}, {"unknown enum name", `{"hilarity":"DAVE"}`, new(proto3pb.Message)}, }