diff options
| author | Reinhard Tartler <siretart@tauware.de> | 2019-01-09 07:11:17 -0500 |
|---|---|---|
| committer | Reinhard Tartler <siretart@tauware.de> | 2019-01-09 07:11:17 -0500 |
| commit | cf3930246e2e6db473f14d652f21fe25072cd2fe (patch) | |
| tree | 58f77bd258f380e7e5ca1c329039b0b479d3fde6 | |
New upstream version 0.0~git20181028.e517b90
68 files changed, 17448 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2b62c96 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +tests/go.stripe/ff/customer_ffjson.go +tests/goser/ff/goser_ffjson.go +tests/types/ff/everything_ffjson.go +tests/number/ff/number_ffjson.go +tests/ff_ffjson.go diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..7ddac45 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: go + +install: + - A=${PWD#*github.com/};A=${A%/ffjson};cd ../..;mv $A pquerna;cd pquerna/ffjson + - go get -d -v -t ./... + +script: make clean && make lint && make test && make test + +go: + - "1.10.x" + - "1.11.x" + +env: + - GO15VENDOREXPERIMENT=1 @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..88e68da --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ + +all: test install + @echo "Done" + +install: + go install github.com/pquerna/ffjson + +deps: + +fmt: + go fmt github.com/pquerna/ffjson/... + +cov: + # TODO: cleanup this make target. + mkdir -p coverage + rm -f coverage/*.html + # gocov test github.com/pquerna/ffjson/generator | gocov-html > coverage/generator.html + # gocov test github.com/pquerna/ffjson/inception | gocov-html > coverage/inception.html + gocov test github.com/pquerna/ffjson/fflib/v1 | gocov-html > coverage/fflib.html + @echo "coverage written" + +test-core: + go test -v github.com/pquerna/ffjson/fflib/v1 github.com/pquerna/ffjson/generator github.com/pquerna/ffjson/inception + +test: ffize test-core + go test -v github.com/pquerna/ffjson/tests/... + +ffize: install + ffjson -force-regenerate tests/ff.go + ffjson -force-regenerate tests/goser/ff/goser.go + ffjson -force-regenerate tests/go.stripe/ff/customer.go + ffjson -force-regenerate -reset-fields tests/types/ff/everything.go + ffjson -force-regenerate tests/number/ff/number.go + +lint: ffize + go get github.com/golang/lint/golint + golint --set_exit_status tests/... + +bench: ffize all + go test -v -benchmem -bench MarshalJSON github.com/pquerna/ffjson/tests + go test -v -benchmem -bench MarshalJSON github.com/pquerna/ffjson/tests/goser github.com/pquerna/ffjson/tests/go.stripe + go test -v -benchmem -bench UnmarshalJSON github.com/pquerna/ffjson/tests/goser github.com/pquerna/ffjson/tests/go.stripe + +clean: + go clean -i github.com/pquerna/ffjson/... + find . -name '*_ffjson.go' -delete + find . -name 'ffjson-inception*' -delete + +.PHONY: deps clean test fmt install all @@ -0,0 +1,8 @@ +ffjson +Copyright (c) 2014, Paul Querna + +This product includes software developed by +Paul Querna (http://paul.querna.org/). + +Portions of this software were developed as +part of Go, Copyright (c) 2012 The Go Authors.
\ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..30b239f --- /dev/null +++ b/README.md @@ -0,0 +1,232 @@ +# ffjson: faster JSON for Go + +[](https://travis-ci.org/pquerna/ffjson) + +`ffjson` generates static `MarshalJSON` and `UnmarshalJSON` functions for structures in Go. The generated functions reduce the reliance upon runtime reflection to do serialization and are generally 2 to 3 times faster. In cases where `ffjson` doesn't understand a Type involved, it falls back to `encoding/json`, meaning it is a safe drop in replacement. By using `ffjson` your JSON serialization just gets faster with no additional code changes. + +When you change your `struct`, you will need to run `ffjson` again (or make it part of your build tools). + +## Blog Posts + +* 2014-03-31: [First Release and Background](https://journal.paul.querna.org/articles/2014/03/31/ffjson-faster-json-in-go/) + +## Getting Started + +If `myfile.go` contains the `struct` types you would like to be faster, and assuming `GOPATH` is set to a reasonable value for an existing project (meaning that in this particular example if `myfile.go` is in the `myproject` directory, the project should be under `$GOPATH/src/myproject`), you can just run: + + go get -u github.com/pquerna/ffjson + ffjson myfile.go + git add myfile_ffjson.go + + +## Performance Status: + +* `MarshalJSON` is **2x to 3x** faster than `encoding/json`. +* `UnmarshalJSON` is **2x to 3x** faster than `encoding/json`. + +## Features + +* **Unmarshal Support:** Since v0.9, `ffjson` supports Unmarshaling of structures. +* **Drop in Replacement:** Because `ffjson` implements the interfaces already defined by `encoding/json` the performance enhancements are transparent to users of your structures. +* **Supports all types:** `ffjson` has native support for most of Go's types -- for any type it doesn't support with fast paths, it falls back to using `encoding/json`. This means all structures should work out of the box. If they don't, [open a issue!](https://github.com/pquerna/ffjson/issues) +* **ffjson: skip**: If you have a structure you want `ffjson` to ignore, add `ffjson: skip` to the doc string for this structure. +* **Extensive Tests:** `ffjson` contains an extensive test suite including fuzz'ing against the JSON parser. + + +# Using ffjson + +`ffjson` generates code based upon existing `struct` types. For example, `ffjson foo.go` will by default create a new file `foo_ffjson.go` that contains serialization functions for all structs found in `foo.go`. + +``` +Usage of ffjson: + + ffjson [options] [input_file] + +ffjson generates Go code for optimized JSON serialization. + + -go-cmd="": Path to go command; Useful for `goapp` support. + -import-name="": Override import name in case it cannot be detected. + -nodecoder: Do not generate decoder functions + -noencoder: Do not generate encoder functions + -w="": Write generate code to this path instead of ${input}_ffjson.go. +``` + +Your code must be in a compilable state for `ffjson` to work. If you code doesn't compile ffjson will most likely exit with an error. + +## Disabling code generation for structs + +You might not want all your structs to have JSON code generated. To completely disable generation for a struct, add `ffjson: skip` to the struct comment. For example: + +```Go +// ffjson: skip +type Foo struct { + Bar string +} +``` + +You can also choose not to have either the decoder or encoder generated by including `ffjson: nodecoder` or `ffjson: noencoder` in your comment. For instance, this will only generate the encoder (marshal) part for this struct: + +```Go +// ffjson: nodecoder +type Foo struct { + Bar string +} +``` + +You can also disable encoders/decoders entirely for a file by using the `-noencoder`/`-nodecoder` commandline flags. + +## Using ffjson with `go generate` + +`ffjson` is a great fit with `go generate`. It allows you to specify the ffjson command inside your individual go files and run them all at once. This way you don't have to maintain a separate build file with the files you need to generate. + +Add this comment anywhere inside your go files: + +```Go +//go:generate ffjson $GOFILE +``` + +To re-generate ffjson for all files with the tag in a folder, simply execute: + +```sh +go generate +``` + +To generate for the current package and all sub-packages, use: + +```sh +go generate ./... +``` +This is most of what you need to know about go generate, but you can sese more about [go generate on the golang blog](http://blog.golang.org/generate). + +## Should I include ffjson files in VCS? + +That question is really up to you. If you don't, you will have a more complex build process. If you do, you have to keep the generated files updated if you change the content of your structs. + +That said, ffjson operates deterministically, so it will generate the same code every time it run, so unless your code changes, the generated content should not change. Note however that this is only true if you are using the same ffjson version, so if you have several people working on a project, you might need to synchronize your ffjson version. + +## Performance pitfalls + +`ffjson` has a few cases where it will fall back to using the runtime encoder/decoder. Notable cases are: + +* Interface struct members. Since it isn't possible to know the type of these types before runtime, ffjson has to use the reflect based coder. +* Structs with custom marshal/unmarshal. +* Map with a complex value. Simple types like `map[string]int` is fine though. +* Inline struct definitions `type A struct{B struct{ X int} }` are handled by the encoder, but currently has fallback in the decoder. +* Slices of slices / slices of maps are currently falling back when generating the decoder. + +## Reducing Garbage Collection + +`ffjson` already does a lot to help garbage generation. However whenever you go through the json.Marshal you get a new byte slice back. On very high throughput servers this can lead to increased GC pressure. + +### Tip 1: Use ffjson.Marshal() / ffjson.Unmarshal() + +This is probably the easiest optimization for you. Instead of going through encoding/json, you can call ffjson. This will disable the checks that encoding/json does to the json when it receives it from struct functions. + +```Go + import "github.com/pquerna/ffjson/ffjson" + + // BEFORE: + buf, err := json.Marshal(&item) + + // AFTER: + buf, err := ffjson.Marshal(&item) +``` +This simple change is likely to double the speed of your encoding/decoding. + + +[![GoDoc][1]][2] +[1]: https://godoc.org/github.com/pquerna/ffjson/ffjson?status.svg +[2]: https://godoc.org/github.com/pquerna/ffjson/ffjson#Marshal + +### Tip 2: Pooling the buffer + +On servers where you have a lot of concurrent encoding going on, you can hand back the byte buffer you get from json.Marshal once you are done using it. An example could look like this: +```Go +import "github.com/pquerna/ffjson/ffjson" + +func Encode(item interface{}, out io.Writer) { + // Encode + buf, err := ffjson.Marshal(&item) + + // Write the buffer + _,_ = out.Write(buf) + + // We are now no longer need the buffer so we pool it. + ffjson.Pool(buf) +} +``` +Note that the buffers you put back in the pool can still be reclaimed by the garbage collector, so you wont risk your program building up a big memory use by pooling the buffers. + +[![GoDoc][1]][2] +[1]: https://godoc.org/github.com/pquerna/ffjson/ffjson?status.svg +[2]: https://godoc.org/github.com/pquerna/ffjson/ffjson#Pool + +### Tip 3: Creating an Encoder + +There might be cases where you need to encode many objects at once. This could be a server backing up, writing a lot of entries to files, etc. + +To do this, there is an interface similar to `encoding/json`, that allow you to create a re-usable encoder. Here is an example where we want to encode an array of the `Item` type, with a comma between entries: +```Go +import "github.com/pquerna/ffjson/ffjson" + +func EncodeItems(items []Item, out io.Writer) { + // We create an encoder. + enc := ffjson.NewEncoder(out) + + for i, item := range items { + // Encode into the buffer + err := enc.Encode(&item) + + // If err is nil, the content is written to out, so we can write to it as well. + if i != len(items) -1 { + _,_ = out.Write([]byte{','}) + } + } +} +``` + + +Documentation: [![GoDoc][1]][2] +[1]: https://godoc.org/github.com/pquerna/ffjson/ffjson?status.svg +[2]: https://godoc.org/github.com/pquerna/ffjson/ffjson#Encoder + +## Tip 4: Avoid interfaces + +We don't want to dictate how you structure your data, but having interfaces in your code will make ffjson use the golang encoder for these. When ffjson has to do this, it may even become slower than using `json.Marshal` directly. + +To see where that happens, search the generated `_ffjson.go` file for the text `Falling back`, which will indicate where ffjson is unable to generate code for your data structure. + +## Tip 5: `ffjson` all the things! + +You should not only create ffjson code for your main struct, but also any structs that is included/used in your json code. + +So if your struct looks like this: +```Go +type Foo struct { + V Bar +} +``` +You should also make sure that code is generated for `Bar` if it is placed in another file. Also note that currently it requires you to do this in order, since generating code for `Foo` will check if code for `Bar` exists. This is only an issue if `Foo` and `Bar` are placed in different files. We are currently working on allowing simultaneous generation of an entire package. + + +## Improvements, bugs, adding features, and taking ffjson new directions! + +Please [open issues in Github](https://github.com/pquerna/ffjson/issues) for ideas, bugs, and general thoughts. Pull requests are of course preferred :) + +## Similar projects + +* [go-codec](https://github.com/ugorji/go/tree/master/codec#readme). Very good project, that also allows streaming en/decoding, but requires you to call the library to use. +* [megajson](https://github.com/benbjohnson/megajson). This has limited support, and development seems to have almost stopped at the time of writing. + +# Credits + +`ffjson` has recieved significant contributions from: + +* [Klaus Post](https://github.com/klauspost) +* [Paul Querna](https://github.com/pquerna) +* [Erik Dubbelboer](https://github.com/erikdubbelboer) + +## License + +`ffjson` is licensed under the [Apache License, Version 2.0](./LICENSE) + diff --git a/ffjson.go b/ffjson.go new file mode 100644 index 0000000..656a9c3 --- /dev/null +++ b/ffjson.go @@ -0,0 +1,85 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + _ "github.com/pquerna/ffjson/fflib/v1" + "github.com/pquerna/ffjson/generator" + _ "github.com/pquerna/ffjson/inception" + + "flag" + "fmt" + "os" + "path/filepath" + "regexp" +) + +var outputPathFlag = flag.String("w", "", "Write generate code to this path instead of ${input}_ffjson.go.") +var goCmdFlag = flag.String("go-cmd", "", "Path to go command; Useful for `goapp` support.") +var importNameFlag = flag.String("import-name", "", "Override import name in case it cannot be detected.") +var forceRegenerateFlag = flag.Bool("force-regenerate", false, "Regenerate every input file, without checking modification date.") +var resetFields = flag.Bool("reset-fields", false, "When unmarshalling reset all fields missing in the JSON") + +func usage() { + fmt.Fprintf(os.Stderr, "Usage of %s:\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "\t%s [options] [input_file]\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "%s generates Go code for optimized JSON serialization.\n\n", os.Args[0]) + flag.PrintDefaults() + os.Exit(1) +} + +var extRe = regexp.MustCompile(`(.*)(\.go)$`) + +func main() { + flag.Parse() + extra := flag.Args() + + if len(extra) != 1 { + usage() + } + + inputPath := filepath.ToSlash(extra[0]) + + var outputPath string + if outputPathFlag == nil || *outputPathFlag == "" { + outputPath = extRe.ReplaceAllString(inputPath, "${1}_ffjson.go") + } else { + outputPath = *outputPathFlag + } + + var goCmd string + if goCmdFlag == nil || *goCmdFlag == "" { + goCmd = "go" + } else { + goCmd = *goCmdFlag + } + + var importName string + if importNameFlag != nil && *importNameFlag != "" { + importName = *importNameFlag + } + + err := generator.GenerateFiles(goCmd, inputPath, outputPath, importName, *forceRegenerateFlag, *resetFields) + + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %s:\n\n", err) + os.Exit(1) + } + + println(outputPath) +} diff --git a/ffjson/decoder.go b/ffjson/decoder.go new file mode 100644 index 0000000..356215a --- /dev/null +++ b/ffjson/decoder.go @@ -0,0 +1,92 @@ +package ffjson + +/** + * Copyright 2015 Paul Querna, Klaus Post + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import ( + "encoding/json" + "errors" + fflib "github.com/pquerna/ffjson/fflib/v1" + "io" + "io/ioutil" + "reflect" +) + +// This is a reusable decoder. +// This should not be used by more than one goroutine at the time. +type Decoder struct { + fs *fflib.FFLexer +} + +// NewDecoder returns a reusable Decoder. +func NewDecoder() *Decoder { + return &Decoder{} +} + +// Decode the data in the supplied data slice. +func (d *Decoder) Decode(data []byte, v interface{}) error { + f, ok := v.(unmarshalFaster) + if ok { + if d.fs == nil { + d.fs = fflib.NewFFLexer(data) + } else { + d.fs.Reset(data) + } + return f.UnmarshalJSONFFLexer(d.fs, fflib.FFParse_map_start) + } + + um, ok := v.(json.Unmarshaler) + if ok { + return um.UnmarshalJSON(data) + } + return json.Unmarshal(data, v) +} + +// Decode the data from the supplied reader. +// You should expect that data is read into memory before it is decoded. +func (d *Decoder) DecodeReader(r io.Reader, v interface{}) error { + _, ok := v.(unmarshalFaster) + _, ok2 := v.(json.Unmarshaler) + if ok || ok2 { + data, err := ioutil.ReadAll(r) + if err != nil { + return err + } + defer fflib.Pool(data) + return d.Decode(data, v) + } + dec := json.NewDecoder(r) + return dec.Decode(v) +} + +// DecodeFast will unmarshal the data if fast unmarshal is available. +// This function can be used if you want to be sure the fast +// unmarshal is used or in testing. +// If you would like to have fallback to encoding/json you can use the +// regular Decode() method. +func (d *Decoder) DecodeFast(data []byte, v interface{}) error { + f, ok := v.(unmarshalFaster) + if !ok { + return errors.New("ffjson unmarshal not available for type " + reflect.TypeOf(v).String()) + } + if d.fs == nil { + d.fs = fflib.NewFFLexer(data) + } else { + d.fs.Reset(data) + } + return f.UnmarshalJSONFFLexer(d.fs, fflib.FFParse_map_start) +} diff --git a/ffjson/encoder.go b/ffjson/encoder.go new file mode 100644 index 0000000..3e82d61 --- /dev/null +++ b/ffjson/encoder.go @@ -0,0 +1,85 @@ +package ffjson + +/** + * Copyright 2015 Paul Querna, Klaus Post + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import ( + "encoding/json" + "errors" + fflib "github.com/pquerna/ffjson/fflib/v1" + "io" + "reflect" +) + +// This is a reusable encoder. +// It allows to encode many objects to a single writer. +// This should not be used by more than one goroutine at the time. +type Encoder struct { + buf fflib.Buffer + w io.Writer + enc *json.Encoder +} + +// SetEscapeHTML specifies whether problematic HTML characters +// should be escaped inside JSON quoted strings. +// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e +// to avoid certain safety problems that can arise when embedding JSON in HTML. +// +// In non-HTML settings where the escaping interferes with the readability +// of the output, SetEscapeHTML(false) disables this behavior. +func (enc *Encoder) SetEscapeHTML(on bool) { + enc.enc.SetEscapeHTML(on) +} + +// NewEncoder returns a reusable Encoder. +// Output will be written to the supplied writer. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w, enc: json.NewEncoder(w)} +} + +// Encode the data in the supplied value to the stream +// given on creation. +// When the function returns the output has been +// written to the stream. +func (e *Encoder) Encode(v interface{}) error { + f, ok := v.(marshalerFaster) + if ok { + e.buf.Reset() + err := f.MarshalJSONBuf(&e.buf) + if err != nil { + return err + } + + _, err = io.Copy(e.w, &e.buf) + return err + } + + return e.enc.Encode(v) +} + +// EncodeFast will unmarshal the data if fast marshall is available. +// This function can be used if you want to be sure the fast +// marshal is used or in testing. +// If you would like to have fallback to encoding/json you can use the +// regular Encode() method. +func (e *Encoder) EncodeFast(v interface{}) error { + _, ok := v.(marshalerFaster) + if !ok { + return errors.New("ffjson marshal not available for type " + reflect.TypeOf(v).String()) + } + return e.Encode(v) +} diff --git a/ffjson/marshal.go b/ffjson/marshal.go new file mode 100644 index 0000000..ff0685e --- /dev/null +++ b/ffjson/marshal.go @@ -0,0 +1,109 @@ +package ffjson + +/** + * Copyright 2015 Paul Querna, Klaus Post + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import ( + "encoding/json" + "errors" + fflib "github.com/pquerna/ffjson/fflib/v1" + "reflect" +) + +type marshalerFaster interface { + MarshalJSONBuf(buf fflib.EncodingBuffer) error +} + +type unmarshalFaster interface { + UnmarshalJSONFFLexer(l *fflib.FFLexer, state fflib.FFParseState) error +} + +// Marshal will act the same way as json.Marshal, except +// it will choose the ffjson marshal function before falling +// back to using json.Marshal. +// Using this function will bypass the internal copying and parsing +// the json library normally does, which greatly speeds up encoding time. +// It is ok to call this function even if no ffjson code has been +// generated for the data type you pass in the interface. +func Marshal(v interface{}) ([]byte, error) { + f, ok := v.(marshalerFaster) + if ok { + buf := fflib.Buffer{} + err := f.MarshalJSONBuf(&buf) + b := buf.Bytes() + if err != nil { + if len(b) > 0 { + Pool(b) + } + return nil, err + } + return b, nil + } + + j, ok := v.(json.Marshaler) + if ok { + return j.MarshalJSON() + } + return json.Marshal(v) +} + +// MarshalFast will marshal the data if fast marshal is available. +// This function can be used if you want to be sure the fast +// marshal is used or in testing. +// If you would like to have fallback to encoding/json you can use the +// Marshal() method. +func MarshalFast(v interface{}) ([]byte, error) { + _, ok := v.(marshalerFaster) + if !ok { + return nil, errors.New("ffjson marshal not available for type " + reflect.TypeOf(v).String()) + } + return Marshal(v) +} + +// Unmarshal will act the same way as json.Unmarshal, except +// it will choose the ffjson unmarshal function before falling +// back to using json.Unmarshal. +// The overhead of unmarshal is lower than on Marshal, +// however this should still provide a speedup for your encoding. +// It is ok to call this function even if no ffjson code has been +// generated for the data type you pass in the interface. +func Unmarshal(data []byte, v interface{}) error { + f, ok := v.(unmarshalFaster) + if ok { + fs := fflib.NewFFLexer(data) + return f.UnmarshalJSONFFLexer(fs, fflib.FFParse_map_start) + } + + j, ok := v.(json.Unmarshaler) + if ok { + return j.UnmarshalJSON(data) + } + return json.Unmarshal(data, v) +} + +// UnmarshalFast will unmarshal the data if fast marshall is available. +// This function can be used if you want to be sure the fast +// unmarshal is used or in testing. +// If you would like to have fallback to encoding/json you can use the +// Unmarshal() method. +func UnmarshalFast(data []byte, v interface{}) error { + _, ok := v.(unmarshalFaster) + if !ok { + return errors.New("ffjson unmarshal not available for type " + reflect.TypeOf(v).String()) + } + return Unmarshal(data, v) +} diff --git a/ffjson/pool.go b/ffjson/pool.go new file mode 100644 index 0000000..b9f127a --- /dev/null +++ b/ffjson/pool.go @@ -0,0 +1,33 @@ +package ffjson + +/** + * Copyright 2015 Paul Querna, Klaus Post + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import ( + fflib "github.com/pquerna/ffjson/fflib/v1" +) + +// Send a buffer to the Pool to reuse for other instances. +// +// On servers where you have a lot of concurrent encoding going on, +// you can hand back the byte buffer you get marshalling once you are done using it. +// +// You may no longer utilize the content of the buffer, since it may be used +// by other goroutines. +func Pool(b []byte) { + fflib.Pool(b) +} diff --git a/fflib/v1/buffer.go b/fflib/v1/buffer.go new file mode 100644 index 0000000..7f63a85 --- /dev/null +++ b/fflib/v1/buffer.go @@ -0,0 +1,421 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package v1 + +// Simple byte buffer for marshaling data. + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "unicode/utf8" +) + +type grower interface { + Grow(n int) +} + +type truncater interface { + Truncate(n int) + Reset() +} + +type bytesReader interface { + Bytes() []byte + String() string +} + +type runeWriter interface { + WriteRune(r rune) (n int, err error) +} + +type stringWriter interface { + WriteString(s string) (n int, err error) +} + +type lener interface { + Len() int +} + +type rewinder interface { + Rewind(n int) (err error) +} + +type encoder interface { + Encode(interface{}) error +} + +// TODO(pquerna): continue to reduce these interfaces + +type EncodingBuffer interface { + io.Writer + io.WriterTo + io.ByteWriter + stringWriter + truncater + grower + rewinder + encoder +} + +type DecodingBuffer interface { + io.ReadWriter + io.ByteWriter + stringWriter + runeWriter + truncater + grower + bytesReader + lener +} + +// A Buffer is a variable-sized buffer of bytes with Read and Write methods. +// The zero value for Buffer is an empty buffer ready to use. +type Buffer struct { + buf []byte // contents are the bytes buf[off : len(buf)] + off int // read at &buf[off], write at &buf[len(buf)] + runeBytes [utf8.UTFMax]byte // avoid allocation of slice on each WriteByte or Rune + encoder *json.Encoder + skipTrailingByte bool +} + +// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer. +var ErrTooLarge = errors.New("fflib.v1.Buffer: too large") + +// Bytes returns a slice of the contents of the unread portion of the buffer; +// len(b.Bytes()) == b.Len(). If the caller changes the contents of the +// returned slice, the contents of the buffer will change provided there +// are no intervening method calls on the Buffer. +func (b *Buffer) Bytes() []byte { return b.buf[b.off:] } + +// String returns the contents of the unread portion of the buffer +// as a string. If the Buffer is a nil pointer, it returns "<nil>". +func (b *Buffer) String() string { + if b == nil { + // Special case, useful in debugging. + return "<nil>" + } + return string(b.buf[b.off:]) +} + +// Len returns the number of bytes of the unread portion of the buffer; +// b.Len() == len(b.Bytes()). +func (b *Buffer) Len() int { return len(b.buf) - b.off } + +// Truncate discards all but the first n unread bytes from the buffer. +// It panics if n is negative or greater than the length of the buffer. +func (b *Buffer) Truncate(n int) { + if n == 0 { + b.off = 0 + b.buf = b.buf[0:0] + } else { + b.buf = b.buf[0 : b.off+n] + } +} + +// Reset resets the buffer so it has no content. +// b.Reset() is the same as b.Truncate(0). +func (b *Buffer) Reset() { b.Truncate(0) } + +// grow grows the buffer to guarantee space for n more bytes. +// It returns the index where bytes should be written. +// If the buffer can't grow it will panic with ErrTooLarge. +func (b *Buffer) grow(n int) int { + // If we have no buffer, get one from the pool + m := b.Len() + if m == 0 { + if b.buf == nil { + b.buf = makeSlice(2 * n) + b.off = 0 + } else if b.off != 0 { + // If buffer is empty, reset to recover space. + b.Truncate(0) + } + } + if len(b.buf)+n > cap(b.buf) { + var buf []byte + if m+n <= cap(b.buf)/2 { + // We can slide things down instead of allocating a new + // slice. We only need m+n <= cap(b.buf) to slide, but + // we instead let capacity get twice as large so we + // don't spend all our time copying. + copy(b.buf[:], b.buf[b.off:]) + buf = b.buf[:m] + } else { + // not enough space anywhere + buf = makeSlice(2*cap(b.buf) + n) + copy(buf, b.buf[b.off:]) + Pool(b.buf) + b.buf = buf + } + b.off = 0 + } + b.buf = b.buf[0 : b.off+m+n] + return b.off + m +} + +// Grow grows the buffer's capacity, if necessary, to guarantee space for +// another n bytes. After Grow(n), at least n bytes can be written to the +// buffer without another allocation. +// If n is negative, Grow will panic. +// If the buffer can't grow it will panic with ErrTooLarge. +func (b *Buffer) Grow(n int) { + if n < 0 { + panic("bytes.Buffer.Grow: negative count") + } + m := b.grow(n) + b.buf = b.buf[0:m] +} + +// Write appends the contents of p to the buffer, growing the buffer as +// needed. The return value n is the length of p; err is always nil. If the +// buffer becomes too large, Write will panic with ErrTooLarge. +func (b *Buffer) Write(p []byte) (n int, err error) { + if b.skipTrailingByte { + p = p[:len(p)-1] + } + m := b.grow(len(p)) + return copy(b.buf[m:], p), nil +} + +// WriteString appends the contents of s to the buffer, growing the buffer as +// needed. The return value n is the length of s; err is always nil. If the +// buffer becomes too large, WriteString will panic with ErrTooLarge. +func (b *Buffer) WriteString(s string) (n int, err error) { + m := b.grow(len(s)) + return copy(b.buf[m:], s), nil +} + +// MinRead is the minimum slice size passed to a Read call by +// Buffer.ReadFrom. As long as the Buffer has at least MinRead bytes beyond +// what is required to hold the contents of r, ReadFrom will not grow the +// underlying buffer. +const minRead = 512 + +// ReadFrom reads data from r until EOF and appends it to the buffer, growing +// the buffer as needed. The return value n is the number of bytes read. Any +// error except io.EOF encountered during the read is also returned. If the +// buffer becomes too large, ReadFrom will panic with ErrTooLarge. +func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) { + // If buffer is empty, reset to recover space. + if b.off >= len(b.buf) { + b.Truncate(0) + } + for { + if free := cap(b.buf) - len(b.buf); free < minRead { + // not enough space at end + newBuf := b.buf + if b.off+free < minRead { + // not enough space using beginning of buffer; + // double buffer capacity + newBuf = makeSlice(2*cap(b.buf) + minRead) + } + copy(newBuf, b.buf[b.off:]) + Pool(b.buf) + b.buf = newBuf[:len(b.buf)-b.off] + b.off = 0 + } + m, e := r.Read(b.buf[len(b.buf):cap(b.buf)]) + b.buf = b.buf[0 : len(b.buf)+m] + n += int64(m) + if e == io.EOF { + break + } + if e != nil { + return n, e + } + } + return n, nil // err is EOF, so return nil explicitly +} + +// WriteTo writes data to w until the buffer is drained or an error occurs. +// The return value n is the number of bytes written; it always fits into an +// int, but it is int64 to match the io.WriterTo interface. Any error +// encountered during the write is also returned. +func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) { + if b.off < len(b.buf) { + nBytes := b.Len() + m, e := w.Write(b.buf[b.off:]) + if m > nBytes { + panic("bytes.Buffer.WriteTo: invalid Write count") + } + b.off += m + n = int64(m) + if e != nil { + return n, e + } + // all bytes should have been written, by definition of + // Write method in io.Writer + if m != nBytes { + return n, io.ErrShortWrite + } + } + // Buffer is now empty; reset. + b.Truncate(0) + return +} + +// WriteByte appends the byte c to the buffer, growing the buffer as needed. +// The returned error is always nil, but is included to match bufio.Writer's +// WriteByte. If the buffer becomes too large, WriteByte will panic with +// ErrTooLarge. +func (b *Buffer) WriteByte(c byte) error { + m := b.grow(1) + b.buf[m] = c + return nil +} + +func (b *Buffer) Rewind(n int) error { + b.buf = b.buf[:len(b.buf)-n] + return nil +} + +func (b *Buffer) Encode(v interface{}) error { + if b.encoder == nil { + b.encoder = json.NewEncoder(b) + } + b.skipTrailingByte = true + err := b.encoder.Encode(v) + b.skipTrailingByte = false + return err +} + +// WriteRune appends the UTF-8 encoding of Unicode code point r to the +// buffer, returning its length and an error, which is always nil but is +// included to match bufio.Writer's WriteRune. The buffer is grown as needed; +// if it becomes too large, WriteRune will panic with ErrTooLarge. +func (b *Buffer) WriteRune(r rune) (n int, err error) { + if r < utf8.RuneSelf { + b.WriteByte(byte(r)) + return 1, nil + } + n = utf8.EncodeRune(b.runeBytes[0:], r) + b.Write(b.runeBytes[0:n]) + return n, nil +} + +// Read reads the next len(p) bytes from the buffer or until the buffer +// is drained. The return value n is the number of bytes read. If the +// buffer has no data to return, err is io.EOF (unless len(p) is zero); +// otherwise it is nil. +func (b *Buffer) Read(p []byte) (n int, err error) { + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + if len(p) == 0 { + return + } + return 0, io.EOF + } + n = copy(p, b.buf[b.off:]) + b.off += n + return +} + +// Next returns a slice containing the next n bytes from the buffer, +// advancing the buffer as if the bytes had been returned by Read. +// If there are fewer than n bytes in the buffer, Next returns the entire buffer. +// The slice is only valid until the next call to a read or write method. +func (b *Buffer) Next(n int) []byte { + m := b.Len() + if n > m { + n = m + } + data := b.buf[b.off : b.off+n] + b.off += n + return data +} + +// ReadByte reads and returns the next byte from the buffer. +// If no byte is available, it returns error io.EOF. +func (b *Buffer) ReadByte() (c byte, err error) { + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, io.EOF + } + c = b.buf[b.off] + b.off++ + return c, nil +} + +// ReadRune reads and returns the next UTF-8-encoded +// Unicode code point from the buffer. +// If no bytes are available, the error returned is io.EOF. +// If the bytes are an erroneous UTF-8 encoding, it +// consumes one byte and returns U+FFFD, 1. +func (b *Buffer) ReadRune() (r rune, size int, err error) { + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, 0, io.EOF + } + c := b.buf[b.off] + if c < utf8.RuneSelf { + b.off++ + return rune(c), 1, nil + } + r, n := utf8.DecodeRune(b.buf[b.off:]) + b.off += n + return r, n, nil +} + +// ReadBytes reads until the first occurrence of delim in the input, +// returning a slice containing the data up to and including the delimiter. +// If ReadBytes encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often io.EOF). +// ReadBytes returns err != nil if and only if the returned data does not end in +// delim. +func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) { + slice, err := b.readSlice(delim) + // return a copy of slice. The buffer's backing array may + // be overwritten by later calls. + line = append(line, slice...) + return +} + +// readSlice is like ReadBytes but returns a reference to internal buffer data. +func (b *Buffer) readSlice(delim byte) (line []byte, err error) { + i := bytes.IndexByte(b.buf[b.off:], delim) + end := b.off + i + 1 + if i < 0 { + end = len(b.buf) + err = io.EOF + } + line = b.buf[b.off:end] + b.off = end + return line, err +} + +// ReadString reads until the first occurrence of delim in the input, +// returning a string containing the data up to and including the delimiter. +// If ReadString encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often io.EOF). +// ReadString returns err != nil if and only if the returned data does not end +// in delim. +func (b *Buffer) ReadString(delim byte) (line string, err error) { + slice, err := b.readSlice(delim) + return string(slice), err +} + +// NewBuffer creates and initializes a new Buffer using buf as its initial +// contents. It is intended to prepare a Buffer to read existing data. It +// can also be used to size the internal buffer for writing. To do that, +// buf should have the desired capacity but a length of zero. +// +// In most cases, new(Buffer) (or just declaring a Buffer variable) is +// sufficient to initialize a Buffer. +func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} } + +// NewBufferString creates and initializes a new Buffer using string s as its +// initial contents. It is intended to prepare a buffer to read an existing +// string. +// +// In most cases, new(Buffer) (or just declaring a Buffer variable) is +// sufficient to initialize a Buffer. +func NewBufferString(s string) *Buffer { + return &Buffer{buf: []byte(s)} +} diff --git a/fflib/v1/buffer_nopool.go b/fflib/v1/buffer_nopool.go new file mode 100644 index 0000000..b84af6f --- /dev/null +++ b/fflib/v1/buffer_nopool.go @@ -0,0 +1,11 @@ +// +build !go1.3 + +package v1 + +// Stub version of buffer_pool.go for Go 1.2, which doesn't have sync.Pool. + +func Pool(b []byte) {} + +func makeSlice(n int) []byte { + return make([]byte, n) +} diff --git a/fflib/v1/buffer_pool.go b/fflib/v1/buffer_pool.go new file mode 100644 index 0000000..a021c57 --- /dev/null +++ b/fflib/v1/buffer_pool.go @@ -0,0 +1,105 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.3 + +package v1 + +// Allocation pools for Buffers. + +import "sync" + +var pools [14]sync.Pool +var pool64 *sync.Pool + +func init() { + var i uint + // TODO(pquerna): add science here around actual pool sizes. + for i = 6; i < 20; i++ { + n := 1 << i + pools[poolNum(n)].New = func() interface{} { return make([]byte, 0, n) } + } + pool64 = &pools[0] +} + +// This returns the pool number that will give a buffer of +// at least 'i' bytes. +func poolNum(i int) int { + // TODO(pquerna): convert to log2 w/ bsr asm instruction: + // <https://groups.google.com/forum/#!topic/golang-nuts/uAb5J1_y7ns> + if i <= 64 { + return 0 + } else if i <= 128 { + return 1 + } else if i <= 256 { + return 2 + } else if i <= 512 { + return 3 + } else if i <= 1024 { + return 4 + } else if i <= 2048 { + return 5 + } else if i <= 4096 { + return 6 + } else if i <= 8192 { + return 7 + } else if i <= 16384 { + return 8 + } else if i <= 32768 { + return 9 + } else if i <= 65536 { + return 10 + } else if i <= 131072 { + return 11 + } else if i <= 262144 { + return 12 + } else if i <= 524288 { + return 13 + } else { + return -1 + } +} + +// Send a buffer to the Pool to reuse for other instances. +// You may no longer utilize the content of the buffer, since it may be used +// by other goroutines. +func Pool(b []byte) { + if b == nil { + return + } + c := cap(b) + + // Our smallest buffer is 64 bytes, so we discard smaller buffers. + if c < 64 { + return + } + + // We need to put the incoming buffer into the NEXT buffer, + // since a buffer guarantees AT LEAST the number of bytes available + // that is the top of this buffer. + // That is the reason for dividing the cap by 2, so it gets into the NEXT bucket. + // We add 2 to avoid rounding down if size is exactly power of 2. + pn := poolNum((c + 2) >> 1) + if pn != -1 { + pools[pn].Put(b[0:0]) + } + // if we didn't have a slot for this []byte, we just drop it and let the GC + // take care of it. +} + +// makeSlice allocates a slice of size n -- it will attempt to use a pool'ed +// instance whenever possible. +func makeSlice(n int) []byte { + if n <= 64 { + return pool64.Get().([]byte)[0:n] + } + + pn := poolNum(n) + + if pn != -1 { + return pools[pn].Get().([]byte)[0:n] + } else { + return make([]byte, n) + } +} diff --git a/fflib/v1/bytenum.go b/fflib/v1/bytenum.go new file mode 100644 index 0000000..0847740 --- /dev/null +++ b/fflib/v1/bytenum.go @@ -0,0 +1,88 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Portions of this file are on Go stdlib's strconv/iota.go */ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package v1 + +import ( + "github.com/pquerna/ffjson/fflib/v1/internal" +) + +func ParseFloat(s []byte, bitSize int) (f float64, err error) { + return internal.ParseFloat(s, bitSize) +} + +// ParseUint is like ParseInt but for unsigned numbers, and oeprating on []byte +func ParseUint(s []byte, base int, bitSize int) (n uint64, err error) { + if len(s) == 1 { + switch s[0] { + case '0': + return 0, nil + case '1': + return 1, nil + case '2': + return 2, nil + case '3': + return 3, nil + case '4': + return 4, nil + case '5': + return 5, nil + case '6': + return 6, nil + case '7': + return 7, nil + case '8': + return 8, nil + case '9': + return 9, nil + } + } + return internal.ParseUint(s, base, bitSize) +} + +func ParseInt(s []byte, base int, bitSize int) (i int64, err error) { + if len(s) == 1 { + switch s[0] { + case '0': + return 0, nil + case '1': + return 1, nil + case '2': + return 2, nil + case '3': + return 3, nil + case '4': + return 4, nil + case '5': + return 5, nil + case '6': + return 6, nil + case '7': + return 7, nil + case '8': + return 8, nil + case '9': + return 9, nil + } + } + return internal.ParseInt(s, base, bitSize) +} diff --git a/fflib/v1/decimal.go b/fflib/v1/decimal.go new file mode 100644 index 0000000..069df7a --- /dev/null +++ b/fflib/v1/decimal.go @@ -0,0 +1,378 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Multiprecision decimal numbers. +// For floating-point formatting only; not general purpose. +// Only operations are assign and (binary) left/right shift. +// Can do binary floating point in multiprecision decimal precisely +// because 2 divides 10; cannot do decimal floating point +// in multiprecision binary precisely. + +package v1 + +type decimal struct { + d [800]byte // digits + nd int // number of digits used + dp int // decimal point + neg bool + trunc bool // discarded nonzero digits beyond d[:nd] +} + +func (a *decimal) String() string { + n := 10 + a.nd + if a.dp > 0 { + n += a.dp + } + if a.dp < 0 { + n += -a.dp + } + + buf := make([]byte, n) + w := 0 + switch { + case a.nd == 0: + return "0" + + case a.dp <= 0: + // zeros fill space between decimal point and digits + buf[w] = '0' + w++ + buf[w] = '.' + w++ + w += digitZero(buf[w : w+-a.dp]) + w += copy(buf[w:], a.d[0:a.nd]) + + case a.dp < a.nd: + // decimal point in middle of digits + w += copy(buf[w:], a.d[0:a.dp]) + buf[w] = '.' + w++ + w += copy(buf[w:], a.d[a.dp:a.nd]) + + default: + // zeros fill space between digits and decimal point + w += copy(buf[w:], a.d[0:a.nd]) + w += digitZero(buf[w : w+a.dp-a.nd]) + } + return string(buf[0:w]) +} + +func digitZero(dst []byte) int { + for i := range dst { + dst[i] = '0' + } + return len(dst) +} + +// trim trailing zeros from number. +// (They are meaningless; the decimal point is tracked +// independent of the number of digits.) +func trim(a *decimal) { + for a.nd > 0 && a.d[a.nd-1] == '0' { + a.nd-- + } + if a.nd == 0 { + a.dp = 0 + } +} + +// Assign v to a. +func (a *decimal) Assign(v uint64) { + var buf [24]byte + + // Write reversed decimal in buf. + n := 0 + for v > 0 { + v1 := v / 10 + v -= 10 * v1 + buf[n] = byte(v + '0') + n++ + v = v1 + } + + // Reverse again to produce forward decimal in a.d. + a.nd = 0 + for n--; n >= 0; n-- { + a.d[a.nd] = buf[n] + a.nd++ + } + a.dp = a.nd + trim(a) +} + +// Maximum shift that we can do in one pass without overflow. +// Signed int has 31 bits, and we have to be able to accommodate 9<<k. +const maxShift = 27 + +// Binary shift right (* 2) by k bits. k <= maxShift to avoid overflow. +func rightShift(a *decimal, k uint) { + r := 0 // read pointer + w := 0 // write pointer + + // Pick up enough leading digits to cover first shift. + n := 0 + for ; n>>k == 0; r++ { + if r >= a.nd { + if n == 0 { + // a == 0; shouldn't get here, but handle anyway. + a.nd = 0 + return + } + for n>>k == 0 { + n = n * 10 + r++ + } + break + } + c := int(a.d[r]) + n = n*10 + c - '0' + } + a.dp -= r - 1 + + // Pick up a digit, put down a digit. + for ; r < a.nd; r++ { + c := int(a.d[r]) + dig := n >> k + n -= dig << k + a.d[w] = byte(dig + '0') + w++ + n = n*10 + c - '0' + } + + // Put down extra digits. + for n > 0 { + dig := n >> k + n -= dig << k + if w < len(a.d) { + a.d[w] = byte(dig + '0') + w++ + } else if dig > 0 { + a.trunc = true + } + n = n * 10 + } + + a.nd = w + trim(a) +} + +// Cheat sheet for left shift: table indexed by shift count giving +// number of new digits that will be introduced by that shift. +// +// For example, leftcheats[4] = {2, "625"}. That means that +// if we are shifting by 4 (multiplying by 16), it will add 2 digits +// when the string prefix is "625" through "999", and one fewer digit +// if the string prefix is "000" through "624". +// +// Credit for this trick goes to Ken. + +type leftCheat struct { + delta int // number of new digits + cutoff string // minus one digit if original < a. +} + +var leftcheats = []leftCheat{ + // Leading digits of 1/2^i = 5^i. + // 5^23 is not an exact 64-bit floating point number, + // so have to use bc for the math. + /* + seq 27 | sed 's/^/5^/' | bc | + awk 'BEGIN{ print "\tleftCheat{ 0, \"\" }," } + { + log2 = log(2)/log(10) + printf("\tleftCheat{ %d, \"%s\" },\t// * %d\n", + int(log2*NR+1), $0, 2**NR) + }' + */ + {0, ""}, + {1, "5"}, // * 2 + {1, "25"}, // * 4 + {1, "125"}, // * 8 + {2, "625"}, // * 16 + {2, "3125"}, // * 32 + {2, "15625"}, // * 64 + {3, "78125"}, // * 128 + {3, "390625"}, // * 256 + {3, "1953125"}, // * 512 + {4, "9765625"}, // * 1024 + {4, "48828125"}, // * 2048 + {4, "244140625"}, // * 4096 + {4, "1220703125"}, // * 8192 + {5, "6103515625"}, // * 16384 + {5, "30517578125"}, // * 32768 + {5, "152587890625"}, // * 65536 + {6, "762939453125"}, // * 131072 + {6, "3814697265625"}, // * 262144 + {6, "19073486328125"}, // * 524288 + {7, "95367431640625"}, // * 1048576 + {7, "476837158203125"}, // * 2097152 + {7, "2384185791015625"}, // * 4194304 + {7, "11920928955078125"}, // * 8388608 + {8, "59604644775390625"}, // * 16777216 + {8, "298023223876953125"}, // * 33554432 + {8, "1490116119384765625"}, // * 67108864 + {9, "7450580596923828125"}, // * 134217728 +} + +// Is the leading prefix of b lexicographically less than s? +func prefixIsLessThan(b []byte, s string) bool { + for i := 0; i < len(s); i++ { + if i >= len(b) { + return true + } + if b[i] != s[i] { + return b[i] < s[i] + } + } + return false +} + +// Binary shift left (/ 2) by k bits. k <= maxShift to avoid overflow. +func leftShift(a *decimal, k uint) { + delta := leftcheats[k].delta + if prefixIsLessThan(a.d[0:a.nd], leftcheats[k].cutoff) { + delta-- + } + + r := a.nd // read index + w := a.nd + delta // write index + n := 0 + + // Pick up a digit, put down a digit. + for r--; r >= 0; r-- { + n += (int(a.d[r]) - '0') << k + quo := n / 10 + rem := n - 10*quo + w-- + if w < len(a.d) { + a.d[w] = byte(rem + '0') + } else if rem != 0 { + a.trunc = true + } + n = quo + } + + // Put down extra digits. + for n > 0 { + quo := n / 10 + rem := n - 10*quo + w-- + if w < len(a.d) { + a.d[w] = byte(rem + '0') + } else if rem != 0 { + a.trunc = true + } + n = quo + } + + a.nd += delta + if a.nd >= len(a.d) { + a.nd = len(a.d) + } + a.dp += delta + trim(a) +} + +// Binary shift left (k > 0) or right (k < 0). +func (a *decimal) Shift(k int) { + switch { + case a.nd == 0: + // nothing to do: a == 0 + case k > 0: + for k > maxShift { + leftShift(a, maxShift) + k -= maxShift + } + leftShift(a, uint(k)) + case k < 0: + for k < -maxShift { + rightShift(a, maxShift) + k += maxShift + } + rightShift(a, uint(-k)) + } +} + +// If we chop a at nd digits, should we round up? +func shouldRoundUp(a *decimal, nd int) bool { + if nd < 0 || nd >= a.nd { + return false + } + if a.d[nd] == '5' && nd+1 == a.nd { // exactly halfway - round to even + // if we truncated, a little higher than what's recorded - always round up + if a.trunc { + return true + } + return nd > 0 && (a.d[nd-1]-'0')%2 != 0 + } + // not halfway - digit tells all + return a.d[nd] >= '5' +} + +// Round a to nd digits (or fewer). +// If nd is zero, it means we're rounding +// just to the left of the digits, as in +// 0.09 -> 0.1. +func (a *decimal) Round(nd int) { + if nd < 0 || nd >= a.nd { + return + } + if shouldRoundUp(a, nd) { + a.RoundUp(nd) + } else { + a.RoundDown(nd) + } +} + +// Round a down to nd digits (or fewer). +func (a *decimal) RoundDown(nd int) { + if nd < 0 || nd >= a.nd { + return + } + a.nd = nd + trim(a) +} + +// Round a up to nd digits (or fewer). +func (a *decimal) RoundUp(nd int) { + if nd < 0 || nd >= a.nd { + return + } + + // round up + for i := nd - 1; i >= 0; i-- { + c := a.d[i] + if c < '9' { // can stop after this digit + a.d[i]++ + a.nd = i + 1 + return + } + } + + // Number is all 9s. + // Change to single 1 with adjusted decimal point. + a.d[0] = '1' + a.nd = 1 + a.dp++ +} + +// Extract integer part, rounded appropriately. +// No guarantees about overflow. +func (a *decimal) RoundedInteger() uint64 { + if a.dp > 20 { + return 0xFFFFFFFFFFFFFFFF + } + var i int + n := uint64(0) + for i = 0; i < a.dp && i < a.nd; i++ { + n = n*10 + uint64(a.d[i]-'0') + } + for ; i < a.dp; i++ { + n *= 10 + } + if shouldRoundUp(a, a.dp) { + n++ + } + return n +} diff --git a/fflib/v1/extfloat.go b/fflib/v1/extfloat.go new file mode 100644 index 0000000..508ddc6 --- /dev/null +++ b/fflib/v1/extfloat.go @@ -0,0 +1,668 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package v1 + +// An extFloat represents an extended floating-point number, with more +// precision than a float64. It does not try to save bits: the +// number represented by the structure is mant*(2^exp), with a negative +// sign if neg is true. +type extFloat struct { + mant uint64 + exp int + neg bool +} + +// Powers of ten taken from double-conversion library. +// http://code.google.com/p/double-conversion/ +const ( + firstPowerOfTen = -348 + stepPowerOfTen = 8 +) + +var smallPowersOfTen = [...]extFloat{ + {1 << 63, -63, false}, // 1 + {0xa << 60, -60, false}, // 1e1 + {0x64 << 57, -57, false}, // 1e2 + {0x3e8 << 54, -54, false}, // 1e3 + {0x2710 << 50, -50, false}, // 1e4 + {0x186a0 << 47, -47, false}, // 1e5 + {0xf4240 << 44, -44, false}, // 1e6 + {0x989680 << 40, -40, false}, // 1e7 +} + +var powersOfTen = [...]extFloat{ + {0xfa8fd5a0081c0288, -1220, false}, // 10^-348 + {0xbaaee17fa23ebf76, -1193, false}, // 10^-340 + {0x8b16fb203055ac76, -1166, false}, // 10^-332 + {0xcf42894a5dce35ea, -1140, false}, // 10^-324 + {0x9a6bb0aa55653b2d, -1113, false}, // 10^-316 + {0xe61acf033d1a45df, -1087, false}, // 10^-308 + {0xab70fe17c79ac6ca, -1060, false}, // 10^-300 + {0xff77b1fcbebcdc4f, -1034, false}, // 10^-292 + {0xbe5691ef416bd60c, -1007, false}, // 10^-284 + {0x8dd01fad907ffc3c, -980, false}, // 10^-276 + {0xd3515c2831559a83, -954, false}, // 10^-268 + {0x9d71ac8fada6c9b5, -927, false}, // 10^-260 + {0xea9c227723ee8bcb, -901, false}, // 10^-252 + {0xaecc49914078536d, -874, false}, // 10^-244 + {0x823c12795db6ce57, -847, false}, // 10^-236 + {0xc21094364dfb5637, -821, false}, // 10^-228 + {0x9096ea6f3848984f, -794, false}, // 10^-220 + {0xd77485cb25823ac7, -768, false}, // 10^-212 + {0xa086cfcd97bf97f4, -741, false}, // 10^-204 + {0xef340a98172aace5, -715, false}, // 10^-196 + {0xb23867fb2a35b28e, -688, false}, // 10^-188 + {0x84c8d4dfd2c63f3b, -661, false}, // 10^-180 + {0xc5dd44271ad3cdba, -635, false}, // 10^-172 + {0x936b9fcebb25c996, -608, false}, // 10^-164 + {0xdbac6c247d62a584, -582, false}, // 10^-156 + {0xa3ab66580d5fdaf6, -555, false}, // 10^-148 + {0xf3e2f893dec3f126, -529, false}, // 10^-140 + {0xb5b5ada8aaff80b8, -502, false}, // 10^-132 + {0x87625f056c7c4a8b, -475, false}, // 10^-124 + {0xc9bcff6034c13053, -449, false}, // 10^-116 + {0x964e858c91ba2655, -422, false}, // 10^-108 + {0xdff9772470297ebd, -396, false}, // 10^-100 + {0xa6dfbd9fb8e5b88f, -369, false}, // 10^-92 + {0xf8a95fcf88747d94, -343, false}, // 10^-84 + {0xb94470938fa89bcf, -316, false}, // 10^-76 + {0x8a08f0f8bf0f156b, -289, false}, // 10^-68 + {0xcdb02555653131b6, -263, false}, // 10^-60 + {0x993fe2c6d07b7fac, -236, false}, // 10^-52 + {0xe45c10c42a2b3b06, -210, false}, // 10^-44 + {0xaa242499697392d3, -183, false}, // 10^-36 + {0xfd87b5f28300ca0e, -157, false}, // 10^-28 + {0xbce5086492111aeb, -130, false}, // 10^-20 + {0x8cbccc096f5088cc, -103, false}, // 10^-12 + {0xd1b71758e219652c, -77, false}, // 10^-4 + {0x9c40000000000000, -50, false}, // 10^4 + {0xe8d4a51000000000, -24, false}, // 10^12 + {0xad78ebc5ac620000, 3, false}, // 10^20 + {0x813f3978f8940984, 30, false}, // 10^28 + {0xc097ce7bc90715b3, 56, false}, // 10^36 + {0x8f7e32ce7bea5c70, 83, false}, // 10^44 + {0xd5d238a4abe98068, 109, false}, // 10^52 + {0x9f4f2726179a2245, 136, false}, // 10^60 + {0xed63a231d4c4fb27, 162, false}, // 10^68 + {0xb0de65388cc8ada8, 189, false}, // 10^76 + {0x83c7088e1aab65db, 216, false}, // 10^84 + {0xc45d1df942711d9a, 242, false}, // 10^92 + {0x924d692ca61be758, 269, false}, // 10^100 + {0xda01ee641a708dea, 295, false}, // 10^108 + {0xa26da3999aef774a, 322, false}, // 10^116 + {0xf209787bb47d6b85, 348, false}, // 10^124 + {0xb454e4a179dd1877, 375, false}, // 10^132 + {0x865b86925b9bc5c2, 402, false}, // 10^140 + {0xc83553c5c8965d3d, 428, false}, // 10^148 + {0x952ab45cfa97a0b3, 455, false}, // 10^156 + {0xde469fbd99a05fe3, 481, false}, // 10^164 + {0xa59bc234db398c25, 508, false}, // 10^172 + {0xf6c69a72a3989f5c, 534, false}, // 10^180 + {0xb7dcbf5354e9bece, 561, false}, // 10^188 + {0x88fcf317f22241e2, 588, false}, // 10^196 + {0xcc20ce9bd35c78a5, 614, false}, // 10^204 + {0x98165af37b2153df, 641, false}, // 10^212 + {0xe2a0b5dc971f303a, 667, false}, // 10^220 + {0xa8d9d1535ce3b396, 694, false}, // 10^228 + {0xfb9b7cd9a4a7443c, 720, false}, // 10^236 + {0xbb764c4ca7a44410, 747, false}, // 10^244 + {0x8bab8eefb6409c1a, 774, false}, // 10^252 + {0xd01fef10a657842c, 800, false}, // 10^260 + {0x9b10a4e5e9913129, 827, false}, // 10^268 + {0xe7109bfba19c0c9d, 853, false}, // 10^276 + {0xac2820d9623bf429, 880, false}, // 10^284 + {0x80444b5e7aa7cf85, 907, false}, // 10^292 + {0xbf21e44003acdd2d, 933, false}, // 10^300 + {0x8e679c2f5e44ff8f, 960, false}, // 10^308 + {0xd433179d9c8cb841, 986, false}, // 10^316 + {0x9e19db92b4e31ba9, 1013, false}, // 10^324 + {0xeb96bf6ebadf77d9, 1039, false}, // 10^332 + {0xaf87023b9bf0ee6b, 1066, false}, // 10^340 +} + +// floatBits returns the bits of the float64 that best approximates +// the extFloat passed as receiver. Overflow is set to true if +// the resulting float64 is ±Inf. +func (f *extFloat) floatBits(flt *floatInfo) (bits uint64, overflow bool) { + f.Normalize() + + exp := f.exp + 63 + + // Exponent too small. + if exp < flt.bias+1 { + n := flt.bias + 1 - exp + f.mant >>= uint(n) + exp += n + } + + // Extract 1+flt.mantbits bits from the 64-bit mantissa. + mant := f.mant >> (63 - flt.mantbits) + if f.mant&(1<<(62-flt.mantbits)) != 0 { + // Round up. + mant += 1 + } + + // Rounding might have added a bit; shift down. + if mant == 2<<flt.mantbits { + mant >>= 1 + exp++ + } + + // Infinities. + if exp-flt.bias >= 1<<flt.expbits-1 { + // ±Inf + mant = 0 + exp = 1<<flt.expbits - 1 + flt.bias + overflow = true + } else if mant&(1<<flt.mantbits) == 0 { + // Denormalized? + exp = flt.bias + } + // Assemble bits. + bits = mant & (uint64(1)<<flt.mantbits - 1) + bits |= uint64((exp-flt.bias)&(1<<flt.expbits-1)) << flt.mantbits + if f.neg { + bits |= 1 << (flt.mantbits + flt.expbits) + } + return +} + +// AssignComputeBounds sets f to the floating point value +// defined by mant, exp and precision given by flt. It returns +// lower, upper such that any number in the closed interval +// [lower, upper] is converted back to the same floating point number. +func (f *extFloat) AssignComputeBounds(mant uint64, exp int, neg bool, flt *floatInfo) (lower, upper extFloat) { + f.mant = mant + f.exp = exp - int(flt.mantbits) + f.neg = neg + if f.exp <= 0 && mant == (mant>>uint(-f.exp))<<uint(-f.exp) { + // An exact integer + f.mant >>= uint(-f.exp) + f.exp = 0 + return *f, *f + } + expBiased := exp - flt.bias + + upper = extFloat{mant: 2*f.mant + 1, exp: f.exp - 1, neg: f.neg} + if mant != 1<<flt.mantbits || expBiased == 1 { + lower = extFloat{mant: 2*f.mant - 1, exp: f.exp - 1, neg: f.neg} + } else { + lower = extFloat{mant: 4*f.mant - 1, exp: f.exp - 2, neg: f.neg} + } + return +} + +// Normalize normalizes f so that the highest bit of the mantissa is +// set, and returns the number by which the mantissa was left-shifted. +func (f *extFloat) Normalize() (shift uint) { + mant, exp := f.mant, f.exp + if mant == 0 { + return 0 + } + if mant>>(64-32) == 0 { + mant <<= 32 + exp -= 32 + } + if mant>>(64-16) == 0 { + mant <<= 16 + exp -= 16 + } + if mant>>(64-8) == 0 { + mant <<= 8 + exp -= 8 + } + if mant>>(64-4) == 0 { + mant <<= 4 + exp -= 4 + } + if mant>>(64-2) == 0 { + mant <<= 2 + exp -= 2 + } + if mant>>(64-1) == 0 { + mant <<= 1 + exp -= 1 + } + shift = uint(f.exp - exp) + f.mant, f.exp = mant, exp + return +} + +// Multiply sets f to the product f*g: the result is correctly rounded, +// but not normalized. +func (f *extFloat) Multiply(g extFloat) { + fhi, flo := f.mant>>32, uint64(uint32(f.mant)) + ghi, glo := g.mant>>32, uint64(uint32(g.mant)) + + // Cross products. + cross1 := fhi * glo + cross2 := flo * ghi + + // f.mant*g.mant is fhi*ghi << 64 + (cross1+cross2) << 32 + flo*glo + f.mant = fhi*ghi + (cross1 >> 32) + (cross2 >> 32) + rem := uint64(uint32(cross1)) + uint64(uint32(cross2)) + ((flo * glo) >> 32) + // Round up. + rem += (1 << 31) + + f.mant += (rem >> 32) + f.exp = f.exp + g.exp + 64 +} + +var uint64pow10 = [...]uint64{ + 1, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, +} + +// AssignDecimal sets f to an approximate value mantissa*10^exp. It +// returns true if the value represented by f is guaranteed to be the +// best approximation of d after being rounded to a float64 or +// float32 depending on flt. +func (f *extFloat) AssignDecimal(mantissa uint64, exp10 int, neg bool, trunc bool, flt *floatInfo) (ok bool) { + const uint64digits = 19 + const errorscale = 8 + errors := 0 // An upper bound for error, computed in errorscale*ulp. + if trunc { + // the decimal number was truncated. + errors += errorscale / 2 + } + + f.mant = mantissa + f.exp = 0 + f.neg = neg + + // Multiply by powers of ten. + i := (exp10 - firstPowerOfTen) / stepPowerOfTen + if exp10 < firstPowerOfTen || i >= len(powersOfTen) { + return false + } + adjExp := (exp10 - firstPowerOfTen) % stepPowerOfTen + + // We multiply by exp%step + if adjExp < uint64digits && mantissa < uint64pow10[uint64digits-adjExp] { + // We can multiply the mantissa exactly. + f.mant *= uint64pow10[adjExp] + f.Normalize() + } else { + f.Normalize() + f.Multiply(smallPowersOfTen[adjExp]) + errors += errorscale / 2 + } + + // We multiply by 10 to the exp - exp%step. + f.Multiply(powersOfTen[i]) + if errors > 0 { + errors += 1 + } + errors += errorscale / 2 + + // Normalize + shift := f.Normalize() + errors <<= shift + + // Now f is a good approximation of the decimal. + // Check whether the error is too large: that is, if the mantissa + // is perturbated by the error, the resulting float64 will change. + // The 64 bits mantissa is 1 + 52 bits for float64 + 11 extra bits. + // + // In many cases the approximation will be good enough. + denormalExp := flt.bias - 63 + var extrabits uint + if f.exp <= denormalExp { + // f.mant * 2^f.exp is smaller than 2^(flt.bias+1). + extrabits = uint(63 - flt.mantbits + 1 + uint(denormalExp-f.exp)) + } else { + extrabits = uint(63 - flt.mantbits) + } + + halfway := uint64(1) << (extrabits - 1) + mant_extra := f.mant & (1<<extrabits - 1) + + // Do a signed comparison here! If the error estimate could make + // the mantissa round differently for the conversion to double, + // then we can't give a definite answer. + if int64(halfway)-int64(errors) < int64(mant_extra) && + int64(mant_extra) < int64(halfway)+int64(errors) { + return false + } + return true +} + +// Frexp10 is an analogue of math.Frexp for decimal powers. It scales +// f by an approximate power of ten 10^-exp, and returns exp10, so +// that f*10^exp10 has the same value as the old f, up to an ulp, +// as well as the index of 10^-exp in the powersOfTen table. +func (f *extFloat) frexp10() (exp10, index int) { + // The constants expMin and expMax constrain the final value of the + // binary exponent of f. We want a small integral part in the result + // because finding digits of an integer requires divisions, whereas + // digits of the fractional part can be found by repeatedly multiplying + // by 10. + const expMin = -60 + const expMax = -32 + // Find power of ten such that x * 10^n has a binary exponent + // between expMin and expMax. + approxExp10 := ((expMin+expMax)/2 - f.exp) * 28 / 93 // log(10)/log(2) is close to 93/28. + i := (approxExp10 - firstPowerOfTen) / stepPowerOfTen +Loop: + for { + exp := f.exp + powersOfTen[i].exp + 64 + switch { + case exp < expMin: + i++ + case exp > expMax: + i-- + default: + break Loop + } + } + // Apply the desired decimal shift on f. It will have exponent + // in the desired range. This is multiplication by 10^-exp10. + f.Multiply(powersOfTen[i]) + + return -(firstPowerOfTen + i*stepPowerOfTen), i +} + +// frexp10Many applies a common shift by a power of ten to a, b, c. +func frexp10Many(a, b, c *extFloat) (exp10 int) { + exp10, i := c.frexp10() + a.Multiply(powersOfTen[i]) + b.Multiply(powersOfTen[i]) + return +} + +// FixedDecimal stores in d the first n significant digits +// of the decimal representation of f. It returns false +// if it cannot be sure of the answer. +func (f *extFloat) FixedDecimal(d *decimalSlice, n int) bool { + if f.mant == 0 { + d.nd = 0 + d.dp = 0 + d.neg = f.neg + return true + } + if n == 0 { + panic("strconv: internal error: extFloat.FixedDecimal called with n == 0") + } + // Multiply by an appropriate power of ten to have a reasonable + // number to process. + f.Normalize() + exp10, _ := f.frexp10() + + shift := uint(-f.exp) + integer := uint32(f.mant >> shift) + fraction := f.mant - (uint64(integer) << shift) + ε := uint64(1) // ε is the uncertainty we have on the mantissa of f. + + // Write exactly n digits to d. + needed := n // how many digits are left to write. + integerDigits := 0 // the number of decimal digits of integer. + pow10 := uint64(1) // the power of ten by which f was scaled. + for i, pow := 0, uint64(1); i < 20; i++ { + if pow > uint64(integer) { + integerDigits = i + break + } + pow *= 10 + } + rest := integer + if integerDigits > needed { + // the integral part is already large, trim the last digits. + pow10 = uint64pow10[integerDigits-needed] + integer /= uint32(pow10) + rest -= integer * uint32(pow10) + } else { + rest = 0 + } + + // Write the digits of integer: the digits of rest are omitted. + var buf [32]byte + pos := len(buf) + for v := integer; v > 0; { + v1 := v / 10 + v -= 10 * v1 + pos-- + buf[pos] = byte(v + '0') + v = v1 + } + for i := pos; i < len(buf); i++ { + d.d[i-pos] = buf[i] + } + nd := len(buf) - pos + d.nd = nd + d.dp = integerDigits + exp10 + needed -= nd + + if needed > 0 { + if rest != 0 || pow10 != 1 { + panic("strconv: internal error, rest != 0 but needed > 0") + } + // Emit digits for the fractional part. Each time, 10*fraction + // fits in a uint64 without overflow. + for needed > 0 { + fraction *= 10 + ε *= 10 // the uncertainty scales as we multiply by ten. + if 2*ε > 1<<shift { + // the error is so large it could modify which digit to write, abort. + return false + } + digit := fraction >> shift + d.d[nd] = byte(digit + '0') + fraction -= digit << shift + nd++ + needed-- + } + d.nd = nd + } + + // We have written a truncation of f (a numerator / 10^d.dp). The remaining part + // can be interpreted as a small number (< 1) to be added to the last digit of the + // numerator. + // + // If rest > 0, the amount is: + // (rest<<shift | fraction) / (pow10 << shift) + // fraction being known with a ±ε uncertainty. + // The fact that n > 0 guarantees that pow10 << shift does not overflow a uint64. + // + // If rest = 0, pow10 == 1 and the amount is + // fraction / (1 << shift) + // fraction being known with a ±ε uncertainty. + // + // We pass this information to the rounding routine for adjustment. + + ok := adjustLastDigitFixed(d, uint64(rest)<<shift|fraction, pow10, shift, ε) + if !ok { + return false + } + // Trim trailing zeros. + for i := d.nd - 1; i >= 0; i-- { + if d.d[i] != '0' { + d.nd = i + 1 + break + } + } + return true +} + +// adjustLastDigitFixed assumes d contains the representation of the integral part +// of some number, whose fractional part is num / (den << shift). The numerator +// num is only known up to an uncertainty of size ε, assumed to be less than +// (den << shift)/2. +// +// It will increase the last digit by one to account for correct rounding, typically +// when the fractional part is greater than 1/2, and will return false if ε is such +// that no correct answer can be given. +func adjustLastDigitFixed(d *decimalSlice, num, den uint64, shift uint, ε uint64) bool { + if num > den<<shift { + panic("strconv: num > den<<shift in adjustLastDigitFixed") + } + if 2*ε > den<<shift { + panic("strconv: ε > (den<<shift)/2") + } + if 2*(num+ε) < den<<shift { + return true + } + if 2*(num-ε) > den<<shift { + // increment d by 1. + i := d.nd - 1 + for ; i >= 0; i-- { + if d.d[i] == '9' { + d.nd-- + } else { + break + } + } + if i < 0 { + d.d[0] = '1' + d.nd = 1 + d.dp++ + } else { + d.d[i]++ + } + return true + } + return false +} + +// ShortestDecimal stores in d the shortest decimal representation of f +// which belongs to the open interval (lower, upper), where f is supposed +// to lie. It returns false whenever the result is unsure. The implementation +// uses the Grisu3 algorithm. +func (f *extFloat) ShortestDecimal(d *decimalSlice, lower, upper *extFloat) bool { + if f.mant == 0 { + d.nd = 0 + d.dp = 0 + d.neg = f.neg + return true + } + if f.exp == 0 && *lower == *f && *lower == *upper { + // an exact integer. + var buf [24]byte + n := len(buf) - 1 + for v := f.mant; v > 0; { + v1 := v / 10 + v -= 10 * v1 + buf[n] = byte(v + '0') + n-- + v = v1 + } + nd := len(buf) - n - 1 + for i := 0; i < nd; i++ { + d.d[i] = buf[n+1+i] + } + d.nd, d.dp = nd, nd + for d.nd > 0 && d.d[d.nd-1] == '0' { + d.nd-- + } + if d.nd == 0 { + d.dp = 0 + } + d.neg = f.neg + return true + } + upper.Normalize() + // Uniformize exponents. + if f.exp > upper.exp { + f.mant <<= uint(f.exp - upper.exp) + f.exp = upper.exp + } + if lower.exp > upper.exp { + lower.mant <<= uint(lower.exp - upper.exp) + lower.exp = upper.exp + } + + exp10 := frexp10Many(lower, f, upper) + // Take a safety margin due to rounding in frexp10Many, but we lose precision. + upper.mant++ + lower.mant-- + + // The shortest representation of f is either rounded up or down, but + // in any case, it is a truncation of upper. + shift := uint(-upper.exp) + integer := uint32(upper.mant >> shift) + fraction := upper.mant - (uint64(integer) << shift) + + // How far we can go down from upper until the result is wrong. + allowance := upper.mant - lower.mant + // How far we should go to get a very precise result. + targetDiff := upper.mant - f.mant + + // Count integral digits: there are at most 10. + var integerDigits int + for i, pow := 0, uint64(1); i < 20; i++ { + if pow > uint64(integer) { + integerDigits = i + break + } + pow *= 10 + } + for i := 0; i < integerDigits; i++ { + pow := uint64pow10[integerDigits-i-1] + digit := integer / uint32(pow) + d.d[i] = byte(digit + '0') + integer -= digit * uint32(pow) + // evaluate whether we should stop. + if currentDiff := uint64(integer)<<shift + fraction; currentDiff < allowance { + d.nd = i + 1 + d.dp = integerDigits + exp10 + d.neg = f.neg + // Sometimes allowance is so large the last digit might need to be + // decremented to get closer to f. + return adjustLastDigit(d, currentDiff, targetDiff, allowance, pow<<shift, 2) + } + } + d.nd = integerDigits + d.dp = d.nd + exp10 + d.neg = f.neg + + // Compute digits of the fractional part. At each step fraction does not + // overflow. The choice of minExp implies that fraction is less than 2^60. + var digit int + multiplier := uint64(1) + for { + fraction *= 10 + multiplier *= 10 + digit = int(fraction >> shift) + d.d[d.nd] = byte(digit + '0') + d.nd++ + fraction -= uint64(digit) << shift + if fraction < allowance*multiplier { + // We are in the admissible range. Note that if allowance is about to + // overflow, that is, allowance > 2^64/10, the condition is automatically + // true due to the limited range of fraction. + return adjustLastDigit(d, + fraction, targetDiff*multiplier, allowance*multiplier, + 1<<shift, multiplier*2) + } + } +} + +// adjustLastDigit modifies d = x-currentDiff*ε, to get closest to +// d = x-targetDiff*ε, without becoming smaller than x-maxDiff*ε. +// It assumes that a decimal digit is worth ulpDecimal*ε, and that +// all data is known with a error estimate of ulpBinary*ε. +func adjustLastDigit(d *decimalSlice, currentDiff, targetDiff, maxDiff, ulpDecimal, ulpBinary uint64) bool { + if ulpDecimal < 2*ulpBinary { + // Approximation is too wide. + return false + } + for currentDiff+ulpDecimal/2+ulpBinary < targetDiff { + d.d[d.nd-1]-- + currentDiff += ulpDecimal + } + if currentDiff+ulpDecimal <= targetDiff+ulpDecimal/2+ulpBinary { + // we have two choices, and don't know what to do. + return false + } + if currentDiff < ulpBinary || currentDiff > maxDiff-ulpBinary { + // we went too far + return false + } + if d.nd == 1 && d.d[0] == '0' { + // the number has actually reached zero. + d.nd = 0 + d.dp = 0 + } + return true +} diff --git a/fflib/v1/fold.go b/fflib/v1/fold.go new file mode 100644 index 0000000..4d33e6f --- /dev/null +++ b/fflib/v1/fold.go @@ -0,0 +1,121 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Portions of this file are on Go stdlib's encoding/json/fold.go */ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package v1 + +import ( + "unicode/utf8" +) + +const ( + caseMask = ^byte(0x20) // Mask to ignore case in ASCII. + kelvin = '\u212a' + smallLongEss = '\u017f' +) + +// equalFoldRight is a specialization of bytes.EqualFold when s is +// known to be all ASCII (including punctuation), but contains an 's', +// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. +// See comments on foldFunc. +func EqualFoldRight(s, t []byte) bool { + for _, sb := range s { + if len(t) == 0 { + return false + } + tb := t[0] + if tb < utf8.RuneSelf { + if sb != tb { + sbUpper := sb & caseMask + if 'A' <= sbUpper && sbUpper <= 'Z' { + if sbUpper != tb&caseMask { + return false + } + } else { + return false + } + } + t = t[1:] + continue + } + // sb is ASCII and t is not. t must be either kelvin + // sign or long s; sb must be s, S, k, or K. + tr, size := utf8.DecodeRune(t) + switch sb { + case 's', 'S': + if tr != smallLongEss { + return false + } + case 'k', 'K': + if tr != kelvin { + return false + } + default: + return false + } + t = t[size:] + + } + if len(t) > 0 { + return false + } + return true +} + +// asciiEqualFold is a specialization of bytes.EqualFold for use when +// s is all ASCII (but may contain non-letters) and contains no +// special-folding letters. +// See comments on foldFunc. +func AsciiEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, sb := range s { + tb := t[i] + if sb == tb { + continue + } + if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { + if sb&caseMask != tb&caseMask { + return false + } + } else { + return false + } + } + return true +} + +// simpleLetterEqualFold is a specialization of bytes.EqualFold for +// use when s is all ASCII letters (no underscores, etc) and also +// doesn't contain 'k', 'K', 's', or 'S'. +// See comments on foldFunc. +func SimpleLetterEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, b := range s { + if b&caseMask != t[i]&caseMask { + return false + } + } + return true +} diff --git a/fflib/v1/ftoa.go b/fflib/v1/ftoa.go new file mode 100644 index 0000000..360d6db --- /dev/null +++ b/fflib/v1/ftoa.go @@ -0,0 +1,542 @@ +package v1 + +/** + * Copyright 2015 Paul Querna, Klaus Post + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Most of this file are on Go stdlib's strconv/ftoa.go */ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +import "math" + +// TODO: move elsewhere? +type floatInfo struct { + mantbits uint + expbits uint + bias int +} + +var optimize = true // can change for testing + +var float32info = floatInfo{23, 8, -127} +var float64info = floatInfo{52, 11, -1023} + +// AppendFloat appends the string form of the floating-point number f, +// as generated by FormatFloat +func AppendFloat(dst EncodingBuffer, val float64, fmt byte, prec, bitSize int) { + var bits uint64 + var flt *floatInfo + switch bitSize { + case 32: + bits = uint64(math.Float32bits(float32(val))) + flt = &float32info + case 64: + bits = math.Float64bits(val) + flt = &float64info + default: + panic("strconv: illegal AppendFloat/FormatFloat bitSize") + } + + neg := bits>>(flt.expbits+flt.mantbits) != 0 + exp := int(bits>>flt.mantbits) & (1<<flt.expbits - 1) + mant := bits & (uint64(1)<<flt.mantbits - 1) + + switch exp { + case 1<<flt.expbits - 1: + // Inf, NaN + var s string + switch { + case mant != 0: + s = "NaN" + case neg: + s = "-Inf" + default: + s = "+Inf" + } + dst.WriteString(s) + return + + case 0: + // denormalized + exp++ + + default: + // add implicit top bit + mant |= uint64(1) << flt.mantbits + } + exp += flt.bias + + // Pick off easy binary format. + if fmt == 'b' { + fmtB(dst, neg, mant, exp, flt) + return + } + + if !optimize { + bigFtoa(dst, prec, fmt, neg, mant, exp, flt) + return + } + + var digs decimalSlice + ok := false + // Negative precision means "only as much as needed to be exact." + shortest := prec < 0 + if shortest { + // Try Grisu3 algorithm. + f := new(extFloat) + lower, upper := f.AssignComputeBounds(mant, exp, neg, flt) + var buf [32]byte + digs.d = buf[:] + ok = f.ShortestDecimal(&digs, &lower, &upper) + if !ok { + bigFtoa(dst, prec, fmt, neg, mant, exp, flt) + return + } + // Precision for shortest representation mode. + switch fmt { + case 'e', 'E': + prec = max(digs.nd-1, 0) + case 'f': + prec = max(digs.nd-digs.dp, 0) + case 'g', 'G': + prec = digs.nd + } + } else if fmt != 'f' { + // Fixed number of digits. + digits := prec + switch fmt { + case 'e', 'E': + digits++ + case 'g', 'G': + if prec == 0 { + prec = 1 + } + digits = prec + } + if digits <= 15 { + // try fast algorithm when the number of digits is reasonable. + var buf [24]byte + digs.d = buf[:] + f := extFloat{mant, exp - int(flt.mantbits), neg} + ok = f.FixedDecimal(&digs, digits) + } + } + if !ok { + bigFtoa(dst, prec, fmt, neg, mant, exp, flt) + return + } + formatDigits(dst, shortest, neg, digs, prec, fmt) + return +} + +// bigFtoa uses multiprecision computations to format a float. +func bigFtoa(dst EncodingBuffer, prec int, fmt byte, neg bool, mant uint64, exp int, flt *floatInfo) { + d := new(decimal) + d.Assign(mant) + d.Shift(exp - int(flt.mantbits)) + var digs decimalSlice + shortest := prec < 0 + if shortest { + roundShortest(d, mant, exp, flt) + digs = decimalSlice{d: d.d[:], nd: d.nd, dp: d.dp} + // Precision for shortest representation mode. + switch fmt { + case 'e', 'E': + prec = digs.nd - 1 + case 'f': + prec = max(digs.nd-digs.dp, 0) + case 'g', 'G': + prec = digs.nd + } + } else { + // Round appropriately. + switch fmt { + case 'e', 'E': + d.Round(prec + 1) + case 'f': + d.Round(d.dp + prec) + case 'g', 'G': + if prec == 0 { + prec = 1 + } + d.Round(prec) + } + digs = decimalSlice{d: d.d[:], nd: d.nd, dp: d.dp} + } + formatDigits(dst, shortest, neg, digs, prec, fmt) + return +} + +func formatDigits(dst EncodingBuffer, shortest bool, neg bool, digs decimalSlice, prec int, fmt byte) { + switch fmt { + case 'e', 'E': + fmtE(dst, neg, digs, prec, fmt) + return + case 'f': + fmtF(dst, neg, digs, prec) + return + case 'g', 'G': + // trailing fractional zeros in 'e' form will be trimmed. + eprec := prec + if eprec > digs.nd && digs.nd >= digs.dp { + eprec = digs.nd + } + // %e is used if the exponent from the conversion + // is less than -4 or greater than or equal to the precision. + // if precision was the shortest possible, use precision 6 for this decision. + if shortest { + eprec = 6 + } + exp := digs.dp - 1 + if exp < -4 || exp >= eprec { + if prec > digs.nd { + prec = digs.nd + } + fmtE(dst, neg, digs, prec-1, fmt+'e'-'g') + return + } + if prec > digs.dp { + prec = digs.nd + } + fmtF(dst, neg, digs, max(prec-digs.dp, 0)) + return + } + + // unknown format + dst.Write([]byte{'%', fmt}) + return +} + +// Round d (= mant * 2^exp) to the shortest number of digits +// that will let the original floating point value be precisely +// reconstructed. Size is original floating point size (64 or 32). +func roundShortest(d *decimal, mant uint64, exp int, flt *floatInfo) { + // If mantissa is zero, the number is zero; stop now. + if mant == 0 { + d.nd = 0 + return + } + + // Compute upper and lower such that any decimal number + // between upper and lower (possibly inclusive) + // will round to the original floating point number. + + // We may see at once that the number is already shortest. + // + // Suppose d is not denormal, so that 2^exp <= d < 10^dp. + // The closest shorter number is at least 10^(dp-nd) away. + // The lower/upper bounds computed below are at distance + // at most 2^(exp-mantbits). + // + // So the number is already shortest if 10^(dp-nd) > 2^(exp-mantbits), + // or equivalently log2(10)*(dp-nd) > exp-mantbits. + // It is true if 332/100*(dp-nd) >= exp-mantbits (log2(10) > 3.32). + minexp := flt.bias + 1 // minimum possible exponent + if exp > minexp && 332*(d.dp-d.nd) >= 100*(exp-int(flt.mantbits)) { + // The number is already shortest. + return + } + + // d = mant << (exp - mantbits) + // Next highest floating point number is mant+1 << exp-mantbits. + // Our upper bound is halfway between, mant*2+1 << exp-mantbits-1. + upper := new(decimal) + upper.Assign(mant*2 + 1) + upper.Shift(exp - int(flt.mantbits) - 1) + + // d = mant << (exp - mantbits) + // Next lowest floating point number is mant-1 << exp-mantbits, + // unless mant-1 drops the significant bit and exp is not the minimum exp, + // in which case the next lowest is mant*2-1 << exp-mantbits-1. + // Either way, call it mantlo << explo-mantbits. + // Our lower bound is halfway between, mantlo*2+1 << explo-mantbits-1. + var mantlo uint64 + var explo int + if mant > 1<<flt.mantbits || exp == minexp { + mantlo = mant - 1 + explo = exp + } else { + mantlo = mant*2 - 1 + explo = exp - 1 + } + lower := new(decimal) + lower.Assign(mantlo*2 + 1) + lower.Shift(explo - int(flt.mantbits) - 1) + + // The upper and lower bounds are possible outputs only if + // the original mantissa is even, so that IEEE round-to-even + // would round to the original mantissa and not the neighbors. + inclusive := mant%2 == 0 + + // Now we can figure out the minimum number of digits required. + // Walk along until d has distinguished itself from upper and lower. + for i := 0; i < d.nd; i++ { + var l, m, u byte // lower, middle, upper digits + if i < lower.nd { + l = lower.d[i] + } else { + l = '0' + } + m = d.d[i] + if i < upper.nd { + u = upper.d[i] + } else { + u = '0' + } + + // Okay to round down (truncate) if lower has a different digit + // or if lower is inclusive and is exactly the result of rounding down. + okdown := l != m || (inclusive && l == m && i+1 == lower.nd) + + // Okay to round up if upper has a different digit and + // either upper is inclusive or upper is bigger than the result of rounding up. + okup := m != u && (inclusive || m+1 < u || i+1 < upper.nd) + + // If it's okay to do either, then round to the nearest one. + // If it's okay to do only one, do it. + switch { + case okdown && okup: + d.Round(i + 1) + return + case okdown: + d.RoundDown(i + 1) + return + case okup: + d.RoundUp(i + 1) + return + } + } +} + +type decimalSlice struct { + d []byte + nd, dp int + neg bool +} + +// %e: -d.ddddde±dd +func fmtE(dst EncodingBuffer, neg bool, d decimalSlice, prec int, fmt byte) { + // sign + if neg { + dst.WriteByte('-') + } + + // first digit + ch := byte('0') + if d.nd != 0 { + ch = d.d[0] + } + dst.WriteByte(ch) + + // .moredigits + if prec > 0 { + dst.WriteByte('.') + i := 1 + m := min(d.nd, prec+1) + if i < m { + dst.Write(d.d[i:m]) + i = m + } + for i <= prec { + dst.WriteByte('0') + i++ + } + } + + // e± + dst.WriteByte(fmt) + exp := d.dp - 1 + if d.nd == 0 { // special case: 0 has exponent 0 + exp = 0 + } + if exp < 0 { + ch = '-' + exp = -exp + } else { + ch = '+' + } + dst.WriteByte(ch) + + // dd or ddd + switch { + case exp < 10: + dst.WriteByte('0') + dst.WriteByte(byte(exp) + '0') + case exp < 100: + dst.WriteByte(byte(exp/10) + '0') + dst.WriteByte(byte(exp%10) + '0') + default: + dst.WriteByte(byte(exp/100) + '0') + dst.WriteByte(byte(exp/10)%10 + '0') + dst.WriteByte(byte(exp%10) + '0') + } + + return +} + +// %f: -ddddddd.ddddd +func fmtF(dst EncodingBuffer, neg bool, d decimalSlice, prec int) { + // sign + if neg { + dst.WriteByte('-') + } + + // integer, padded with zeros as needed. + if d.dp > 0 { + m := min(d.nd, d.dp) + dst.Write(d.d[:m]) + for ; m < d.dp; m++ { + dst.WriteByte('0') + } + } else { + dst.WriteByte('0') + } + + // fraction + if prec > 0 { + dst.WriteByte('.') + for i := 0; i < prec; i++ { + ch := byte('0') + if j := d.dp + i; 0 <= j && j < d.nd { + ch = d.d[j] + } + dst.WriteByte(ch) + } + } + + return +} + +// %b: -ddddddddp±ddd +func fmtB(dst EncodingBuffer, neg bool, mant uint64, exp int, flt *floatInfo) { + // sign + if neg { + dst.WriteByte('-') + } + + // mantissa + formatBits(dst, mant, 10, false) + + // p + dst.WriteByte('p') + + // ±exponent + exp -= int(flt.mantbits) + if exp >= 0 { + dst.WriteByte('+') + } + formatBits(dst, uint64(exp), 10, exp < 0) + + return +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +// formatBits computes the string representation of u in the given base. +// If neg is set, u is treated as negative int64 value. +func formatBits(dst EncodingBuffer, u uint64, base int, neg bool) { + if base < 2 || base > len(digits) { + panic("strconv: illegal AppendInt/FormatInt base") + } + // 2 <= base && base <= len(digits) + + var a [64 + 1]byte // +1 for sign of 64bit value in base 2 + i := len(a) + + if neg { + u = -u + } + + // convert bits + if base == 10 { + // common case: use constants for / because + // the compiler can optimize it into a multiply+shift + + if ^uintptr(0)>>32 == 0 { + for u > uint64(^uintptr(0)) { + q := u / 1e9 + us := uintptr(u - q*1e9) // us % 1e9 fits into a uintptr + for j := 9; j > 0; j-- { + i-- + qs := us / 10 + a[i] = byte(us - qs*10 + '0') + us = qs + } + u = q + } + } + + // u guaranteed to fit into a uintptr + us := uintptr(u) + for us >= 10 { + i-- + q := us / 10 + a[i] = byte(us - q*10 + '0') + us = q + } + // u < 10 + i-- + a[i] = byte(us + '0') + + } else if s := shifts[base]; s > 0 { + // base is power of 2: use shifts and masks instead of / and % + b := uint64(base) + m := uintptr(b) - 1 // == 1<<s - 1 + for u >= b { + i-- + a[i] = digits[uintptr(u)&m] + u >>= s + } + // u < base + i-- + a[i] = digits[uintptr(u)] + + } else { + // general case + b := uint64(base) + for u >= b { + i-- + q := u / b + a[i] = digits[uintptr(u-q*b)] + u = q + } + // u < base + i-- + a[i] = digits[uintptr(u)] + } + + // add sign, if any + if neg { + i-- + a[i] = '-' + } + + dst.Write(a[i:]) +} diff --git a/fflib/v1/internal/atof.go b/fflib/v1/internal/atof.go new file mode 100644 index 0000000..46c1289 --- /dev/null +++ b/fflib/v1/internal/atof.go @@ -0,0 +1,936 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Portions of this file are on Go stdlib's strconv/atof.go */ + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +// decimal to binary floating point conversion. +// Algorithm: +// 1) Store input in multiprecision decimal. +// 2) Multiply/divide decimal by powers of two until in range [0.5, 1) +// 3) Multiply by 2^precision and round to get mantissa. + +import "math" + +var optimize = true // can change for testing + +func equalIgnoreCase(s1 []byte, s2 []byte) bool { + if len(s1) != len(s2) { + return false + } + for i := 0; i < len(s1); i++ { + c1 := s1[i] + if 'A' <= c1 && c1 <= 'Z' { + c1 += 'a' - 'A' + } + c2 := s2[i] + if 'A' <= c2 && c2 <= 'Z' { + c2 += 'a' - 'A' + } + if c1 != c2 { + return false + } + } + return true +} + +func special(s []byte) (f float64, ok bool) { + if len(s) == 0 { + return + } + switch s[0] { + default: + return + case '+': + if equalIgnoreCase(s, []byte("+inf")) || equalIgnoreCase(s, []byte("+infinity")) { + return math.Inf(1), true + } + case '-': + if equalIgnoreCase(s, []byte("-inf")) || equalIgnoreCase(s, []byte("-infinity")) { + return math.Inf(-1), true + } + case 'n', 'N': + if equalIgnoreCase(s, []byte("nan")) { + return math.NaN(), true + } + case 'i', 'I': + if equalIgnoreCase(s, []byte("inf")) || equalIgnoreCase(s, []byte("infinity")) { + return math.Inf(1), true + } + } + return +} + +func (b *decimal) set(s []byte) (ok bool) { + i := 0 + b.neg = false + b.trunc = false + + // optional sign + if i >= len(s) { + return + } + switch { + case s[i] == '+': + i++ + case s[i] == '-': + b.neg = true + i++ + } + + // digits + sawdot := false + sawdigits := false + for ; i < len(s); i++ { + switch { + case s[i] == '.': + if sawdot { + return + } + sawdot = true + b.dp = b.nd + continue + + case '0' <= s[i] && s[i] <= '9': + sawdigits = true + if s[i] == '0' && b.nd == 0 { // ignore leading zeros + b.dp-- + continue + } + if b.nd < len(b.d) { + b.d[b.nd] = s[i] + b.nd++ + } else if s[i] != '0' { + b.trunc = true + } + continue + } + break + } + if !sawdigits { + return + } + if !sawdot { + b.dp = b.nd + } + + // optional exponent moves decimal point. + // if we read a very large, very long number, + // just be sure to move the decimal point by + // a lot (say, 100000). it doesn't matter if it's + // not the exact number. + if i < len(s) && (s[i] == 'e' || s[i] == 'E') { + i++ + if i >= len(s) { + return + } + esign := 1 + if s[i] == '+' { + i++ + } else if s[i] == '-' { + i++ + esign = -1 + } + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + e := 0 + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { + if e < 10000 { + e = e*10 + int(s[i]) - '0' + } + } + b.dp += e * esign + } + + if i != len(s) { + return + } + + ok = true + return +} + +// readFloat reads a decimal mantissa and exponent from a float +// string representation. It sets ok to false if the number could +// not fit return types or is invalid. +func readFloat(s []byte) (mantissa uint64, exp int, neg, trunc, ok bool) { + const uint64digits = 19 + i := 0 + + // optional sign + if i >= len(s) { + return + } + switch { + case s[i] == '+': + i++ + case s[i] == '-': + neg = true + i++ + } + + // digits + sawdot := false + sawdigits := false + nd := 0 + ndMant := 0 + dp := 0 + for ; i < len(s); i++ { + switch c := s[i]; true { + case c == '.': + if sawdot { + return + } + sawdot = true + dp = nd + continue + + case '0' <= c && c <= '9': + sawdigits = true + if c == '0' && nd == 0 { // ignore leading zeros + dp-- + continue + } + nd++ + if ndMant < uint64digits { + mantissa *= 10 + mantissa += uint64(c - '0') + ndMant++ + } else if s[i] != '0' { + trunc = true + } + continue + } + break + } + if !sawdigits { + return + } + if !sawdot { + dp = nd + } + + // optional exponent moves decimal point. + // if we read a very large, very long number, + // just be sure to move the decimal point by + // a lot (say, 100000). it doesn't matter if it's + // not the exact number. + if i < len(s) && (s[i] == 'e' || s[i] == 'E') { + i++ + if i >= len(s) { + return + } + esign := 1 + if s[i] == '+' { + i++ + } else if s[i] == '-' { + i++ + esign = -1 + } + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + e := 0 + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { + if e < 10000 { + e = e*10 + int(s[i]) - '0' + } + } + dp += e * esign + } + + if i != len(s) { + return + } + + exp = dp - ndMant + ok = true + return + +} + +// decimal power of ten to binary power of two. +var powtab = []int{1, 3, 6, 9, 13, 16, 19, 23, 26} + +func (d *decimal) floatBits(flt *floatInfo) (b uint64, overflow bool) { + var exp int + var mant uint64 + + // Zero is always a special case. + if d.nd == 0 { + mant = 0 + exp = flt.bias + goto out + } + + // Obvious overflow/underflow. + // These bounds are for 64-bit floats. + // Will have to change if we want to support 80-bit floats in the future. + if d.dp > 310 { + goto overflow + } + if d.dp < -330 { + // zero + mant = 0 + exp = flt.bias + goto out + } + + // Scale by powers of two until in range [0.5, 1.0) + exp = 0 + for d.dp > 0 { + var n int + if d.dp >= len(powtab) { + n = 27 + } else { + n = powtab[d.dp] + } + d.Shift(-n) + exp += n + } + for d.dp < 0 || d.dp == 0 && d.d[0] < '5' { + var n int + if -d.dp >= len(powtab) { + n = 27 + } else { + n = powtab[-d.dp] + } + d.Shift(n) + exp -= n + } + + // Our range is [0.5,1) but floating point range is [1,2). + exp-- + + // Minimum representable exponent is flt.bias+1. + // If the exponent is smaller, move it up and + // adjust d accordingly. + if exp < flt.bias+1 { + n := flt.bias + 1 - exp + d.Shift(-n) + exp += n + } + + if exp-flt.bias >= 1<<flt.expbits-1 { + goto overflow + } + + // Extract 1+flt.mantbits bits. + d.Shift(int(1 + flt.mantbits)) + mant = d.RoundedInteger() + + // Rounding might have added a bit; shift down. + if mant == 2<<flt.mantbits { + mant >>= 1 + exp++ + if exp-flt.bias >= 1<<flt.expbits-1 { + goto overflow + } + } + + // Denormalized? + if mant&(1<<flt.mantbits) == 0 { + exp = flt.bias + } + goto out + +overflow: + // ±Inf + mant = 0 + exp = 1<<flt.expbits - 1 + flt.bias + overflow = true + +out: + // Assemble bits. + bits := mant & (uint64(1)<<flt.mantbits - 1) + bits |= uint64((exp-flt.bias)&(1<<flt.expbits-1)) << flt.mantbits + if d.neg { + bits |= 1 << flt.mantbits << flt.expbits + } + return bits, overflow +} + +// Exact powers of 10. +var float64pow10 = []float64{ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, + 1e20, 1e21, 1e22, +} +var float32pow10 = []float32{1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10} + +// If possible to convert decimal representation to 64-bit float f exactly, +// entirely in floating-point math, do so, avoiding the expense of decimalToFloatBits. +// Three common cases: +// value is exact integer +// value is exact integer * exact power of ten +// value is exact integer / exact power of ten +// These all produce potentially inexact but correctly rounded answers. +func atof64exact(mantissa uint64, exp int, neg bool) (f float64, ok bool) { + if mantissa>>float64info.mantbits != 0 { + return + } + f = float64(mantissa) + if neg { + f = -f + } + switch { + case exp == 0: + // an integer. + return f, true + // Exact integers are <= 10^15. + // Exact powers of ten are <= 10^22. + case exp > 0 && exp <= 15+22: // int * 10^k + // If exponent is big but number of digits is not, + // can move a few zeros into the integer part. + if exp > 22 { + f *= float64pow10[exp-22] + exp = 22 + } + if f > 1e15 || f < -1e15 { + // the exponent was really too large. + return + } + return f * float64pow10[exp], true + case exp < 0 && exp >= -22: // int / 10^k + return f / float64pow10[-exp], true + } + return +} + +// If possible to compute mantissa*10^exp to 32-bit float f exactly, +// entirely in floating-point math, do so, avoiding the machinery above. +func atof32exact(mantissa uint64, exp int, neg bool) (f float32, ok bool) { + if mantissa>>float32info.mantbits != 0 { + return + } + f = float32(mantissa) + if neg { + f = -f + } + switch { + case exp == 0: + return f, true + // Exact integers are <= 10^7. + // Exact powers of ten are <= 10^10. + case exp > 0 && exp <= 7+10: // int * 10^k + // If exponent is big but number of digits is not, + // can move a few zeros into the integer part. + if exp > 10 { + f *= float32pow10[exp-10] + exp = 10 + } + if f > 1e7 || f < -1e7 { + // the exponent was really too large. + return + } + return f * float32pow10[exp], true + case exp < 0 && exp >= -10: // int / 10^k + return f / float32pow10[-exp], true + } + return +} + +const fnParseFloat = "ParseFloat" + +func atof32(s []byte) (f float32, err error) { + if val, ok := special(s); ok { + return float32(val), nil + } + + if optimize { + // Parse mantissa and exponent. + mantissa, exp, neg, trunc, ok := readFloat(s) + if ok { + // Try pure floating-point arithmetic conversion. + if !trunc { + if f, ok := atof32exact(mantissa, exp, neg); ok { + return f, nil + } + } + // Try another fast path. + ext := new(extFloat) + if ok := ext.AssignDecimal(mantissa, exp, neg, trunc, &float32info); ok { + b, ovf := ext.floatBits(&float32info) + f = math.Float32frombits(uint32(b)) + if ovf { + err = rangeError(fnParseFloat, string(s)) + } + return f, err + } + } + } + var d decimal + if !d.set(s) { + return 0, syntaxError(fnParseFloat, string(s)) + } + b, ovf := d.floatBits(&float32info) + f = math.Float32frombits(uint32(b)) + if ovf { + err = rangeError(fnParseFloat, string(s)) + } + return f, err +} + +func atof64(s []byte) (f float64, err error) { + if val, ok := special(s); ok { + return val, nil + } + + if optimize { + // Parse mantissa and exponent. + mantissa, exp, neg, trunc, ok := readFloat(s) + if ok { + // Try pure floating-point arithmetic conversion. + if !trunc { + if f, ok := atof64exact(mantissa, exp, neg); ok { + return f, nil + } + } + // Try another fast path. + ext := new(extFloat) + if ok := ext.AssignDecimal(mantissa, exp, neg, trunc, &float64info); ok { + b, ovf := ext.floatBits(&float64info) + f = math.Float64frombits(b) + if ovf { + err = rangeError(fnParseFloat, string(s)) + } + return f, err + } + } + } + var d decimal + if !d.set(s) { + return 0, syntaxError(fnParseFloat, string(s)) + } + b, ovf := d.floatBits(&float64info) + f = math.Float64frombits(b) + if ovf { + err = rangeError(fnParseFloat, string(s)) + } + return f, err +} + +// ParseFloat converts the string s to a floating-point number +// with the precision specified by bitSize: 32 for float32, or 64 for float64. +// When bitSize=32, the result still has type float64, but it will be +// convertible to float32 without changing its value. +// +// If s is well-formed and near a valid floating point number, +// ParseFloat returns the nearest floating point number rounded +// using IEEE754 unbiased rounding. +// +// The errors that ParseFloat returns have concrete type *NumError +// and include err.Num = s. +// +// If s is not syntactically well-formed, ParseFloat returns err.Err = ErrSyntax. +// +// If s is syntactically well-formed but is more than 1/2 ULP +// away from the largest floating point number of the given size, +// ParseFloat returns f = ±Inf, err.Err = ErrRange. +func ParseFloat(s []byte, bitSize int) (f float64, err error) { + if bitSize == 32 { + f1, err1 := atof32(s) + return float64(f1), err1 + } + f1, err1 := atof64(s) + return f1, err1 +} + +// oroginal: strconv/decimal.go, but not exported, and needed for PareFloat. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Multiprecision decimal numbers. +// For floating-point formatting only; not general purpose. +// Only operations are assign and (binary) left/right shift. +// Can do binary floating point in multiprecision decimal precisely +// because 2 divides 10; cannot do decimal floating point +// in multiprecision binary precisely. + +type decimal struct { + d [800]byte // digits + nd int // number of digits used + dp int // decimal point + neg bool + trunc bool // discarded nonzero digits beyond d[:nd] +} + +func (a *decimal) String() string { + n := 10 + a.nd + if a.dp > 0 { + n += a.dp + } + if a.dp < 0 { + n += -a.dp + } + + buf := make([]byte, n) + w := 0 + switch { + case a.nd == 0: + return "0" + + case a.dp <= 0: + // zeros fill space between decimal point and digits + buf[w] = '0' + w++ + buf[w] = '.' + w++ + w += digitZero(buf[w : w+-a.dp]) + w += copy(buf[w:], a.d[0:a.nd]) + + case a.dp < a.nd: + // decimal point in middle of digits + w += copy(buf[w:], a.d[0:a.dp]) + buf[w] = '.' + w++ + w += copy(buf[w:], a.d[a.dp:a.nd]) + + default: + // zeros fill space between digits and decimal point + w += copy(buf[w:], a.d[0:a.nd]) + w += digitZero(buf[w : w+a.dp-a.nd]) + } + return string(buf[0:w]) +} + +func digitZero(dst []byte) int { + for i := range dst { + dst[i] = '0' + } + return len(dst) +} + +// trim trailing zeros from number. +// (They are meaningless; the decimal point is tracked +// independent of the number of digits.) +func trim(a *decimal) { + for a.nd > 0 && a.d[a.nd-1] == '0' { + a.nd-- + } + if a.nd == 0 { + a.dp = 0 + } +} + +// Assign v to a. +func (a *decimal) Assign(v uint64) { + var buf [24]byte + + // Write reversed decimal in buf. + n := 0 + for v > 0 { + v1 := v / 10 + v -= 10 * v1 + buf[n] = byte(v + '0') + n++ + v = v1 + } + + // Reverse again to produce forward decimal in a.d. + a.nd = 0 + for n--; n >= 0; n-- { + a.d[a.nd] = buf[n] + a.nd++ + } + a.dp = a.nd + trim(a) +} + +// Maximum shift that we can do in one pass without overflow. +// Signed int has 31 bits, and we have to be able to accommodate 9<<k. +const maxShift = 27 + +// Binary shift right (* 2) by k bits. k <= maxShift to avoid overflow. +func rightShift(a *decimal, k uint) { + r := 0 // read pointer + w := 0 // write pointer + + // Pick up enough leading digits to cover first shift. + n := 0 + for ; n>>k == 0; r++ { + if r >= a.nd { + if n == 0 { + // a == 0; shouldn't get here, but handle anyway. + a.nd = 0 + return + } + for n>>k == 0 { + n = n * 10 + r++ + } + break + } + c := int(a.d[r]) + n = n*10 + c - '0' + } + a.dp -= r - 1 + + // Pick up a digit, put down a digit. + for ; r < a.nd; r++ { + c := int(a.d[r]) + dig := n >> k + n -= dig << k + a.d[w] = byte(dig + '0') + w++ + n = n*10 + c - '0' + } + + // Put down extra digits. + for n > 0 { + dig := n >> k + n -= dig << k + if w < len(a.d) { + a.d[w] = byte(dig + '0') + w++ + } else if dig > 0 { + a.trunc = true + } + n = n * 10 + } + + a.nd = w + trim(a) +} + +// Cheat sheet for left shift: table indexed by shift count giving +// number of new digits that will be introduced by that shift. +// +// For example, leftcheats[4] = {2, "625"}. That means that +// if we are shifting by 4 (multiplying by 16), it will add 2 digits +// when the string prefix is "625" through "999", and one fewer digit +// if the string prefix is "000" through "624". +// +// Credit for this trick goes to Ken. + +type leftCheat struct { + delta int // number of new digits + cutoff string // minus one digit if original < a. +} + +var leftcheats = []leftCheat{ + // Leading digits of 1/2^i = 5^i. + // 5^23 is not an exact 64-bit floating point number, + // so have to use bc for the math. + /* + seq 27 | sed 's/^/5^/' | bc | + awk 'BEGIN{ print "\tleftCheat{ 0, \"\" }," } + { + log2 = log(2)/log(10) + printf("\tleftCheat{ %d, \"%s\" },\t// * %d\n", + int(log2*NR+1), $0, 2**NR) + }' + */ + {0, ""}, + {1, "5"}, // * 2 + {1, "25"}, // * 4 + {1, "125"}, // * 8 + {2, "625"}, // * 16 + {2, "3125"}, // * 32 + {2, "15625"}, // * 64 + {3, "78125"}, // * 128 + {3, "390625"}, // * 256 + {3, "1953125"}, // * 512 + {4, "9765625"}, // * 1024 + {4, "48828125"}, // * 2048 + {4, "244140625"}, // * 4096 + {4, "1220703125"}, // * 8192 + {5, "6103515625"}, // * 16384 + {5, "30517578125"}, // * 32768 + {5, "152587890625"}, // * 65536 + {6, "762939453125"}, // * 131072 + {6, "3814697265625"}, // * 262144 + {6, "19073486328125"}, // * 524288 + {7, "95367431640625"}, // * 1048576 + {7, "476837158203125"}, // * 2097152 + {7, "2384185791015625"}, // * 4194304 + {7, "11920928955078125"}, // * 8388608 + {8, "59604644775390625"}, // * 16777216 + {8, "298023223876953125"}, // * 33554432 + {8, "1490116119384765625"}, // * 67108864 + {9, "7450580596923828125"}, // * 134217728 +} + +// Is the leading prefix of b lexicographically less than s? +func prefixIsLessThan(b []byte, s string) bool { + for i := 0; i < len(s); i++ { + if i >= len(b) { + return true + } + if b[i] != s[i] { + return b[i] < s[i] + } + } + return false +} + +// Binary shift left (/ 2) by k bits. k <= maxShift to avoid overflow. +func leftShift(a *decimal, k uint) { + delta := leftcheats[k].delta + if prefixIsLessThan(a.d[0:a.nd], leftcheats[k].cutoff) { + delta-- + } + + r := a.nd // read index + w := a.nd + delta // write index + n := 0 + + // Pick up a digit, put down a digit. + for r--; r >= 0; r-- { + n += (int(a.d[r]) - '0') << k + quo := n / 10 + rem := n - 10*quo + w-- + if w < len(a.d) { + a.d[w] = byte(rem + '0') + } else if rem != 0 { + a.trunc = true + } + n = quo + } + + // Put down extra digits. + for n > 0 { + quo := n / 10 + rem := n - 10*quo + w-- + if w < len(a.d) { + a.d[w] = byte(rem + '0') + } else if rem != 0 { + a.trunc = true + } + n = quo + } + + a.nd += delta + if a.nd >= len(a.d) { + a.nd = len(a.d) + } + a.dp += delta + trim(a) +} + +// Binary shift left (k > 0) or right (k < 0). +func (a *decimal) Shift(k int) { + switch { + case a.nd == 0: + // nothing to do: a == 0 + case k > 0: + for k > maxShift { + leftShift(a, maxShift) + k -= maxShift + } + leftShift(a, uint(k)) + case k < 0: + for k < -maxShift { + rightShift(a, maxShift) + k += maxShift + } + rightShift(a, uint(-k)) + } +} + +// If we chop a at nd digits, should we round up? +func shouldRoundUp(a *decimal, nd int) bool { + if nd < 0 || nd >= a.nd { + return false + } + if a.d[nd] == '5' && nd+1 == a.nd { // exactly halfway - round to even + // if we truncated, a little higher than what's recorded - always round up + if a.trunc { + return true + } + return nd > 0 && (a.d[nd-1]-'0')%2 != 0 + } + // not halfway - digit tells all + return a.d[nd] >= '5' +} + +// Round a to nd digits (or fewer). +// If nd is zero, it means we're rounding +// just to the left of the digits, as in +// 0.09 -> 0.1. +func (a *decimal) Round(nd int) { + if nd < 0 || nd >= a.nd { + return + } + if shouldRoundUp(a, nd) { + a.RoundUp(nd) + } else { + a.RoundDown(nd) + } +} + +// Round a down to nd digits (or fewer). +func (a *decimal) RoundDown(nd int) { + if nd < 0 || nd >= a.nd { + return + } + a.nd = nd + trim(a) +} + +// Round a up to nd digits (or fewer). +func (a *decimal) RoundUp(nd int) { + if nd < 0 || nd >= a.nd { + return + } + + // round up + for i := nd - 1; i >= 0; i-- { + c := a.d[i] + if c < '9' { // can stop after this digit + a.d[i]++ + a.nd = i + 1 + return + } + } + + // Number is all 9s. + // Change to single 1 with adjusted decimal point. + a.d[0] = '1' + a.nd = 1 + a.dp++ +} + +// Extract integer part, rounded appropriately. +// No guarantees about overflow. +func (a *decimal) RoundedInteger() uint64 { + if a.dp > 20 { + return 0xFFFFFFFFFFFFFFFF + } + var i int + n := uint64(0) + for i = 0; i < a.dp && i < a.nd; i++ { + n = n*10 + uint64(a.d[i]-'0') + } + for ; i < a.dp; i++ { + n *= 10 + } + if shouldRoundUp(a, a.dp) { + n++ + } + return n +} diff --git a/fflib/v1/internal/atoi.go b/fflib/v1/internal/atoi.go new file mode 100644 index 0000000..06eb2ec --- /dev/null +++ b/fflib/v1/internal/atoi.go @@ -0,0 +1,213 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Portions of this file are on Go stdlib's strconv/atoi.go */ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +import ( + "errors" + "strconv" +) + +// ErrRange indicates that a value is out of range for the target type. +var ErrRange = errors.New("value out of range") + +// ErrSyntax indicates that a value does not have the right syntax for the target type. +var ErrSyntax = errors.New("invalid syntax") + +// A NumError records a failed conversion. +type NumError struct { + Func string // the failing function (ParseBool, ParseInt, ParseUint, ParseFloat) + Num string // the input + Err error // the reason the conversion failed (ErrRange, ErrSyntax) +} + +func (e *NumError) Error() string { + return "strconv." + e.Func + ": " + "parsing " + strconv.Quote(e.Num) + ": " + e.Err.Error() +} + +func syntaxError(fn, str string) *NumError { + return &NumError{fn, str, ErrSyntax} +} + +func rangeError(fn, str string) *NumError { + return &NumError{fn, str, ErrRange} +} + +const intSize = 32 << uint(^uint(0)>>63) + +// IntSize is the size in bits of an int or uint value. +const IntSize = intSize + +// Return the first number n such that n*base >= 1<<64. +func cutoff64(base int) uint64 { + if base < 2 { + return 0 + } + return (1<<64-1)/uint64(base) + 1 +} + +// ParseUint is like ParseInt but for unsigned numbers, and oeprating on []byte +func ParseUint(s []byte, base int, bitSize int) (n uint64, err error) { + var cutoff, maxVal uint64 + + if bitSize == 0 { + bitSize = int(IntSize) + } + + s0 := s + switch { + case len(s) < 1: + err = ErrSyntax + goto Error + + case 2 <= base && base <= 36: + // valid base; nothing to do + + case base == 0: + // Look for octal, hex prefix. + switch { + case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'): + base = 16 + s = s[2:] + if len(s) < 1 { + err = ErrSyntax + goto Error + } + case s[0] == '0': + base = 8 + default: + base = 10 + } + + default: + err = errors.New("invalid base " + strconv.Itoa(base)) + goto Error + } + + n = 0 + cutoff = cutoff64(base) + maxVal = 1<<uint(bitSize) - 1 + + for i := 0; i < len(s); i++ { + var v byte + d := s[i] + switch { + case '0' <= d && d <= '9': + v = d - '0' + case 'a' <= d && d <= 'z': + v = d - 'a' + 10 + case 'A' <= d && d <= 'Z': + v = d - 'A' + 10 + default: + n = 0 + err = ErrSyntax + goto Error + } + if int(v) >= base { + n = 0 + err = ErrSyntax + goto Error + } + + if n >= cutoff { + // n*base overflows + n = 1<<64 - 1 + err = ErrRange + goto Error + } + n *= uint64(base) + + n1 := n + uint64(v) + if n1 < n || n1 > maxVal { + // n+v overflows + n = 1<<64 - 1 + err = ErrRange + goto Error + } + n = n1 + } + + return n, nil + +Error: + return n, &NumError{"ParseUint", string(s0), err} +} + +// ParseInt interprets a string s in the given base (2 to 36) and +// returns the corresponding value i. If base == 0, the base is +// implied by the string's prefix: base 16 for "0x", base 8 for +// "0", and base 10 otherwise. +// +// The bitSize argument specifies the integer type +// that the result must fit into. Bit sizes 0, 8, 16, 32, and 64 +// correspond to int, int8, int16, int32, and int64. +// +// The errors that ParseInt returns have concrete type *NumError +// and include err.Num = s. If s is empty or contains invalid +// digits, err.Err = ErrSyntax and the returned value is 0; +// if the value corresponding to s cannot be represented by a +// signed integer of the given size, err.Err = ErrRange and the +// returned value is the maximum magnitude integer of the +// appropriate bitSize and sign. +func ParseInt(s []byte, base int, bitSize int) (i int64, err error) { + const fnParseInt = "ParseInt" + + if bitSize == 0 { + bitSize = int(IntSize) + } + + // Empty string bad. + if len(s) == 0 { + return 0, syntaxError(fnParseInt, string(s)) + } + + // Pick off leading sign. + s0 := s + neg := false + if s[0] == '+' { + s = s[1:] + } else if s[0] == '-' { + neg = true + s = s[1:] + } + + // Convert unsigned and check range. + var un uint64 + un, err = ParseUint(s, base, bitSize) + if err != nil && err.(*NumError).Err != ErrRange { + err.(*NumError).Func = fnParseInt + err.(*NumError).Num = string(s0) + return 0, err + } + cutoff := uint64(1 << uint(bitSize-1)) + if !neg && un >= cutoff { + return int64(cutoff - 1), rangeError(fnParseInt, string(s0)) + } + if neg && un > cutoff { + return -int64(cutoff), rangeError(fnParseInt, string(s0)) + } + n := int64(un) + if neg { + n = -n + } + return n, nil +} diff --git a/fflib/v1/internal/extfloat.go b/fflib/v1/internal/extfloat.go new file mode 100644 index 0000000..ab79108 --- /dev/null +++ b/fflib/v1/internal/extfloat.go @@ -0,0 +1,668 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +// An extFloat represents an extended floating-point number, with more +// precision than a float64. It does not try to save bits: the +// number represented by the structure is mant*(2^exp), with a negative +// sign if neg is true. +type extFloat struct { + mant uint64 + exp int + neg bool +} + +// Powers of ten taken from double-conversion library. +// http://code.google.com/p/double-conversion/ +const ( + firstPowerOfTen = -348 + stepPowerOfTen = 8 +) + +var smallPowersOfTen = [...]extFloat{ + {1 << 63, -63, false}, // 1 + {0xa << 60, -60, false}, // 1e1 + {0x64 << 57, -57, false}, // 1e2 + {0x3e8 << 54, -54, false}, // 1e3 + {0x2710 << 50, -50, false}, // 1e4 + {0x186a0 << 47, -47, false}, // 1e5 + {0xf4240 << 44, -44, false}, // 1e6 + {0x989680 << 40, -40, false}, // 1e7 +} + +var powersOfTen = [...]extFloat{ + {0xfa8fd5a0081c0288, -1220, false}, // 10^-348 + {0xbaaee17fa23ebf76, -1193, false}, // 10^-340 + {0x8b16fb203055ac76, -1166, false}, // 10^-332 + {0xcf42894a5dce35ea, -1140, false}, // 10^-324 + {0x9a6bb0aa55653b2d, -1113, false}, // 10^-316 + {0xe61acf033d1a45df, -1087, false}, // 10^-308 + {0xab70fe17c79ac6ca, -1060, false}, // 10^-300 + {0xff77b1fcbebcdc4f, -1034, false}, // 10^-292 + {0xbe5691ef416bd60c, -1007, false}, // 10^-284 + {0x8dd01fad907ffc3c, -980, false}, // 10^-276 + {0xd3515c2831559a83, -954, false}, // 10^-268 + {0x9d71ac8fada6c9b5, -927, false}, // 10^-260 + {0xea9c227723ee8bcb, -901, false}, // 10^-252 + {0xaecc49914078536d, -874, false}, // 10^-244 + {0x823c12795db6ce57, -847, false}, // 10^-236 + {0xc21094364dfb5637, -821, false}, // 10^-228 + {0x9096ea6f3848984f, -794, false}, // 10^-220 + {0xd77485cb25823ac7, -768, false}, // 10^-212 + {0xa086cfcd97bf97f4, -741, false}, // 10^-204 + {0xef340a98172aace5, -715, false}, // 10^-196 + {0xb23867fb2a35b28e, -688, false}, // 10^-188 + {0x84c8d4dfd2c63f3b, -661, false}, // 10^-180 + {0xc5dd44271ad3cdba, -635, false}, // 10^-172 + {0x936b9fcebb25c996, -608, false}, // 10^-164 + {0xdbac6c247d62a584, -582, false}, // 10^-156 + {0xa3ab66580d5fdaf6, -555, false}, // 10^-148 + {0xf3e2f893dec3f126, -529, false}, // 10^-140 + {0xb5b5ada8aaff80b8, -502, false}, // 10^-132 + {0x87625f056c7c4a8b, -475, false}, // 10^-124 + {0xc9bcff6034c13053, -449, false}, // 10^-116 + {0x964e858c91ba2655, -422, false}, // 10^-108 + {0xdff9772470297ebd, -396, false}, // 10^-100 + {0xa6dfbd9fb8e5b88f, -369, false}, // 10^-92 + {0xf8a95fcf88747d94, -343, false}, // 10^-84 + {0xb94470938fa89bcf, -316, false}, // 10^-76 + {0x8a08f0f8bf0f156b, -289, false}, // 10^-68 + {0xcdb02555653131b6, -263, false}, // 10^-60 + {0x993fe2c6d07b7fac, -236, false}, // 10^-52 + {0xe45c10c42a2b3b06, -210, false}, // 10^-44 + {0xaa242499697392d3, -183, false}, // 10^-36 + {0xfd87b5f28300ca0e, -157, false}, // 10^-28 + {0xbce5086492111aeb, -130, false}, // 10^-20 + {0x8cbccc096f5088cc, -103, false}, // 10^-12 + {0xd1b71758e219652c, -77, false}, // 10^-4 + {0x9c40000000000000, -50, false}, // 10^4 + {0xe8d4a51000000000, -24, false}, // 10^12 + {0xad78ebc5ac620000, 3, false}, // 10^20 + {0x813f3978f8940984, 30, false}, // 10^28 + {0xc097ce7bc90715b3, 56, false}, // 10^36 + {0x8f7e32ce7bea5c70, 83, false}, // 10^44 + {0xd5d238a4abe98068, 109, false}, // 10^52 + {0x9f4f2726179a2245, 136, false}, // 10^60 + {0xed63a231d4c4fb27, 162, false}, // 10^68 + {0xb0de65388cc8ada8, 189, false}, // 10^76 + {0x83c7088e1aab65db, 216, false}, // 10^84 + {0xc45d1df942711d9a, 242, false}, // 10^92 + {0x924d692ca61be758, 269, false}, // 10^100 + {0xda01ee641a708dea, 295, false}, // 10^108 + {0xa26da3999aef774a, 322, false}, // 10^116 + {0xf209787bb47d6b85, 348, false}, // 10^124 + {0xb454e4a179dd1877, 375, false}, // 10^132 + {0x865b86925b9bc5c2, 402, false}, // 10^140 + {0xc83553c5c8965d3d, 428, false}, // 10^148 + {0x952ab45cfa97a0b3, 455, false}, // 10^156 + {0xde469fbd99a05fe3, 481, false}, // 10^164 + {0xa59bc234db398c25, 508, false}, // 10^172 + {0xf6c69a72a3989f5c, 534, false}, // 10^180 + {0xb7dcbf5354e9bece, 561, false}, // 10^188 + {0x88fcf317f22241e2, 588, false}, // 10^196 + {0xcc20ce9bd35c78a5, 614, false}, // 10^204 + {0x98165af37b2153df, 641, false}, // 10^212 + {0xe2a0b5dc971f303a, 667, false}, // 10^220 + {0xa8d9d1535ce3b396, 694, false}, // 10^228 + {0xfb9b7cd9a4a7443c, 720, false}, // 10^236 + {0xbb764c4ca7a44410, 747, false}, // 10^244 + {0x8bab8eefb6409c1a, 774, false}, // 10^252 + {0xd01fef10a657842c, 800, false}, // 10^260 + {0x9b10a4e5e9913129, 827, false}, // 10^268 + {0xe7109bfba19c0c9d, 853, false}, // 10^276 + {0xac2820d9623bf429, 880, false}, // 10^284 + {0x80444b5e7aa7cf85, 907, false}, // 10^292 + {0xbf21e44003acdd2d, 933, false}, // 10^300 + {0x8e679c2f5e44ff8f, 960, false}, // 10^308 + {0xd433179d9c8cb841, 986, false}, // 10^316 + {0x9e19db92b4e31ba9, 1013, false}, // 10^324 + {0xeb96bf6ebadf77d9, 1039, false}, // 10^332 + {0xaf87023b9bf0ee6b, 1066, false}, // 10^340 +} + +// floatBits returns the bits of the float64 that best approximates +// the extFloat passed as receiver. Overflow is set to true if +// the resulting float64 is ±Inf. +func (f *extFloat) floatBits(flt *floatInfo) (bits uint64, overflow bool) { + f.Normalize() + + exp := f.exp + 63 + + // Exponent too small. + if exp < flt.bias+1 { + n := flt.bias + 1 - exp + f.mant >>= uint(n) + exp += n + } + + // Extract 1+flt.mantbits bits from the 64-bit mantissa. + mant := f.mant >> (63 - flt.mantbits) + if f.mant&(1<<(62-flt.mantbits)) != 0 { + // Round up. + mant += 1 + } + + // Rounding might have added a bit; shift down. + if mant == 2<<flt.mantbits { + mant >>= 1 + exp++ + } + + // Infinities. + if exp-flt.bias >= 1<<flt.expbits-1 { + // ±Inf + mant = 0 + exp = 1<<flt.expbits - 1 + flt.bias + overflow = true + } else if mant&(1<<flt.mantbits) == 0 { + // Denormalized? + exp = flt.bias + } + // Assemble bits. + bits = mant & (uint64(1)<<flt.mantbits - 1) + bits |= uint64((exp-flt.bias)&(1<<flt.expbits-1)) << flt.mantbits + if f.neg { + bits |= 1 << (flt.mantbits + flt.expbits) + } + return +} + +// AssignComputeBounds sets f to the floating point value +// defined by mant, exp and precision given by flt. It returns +// lower, upper such that any number in the closed interval +// [lower, upper] is converted back to the same floating point number. +func (f *extFloat) AssignComputeBounds(mant uint64, exp int, neg bool, flt *floatInfo) (lower, upper extFloat) { + f.mant = mant + f.exp = exp - int(flt.mantbits) + f.neg = neg + if f.exp <= 0 && mant == (mant>>uint(-f.exp))<<uint(-f.exp) { + // An exact integer + f.mant >>= uint(-f.exp) + f.exp = 0 + return *f, *f + } + expBiased := exp - flt.bias + + upper = extFloat{mant: 2*f.mant + 1, exp: f.exp - 1, neg: f.neg} + if mant != 1<<flt.mantbits || expBiased == 1 { + lower = extFloat{mant: 2*f.mant - 1, exp: f.exp - 1, neg: f.neg} + } else { + lower = extFloat{mant: 4*f.mant - 1, exp: f.exp - 2, neg: f.neg} + } + return +} + +// Normalize normalizes f so that the highest bit of the mantissa is +// set, and returns the number by which the mantissa was left-shifted. +func (f *extFloat) Normalize() (shift uint) { + mant, exp := f.mant, f.exp + if mant == 0 { + return 0 + } + if mant>>(64-32) == 0 { + mant <<= 32 + exp -= 32 + } + if mant>>(64-16) == 0 { + mant <<= 16 + exp -= 16 + } + if mant>>(64-8) == 0 { + mant <<= 8 + exp -= 8 + } + if mant>>(64-4) == 0 { + mant <<= 4 + exp -= 4 + } + if mant>>(64-2) == 0 { + mant <<= 2 + exp -= 2 + } + if mant>>(64-1) == 0 { + mant <<= 1 + exp -= 1 + } + shift = uint(f.exp - exp) + f.mant, f.exp = mant, exp + return +} + +// Multiply sets f to the product f*g: the result is correctly rounded, +// but not normalized. +func (f *extFloat) Multiply(g extFloat) { + fhi, flo := f.mant>>32, uint64(uint32(f.mant)) + ghi, glo := g.mant>>32, uint64(uint32(g.mant)) + + // Cross products. + cross1 := fhi * glo + cross2 := flo * ghi + + // f.mant*g.mant is fhi*ghi << 64 + (cross1+cross2) << 32 + flo*glo + f.mant = fhi*ghi + (cross1 >> 32) + (cross2 >> 32) + rem := uint64(uint32(cross1)) + uint64(uint32(cross2)) + ((flo * glo) >> 32) + // Round up. + rem += (1 << 31) + + f.mant += (rem >> 32) + f.exp = f.exp + g.exp + 64 +} + +var uint64pow10 = [...]uint64{ + 1, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, +} + +// AssignDecimal sets f to an approximate value mantissa*10^exp. It +// returns true if the value represented by f is guaranteed to be the +// best approximation of d after being rounded to a float64 or +// float32 depending on flt. +func (f *extFloat) AssignDecimal(mantissa uint64, exp10 int, neg bool, trunc bool, flt *floatInfo) (ok bool) { + const uint64digits = 19 + const errorscale = 8 + errors := 0 // An upper bound for error, computed in errorscale*ulp. + if trunc { + // the decimal number was truncated. + errors += errorscale / 2 + } + + f.mant = mantissa + f.exp = 0 + f.neg = neg + + // Multiply by powers of ten. + i := (exp10 - firstPowerOfTen) / stepPowerOfTen + if exp10 < firstPowerOfTen || i >= len(powersOfTen) { + return false + } + adjExp := (exp10 - firstPowerOfTen) % stepPowerOfTen + + // We multiply by exp%step + if adjExp < uint64digits && mantissa < uint64pow10[uint64digits-adjExp] { + // We can multiply the mantissa exactly. + f.mant *= uint64pow10[adjExp] + f.Normalize() + } else { + f.Normalize() + f.Multiply(smallPowersOfTen[adjExp]) + errors += errorscale / 2 + } + + // We multiply by 10 to the exp - exp%step. + f.Multiply(powersOfTen[i]) + if errors > 0 { + errors += 1 + } + errors += errorscale / 2 + + // Normalize + shift := f.Normalize() + errors <<= shift + + // Now f is a good approximation of the decimal. + // Check whether the error is too large: that is, if the mantissa + // is perturbated by the error, the resulting float64 will change. + // The 64 bits mantissa is 1 + 52 bits for float64 + 11 extra bits. + // + // In many cases the approximation will be good enough. + denormalExp := flt.bias - 63 + var extrabits uint + if f.exp <= denormalExp { + // f.mant * 2^f.exp is smaller than 2^(flt.bias+1). + extrabits = uint(63 - flt.mantbits + 1 + uint(denormalExp-f.exp)) + } else { + extrabits = uint(63 - flt.mantbits) + } + + halfway := uint64(1) << (extrabits - 1) + mant_extra := f.mant & (1<<extrabits - 1) + + // Do a signed comparison here! If the error estimate could make + // the mantissa round differently for the conversion to double, + // then we can't give a definite answer. + if int64(halfway)-int64(errors) < int64(mant_extra) && + int64(mant_extra) < int64(halfway)+int64(errors) { + return false + } + return true +} + +// Frexp10 is an analogue of math.Frexp for decimal powers. It scales +// f by an approximate power of ten 10^-exp, and returns exp10, so +// that f*10^exp10 has the same value as the old f, up to an ulp, +// as well as the index of 10^-exp in the powersOfTen table. +func (f *extFloat) frexp10() (exp10, index int) { + // The constants expMin and expMax constrain the final value of the + // binary exponent of f. We want a small integral part in the result + // because finding digits of an integer requires divisions, whereas + // digits of the fractional part can be found by repeatedly multiplying + // by 10. + const expMin = -60 + const expMax = -32 + // Find power of ten such that x * 10^n has a binary exponent + // between expMin and expMax. + approxExp10 := ((expMin+expMax)/2 - f.exp) * 28 / 93 // log(10)/log(2) is close to 93/28. + i := (approxExp10 - firstPowerOfTen) / stepPowerOfTen +Loop: + for { + exp := f.exp + powersOfTen[i].exp + 64 + switch { + case exp < expMin: + i++ + case exp > expMax: + i-- + default: + break Loop + } + } + // Apply the desired decimal shift on f. It will have exponent + // in the desired range. This is multiplication by 10^-exp10. + f.Multiply(powersOfTen[i]) + + return -(firstPowerOfTen + i*stepPowerOfTen), i +} + +// frexp10Many applies a common shift by a power of ten to a, b, c. +func frexp10Many(a, b, c *extFloat) (exp10 int) { + exp10, i := c.frexp10() + a.Multiply(powersOfTen[i]) + b.Multiply(powersOfTen[i]) + return +} + +// FixedDecimal stores in d the first n significant digits +// of the decimal representation of f. It returns false +// if it cannot be sure of the answer. +func (f *extFloat) FixedDecimal(d *decimalSlice, n int) bool { + if f.mant == 0 { + d.nd = 0 + d.dp = 0 + d.neg = f.neg + return true + } + if n == 0 { + panic("strconv: internal error: extFloat.FixedDecimal called with n == 0") + } + // Multiply by an appropriate power of ten to have a reasonable + // number to process. + f.Normalize() + exp10, _ := f.frexp10() + + shift := uint(-f.exp) + integer := uint32(f.mant >> shift) + fraction := f.mant - (uint64(integer) << shift) + ε := uint64(1) // ε is the uncertainty we have on the mantissa of f. + + // Write exactly n digits to d. + needed := n // how many digits are left to write. + integerDigits := 0 // the number of decimal digits of integer. + pow10 := uint64(1) // the power of ten by which f was scaled. + for i, pow := 0, uint64(1); i < 20; i++ { + if pow > uint64(integer) { + integerDigits = i + break + } + pow *= 10 + } + rest := integer + if integerDigits > needed { + // the integral part is already large, trim the last digits. + pow10 = uint64pow10[integerDigits-needed] + integer /= uint32(pow10) + rest -= integer * uint32(pow10) + } else { + rest = 0 + } + + // Write the digits of integer: the digits of rest are omitted. + var buf [32]byte + pos := len(buf) + for v := integer; v > 0; { + v1 := v / 10 + v -= 10 * v1 + pos-- + buf[pos] = byte(v + '0') + v = v1 + } + for i := pos; i < len(buf); i++ { + d.d[i-pos] = buf[i] + } + nd := len(buf) - pos + d.nd = nd + d.dp = integerDigits + exp10 + needed -= nd + + if needed > 0 { + if rest != 0 || pow10 != 1 { + panic("strconv: internal error, rest != 0 but needed > 0") + } + // Emit digits for the fractional part. Each time, 10*fraction + // fits in a uint64 without overflow. + for needed > 0 { + fraction *= 10 + ε *= 10 // the uncertainty scales as we multiply by ten. + if 2*ε > 1<<shift { + // the error is so large it could modify which digit to write, abort. + return false + } + digit := fraction >> shift + d.d[nd] = byte(digit + '0') + fraction -= digit << shift + nd++ + needed-- + } + d.nd = nd + } + + // We have written a truncation of f (a numerator / 10^d.dp). The remaining part + // can be interpreted as a small number (< 1) to be added to the last digit of the + // numerator. + // + // If rest > 0, the amount is: + // (rest<<shift | fraction) / (pow10 << shift) + // fraction being known with a ±ε uncertainty. + // The fact that n > 0 guarantees that pow10 << shift does not overflow a uint64. + // + // If rest = 0, pow10 == 1 and the amount is + // fraction / (1 << shift) + // fraction being known with a ±ε uncertainty. + // + // We pass this information to the rounding routine for adjustment. + + ok := adjustLastDigitFixed(d, uint64(rest)<<shift|fraction, pow10, shift, ε) + if !ok { + return false + } + // Trim trailing zeros. + for i := d.nd - 1; i >= 0; i-- { + if d.d[i] != '0' { + d.nd = i + 1 + break + } + } + return true +} + +// adjustLastDigitFixed assumes d contains the representation of the integral part +// of some number, whose fractional part is num / (den << shift). The numerator +// num is only known up to an uncertainty of size ε, assumed to be less than +// (den << shift)/2. +// +// It will increase the last digit by one to account for correct rounding, typically +// when the fractional part is greater than 1/2, and will return false if ε is such +// that no correct answer can be given. +func adjustLastDigitFixed(d *decimalSlice, num, den uint64, shift uint, ε uint64) bool { + if num > den<<shift { + panic("strconv: num > den<<shift in adjustLastDigitFixed") + } + if 2*ε > den<<shift { + panic("strconv: ε > (den<<shift)/2") + } + if 2*(num+ε) < den<<shift { + return true + } + if 2*(num-ε) > den<<shift { + // increment d by 1. + i := d.nd - 1 + for ; i >= 0; i-- { + if d.d[i] == '9' { + d.nd-- + } else { + break + } + } + if i < 0 { + d.d[0] = '1' + d.nd = 1 + d.dp++ + } else { + d.d[i]++ + } + return true + } + return false +} + +// ShortestDecimal stores in d the shortest decimal representation of f +// which belongs to the open interval (lower, upper), where f is supposed +// to lie. It returns false whenever the result is unsure. The implementation +// uses the Grisu3 algorithm. +func (f *extFloat) ShortestDecimal(d *decimalSlice, lower, upper *extFloat) bool { + if f.mant == 0 { + d.nd = 0 + d.dp = 0 + d.neg = f.neg + return true + } + if f.exp == 0 && *lower == *f && *lower == *upper { + // an exact integer. + var buf [24]byte + n := len(buf) - 1 + for v := f.mant; v > 0; { + v1 := v / 10 + v -= 10 * v1 + buf[n] = byte(v + '0') + n-- + v = v1 + } + nd := len(buf) - n - 1 + for i := 0; i < nd; i++ { + d.d[i] = buf[n+1+i] + } + d.nd, d.dp = nd, nd + for d.nd > 0 && d.d[d.nd-1] == '0' { + d.nd-- + } + if d.nd == 0 { + d.dp = 0 + } + d.neg = f.neg + return true + } + upper.Normalize() + // Uniformize exponents. + if f.exp > upper.exp { + f.mant <<= uint(f.exp - upper.exp) + f.exp = upper.exp + } + if lower.exp > upper.exp { + lower.mant <<= uint(lower.exp - upper.exp) + lower.exp = upper.exp + } + + exp10 := frexp10Many(lower, f, upper) + // Take a safety margin due to rounding in frexp10Many, but we lose precision. + upper.mant++ + lower.mant-- + + // The shortest representation of f is either rounded up or down, but + // in any case, it is a truncation of upper. + shift := uint(-upper.exp) + integer := uint32(upper.mant >> shift) + fraction := upper.mant - (uint64(integer) << shift) + + // How far we can go down from upper until the result is wrong. + allowance := upper.mant - lower.mant + // How far we should go to get a very precise result. + targetDiff := upper.mant - f.mant + + // Count integral digits: there are at most 10. + var integerDigits int + for i, pow := 0, uint64(1); i < 20; i++ { + if pow > uint64(integer) { + integerDigits = i + break + } + pow *= 10 + } + for i := 0; i < integerDigits; i++ { + pow := uint64pow10[integerDigits-i-1] + digit := integer / uint32(pow) + d.d[i] = byte(digit + '0') + integer -= digit * uint32(pow) + // evaluate whether we should stop. + if currentDiff := uint64(integer)<<shift + fraction; currentDiff < allowance { + d.nd = i + 1 + d.dp = integerDigits + exp10 + d.neg = f.neg + // Sometimes allowance is so large the last digit might need to be + // decremented to get closer to f. + return adjustLastDigit(d, currentDiff, targetDiff, allowance, pow<<shift, 2) + } + } + d.nd = integerDigits + d.dp = d.nd + exp10 + d.neg = f.neg + + // Compute digits of the fractional part. At each step fraction does not + // overflow. The choice of minExp implies that fraction is less than 2^60. + var digit int + multiplier := uint64(1) + for { + fraction *= 10 + multiplier *= 10 + digit = int(fraction >> shift) + d.d[d.nd] = byte(digit + '0') + d.nd++ + fraction -= uint64(digit) << shift + if fraction < allowance*multiplier { + // We are in the admissible range. Note that if allowance is about to + // overflow, that is, allowance > 2^64/10, the condition is automatically + // true due to the limited range of fraction. + return adjustLastDigit(d, + fraction, targetDiff*multiplier, allowance*multiplier, + 1<<shift, multiplier*2) + } + } +} + +// adjustLastDigit modifies d = x-currentDiff*ε, to get closest to +// d = x-targetDiff*ε, without becoming smaller than x-maxDiff*ε. +// It assumes that a decimal digit is worth ulpDecimal*ε, and that +// all data is known with a error estimate of ulpBinary*ε. +func adjustLastDigit(d *decimalSlice, currentDiff, targetDiff, maxDiff, ulpDecimal, ulpBinary uint64) bool { + if ulpDecimal < 2*ulpBinary { + // Approximation is too wide. + return false + } + for currentDiff+ulpDecimal/2+ulpBinary < targetDiff { + d.d[d.nd-1]-- + currentDiff += ulpDecimal + } + if currentDiff+ulpDecimal <= targetDiff+ulpDecimal/2+ulpBinary { + // we have two choices, and don't know what to do. + return false + } + if currentDiff < ulpBinary || currentDiff > maxDiff-ulpBinary { + // we went too far + return false + } + if d.nd == 1 && d.d[0] == '0' { + // the number has actually reached zero. + d.nd = 0 + d.dp = 0 + } + return true +} diff --git a/fflib/v1/internal/ftoa.go b/fflib/v1/internal/ftoa.go new file mode 100644 index 0000000..253f83b --- /dev/null +++ b/fflib/v1/internal/ftoa.go @@ -0,0 +1,475 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Binary to decimal floating point conversion. +// Algorithm: +// 1) store mantissa in multiprecision decimal +// 2) shift decimal by exponent +// 3) read digits out & format + +package internal + +import "math" + +// TODO: move elsewhere? +type floatInfo struct { + mantbits uint + expbits uint + bias int +} + +var float32info = floatInfo{23, 8, -127} +var float64info = floatInfo{52, 11, -1023} + +// FormatFloat converts the floating-point number f to a string, +// according to the format fmt and precision prec. It rounds the +// result assuming that the original was obtained from a floating-point +// value of bitSize bits (32 for float32, 64 for float64). +// +// The format fmt is one of +// 'b' (-ddddp±ddd, a binary exponent), +// 'e' (-d.dddde±dd, a decimal exponent), +// 'E' (-d.ddddE±dd, a decimal exponent), +// 'f' (-ddd.dddd, no exponent), +// 'g' ('e' for large exponents, 'f' otherwise), or +// 'G' ('E' for large exponents, 'f' otherwise). +// +// The precision prec controls the number of digits +// (excluding the exponent) printed by the 'e', 'E', 'f', 'g', and 'G' formats. +// For 'e', 'E', and 'f' it is the number of digits after the decimal point. +// For 'g' and 'G' it is the total number of digits. +// The special precision -1 uses the smallest number of digits +// necessary such that ParseFloat will return f exactly. +func formatFloat(f float64, fmt byte, prec, bitSize int) string { + return string(genericFtoa(make([]byte, 0, max(prec+4, 24)), f, fmt, prec, bitSize)) +} + +// AppendFloat appends the string form of the floating-point number f, +// as generated by FormatFloat, to dst and returns the extended buffer. +func appendFloat(dst []byte, f float64, fmt byte, prec int, bitSize int) []byte { + return genericFtoa(dst, f, fmt, prec, bitSize) +} + +func genericFtoa(dst []byte, val float64, fmt byte, prec, bitSize int) []byte { + var bits uint64 + var flt *floatInfo + switch bitSize { + case 32: + bits = uint64(math.Float32bits(float32(val))) + flt = &float32info + case 64: + bits = math.Float64bits(val) + flt = &float64info + default: + panic("strconv: illegal AppendFloat/FormatFloat bitSize") + } + + neg := bits>>(flt.expbits+flt.mantbits) != 0 + exp := int(bits>>flt.mantbits) & (1<<flt.expbits - 1) + mant := bits & (uint64(1)<<flt.mantbits - 1) + + switch exp { + case 1<<flt.expbits - 1: + // Inf, NaN + var s string + switch { + case mant != 0: + s = "NaN" + case neg: + s = "-Inf" + default: + s = "+Inf" + } + return append(dst, s...) + + case 0: + // denormalized + exp++ + + default: + // add implicit top bit + mant |= uint64(1) << flt.mantbits + } + exp += flt.bias + + // Pick off easy binary format. + if fmt == 'b' { + return fmtB(dst, neg, mant, exp, flt) + } + + if !optimize { + return bigFtoa(dst, prec, fmt, neg, mant, exp, flt) + } + + var digs decimalSlice + ok := false + // Negative precision means "only as much as needed to be exact." + shortest := prec < 0 + if shortest { + // Try Grisu3 algorithm. + f := new(extFloat) + lower, upper := f.AssignComputeBounds(mant, exp, neg, flt) + var buf [32]byte + digs.d = buf[:] + ok = f.ShortestDecimal(&digs, &lower, &upper) + if !ok { + return bigFtoa(dst, prec, fmt, neg, mant, exp, flt) + } + // Precision for shortest representation mode. + switch fmt { + case 'e', 'E': + prec = digs.nd - 1 + case 'f': + prec = max(digs.nd-digs.dp, 0) + case 'g', 'G': + prec = digs.nd + } + } else if fmt != 'f' { + // Fixed number of digits. + digits := prec + switch fmt { + case 'e', 'E': + digits++ + case 'g', 'G': + if prec == 0 { + prec = 1 + } + digits = prec + } + if digits <= 15 { + // try fast algorithm when the number of digits is reasonable. + var buf [24]byte + digs.d = buf[:] + f := extFloat{mant, exp - int(flt.mantbits), neg} + ok = f.FixedDecimal(&digs, digits) + } + } + if !ok { + return bigFtoa(dst, prec, fmt, neg, mant, exp, flt) + } + return formatDigits(dst, shortest, neg, digs, prec, fmt) +} + +// bigFtoa uses multiprecision computations to format a float. +func bigFtoa(dst []byte, prec int, fmt byte, neg bool, mant uint64, exp int, flt *floatInfo) []byte { + d := new(decimal) + d.Assign(mant) + d.Shift(exp - int(flt.mantbits)) + var digs decimalSlice + shortest := prec < 0 + if shortest { + roundShortest(d, mant, exp, flt) + digs = decimalSlice{d: d.d[:], nd: d.nd, dp: d.dp} + // Precision for shortest representation mode. + switch fmt { + case 'e', 'E': + prec = digs.nd - 1 + case 'f': + prec = max(digs.nd-digs.dp, 0) + case 'g', 'G': + prec = digs.nd + } + } else { + // Round appropriately. + switch fmt { + case 'e', 'E': + d.Round(prec + 1) + case 'f': + d.Round(d.dp + prec) + case 'g', 'G': + if prec == 0 { + prec = 1 + } + d.Round(prec) + } + digs = decimalSlice{d: d.d[:], nd: d.nd, dp: d.dp} + } + return formatDigits(dst, shortest, neg, digs, prec, fmt) +} + +func formatDigits(dst []byte, shortest bool, neg bool, digs decimalSlice, prec int, fmt byte) []byte { + switch fmt { + case 'e', 'E': + return fmtE(dst, neg, digs, prec, fmt) + case 'f': + return fmtF(dst, neg, digs, prec) + case 'g', 'G': + // trailing fractional zeros in 'e' form will be trimmed. + eprec := prec + if eprec > digs.nd && digs.nd >= digs.dp { + eprec = digs.nd + } + // %e is used if the exponent from the conversion + // is less than -4 or greater than or equal to the precision. + // if precision was the shortest possible, use precision 6 for this decision. + if shortest { + eprec = 6 + } + exp := digs.dp - 1 + if exp < -4 || exp >= eprec { + if prec > digs.nd { + prec = digs.nd + } + return fmtE(dst, neg, digs, prec-1, fmt+'e'-'g') + } + if prec > digs.dp { + prec = digs.nd + } + return fmtF(dst, neg, digs, max(prec-digs.dp, 0)) + } + + // unknown format + return append(dst, '%', fmt) +} + +// Round d (= mant * 2^exp) to the shortest number of digits +// that will let the original floating point value be precisely +// reconstructed. Size is original floating point size (64 or 32). +func roundShortest(d *decimal, mant uint64, exp int, flt *floatInfo) { + // If mantissa is zero, the number is zero; stop now. + if mant == 0 { + d.nd = 0 + return + } + + // Compute upper and lower such that any decimal number + // between upper and lower (possibly inclusive) + // will round to the original floating point number. + + // We may see at once that the number is already shortest. + // + // Suppose d is not denormal, so that 2^exp <= d < 10^dp. + // The closest shorter number is at least 10^(dp-nd) away. + // The lower/upper bounds computed below are at distance + // at most 2^(exp-mantbits). + // + // So the number is already shortest if 10^(dp-nd) > 2^(exp-mantbits), + // or equivalently log2(10)*(dp-nd) > exp-mantbits. + // It is true if 332/100*(dp-nd) >= exp-mantbits (log2(10) > 3.32). + minexp := flt.bias + 1 // minimum possible exponent + if exp > minexp && 332*(d.dp-d.nd) >= 100*(exp-int(flt.mantbits)) { + // The number is already shortest. + return + } + + // d = mant << (exp - mantbits) + // Next highest floating point number is mant+1 << exp-mantbits. + // Our upper bound is halfway between, mant*2+1 << exp-mantbits-1. + upper := new(decimal) + upper.Assign(mant*2 + 1) + upper.Shift(exp - int(flt.mantbits) - 1) + + // d = mant << (exp - mantbits) + // Next lowest floating point number is mant-1 << exp-mantbits, + // unless mant-1 drops the significant bit and exp is not the minimum exp, + // in which case the next lowest is mant*2-1 << exp-mantbits-1. + // Either way, call it mantlo << explo-mantbits. + // Our lower bound is halfway between, mantlo*2+1 << explo-mantbits-1. + var mantlo uint64 + var explo int + if mant > 1<<flt.mantbits || exp == minexp { + mantlo = mant - 1 + explo = exp + } else { + mantlo = mant*2 - 1 + explo = exp - 1 + } + lower := new(decimal) + lower.Assign(mantlo*2 + 1) + lower.Shift(explo - int(flt.mantbits) - 1) + + // The upper and lower bounds are possible outputs only if + // the original mantissa is even, so that IEEE round-to-even + // would round to the original mantissa and not the neighbors. + inclusive := mant%2 == 0 + + // Now we can figure out the minimum number of digits required. + // Walk along until d has distinguished itself from upper and lower. + for i := 0; i < d.nd; i++ { + var l, m, u byte // lower, middle, upper digits + if i < lower.nd { + l = lower.d[i] + } else { + l = '0' + } + m = d.d[i] + if i < upper.nd { + u = upper.d[i] + } else { + u = '0' + } + + // Okay to round down (truncate) if lower has a different digit + // or if lower is inclusive and is exactly the result of rounding down. + okdown := l != m || (inclusive && l == m && i+1 == lower.nd) + + // Okay to round up if upper has a different digit and + // either upper is inclusive or upper is bigger than the result of rounding up. + okup := m != u && (inclusive || m+1 < u || i+1 < upper.nd) + + // If it's okay to do either, then round to the nearest one. + // If it's okay to do only one, do it. + switch { + case okdown && okup: + d.Round(i + 1) + return + case okdown: + d.RoundDown(i + 1) + return + case okup: + d.RoundUp(i + 1) + return + } + } +} + +type decimalSlice struct { + d []byte + nd, dp int + neg bool +} + +// %e: -d.ddddde±dd +func fmtE(dst []byte, neg bool, d decimalSlice, prec int, fmt byte) []byte { + // sign + if neg { + dst = append(dst, '-') + } + + // first digit + ch := byte('0') + if d.nd != 0 { + ch = d.d[0] + } + dst = append(dst, ch) + + // .moredigits + if prec > 0 { + dst = append(dst, '.') + i := 1 + m := d.nd + prec + 1 - max(d.nd, prec+1) + for i < m { + dst = append(dst, d.d[i]) + i++ + } + for i <= prec { + dst = append(dst, '0') + i++ + } + } + + // e± + dst = append(dst, fmt) + exp := d.dp - 1 + if d.nd == 0 { // special case: 0 has exponent 0 + exp = 0 + } + if exp < 0 { + ch = '-' + exp = -exp + } else { + ch = '+' + } + dst = append(dst, ch) + + // dddd + var buf [3]byte + i := len(buf) + for exp >= 10 { + i-- + buf[i] = byte(exp%10 + '0') + exp /= 10 + } + // exp < 10 + i-- + buf[i] = byte(exp + '0') + + switch i { + case 0: + dst = append(dst, buf[0], buf[1], buf[2]) + case 1: + dst = append(dst, buf[1], buf[2]) + case 2: + // leading zeroes + dst = append(dst, '0', buf[2]) + } + return dst +} + +// %f: -ddddddd.ddddd +func fmtF(dst []byte, neg bool, d decimalSlice, prec int) []byte { + // sign + if neg { + dst = append(dst, '-') + } + + // integer, padded with zeros as needed. + if d.dp > 0 { + var i int + for i = 0; i < d.dp && i < d.nd; i++ { + dst = append(dst, d.d[i]) + } + for ; i < d.dp; i++ { + dst = append(dst, '0') + } + } else { + dst = append(dst, '0') + } + + // fraction + if prec > 0 { + dst = append(dst, '.') + for i := 0; i < prec; i++ { + ch := byte('0') + if j := d.dp + i; 0 <= j && j < d.nd { + ch = d.d[j] + } + dst = append(dst, ch) + } + } + + return dst +} + +// %b: -ddddddddp+ddd +func fmtB(dst []byte, neg bool, mant uint64, exp int, flt *floatInfo) []byte { + var buf [50]byte + w := len(buf) + exp -= int(flt.mantbits) + esign := byte('+') + if exp < 0 { + esign = '-' + exp = -exp + } + n := 0 + for exp > 0 || n < 1 { + n++ + w-- + buf[w] = byte(exp%10 + '0') + exp /= 10 + } + w-- + buf[w] = esign + w-- + buf[w] = 'p' + n = 0 + for mant > 0 || n < 1 { + n++ + w-- + buf[w] = byte(mant%10 + '0') + mant /= 10 + } + if neg { + w-- + buf[w] = '-' + } + return append(dst, buf[w:]...) +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/fflib/v1/iota.go b/fflib/v1/iota.go new file mode 100644 index 0000000..3e50f0c --- /dev/null +++ b/fflib/v1/iota.go @@ -0,0 +1,161 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Portions of this file are on Go stdlib's strconv/iota.go */ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package v1 + +import ( + "io" +) + +const ( + digits = "0123456789abcdefghijklmnopqrstuvwxyz" + digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" +) + +var shifts = [len(digits) + 1]uint{ + 1 << 1: 1, + 1 << 2: 2, + 1 << 3: 3, + 1 << 4: 4, + 1 << 5: 5, +} + +var smallNumbers = [][]byte{ + []byte("0"), + []byte("1"), + []byte("2"), + []byte("3"), + []byte("4"), + []byte("5"), + []byte("6"), + []byte("7"), + []byte("8"), + []byte("9"), + []byte("10"), +} + +type FormatBitsWriter interface { + io.Writer + io.ByteWriter +} + +type FormatBitsScratch struct{} + +// +// DEPRECIATED: `scratch` is no longer used, FormatBits2 is available. +// +// FormatBits computes the string representation of u in the given base. +// If neg is set, u is treated as negative int64 value. If append_ is +// set, the string is appended to dst and the resulting byte slice is +// returned as the first result value; otherwise the string is returned +// as the second result value. +// +func FormatBits(scratch *FormatBitsScratch, dst FormatBitsWriter, u uint64, base int, neg bool) { + FormatBits2(dst, u, base, neg) +} + +// FormatBits2 computes the string representation of u in the given base. +// If neg is set, u is treated as negative int64 value. If append_ is +// set, the string is appended to dst and the resulting byte slice is +// returned as the first result value; otherwise the string is returned +// as the second result value. +// +func FormatBits2(dst FormatBitsWriter, u uint64, base int, neg bool) { + if base < 2 || base > len(digits) { + panic("strconv: illegal AppendInt/FormatInt base") + } + // fast path for small common numbers + if u <= 10 { + if neg { + dst.WriteByte('-') + } + dst.Write(smallNumbers[u]) + return + } + + // 2 <= base && base <= len(digits) + + var a = makeSlice(65) + // var a [64 + 1]byte // +1 for sign of 64bit value in base 2 + i := len(a) + + if neg { + u = -u + } + + // convert bits + if base == 10 { + // common case: use constants for / and % because + // the compiler can optimize it into a multiply+shift, + // and unroll loop + for u >= 100 { + i -= 2 + q := u / 100 + j := uintptr(u - q*100) + a[i+1] = digits01[j] + a[i+0] = digits10[j] + u = q + } + if u >= 10 { + i-- + q := u / 10 + a[i] = digits[uintptr(u-q*10)] + u = q + } + + } else if s := shifts[base]; s > 0 { + // base is power of 2: use shifts and masks instead of / and % + b := uint64(base) + m := uintptr(b) - 1 // == 1<<s - 1 + for u >= b { + i-- + a[i] = digits[uintptr(u)&m] + u >>= s + } + + } else { + // general case + b := uint64(base) + for u >= b { + i-- + a[i] = digits[uintptr(u%b)] + u /= b + } + } + + // u < base + i-- + a[i] = digits[uintptr(u)] + + // add sign, if any + if neg { + i-- + a[i] = '-' + } + + dst.Write(a[i:]) + + Pool(a) + + return +} diff --git a/fflib/v1/jsonstring.go b/fflib/v1/jsonstring.go new file mode 100644 index 0000000..513b45d --- /dev/null +++ b/fflib/v1/jsonstring.go @@ -0,0 +1,512 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Portions of this file are on Go stdlib's encoding/json/encode.go */ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package v1 + +import ( + "io" + "unicode/utf8" + "strconv" + "unicode/utf16" + "unicode" +) + +const hex = "0123456789abcdef" + +type JsonStringWriter interface { + io.Writer + io.ByteWriter + stringWriter +} + +func WriteJsonString(buf JsonStringWriter, s string) { + WriteJson(buf, []byte(s)) +} + +/** + * Function ported from encoding/json: func (e *encodeState) string(s string) (int, error) + */ +func WriteJson(buf JsonStringWriter, s []byte) { + buf.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + /* + if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { + i++ + continue + } + */ + if lt[b] == true { + i++ + continue + } + + if start < i { + buf.Write(s[start:i]) + } + switch b { + case '\\', '"': + buf.WriteByte('\\') + buf.WriteByte(b) + case '\n': + buf.WriteByte('\\') + buf.WriteByte('n') + case '\r': + buf.WriteByte('\\') + buf.WriteByte('r') + default: + // This encodes bytes < 0x20 except for \n and \r, + // as well as < and >. The latter are escaped because they + // can lead to security holes when user-controlled strings + // are rendered into JSON and served to some browsers. + buf.WriteString(`\u00`) + buf.WriteByte(hex[b>>4]) + buf.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRune(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + buf.Write(s[start:i]) + } + buf.WriteString(`\ufffd`) + i += size + start = i + continue + } + // U+2028 is LINE SEPARATOR. + // U+2029 is PARAGRAPH SEPARATOR. + // They are both technically valid characters in JSON strings, + // but don't work in JSONP, which has to be evaluated as JavaScript, + // and can lead to security holes there. It is valid JSON to + // escape them, so we do so unconditionally. + // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. + if c == '\u2028' || c == '\u2029' { + if start < i { + buf.Write(s[start:i]) + } + buf.WriteString(`\u202`) + buf.WriteByte(hex[c&0xF]) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + buf.Write(s[start:]) + } + buf.WriteByte('"') +} + +// UnquoteBytes will decode []byte containing json string to go string +// ported from encoding/json/decode.go +func UnquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(s[r:]) + if rr < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return b[0:w], true +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + r, err := strconv.ParseUint(string(s[2:6]), 16, 64) + if err != nil { + return -1 + } + return rune(r) +} + +// TODO(pquerna): consider combining wibth the normal byte mask. +var lt [256]bool = [256]bool{ + false, /* 0 */ + false, /* 1 */ + false, /* 2 */ + false, /* 3 */ + false, /* 4 */ + false, /* 5 */ + false, /* 6 */ + false, /* 7 */ + false, /* 8 */ + false, /* 9 */ + false, /* 10 */ + false, /* 11 */ + false, /* 12 */ + false, /* 13 */ + false, /* 14 */ + false, /* 15 */ + false, /* 16 */ + false, /* 17 */ + false, /* 18 */ + false, /* 19 */ + false, /* 20 */ + false, /* 21 */ + false, /* 22 */ + false, /* 23 */ + false, /* 24 */ + false, /* 25 */ + false, /* 26 */ + false, /* 27 */ + false, /* 28 */ + false, /* 29 */ + false, /* 30 */ + false, /* 31 */ + true, /* 32 */ + true, /* 33 */ + false, /* 34 */ + true, /* 35 */ + true, /* 36 */ + true, /* 37 */ + false, /* 38 */ + true, /* 39 */ + true, /* 40 */ + true, /* 41 */ + true, /* 42 */ + true, /* 43 */ + true, /* 44 */ + true, /* 45 */ + true, /* 46 */ + true, /* 47 */ + true, /* 48 */ + true, /* 49 */ + true, /* 50 */ + true, /* 51 */ + true, /* 52 */ + true, /* 53 */ + true, /* 54 */ + true, /* 55 */ + true, /* 56 */ + true, /* 57 */ + true, /* 58 */ + true, /* 59 */ + false, /* 60 */ + true, /* 61 */ + false, /* 62 */ + true, /* 63 */ + true, /* 64 */ + true, /* 65 */ + true, /* 66 */ + true, /* 67 */ + true, /* 68 */ + true, /* 69 */ + true, /* 70 */ + true, /* 71 */ + true, /* 72 */ + true, /* 73 */ + true, /* 74 */ + true, /* 75 */ + true, /* 76 */ + true, /* 77 */ + true, /* 78 */ + true, /* 79 */ + true, /* 80 */ + true, /* 81 */ + true, /* 82 */ + true, /* 83 */ + true, /* 84 */ + true, /* 85 */ + true, /* 86 */ + true, /* 87 */ + true, /* 88 */ + true, /* 89 */ + true, /* 90 */ + true, /* 91 */ + false, /* 92 */ + true, /* 93 */ + true, /* 94 */ + true, /* 95 */ + true, /* 96 */ + true, /* 97 */ + true, /* 98 */ + true, /* 99 */ + true, /* 100 */ + true, /* 101 */ + true, /* 102 */ + true, /* 103 */ + true, /* 104 */ + true, /* 105 */ + true, /* 106 */ + true, /* 107 */ + true, /* 108 */ + true, /* 109 */ + true, /* 110 */ + true, /* 111 */ + true, /* 112 */ + true, /* 113 */ + true, /* 114 */ + true, /* 115 */ + true, /* 116 */ + true, /* 117 */ + true, /* 118 */ + true, /* 119 */ + true, /* 120 */ + true, /* 121 */ + true, /* 122 */ + true, /* 123 */ + true, /* 124 */ + true, /* 125 */ + true, /* 126 */ + true, /* 127 */ + true, /* 128 */ + true, /* 129 */ + true, /* 130 */ + true, /* 131 */ + true, /* 132 */ + true, /* 133 */ + true, /* 134 */ + true, /* 135 */ + true, /* 136 */ + true, /* 137 */ + true, /* 138 */ + true, /* 139 */ + true, /* 140 */ + true, /* 141 */ + true, /* 142 */ + true, /* 143 */ + true, /* 144 */ + true, /* 145 */ + true, /* 146 */ + true, /* 147 */ + true, /* 148 */ + true, /* 149 */ + true, /* 150 */ + true, /* 151 */ + true, /* 152 */ + true, /* 153 */ + true, /* 154 */ + true, /* 155 */ + true, /* 156 */ + true, /* 157 */ + true, /* 158 */ + true, /* 159 */ + true, /* 160 */ + true, /* 161 */ + true, /* 162 */ + true, /* 163 */ + true, /* 164 */ + true, /* 165 */ + true, /* 166 */ + true, /* 167 */ + true, /* 168 */ + true, /* 169 */ + true, /* 170 */ + true, /* 171 */ + true, /* 172 */ + true, /* 173 */ + true, /* 174 */ + true, /* 175 */ + true, /* 176 */ + true, /* 177 */ + true, /* 178 */ + true, /* 179 */ + true, /* 180 */ + true, /* 181 */ + true, /* 182 */ + true, /* 183 */ + true, /* 184 */ + true, /* 185 */ + true, /* 186 */ + true, /* 187 */ + true, /* 188 */ + true, /* 189 */ + true, /* 190 */ + true, /* 191 */ + true, /* 192 */ + true, /* 193 */ + true, /* 194 */ + true, /* 195 */ + true, /* 196 */ + true, /* 197 */ + true, /* 198 */ + true, /* 199 */ + true, /* 200 */ + true, /* 201 */ + true, /* 202 */ + true, /* 203 */ + true, /* 204 */ + true, /* 205 */ + true, /* 206 */ + true, /* 207 */ + true, /* 208 */ + true, /* 209 */ + true, /* 210 */ + true, /* 211 */ + true, /* 212 */ + true, /* 213 */ + true, /* 214 */ + true, /* 215 */ + true, /* 216 */ + true, /* 217 */ + true, /* 218 */ + true, /* 219 */ + true, /* 220 */ + true, /* 221 */ + true, /* 222 */ + true, /* 223 */ + true, /* 224 */ + true, /* 225 */ + true, /* 226 */ + true, /* 227 */ + true, /* 228 */ + true, /* 229 */ + true, /* 230 */ + true, /* 231 */ + true, /* 232 */ + true, /* 233 */ + true, /* 234 */ + true, /* 235 */ + true, /* 236 */ + true, /* 237 */ + true, /* 238 */ + true, /* 239 */ + true, /* 240 */ + true, /* 241 */ + true, /* 242 */ + true, /* 243 */ + true, /* 244 */ + true, /* 245 */ + true, /* 246 */ + true, /* 247 */ + true, /* 248 */ + true, /* 249 */ + true, /* 250 */ + true, /* 251 */ + true, /* 252 */ + true, /* 253 */ + true, /* 254 */ + true, /* 255 */ +} diff --git a/fflib/v1/jsonstring_test.go b/fflib/v1/jsonstring_test.go new file mode 100644 index 0000000..476d336 --- /dev/null +++ b/fflib/v1/jsonstring_test.go @@ -0,0 +1,38 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package v1 + +import ( + "bytes" + "testing" +) + +func TestWriteJsonString(t *testing.T) { + var buf bytes.Buffer + WriteJsonString(&buf, "foo") + if string(buf.Bytes()) != `"foo"` { + t.Fatalf("Expected: %v\nGot: %v", `"foo"`, string(buf.Bytes())) + } + + buf.Reset() + WriteJsonString(&buf, `f"oo`) + if string(buf.Bytes()) != `"f\"oo"` { + t.Fatalf("Expected: %v\nGot: %v", `"f\"oo"`, string(buf.Bytes())) + } + // TODO(pquerna): all them important tests. +} diff --git a/fflib/v1/lexer.go b/fflib/v1/lexer.go new file mode 100644 index 0000000..5589292 --- /dev/null +++ b/fflib/v1/lexer.go @@ -0,0 +1,937 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Portions of this file are on derived from yajl: <https://github.com/lloyd/yajl> */ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package v1 + +import ( + "errors" + "fmt" + "io" +) + +type FFParseState int + +const ( + FFParse_map_start FFParseState = iota + FFParse_want_key + FFParse_want_colon + FFParse_want_value + FFParse_after_value +) + +type FFTok int + +const ( + FFTok_init FFTok = iota + FFTok_bool FFTok = iota + FFTok_colon FFTok = iota + FFTok_comma FFTok = iota + FFTok_eof FFTok = iota + FFTok_error FFTok = iota + FFTok_left_brace FFTok = iota + FFTok_left_bracket FFTok = iota + FFTok_null FFTok = iota + FFTok_right_brace FFTok = iota + FFTok_right_bracket FFTok = iota + + /* we differentiate between integers and doubles to allow the + * parser to interpret the number without re-scanning */ + FFTok_integer FFTok = iota + FFTok_double FFTok = iota + + FFTok_string FFTok = iota + + /* comment tokens are not currently returned to the parser, ever */ + FFTok_comment FFTok = iota +) + +type FFErr int + +const ( + FFErr_e_ok FFErr = iota + FFErr_io FFErr = iota + FFErr_string_invalid_utf8 FFErr = iota + FFErr_string_invalid_escaped_char FFErr = iota + FFErr_string_invalid_json_char FFErr = iota + FFErr_string_invalid_hex_char FFErr = iota + FFErr_invalid_char FFErr = iota + FFErr_invalid_string FFErr = iota + FFErr_missing_integer_after_decimal FFErr = iota + FFErr_missing_integer_after_exponent FFErr = iota + FFErr_missing_integer_after_minus FFErr = iota + FFErr_unallowed_comment FFErr = iota + FFErr_incomplete_comment FFErr = iota + FFErr_unexpected_token_type FFErr = iota // TODO: improve this error +) + +type FFLexer struct { + reader *ffReader + Output DecodingBuffer + Token FFTok + Error FFErr + BigError error + // TODO: convert all of this to an interface + lastCurrentChar int + captureAll bool + buf Buffer +} + +func NewFFLexer(input []byte) *FFLexer { + fl := &FFLexer{ + Token: FFTok_init, + Error: FFErr_e_ok, + reader: newffReader(input), + Output: &Buffer{}, + } + // TODO: guess size? + //fl.Output.Grow(64) + return fl +} + +type LexerError struct { + offset int + line int + char int + err error +} + +// Reset the Lexer and add new input. +func (ffl *FFLexer) Reset(input []byte) { + ffl.Token = FFTok_init + ffl.Error = FFErr_e_ok + ffl.BigError = nil + ffl.reader.Reset(input) + ffl.lastCurrentChar = 0 + ffl.Output.Reset() +} + +func (le *LexerError) Error() string { + return fmt.Sprintf(`ffjson error: (%T)%s offset=%d line=%d char=%d`, + le.err, le.err.Error(), + le.offset, le.line, le.char) +} + +func (ffl *FFLexer) WrapErr(err error) error { + line, char := ffl.reader.PosWithLine() + // TOOD: calcualte lines/characters based on offset + return &LexerError{ + offset: ffl.reader.Pos(), + line: line, + char: char, + err: err, + } +} + +func (ffl *FFLexer) scanReadByte() (byte, error) { + var c byte + var err error + if ffl.captureAll { + c, err = ffl.reader.ReadByte() + } else { + c, err = ffl.reader.ReadByteNoWS() + } + + if err != nil { + ffl.Error = FFErr_io + ffl.BigError = err + return 0, err + } + + return c, nil +} + +func (ffl *FFLexer) readByte() (byte, error) { + + c, err := ffl.reader.ReadByte() + if err != nil { + ffl.Error = FFErr_io + ffl.BigError = err + return 0, err + } + + return c, nil +} + +func (ffl *FFLexer) unreadByte() { + ffl.reader.UnreadByte() +} + +func (ffl *FFLexer) wantBytes(want []byte, iftrue FFTok) FFTok { + startPos := ffl.reader.Pos() + for _, b := range want { + c, err := ffl.readByte() + + if err != nil { + return FFTok_error + } + + if c != b { + ffl.unreadByte() + // fmt.Printf("wanted bytes: %s\n", string(want)) + // TODO(pquerna): thsi is a bad error message + ffl.Error = FFErr_invalid_string + return FFTok_error + } + } + + endPos := ffl.reader.Pos() + ffl.Output.Write(ffl.reader.Slice(startPos, endPos)) + return iftrue +} + +func (ffl *FFLexer) lexComment() FFTok { + c, err := ffl.readByte() + if err != nil { + return FFTok_error + } + + if c == '/' { + // a // comment, scan until line ends. + for { + c, err := ffl.readByte() + if err != nil { + return FFTok_error + } + + if c == '\n' { + return FFTok_comment + } + } + } else if c == '*' { + // a /* */ comment, scan */ + for { + c, err := ffl.readByte() + if err != nil { + return FFTok_error + } + + if c == '*' { + c, err := ffl.readByte() + + if err != nil { + return FFTok_error + } + + if c == '/' { + return FFTok_comment + } + + ffl.Error = FFErr_incomplete_comment + return FFTok_error + } + } + } else { + ffl.Error = FFErr_incomplete_comment + return FFTok_error + } +} + +func (ffl *FFLexer) lexString() FFTok { + if ffl.captureAll { + ffl.buf.Reset() + err := ffl.reader.SliceString(&ffl.buf) + + if err != nil { + ffl.BigError = err + return FFTok_error + } + + WriteJson(ffl.Output, ffl.buf.Bytes()) + + return FFTok_string + } else { + err := ffl.reader.SliceString(ffl.Output) + + if err != nil { + ffl.BigError = err + return FFTok_error + } + + return FFTok_string + } +} + +func (ffl *FFLexer) lexNumber() FFTok { + var numRead int = 0 + tok := FFTok_integer + startPos := ffl.reader.Pos() + + c, err := ffl.readByte() + if err != nil { + return FFTok_error + } + + /* optional leading minus */ + if c == '-' { + c, err = ffl.readByte() + if err != nil { + return FFTok_error + } + } + + /* a single zero, or a series of integers */ + if c == '0' { + c, err = ffl.readByte() + if err != nil { + return FFTok_error + } + } else if c >= '1' && c <= '9' { + for c >= '0' && c <= '9' { + c, err = ffl.readByte() + if err != nil { + return FFTok_error + } + } + } else { + ffl.unreadByte() + ffl.Error = FFErr_missing_integer_after_minus + return FFTok_error + } + + if c == '.' { + numRead = 0 + c, err = ffl.readByte() + if err != nil { + return FFTok_error + } + + for c >= '0' && c <= '9' { + numRead++ + c, err = ffl.readByte() + if err != nil { + return FFTok_error + } + } + + if numRead == 0 { + ffl.unreadByte() + + ffl.Error = FFErr_missing_integer_after_decimal + return FFTok_error + } + + tok = FFTok_double + } + + /* optional exponent (indicates this is floating point) */ + if c == 'e' || c == 'E' { + numRead = 0 + c, err = ffl.readByte() + if err != nil { + return FFTok_error + } + + /* optional sign */ + if c == '+' || c == '-' { + c, err = ffl.readByte() + if err != nil { + return FFTok_error + } + } + + for c >= '0' && c <= '9' { + numRead++ + c, err = ffl.readByte() + if err != nil { + return FFTok_error + } + } + + if numRead == 0 { + ffl.Error = FFErr_missing_integer_after_exponent + return FFTok_error + } + + tok = FFTok_double + } + + ffl.unreadByte() + + endPos := ffl.reader.Pos() + ffl.Output.Write(ffl.reader.Slice(startPos, endPos)) + return tok +} + +var true_bytes = []byte{'r', 'u', 'e'} +var false_bytes = []byte{'a', 'l', 's', 'e'} +var null_bytes = []byte{'u', 'l', 'l'} + +func (ffl *FFLexer) Scan() FFTok { + tok := FFTok_error + if ffl.captureAll == false { + ffl.Output.Reset() + } + ffl.Token = FFTok_init + + for { + c, err := ffl.scanReadByte() + if err != nil { + if err == io.EOF { + return FFTok_eof + } else { + return FFTok_error + } + } + + switch c { + case '{': + tok = FFTok_left_bracket + if ffl.captureAll { + ffl.Output.WriteByte('{') + } + goto lexed + case '}': + tok = FFTok_right_bracket + if ffl.captureAll { + ffl.Output.WriteByte('}') + } + goto lexed + case '[': + tok = FFTok_left_brace + if ffl.captureAll { + ffl.Output.WriteByte('[') + } + goto lexed + case ']': + tok = FFTok_right_brace + if ffl.captureAll { + ffl.Output.WriteByte(']') + } + goto lexed + case ',': + tok = FFTok_comma + if ffl.captureAll { + ffl.Output.WriteByte(',') + } + goto lexed + case ':': + tok = FFTok_colon + if ffl.captureAll { + ffl.Output.WriteByte(':') + } + goto lexed + case '\t', '\n', '\v', '\f', '\r', ' ': + if ffl.captureAll { + ffl.Output.WriteByte(c) + } + case 't': + ffl.Output.WriteByte('t') + tok = ffl.wantBytes(true_bytes, FFTok_bool) + goto lexed + case 'f': + ffl.Output.WriteByte('f') + tok = ffl.wantBytes(false_bytes, FFTok_bool) + goto lexed + case 'n': + ffl.Output.WriteByte('n') + tok = ffl.wantBytes(null_bytes, FFTok_null) + goto lexed + case '"': + tok = ffl.lexString() + goto lexed + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + ffl.unreadByte() + tok = ffl.lexNumber() + goto lexed + case '/': + tok = ffl.lexComment() + goto lexed + default: + tok = FFTok_error + ffl.Error = FFErr_invalid_char + goto lexed + } + } + +lexed: + ffl.Token = tok + return tok +} + +func (ffl *FFLexer) scanField(start FFTok, capture bool) ([]byte, error) { + switch start { + case FFTok_left_brace, + FFTok_left_bracket: + { + end := FFTok_right_brace + if start == FFTok_left_bracket { + end = FFTok_right_bracket + if capture { + ffl.Output.WriteByte('{') + } + } else { + if capture { + ffl.Output.WriteByte('[') + } + } + + depth := 1 + if capture { + ffl.captureAll = true + } + // TODO: work. + scanloop: + for { + tok := ffl.Scan() + //fmt.Printf("capture-token: %v end: %v depth: %v\n", tok, end, depth) + switch tok { + case FFTok_eof: + return nil, errors.New("ffjson: unexpected EOF") + case FFTok_error: + if ffl.BigError != nil { + return nil, ffl.BigError + } + return nil, ffl.Error.ToError() + case end: + depth-- + if depth == 0 { + break scanloop + } + case start: + depth++ + } + } + + if capture { + ffl.captureAll = false + } + + if capture { + return ffl.Output.Bytes(), nil + } else { + return nil, nil + } + } + case FFTok_bool, + FFTok_integer, + FFTok_null, + FFTok_double: + // simple value, return it. + if capture { + return ffl.Output.Bytes(), nil + } else { + return nil, nil + } + + case FFTok_string: + //TODO(pquerna): so, other users expect this to be a quoted string :( + if capture { + ffl.buf.Reset() + WriteJson(&ffl.buf, ffl.Output.Bytes()) + return ffl.buf.Bytes(), nil + } else { + return nil, nil + } + } + + return nil, fmt.Errorf("ffjson: invalid capture type: %v", start) +} + +// Captures an entire field value, including recursive objects, +// and converts them to a []byte suitable to pass to a sub-object's +// UnmarshalJSON +func (ffl *FFLexer) CaptureField(start FFTok) ([]byte, error) { + return ffl.scanField(start, true) +} + +func (ffl *FFLexer) SkipField(start FFTok) error { + _, err := ffl.scanField(start, false) + return err +} + +// TODO(pquerna): return line number and offset. +func (err FFErr) ToError() error { + switch err { + case FFErr_e_ok: + return nil + case FFErr_io: + return errors.New("ffjson: IO error") + case FFErr_string_invalid_utf8: + return errors.New("ffjson: string with invalid UTF-8 sequence") + case FFErr_string_invalid_escaped_char: + return errors.New("ffjson: string with invalid escaped character") + case FFErr_string_invalid_json_char: + return errors.New("ffjson: string with invalid JSON character") + case FFErr_string_invalid_hex_char: + return errors.New("ffjson: string with invalid hex character") + case FFErr_invalid_char: + return errors.New("ffjson: invalid character") + case FFErr_invalid_string: + return errors.New("ffjson: invalid string") + case FFErr_missing_integer_after_decimal: + return errors.New("ffjson: missing integer after decimal") + case FFErr_missing_integer_after_exponent: + return errors.New("ffjson: missing integer after exponent") + case FFErr_missing_integer_after_minus: + return errors.New("ffjson: missing integer after minus") + case FFErr_unallowed_comment: + return errors.New("ffjson: unallowed comment") + case FFErr_incomplete_comment: + return errors.New("ffjson: incomplete comment") + case FFErr_unexpected_token_type: + return errors.New("ffjson: unexpected token sequence") + } + + panic(fmt.Sprintf("unknown error type: %v ", err)) +} + +func (state FFParseState) String() string { + switch state { + case FFParse_map_start: + return "map:start" + case FFParse_want_key: + return "want_key" + case FFParse_want_colon: + return "want_colon" + case FFParse_want_value: + return "want_value" + case FFParse_after_value: + return "after_value" + } + + panic(fmt.Sprintf("unknown parse state: %d", int(state))) +} + +func (tok FFTok) String() string { + switch tok { + case FFTok_init: + return "tok:init" + case FFTok_bool: + return "tok:bool" + case FFTok_colon: + return "tok:colon" + case FFTok_comma: + return "tok:comma" + case FFTok_eof: + return "tok:eof" + case FFTok_error: + return "tok:error" + case FFTok_left_brace: + return "tok:left_brace" + case FFTok_left_bracket: + return "tok:left_bracket" + case FFTok_null: + return "tok:null" + case FFTok_right_brace: + return "tok:right_brace" + case FFTok_right_bracket: + return "tok:right_bracket" + case FFTok_integer: + return "tok:integer" + case FFTok_double: + return "tok:double" + case FFTok_string: + return "tok:string" + case FFTok_comment: + return "comment" + } + + panic(fmt.Sprintf("unknown token: %d", int(tok))) +} + +/* a lookup table which lets us quickly determine three things: + * cVEC - valid escaped control char + * note. the solidus '/' may be escaped or not. + * cIJC - invalid json char + * cVHC - valid hex char + * cNFP - needs further processing (from a string scanning perspective) + * cNUC - needs utf8 checking when enabled (from a string scanning perspective) + */ + +const ( + cVEC int8 = 0x01 + cIJC int8 = 0x02 + cVHC int8 = 0x04 + cNFP int8 = 0x08 + cNUC int8 = 0x10 +) + +var byteLookupTable [256]int8 = [256]int8{ + cIJC, /* 0 */ + cIJC, /* 1 */ + cIJC, /* 2 */ + cIJC, /* 3 */ + cIJC, /* 4 */ + cIJC, /* 5 */ + cIJC, /* 6 */ + cIJC, /* 7 */ + cIJC, /* 8 */ + cIJC, /* 9 */ + cIJC, /* 10 */ + cIJC, /* 11 */ + cIJC, /* 12 */ + cIJC, /* 13 */ + cIJC, /* 14 */ + cIJC, /* 15 */ + cIJC, /* 16 */ + cIJC, /* 17 */ + cIJC, /* 18 */ + cIJC, /* 19 */ + cIJC, /* 20 */ + cIJC, /* 21 */ + cIJC, /* 22 */ + cIJC, /* 23 */ + cIJC, /* 24 */ + cIJC, /* 25 */ + cIJC, /* 26 */ + cIJC, /* 27 */ + cIJC, /* 28 */ + cIJC, /* 29 */ + cIJC, /* 30 */ + cIJC, /* 31 */ + 0, /* 32 */ + 0, /* 33 */ + cVEC | cIJC | cNFP, /* 34 */ + 0, /* 35 */ + 0, /* 36 */ + 0, /* 37 */ + 0, /* 38 */ + 0, /* 39 */ + 0, /* 40 */ + 0, /* 41 */ + 0, /* 42 */ + 0, /* 43 */ + 0, /* 44 */ + 0, /* 45 */ + 0, /* 46 */ + cVEC, /* 47 */ + cVHC, /* 48 */ + cVHC, /* 49 */ + cVHC, /* 50 */ + cVHC, /* 51 */ + cVHC, /* 52 */ + cVHC, /* 53 */ + cVHC, /* 54 */ + cVHC, /* 55 */ + cVHC, /* 56 */ + cVHC, /* 57 */ + 0, /* 58 */ + 0, /* 59 */ + 0, /* 60 */ + 0, /* 61 */ + 0, /* 62 */ + 0, /* 63 */ + 0, /* 64 */ + cVHC, /* 65 */ + cVHC, /* 66 */ + cVHC, /* 67 */ + cVHC, /* 68 */ + cVHC, /* 69 */ + cVHC, /* 70 */ + 0, /* 71 */ + 0, /* 72 */ + 0, /* 73 */ + 0, /* 74 */ + 0, /* 75 */ + 0, /* 76 */ + 0, /* 77 */ + 0, /* 78 */ + 0, /* 79 */ + 0, /* 80 */ + 0, /* 81 */ + 0, /* 82 */ + 0, /* 83 */ + 0, /* 84 */ + 0, /* 85 */ + 0, /* 86 */ + 0, /* 87 */ + 0, /* 88 */ + 0, /* 89 */ + 0, /* 90 */ + 0, /* 91 */ + cVEC | cIJC | cNFP, /* 92 */ + 0, /* 93 */ + 0, /* 94 */ + 0, /* 95 */ + 0, /* 96 */ + cVHC, /* 97 */ + cVEC | cVHC, /* 98 */ + cVHC, /* 99 */ + cVHC, /* 100 */ + cVHC, /* 101 */ + cVEC | cVHC, /* 102 */ + 0, /* 103 */ + 0, /* 104 */ + 0, /* 105 */ + 0, /* 106 */ + 0, /* 107 */ + 0, /* 108 */ + 0, /* 109 */ + cVEC, /* 110 */ + 0, /* 111 */ + 0, /* 112 */ + 0, /* 113 */ + cVEC, /* 114 */ + 0, /* 115 */ + cVEC, /* 116 */ + 0, /* 117 */ + 0, /* 118 */ + 0, /* 119 */ + 0, /* 120 */ + 0, /* 121 */ + 0, /* 122 */ + 0, /* 123 */ + 0, /* 124 */ + 0, /* 125 */ + 0, /* 126 */ + 0, /* 127 */ + cNUC, /* 128 */ + cNUC, /* 129 */ + cNUC, /* 130 */ + cNUC, /* 131 */ + cNUC, /* 132 */ + cNUC, /* 133 */ + cNUC, /* 134 */ + cNUC, /* 135 */ + cNUC, /* 136 */ + cNUC, /* 137 */ + cNUC, /* 138 */ + cNUC, /* 139 */ + cNUC, /* 140 */ + cNUC, /* 141 */ + cNUC, /* 142 */ + cNUC, /* 143 */ + cNUC, /* 144 */ + cNUC, /* 145 */ + cNUC, /* 146 */ + cNUC, /* 147 */ + cNUC, /* 148 */ + cNUC, /* 149 */ + cNUC, /* 150 */ + cNUC, /* 151 */ + cNUC, /* 152 */ + cNUC, /* 153 */ + cNUC, /* 154 */ + cNUC, /* 155 */ + cNUC, /* 156 */ + cNUC, /* 157 */ + cNUC, /* 158 */ + cNUC, /* 159 */ + cNUC, /* 160 */ + cNUC, /* 161 */ + cNUC, /* 162 */ + cNUC, /* 163 */ + cNUC, /* 164 */ + cNUC, /* 165 */ + cNUC, /* 166 */ + cNUC, /* 167 */ + cNUC, /* 168 */ + cNUC, /* 169 */ + cNUC, /* 170 */ + cNUC, /* 171 */ + cNUC, /* 172 */ + cNUC, /* 173 */ + cNUC, /* 174 */ + cNUC, /* 175 */ + cNUC, /* 176 */ + cNUC, /* 177 */ + cNUC, /* 178 */ + cNUC, /* 179 */ + cNUC, /* 180 */ + cNUC, /* 181 */ + cNUC, /* 182 */ + cNUC, /* 183 */ + cNUC, /* 184 */ + cNUC, /* 185 */ + cNUC, /* 186 */ + cNUC, /* 187 */ + cNUC, /* 188 */ + cNUC, /* 189 */ + cNUC, /* 190 */ + cNUC, /* 191 */ + cNUC, /* 192 */ + cNUC, /* 193 */ + cNUC, /* 194 */ + cNUC, /* 195 */ + cNUC, /* 196 */ + cNUC, /* 197 */ + cNUC, /* 198 */ + cNUC, /* 199 */ + cNUC, /* 200 */ + cNUC, /* 201 */ + cNUC, /* 202 */ + cNUC, /* 203 */ + cNUC, /* 204 */ + cNUC, /* 205 */ + cNUC, /* 206 */ + cNUC, /* 207 */ + cNUC, /* 208 */ + cNUC, /* 209 */ + cNUC, /* 210 */ + cNUC, /* 211 */ + cNUC, /* 212 */ + cNUC, /* 213 */ + cNUC, /* 214 */ + cNUC, /* 215 */ + cNUC, /* 216 */ + cNUC, /* 217 */ + cNUC, /* 218 */ + cNUC, /* 219 */ + cNUC, /* 220 */ + cNUC, /* 221 */ + cNUC, /* 222 */ + cNUC, /* 223 */ + cNUC, /* 224 */ + cNUC, /* 225 */ + cNUC, /* 226 */ + cNUC, /* 227 */ + cNUC, /* 228 */ + cNUC, /* 229 */ + cNUC, /* 230 */ + cNUC, /* 231 */ + cNUC, /* 232 */ + cNUC, /* 233 */ + cNUC, /* 234 */ + cNUC, /* 235 */ + cNUC, /* 236 */ + cNUC, /* 237 */ + cNUC, /* 238 */ + cNUC, /* 239 */ + cNUC, /* 240 */ + cNUC, /* 241 */ + cNUC, /* 242 */ + cNUC, /* 243 */ + cNUC, /* 244 */ + cNUC, /* 245 */ + cNUC, /* 246 */ + cNUC, /* 247 */ + cNUC, /* 248 */ + cNUC, /* 249 */ + cNUC, /* 250 */ + cNUC, /* 251 */ + cNUC, /* 252 */ + cNUC, /* 253 */ + cNUC, /* 254 */ + cNUC, /* 255 */ +} diff --git a/fflib/v1/lexer_test.go b/fflib/v1/lexer_test.go new file mode 100644 index 0000000..aa1aba8 --- /dev/null +++ b/fflib/v1/lexer_test.go @@ -0,0 +1,327 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package v1 + +import ( + "bytes" + "errors" + "strconv" + "testing" +) + +func scanAll(ffl *FFLexer) []FFTok { + rv := make([]FFTok, 0, 0) + for { + tok := ffl.Scan() + rv = append(rv, tok) + if tok == FFTok_eof || tok == FFTok_error { + break + } + } + + return rv +} + +func assertTokensEqual(t *testing.T, a []FFTok, b []FFTok) { + + if len(a) != len(b) { + t.Fatalf("Token lists of mixed length: expected=%v found=%v", a, b) + return + } + + for i, v := range a { + if b[i] != v { + t.Fatalf("Invalid Token: expected=%d found=%d token=%d", + v, b, i) + return + } + } +} + +func scanToTok(ffl *FFLexer, targetTok FFTok) error { + _, err := scanToTokCount(ffl, targetTok) + return err +} + +func scanToTokCount(ffl *FFLexer, targetTok FFTok) (int, error) { + c := 0 + for { + tok := ffl.Scan() + c++ + + if tok == targetTok { + return c, nil + } + + if tok == FFTok_error { + return c, errors.New("Hit error before target token") + } + if tok == FFTok_eof { + return c, errors.New("Hit EOF before target token") + } + } +} + +func TestBasicLexing(t *testing.T) { + ffl := NewFFLexer([]byte(`{}`)) + toks := scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_right_bracket, + FFTok_eof, + }, toks) +} + +func TestHelloWorld(t *testing.T) { + ffl := NewFFLexer([]byte(`{"hello":"world"}`)) + toks := scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_string, + FFTok_right_bracket, + FFTok_eof, + }, toks) + + ffl = NewFFLexer([]byte(`{"hello": 1}`)) + toks = scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_integer, + FFTok_right_bracket, + FFTok_eof, + }, toks) + + ffl = NewFFLexer([]byte(`{"hello": 1.0}`)) + toks = scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_double, + FFTok_right_bracket, + FFTok_eof, + }, toks) + + ffl = NewFFLexer([]byte(`{"hello": 1e2}`)) + toks = scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_double, + FFTok_right_bracket, + FFTok_eof, + }, toks) + + ffl = NewFFLexer([]byte(`{"hello": {}}`)) + toks = scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_left_bracket, + FFTok_right_bracket, + FFTok_right_bracket, + FFTok_eof, + }, toks) + + ffl = NewFFLexer([]byte(`{"hello": {"blah": null}}`)) + toks = scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_null, + FFTok_right_bracket, + FFTok_right_bracket, + FFTok_eof, + }, toks) + + ffl = NewFFLexer([]byte(`{"hello": /* comment */ 0}`)) + toks = scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_comment, + FFTok_integer, + FFTok_right_bracket, + FFTok_eof, + }, toks) + + ffl = NewFFLexer([]byte(`{"hello": / comment`)) + toks = scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_error, + }, toks) + + ffl = NewFFLexer([]byte(`{"陫ʋsş\")珷\u003cºɖgȏ哙ȍ":"2ħ籦ö嗏ʑ\u003e季"}`)) + toks = scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_string, + FFTok_right_bracket, + FFTok_eof, + }, toks) + + ffl = NewFFLexer([]byte(`{"X":{"陫ʋsş\")珷\u003cºɖgȏ哙ȍ":"2ħ籦ö嗏ʑ\u003e季"}}`)) + toks = scanAll(ffl) + assertTokensEqual(t, []FFTok{ + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_left_bracket, + FFTok_string, + FFTok_colon, + FFTok_string, + FFTok_right_bracket, + FFTok_right_bracket, + FFTok_eof, + }, toks) +} + +func tDouble(t *testing.T, input string, target float64) { + ffl := NewFFLexer([]byte(input)) + err := scanToTok(ffl, FFTok_double) + if err != nil { + t.Fatalf("scanToTok failed, couldnt find double: %v input: %v", err, input) + } + + f64, err := strconv.ParseFloat(ffl.Output.String(), 64) + if err != nil { + t.Fatalf("ParseFloat failed, shouldnt of: %v input: %v", err, input) + } + + if int64(f64*1000) != int64(target*1000) { + t.Fatalf("ffl.Output: expected f64 '%v', got: %v from: %v input: %v", + target, f64, ffl.Output.String(), input) + } + + err = scanToTok(ffl, FFTok_eof) + if err != nil { + t.Fatalf("Failed to find EOF after double. input: %v", input) + } +} + +func TestDouble(t *testing.T) { + tDouble(t, `{"a": 1.2}`, 1.2) + tDouble(t, `{"a": 1.2e2}`, 1.2e2) + tDouble(t, `{"a": -1.2e2}`, -1.2e2) + tDouble(t, `{"a": 1.2e-2}`, 1.2e-2) + tDouble(t, `{"a": -1.2e-2}`, -1.2e-2) +} + +func tInt(t *testing.T, input string, target int64) { + ffl := NewFFLexer([]byte(input)) + err := scanToTok(ffl, FFTok_integer) + if err != nil { + t.Fatalf("scanToTok failed, couldnt find int: %v input: %v", err, input) + } + + // Bit sizes 0, 8, 16, 32, and 64 correspond to int, int8, int16, int32, and int64. + i64, err := strconv.ParseInt(ffl.Output.String(), 10, 64) + if err != nil { + t.Fatalf("ParseInt failed, shouldnt of: %v input: %v", err, input) + } + + if i64 != target { + t.Fatalf("ffl.Output: expected i64 '%v', got: %v from: %v", target, i64, ffl.Output.String()) + } + + err = scanToTok(ffl, FFTok_eof) + if err != nil { + t.Fatalf("Failed to find EOF after int. input: %v", input) + } +} + +func TestInt(t *testing.T) { + tInt(t, `{"a": 2000}`, 2000) + tInt(t, `{"a": -2000}`, -2000) + tInt(t, `{"a": 0}`, 0) + tInt(t, `{"a": -0}`, -0) +} + +func tError(t *testing.T, input string, targetCount int, targetError FFErr) { + ffl := NewFFLexer([]byte(input)) + count, err := scanToTokCount(ffl, FFTok_error) + if err != nil { + t.Fatalf("scanToTok failed, couldnt find error token: %v input: %v", err, input) + } + + if count != targetCount { + t.Fatalf("Expected error token at offset %v, but found it at %v input: %v", + count, targetCount, input) + } + + if ffl.Error != targetError { + t.Fatalf("Expected error token %v, but got %v input: %v", + targetError, ffl.Error, input) + } + + line, char := ffl.reader.PosWithLine() + if line == 0 || char == 0 { + t.Fatalf("ffl.PosWithLine(): expected >=0 values. line=%v char=%v", + line, char) + } + + berr := ffl.WrapErr(ffl.Error.ToError()) + if berr == nil { + t.Fatalf("expected error") + } + +} + +func TestInvalid(t *testing.T) { + tError(t, `{"a": nul}`, 4, FFErr_invalid_string) + tError(t, `{"a": 1.a}`, 4, FFErr_missing_integer_after_decimal) +} + +func TestCapture(t *testing.T) { + ffl := NewFFLexer([]byte(`{"hello": {"blah": [null, 1]}}`)) + + err := scanToTok(ffl, FFTok_left_bracket) + if err != nil { + t.Fatalf("scanToTok failed: %v", err) + } + + err = scanToTok(ffl, FFTok_left_bracket) + if err != nil { + t.Fatalf("scanToTok failed: %v", err) + } + + buf, err := ffl.CaptureField(FFTok_left_bracket) + if err != nil { + t.Fatalf("CaptureField failed: %v", err) + } + + if bytes.Compare(buf, []byte(`{"blah": [null, 1]}`)) != 0 { + t.Fatalf("didnt capture subfield: buf: %v", string(buf)) + } +} diff --git a/fflib/v1/reader.go b/fflib/v1/reader.go new file mode 100644 index 0000000..0f22c46 --- /dev/null +++ b/fflib/v1/reader.go @@ -0,0 +1,512 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package v1 + +import ( + "fmt" + "io" + "unicode" + "unicode/utf16" +) + +const sliceStringMask = cIJC | cNFP + +type ffReader struct { + s []byte + i int + l int +} + +func newffReader(d []byte) *ffReader { + return &ffReader{ + s: d, + i: 0, + l: len(d), + } +} + +func (r *ffReader) Slice(start, stop int) []byte { + return r.s[start:stop] +} + +func (r *ffReader) Pos() int { + return r.i +} + +// Reset the reader, and add new input. +func (r *ffReader) Reset(d []byte) { + r.s = d + r.i = 0 + r.l = len(d) +} + +// Calcuates the Position with line and line offset, +// because this isn't counted for performance reasons, +// it will iterate the buffer from the beginning, and should +// only be used in error-paths. +func (r *ffReader) PosWithLine() (int, int) { + currentLine := 1 + currentChar := 0 + + for i := 0; i < r.i; i++ { + c := r.s[i] + currentChar++ + if c == '\n' { + currentLine++ + currentChar = 0 + } + } + + return currentLine, currentChar +} + +func (r *ffReader) ReadByteNoWS() (byte, error) { + if r.i >= r.l { + return 0, io.EOF + } + + j := r.i + + for { + c := r.s[j] + j++ + + // inline whitespace parsing gives another ~8% performance boost + // for many kinds of nicely indented JSON. + // ... and using a [255]bool instead of multiple ifs, gives another 2% + /* + if c != '\t' && + c != '\n' && + c != '\v' && + c != '\f' && + c != '\r' && + c != ' ' { + r.i = j + return c, nil + } + */ + if whitespaceLookupTable[c] == false { + r.i = j + return c, nil + } + + if j >= r.l { + return 0, io.EOF + } + } +} + +func (r *ffReader) ReadByte() (byte, error) { + if r.i >= r.l { + return 0, io.EOF + } + + r.i++ + + return r.s[r.i-1], nil +} + +func (r *ffReader) UnreadByte() error { + if r.i <= 0 { + panic("ffReader.UnreadByte: at beginning of slice") + } + r.i-- + return nil +} + +func (r *ffReader) readU4(j int) (rune, error) { + + var u4 [4]byte + for i := 0; i < 4; i++ { + if j >= r.l { + return -1, io.EOF + } + c := r.s[j] + if byteLookupTable[c]&cVHC != 0 { + u4[i] = c + j++ + continue + } else { + // TODO(pquerna): handle errors better. layering violation. + return -1, fmt.Errorf("lex_string_invalid_hex_char: %v %v", c, string(u4[:])) + } + } + + // TODO(pquerna): utf16.IsSurrogate + rr, err := ParseUint(u4[:], 16, 64) + if err != nil { + return -1, err + } + return rune(rr), nil +} + +func (r *ffReader) handleEscaped(c byte, j int, out DecodingBuffer) (int, error) { + if j >= r.l { + return 0, io.EOF + } + + c = r.s[j] + j++ + + if c == 'u' { + ru, err := r.readU4(j) + if err != nil { + return 0, err + } + + if utf16.IsSurrogate(ru) { + ru2, err := r.readU4(j + 6) + if err != nil { + return 0, err + } + out.Write(r.s[r.i : j-2]) + r.i = j + 10 + j = r.i + rval := utf16.DecodeRune(ru, ru2) + if rval != unicode.ReplacementChar { + out.WriteRune(rval) + } else { + return 0, fmt.Errorf("lex_string_invalid_unicode_surrogate: %v %v", ru, ru2) + } + } else { + out.Write(r.s[r.i : j-2]) + r.i = j + 4 + j = r.i + out.WriteRune(ru) + } + return j, nil + } else if byteLookupTable[c]&cVEC == 0 { + return 0, fmt.Errorf("lex_string_invalid_escaped_char: %v", c) + } else { + out.Write(r.s[r.i : j-2]) + r.i = j + j = r.i + + switch c { + case '"': + out.WriteByte('"') + case '\\': + out.WriteByte('\\') + case '/': + out.WriteByte('/') + case 'b': + out.WriteByte('\b') + case 'f': + out.WriteByte('\f') + case 'n': + out.WriteByte('\n') + case 'r': + out.WriteByte('\r') + case 't': + out.WriteByte('\t') + } + } + + return j, nil +} + +func (r *ffReader) SliceString(out DecodingBuffer) error { + var c byte + // TODO(pquerna): string_with_escapes? de-escape here? + j := r.i + + for { + if j >= r.l { + return io.EOF + } + + j, c = scanString(r.s, j) + + if c == '"' { + if j != r.i { + out.Write(r.s[r.i : j-1]) + r.i = j + } + return nil + } else if c == '\\' { + var err error + j, err = r.handleEscaped(c, j, out) + if err != nil { + return err + } + } else if byteLookupTable[c]&cIJC != 0 { + return fmt.Errorf("lex_string_invalid_json_char: %v", c) + } + continue + } +} + +// TODO(pquerna): consider combining wibth the normal byte mask. +var whitespaceLookupTable [256]bool = [256]bool{ + false, /* 0 */ + false, /* 1 */ + false, /* 2 */ + false, /* 3 */ + false, /* 4 */ + false, /* 5 */ + false, /* 6 */ + false, /* 7 */ + false, /* 8 */ + true, /* 9 */ + true, /* 10 */ + true, /* 11 */ + true, /* 12 */ + true, /* 13 */ + false, /* 14 */ + false, /* 15 */ + false, /* 16 */ + false, /* 17 */ + false, /* 18 */ + false, /* 19 */ + false, /* 20 */ + false, /* 21 */ + false, /* 22 */ + false, /* 23 */ + false, /* 24 */ + false, /* 25 */ + false, /* 26 */ + false, /* 27 */ + false, /* 28 */ + false, /* 29 */ + false, /* 30 */ + false, /* 31 */ + true, /* 32 */ + false, /* 33 */ + false, /* 34 */ + false, /* 35 */ + false, /* 36 */ + false, /* 37 */ + false, /* 38 */ + false, /* 39 */ + false, /* 40 */ + false, /* 41 */ + false, /* 42 */ + false, /* 43 */ + false, /* 44 */ + false, /* 45 */ + false, /* 46 */ + false, /* 47 */ + false, /* 48 */ + false, /* 49 */ + false, /* 50 */ + false, /* 51 */ + false, /* 52 */ + false, /* 53 */ + false, /* 54 */ + false, /* 55 */ + false, /* 56 */ + false, /* 57 */ + false, /* 58 */ + false, /* 59 */ + false, /* 60 */ + false, /* 61 */ + false, /* 62 */ + false, /* 63 */ + false, /* 64 */ + false, /* 65 */ + false, /* 66 */ + false, /* 67 */ + false, /* 68 */ + false, /* 69 */ + false, /* 70 */ + false, /* 71 */ + false, /* 72 */ + false, /* 73 */ + false, /* 74 */ + false, /* 75 */ + false, /* 76 */ + false, /* 77 */ + false, /* 78 */ + false, /* 79 */ + false, /* 80 */ + false, /* 81 */ + false, /* 82 */ + false, /* 83 */ + false, /* 84 */ + false, /* 85 */ + false, /* 86 */ + false, /* 87 */ + false, /* 88 */ + false, /* 89 */ + false, /* 90 */ + false, /* 91 */ + false, /* 92 */ + false, /* 93 */ + false, /* 94 */ + false, /* 95 */ + false, /* 96 */ + false, /* 97 */ + false, /* 98 */ + false, /* 99 */ + false, /* 100 */ + false, /* 101 */ + false, /* 102 */ + false, /* 103 */ + false, /* 104 */ + false, /* 105 */ + false, /* 106 */ + false, /* 107 */ + false, /* 108 */ + false, /* 109 */ + false, /* 110 */ + false, /* 111 */ + false, /* 112 */ + false, /* 113 */ + false, /* 114 */ + false, /* 115 */ + false, /* 116 */ + false, /* 117 */ + false, /* 118 */ + false, /* 119 */ + false, /* 120 */ + false, /* 121 */ + false, /* 122 */ + false, /* 123 */ + false, /* 124 */ + false, /* 125 */ + false, /* 126 */ + false, /* 127 */ + false, /* 128 */ + false, /* 129 */ + false, /* 130 */ + false, /* 131 */ + false, /* 132 */ + false, /* 133 */ + false, /* 134 */ + false, /* 135 */ + false, /* 136 */ + false, /* 137 */ + false, /* 138 */ + false, /* 139 */ + false, /* 140 */ + false, /* 141 */ + false, /* 142 */ + false, /* 143 */ + false, /* 144 */ + false, /* 145 */ + false, /* 146 */ + false, /* 147 */ + false, /* 148 */ + false, /* 149 */ + false, /* 150 */ + false, /* 151 */ + false, /* 152 */ + false, /* 153 */ + false, /* 154 */ + false, /* 155 */ + false, /* 156 */ + false, /* 157 */ + false, /* 158 */ + false, /* 159 */ + false, /* 160 */ + false, /* 161 */ + false, /* 162 */ + false, /* 163 */ + false, /* 164 */ + false, /* 165 */ + false, /* 166 */ + false, /* 167 */ + false, /* 168 */ + false, /* 169 */ + false, /* 170 */ + false, /* 171 */ + false, /* 172 */ + false, /* 173 */ + false, /* 174 */ + false, /* 175 */ + false, /* 176 */ + false, /* 177 */ + false, /* 178 */ + false, /* 179 */ + false, /* 180 */ + false, /* 181 */ + false, /* 182 */ + false, /* 183 */ + false, /* 184 */ + false, /* 185 */ + false, /* 186 */ + false, /* 187 */ + false, /* 188 */ + false, /* 189 */ + false, /* 190 */ + false, /* 191 */ + false, /* 192 */ + false, /* 193 */ + false, /* 194 */ + false, /* 195 */ + false, /* 196 */ + false, /* 197 */ + false, /* 198 */ + false, /* 199 */ + false, /* 200 */ + false, /* 201 */ + false, /* 202 */ + false, /* 203 */ + false, /* 204 */ + false, /* 205 */ + false, /* 206 */ + false, /* 207 */ + false, /* 208 */ + false, /* 209 */ + false, /* 210 */ + false, /* 211 */ + false, /* 212 */ + false, /* 213 */ + false, /* 214 */ + false, /* 215 */ + false, /* 216 */ + false, /* 217 */ + false, /* 218 */ + false, /* 219 */ + false, /* 220 */ + false, /* 221 */ + false, /* 222 */ + false, /* 223 */ + false, /* 224 */ + false, /* 225 */ + false, /* 226 */ + false, /* 227 */ + false, /* 228 */ + false, /* 229 */ + false, /* 230 */ + false, /* 231 */ + false, /* 232 */ + false, /* 233 */ + false, /* 234 */ + false, /* 235 */ + false, /* 236 */ + false, /* 237 */ + false, /* 238 */ + false, /* 239 */ + false, /* 240 */ + false, /* 241 */ + false, /* 242 */ + false, /* 243 */ + false, /* 244 */ + false, /* 245 */ + false, /* 246 */ + false, /* 247 */ + false, /* 248 */ + false, /* 249 */ + false, /* 250 */ + false, /* 251 */ + false, /* 252 */ + false, /* 253 */ + false, /* 254 */ + false, /* 255 */ +} diff --git a/fflib/v1/reader_scan_generic.go b/fflib/v1/reader_scan_generic.go new file mode 100644 index 0000000..47c2607 --- /dev/null +++ b/fflib/v1/reader_scan_generic.go @@ -0,0 +1,34 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package v1 + +func scanString(s []byte, j int) (int, byte) { + for { + if j >= len(s) { + return j, 0 + } + + c := s[j] + j++ + if byteLookupTable[c]&sliceStringMask == 0 { + continue + } + + return j, c + } +} diff --git a/fflib/v1/reader_test.go b/fflib/v1/reader_test.go new file mode 100644 index 0000000..b1a1b3e --- /dev/null +++ b/fflib/v1/reader_test.go @@ -0,0 +1,73 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package v1 + +import ( + "testing" +) + +func tsliceString(t *testing.T, expected string, enc string) { + var out Buffer + ffr := newffReader([]byte(enc + `"`)) + err := ffr.SliceString(&out) + if err != nil { + t.Fatalf("unexpect SliceString error: %v from %v", err, enc) + } + + if out.String() != expected { + t.Fatalf(`failed to decode %v into %v, got: %v`, enc, expected, out.String()) + } +} + +func TestUnicode(t *testing.T) { + var testvecs = map[string]string{ + "€": `\u20AC`, + "𐐷": `\uD801\uDC37`, + } + + for k, v := range testvecs { + tsliceString(t, k, v) + } +} + +func TestBadUnicode(t *testing.T) { + var out Buffer + ffr := newffReader([]byte(`\u20--"`)) + err := ffr.SliceString(&out) + if err == nil { + t.Fatalf("expected SliceString hex decode error") + } +} + +func TestNonUnicodeEscape(t *testing.T) { + var out Buffer + ffr := newffReader([]byte(`\t\n\r"`)) + err := ffr.SliceString(&out) + if err != nil { + t.Fatalf("unexpected SliceString error: %v", err) + } +} + +func TestInvalidEscape(t *testing.T) { + var out Buffer + ffr := newffReader([]byte(`\x134"`)) + err := ffr.SliceString(&out) + if err == nil { + t.Fatalf("expected SliceString escape decode error") + } +} diff --git a/generator/generator.go b/generator/generator.go new file mode 100644 index 0000000..1f50380 --- /dev/null +++ b/generator/generator.go @@ -0,0 +1,59 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package generator + +import ( + "errors" + "fmt" + "os" +) + +func GenerateFiles(goCmd string, inputPath string, outputPath string, importName string, forceRegenerate bool, resetFields bool) error { + + if _, StatErr := os.Stat(outputPath); !os.IsNotExist(StatErr) { + inputFileInfo, inputFileErr := os.Stat(inputPath) + outputFileInfo, outputFileErr := os.Stat(outputPath) + + if nil == outputFileErr && nil == inputFileErr { + if !forceRegenerate && inputFileInfo.ModTime().Before(outputFileInfo.ModTime()) { + fmt.Println("File " + outputPath + " already exists.") + + return nil + } + } + } + + packageName, structs, err := ExtractStructs(inputPath) + if err != nil { + return err + } + + im := NewInceptionMain(goCmd, inputPath, outputPath, resetFields) + + err = im.Generate(packageName, structs, importName) + if err != nil { + return errors.New(fmt.Sprintf("error=%v path=%q", err, im.TempMainPath)) + } + + err = im.Run() + if err != nil { + return err + } + + return nil +} diff --git a/generator/inceptionmain.go b/generator/inceptionmain.go new file mode 100644 index 0000000..f5f140b --- /dev/null +++ b/generator/inceptionmain.go @@ -0,0 +1,251 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package generator + +import ( + "bytes" + "errors" + "fmt" + "go/format" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + + "github.com/pquerna/ffjson/shared" +) + +const inceptionMainTemplate = ` +// DO NOT EDIT! +// Code generated by ffjson <https://github.com/pquerna/ffjson> +// DO NOT EDIT! + +package main + +import ( + "github.com/pquerna/ffjson/inception" + importedinceptionpackage "{{.ImportName}}" +) + +func main() { + i := ffjsoninception.NewInception("{{.InputPath}}", "{{.PackageName}}", "{{.OutputPath}}", {{.ResetFields}}) + i.AddMany(importedinceptionpackage.FFJSONExpose()) + i.Execute() +} +` + +const ffjsonExposeTemplate = ` +// Code generated by ffjson <https://github.com/pquerna/ffjson> +// +// This should be automatically deleted by running 'ffjson', +// if leftover, please delete it. + +package {{.PackageName}} + +import ( + ffjsonshared "github.com/pquerna/ffjson/shared" +) + +func FFJSONExpose() []ffjsonshared.InceptionType { + rv := make([]ffjsonshared.InceptionType, 0) +{{range .StructNames}} + rv = append(rv, ffjsonshared.InceptionType{Obj: {{.Name}}{}, Options: ffjson{{printf "%#v" .Options}} } ) +{{end}} + return rv +} +` + +type structName struct { + Name string + Options shared.StructOptions +} + +type templateCtx struct { + StructNames []structName + ImportName string + PackageName string + InputPath string + OutputPath string + ResetFields bool +} + +type InceptionMain struct { + goCmd string + inputPath string + exposePath string + outputPath string + TempMainPath string + tempDir string + tempMain *os.File + tempExpose *os.File + resetFields bool +} + +func NewInceptionMain(goCmd string, inputPath string, outputPath string, resetFields bool) *InceptionMain { + exposePath := getExposePath(inputPath) + return &InceptionMain{ + goCmd: goCmd, + inputPath: inputPath, + outputPath: outputPath, + exposePath: exposePath, + resetFields: resetFields, + } +} + +func getImportName(inputPath string) (string, error) { + p, err := filepath.Abs(inputPath) + if err != nil { + return "", err + } + + dir := filepath.Dir(p) + gopaths := strings.Split(os.Getenv("GOPATH"), string(os.PathListSeparator)) + + for _, path := range gopaths { + gpath, err := filepath.Abs(path) + if err != nil { + continue + } + rel, err := filepath.Rel(filepath.ToSlash(gpath), dir) + if err != nil { + return "", err + } + + if len(rel) < 4 || rel[:4] != "src"+string(os.PathSeparator) { + continue + } + return rel[4:], nil + } + return "", errors.New(fmt.Sprintf("Could not find source directory: GOPATH=%q REL=%q", gopaths, dir)) + +} + +func getExposePath(inputPath string) string { + return inputPath[0:len(inputPath)-3] + "_ffjson_expose.go" +} + +func (im *InceptionMain) renderTpl(f *os.File, t *template.Template, tc *templateCtx) error { + buf := new(bytes.Buffer) + err := t.Execute(buf, tc) + if err != nil { + return err + } + formatted, err := format.Source(buf.Bytes()) + if err != nil { + return err + } + _, err = f.Write(formatted) + return err +} + +func (im *InceptionMain) Generate(packageName string, si []*StructInfo, importName string) error { + var err error + + if importName == "" { + importName, err = getImportName(im.inputPath) + if err != nil { + return err + } + } + + im.tempDir, err = ioutil.TempDir(filepath.Dir(im.inputPath), "ffjson-inception") + if err != nil { + return err + } + + importName = filepath.ToSlash(importName) + // for `go run` to work, we must have a file ending in ".go". + im.tempMain, err = TempFileWithPostfix(im.tempDir, "ffjson-inception", ".go") + if err != nil { + return err + } + + im.TempMainPath = im.tempMain.Name() + sn := make([]structName, len(si)) + for i, st := range si { + sn[i].Name = st.Name + sn[i].Options = st.Options + } + + tc := &templateCtx{ + ImportName: importName, + PackageName: packageName, + StructNames: sn, + InputPath: im.inputPath, + OutputPath: im.outputPath, + ResetFields: im.resetFields, + } + + t := template.Must(template.New("inception.go").Parse(inceptionMainTemplate)) + + err = im.renderTpl(im.tempMain, t, tc) + if err != nil { + return err + } + + im.tempExpose, err = os.Create(im.exposePath) + if err != nil { + return err + } + + t = template.Must(template.New("ffjson_expose.go").Parse(ffjsonExposeTemplate)) + + err = im.renderTpl(im.tempExpose, t, tc) + if err != nil { + return err + } + + return nil +} + +func (im *InceptionMain) Run() error { + var out bytes.Buffer + var errOut bytes.Buffer + + cmd := exec.Command(im.goCmd, "run", "-a", im.TempMainPath) + cmd.Stdout = &out + cmd.Stderr = &errOut + + err := cmd.Run() + + if err != nil { + return errors.New( + fmt.Sprintf("Go Run Failed for: %s\nSTDOUT:\n%s\nSTDERR:\n%s\n", + im.TempMainPath, + string(out.Bytes()), + string(errOut.Bytes()))) + } + + defer func() { + if im.tempExpose != nil { + im.tempExpose.Close() + } + + if im.tempMain != nil { + im.tempMain.Close() + } + + os.Remove(im.TempMainPath) + os.Remove(im.exposePath) + os.Remove(im.tempDir) + }() + + return nil +} diff --git a/generator/parser.go b/generator/parser.go new file mode 100644 index 0000000..76458d6 --- /dev/null +++ b/generator/parser.go @@ -0,0 +1,142 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package generator + +import ( + "flag" + "fmt" + "github.com/pquerna/ffjson/shared" + "go/ast" + "go/doc" + "go/parser" + "go/token" + "regexp" + "strings" +) + +var noEncoder = flag.Bool("noencoder", false, "Do not generate encoder functions") +var noDecoder = flag.Bool("nodecoder", false, "Do not generate decoder functions") + +type StructField struct { + Name string +} + +type StructInfo struct { + Name string + Options shared.StructOptions +} + +func NewStructInfo(name string) *StructInfo { + return &StructInfo{ + Name: name, + Options: shared.StructOptions{ + SkipDecoder: *noDecoder, + SkipEncoder: *noEncoder, + }, + } +} + +var skipre = regexp.MustCompile("(.*)ffjson:(\\s*)((skip)|(ignore))(.*)") +var skipdec = regexp.MustCompile("(.*)ffjson:(\\s*)((skipdecoder)|(nodecoder))(.*)") +var skipenc = regexp.MustCompile("(.*)ffjson:(\\s*)((skipencoder)|(noencoder))(.*)") + +func shouldInclude(d *ast.Object) (bool, error) { + ts, ok := d.Decl.(*ast.TypeSpec) + if !ok { + return false, fmt.Errorf("Unknown type without TypeSec: %v", d) + } + + _, ok = ts.Type.(*ast.StructType) + if !ok { + ident, ok := ts.Type.(*ast.Ident) + if !ok || ident.Name == "" { + return false, nil + } + + // It must be in this package, and not a pointer alias + if strings.Contains(ident.Name, ".") || strings.Contains(ident.Name, "*") { + return false, nil + } + + // if Obj is nil, we have an external type or built-in. + if ident.Obj == nil || ident.Obj.Decl == nil { + return false, nil + } + return shouldInclude(ident.Obj) + } + return true, nil +} + +func ExtractStructs(inputPath string) (string, []*StructInfo, error) { + fset := token.NewFileSet() + + f, err := parser.ParseFile(fset, inputPath, nil, parser.ParseComments) + + if err != nil { + return "", nil, err + } + + packageName := f.Name.String() + structs := make(map[string]*StructInfo) + + for k, d := range f.Scope.Objects { + if d.Kind == ast.Typ { + incl, err := shouldInclude(d) + if err != nil { + return "", nil, err + } + if incl { + stobj := NewStructInfo(k) + + structs[k] = stobj + } + } + } + + files := map[string]*ast.File{ + inputPath: f, + } + + pkg, _ := ast.NewPackage(fset, files, nil, nil) + + d := doc.New(pkg, f.Name.String(), doc.AllDecls) + for _, t := range d.Types { + if skipre.MatchString(t.Doc) { + delete(structs, t.Name) + } else { + if skipdec.MatchString(t.Doc) { + s, ok := structs[t.Name] + if ok { + s.Options.SkipDecoder = true + } + } + if skipenc.MatchString(t.Doc) { + s, ok := structs[t.Name] + if ok { + s.Options.SkipEncoder = true + } + } + } + } + + rv := make([]*StructInfo, 0) + for _, v := range structs { + rv = append(rv, v) + } + return packageName, rv, nil +} diff --git a/generator/tags.go b/generator/tags.go new file mode 100644 index 0000000..d7fca95 --- /dev/null +++ b/generator/tags.go @@ -0,0 +1,59 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package generator + +import ( + "strings" +) + +// from: http://golang.org/src/pkg/encoding/json/tags.go + +// tagOptions is the string following a comma in a struct field's "json" +// tag, or the empty string. It does not include the leading comma. +type tagOptions string + +// parseTag splits a struct field's json tag into its name and +// comma-separated options. +func parseTag(tag string) (string, tagOptions) { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx], tagOptions(tag[idx+1:]) + } + return tag, tagOptions("") +} + +// Contains reports whether a comma-separated list of options +// contains a particular substr flag. substr must be surrounded by a +// string boundary or commas. +func (o tagOptions) Contains(optionName string) bool { + if len(o) == 0 { + return false + } + s := string(o) + for s != "" { + var next string + i := strings.Index(s, ",") + if i >= 0 { + s, next = s[:i], s[i+1:] + } + if s == optionName { + return true + } + s = next + } + return false +} diff --git a/generator/tempfile.go b/generator/tempfile.go new file mode 100644 index 0000000..1d116ad --- /dev/null +++ b/generator/tempfile.go @@ -0,0 +1,65 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package generator + +import ( + "os" + "path/filepath" + "strconv" + "sync" + "time" +) + +// Random number state. +// We generate random temporary file names so that there's a good +// chance the file doesn't exist yet - keeps the number of tries in +// TempFile to a minimum. +var rand uint32 +var randmu sync.Mutex + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} + +func nextSuffix() string { + randmu.Lock() + r := rand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + rand = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +// TempFile creates a new temporary file in the directory dir +// with a name beginning with prefix, opens the file for reading +// and writing, and returns the resulting *os.File. +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the pathname of the file. It is the caller's responsibility +// to remove the file when no longer needed. +func TempFileWithPostfix(dir, prefix string, postfix string) (f *os.File, err error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextSuffix()+postfix) + f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + rand = reseed() + } + continue + } + break + } + return +} diff --git a/inception/decoder.go b/inception/decoder.go new file mode 100644 index 0000000..908347a --- /dev/null +++ b/inception/decoder.go @@ -0,0 +1,323 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "fmt" + "reflect" + "strings" + + "github.com/pquerna/ffjson/shared" +) + +var validValues []string = []string{ + "FFTok_left_brace", + "FFTok_left_bracket", + "FFTok_integer", + "FFTok_double", + "FFTok_string", + "FFTok_bool", + "FFTok_null", +} + +func CreateUnmarshalJSON(ic *Inception, si *StructInfo) error { + out := "" + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + if len(si.Fields) > 0 { + ic.OutputImports[`"bytes"`] = true + } + ic.OutputImports[`"fmt"`] = true + + out += tplStr(decodeTpl["header"], header{ + IC: ic, + SI: si, + }) + + out += tplStr(decodeTpl["ujFunc"], ujFunc{ + SI: si, + IC: ic, + ValidValues: validValues, + ResetFields: ic.ResetFields, + }) + + ic.OutputFuncs = append(ic.OutputFuncs, out) + + return nil +} + +func handleField(ic *Inception, name string, typ reflect.Type, ptr bool, quoted bool) string { + return handleFieldAddr(ic, name, false, typ, ptr, quoted) +} + +func handleFieldAddr(ic *Inception, name string, takeAddr bool, typ reflect.Type, ptr bool, quoted bool) string { + out := fmt.Sprintf("/* handler: %s type=%v kind=%v quoted=%t*/\n", name, typ, typ.Kind(), quoted) + + umlx := typ.Implements(unmarshalFasterType) || typeInInception(ic, typ, shared.MustDecoder) + umlx = umlx || reflect.PtrTo(typ).Implements(unmarshalFasterType) + + umlstd := typ.Implements(unmarshalerType) || reflect.PtrTo(typ).Implements(unmarshalerType) + + out += tplStr(decodeTpl["handleUnmarshaler"], handleUnmarshaler{ + IC: ic, + Name: name, + Typ: typ, + Ptr: reflect.Ptr, + TakeAddr: takeAddr || ptr, + UnmarshalJSONFFLexer: umlx, + Unmarshaler: umlstd, + }) + + if umlx || umlstd { + return out + } + + // TODO(pquerna): generic handling of token type mismatching struct type + switch typ.Kind() { + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64: + + allowed := buildTokens(quoted, "FFTok_string", "FFTok_integer", "FFTok_null") + out += getAllowTokens(typ.Name(), allowed...) + + out += getNumberHandler(ic, name, takeAddr || ptr, typ, "ParseInt") + + case reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64: + + allowed := buildTokens(quoted, "FFTok_string", "FFTok_integer", "FFTok_null") + out += getAllowTokens(typ.Name(), allowed...) + + out += getNumberHandler(ic, name, takeAddr || ptr, typ, "ParseUint") + + case reflect.Float32, + reflect.Float64: + + allowed := buildTokens(quoted, "FFTok_string", "FFTok_double", "FFTok_integer", "FFTok_null") + out += getAllowTokens(typ.Name(), allowed...) + + out += getNumberHandler(ic, name, takeAddr || ptr, typ, "ParseFloat") + + case reflect.Bool: + ic.OutputImports[`"bytes"`] = true + ic.OutputImports[`"errors"`] = true + + allowed := buildTokens(quoted, "FFTok_string", "FFTok_bool", "FFTok_null") + out += getAllowTokens(typ.Name(), allowed...) + + out += tplStr(decodeTpl["handleBool"], handleBool{ + Name: name, + Typ: typ, + TakeAddr: takeAddr || ptr, + }) + + case reflect.Ptr: + out += tplStr(decodeTpl["handlePtr"], handlePtr{ + IC: ic, + Name: name, + Typ: typ, + Quoted: quoted, + }) + + case reflect.Array, + reflect.Slice: + out += getArrayHandler(ic, name, typ, ptr) + + case reflect.String: + // Is it a json.Number? + if typ.PkgPath() == "encoding/json" && typ.Name() == "Number" { + // Fall back to json package to rely on the valid number check. + // See: https://github.com/golang/go/blob/f05c3aa24d815cd3869153750c9875e35fc48a6e/src/encoding/json/decode.go#L897 + ic.OutputImports[`"encoding/json"`] = true + out += tplStr(decodeTpl["handleFallback"], handleFallback{ + Name: name, + Typ: typ, + Kind: typ.Kind(), + }) + } else { + out += tplStr(decodeTpl["handleString"], handleString{ + IC: ic, + Name: name, + Typ: typ, + TakeAddr: takeAddr || ptr, + Quoted: quoted, + }) + } + case reflect.Interface: + ic.OutputImports[`"encoding/json"`] = true + out += tplStr(decodeTpl["handleFallback"], handleFallback{ + Name: name, + Typ: typ, + Kind: typ.Kind(), + }) + case reflect.Map: + out += tplStr(decodeTpl["handleObject"], handleObject{ + IC: ic, + Name: name, + Typ: typ, + Ptr: reflect.Ptr, + TakeAddr: takeAddr || ptr, + }) + default: + ic.OutputImports[`"encoding/json"`] = true + out += tplStr(decodeTpl["handleFallback"], handleFallback{ + Name: name, + Typ: typ, + Kind: typ.Kind(), + }) + } + + return out +} + +func getArrayHandler(ic *Inception, name string, typ reflect.Type, ptr bool) string { + if typ.Kind() == reflect.Slice && typ.Elem().Kind() == reflect.Uint8 { + ic.OutputImports[`"encoding/base64"`] = true + useReflectToSet := false + if typ.Elem().Name() != "byte" { + ic.OutputImports[`"reflect"`] = true + useReflectToSet = true + } + + return tplStr(decodeTpl["handleByteSlice"], handleArray{ + IC: ic, + Name: name, + Typ: typ, + Ptr: reflect.Ptr, + UseReflectToSet: useReflectToSet, + }) + } + + if typ.Elem().Kind() == reflect.Struct && typ.Elem().Name() != "" { + goto sliceOrArray + } + + if (typ.Elem().Kind() == reflect.Struct || typ.Elem().Kind() == reflect.Map) || + typ.Elem().Kind() == reflect.Array || typ.Elem().Kind() == reflect.Slice && + typ.Elem().Name() == "" { + ic.OutputImports[`"encoding/json"`] = true + + return tplStr(decodeTpl["handleFallback"], handleFallback{ + Name: name, + Typ: typ, + Kind: typ.Kind(), + }) + } + +sliceOrArray: + + if typ.Kind() == reflect.Array { + return tplStr(decodeTpl["handleArray"], handleArray{ + IC: ic, + Name: name, + Typ: typ, + IsPtr: ptr, + Ptr: reflect.Ptr, + }) + } + + return tplStr(decodeTpl["handleSlice"], handleArray{ + IC: ic, + Name: name, + Typ: typ, + IsPtr: ptr, + Ptr: reflect.Ptr, + }) +} + +func getAllowTokens(name string, tokens ...string) string { + return tplStr(decodeTpl["allowTokens"], allowTokens{ + Name: name, + Tokens: tokens, + }) +} + +func getNumberHandler(ic *Inception, name string, takeAddr bool, typ reflect.Type, parsefunc string) string { + return tplStr(decodeTpl["handlerNumeric"], handlerNumeric{ + IC: ic, + Name: name, + ParseFunc: parsefunc, + TakeAddr: takeAddr, + Typ: typ, + }) +} + +func getNumberSize(typ reflect.Type) string { + return fmt.Sprintf("%d", typ.Bits()) +} + +func getType(ic *Inception, name string, typ reflect.Type) string { + s := typ.Name() + + if typ.PkgPath() != "" && typ.PkgPath() != ic.PackagePath { + path := removeVendor(typ.PkgPath()) + ic.OutputImports[`"`+path+`"`] = true + s = typ.String() + } + + if s == "" { + return typ.String() + } + + return s +} + +// removeVendor removes everything before and including a '/vendor/' +// substring in the package path. +// This is needed becuase that full path can't be used in the +// import statement. +func removeVendor(path string) string { + i := strings.Index(path, "/vendor/") + if i == -1 { + return path + } + return path[i+8:] +} + +func buildTokens(containsOptional bool, optional string, required ...string) []string { + if containsOptional { + return append(required, optional) + } + + return required +} + +func unquoteField(quoted bool) string { + // The outer quote of a string is already stripped out by + // the lexer. We need to check if the inner string is also + // quoted. If so, we will decode it as json string. If decoding + // fails, we will use the original string + if quoted { + return ` + unquoted, ok := fflib.UnquoteBytes(outBuf) + if ok { + outBuf = unquoted + } + ` + } + return "" +} + +func getTmpVarFor(name string) string { + return "tmp" + strings.Replace(strings.Title(name), ".", "", -1) +} diff --git a/inception/decoder_tpl.go b/inception/decoder_tpl.go new file mode 100644 index 0000000..0985061 --- /dev/null +++ b/inception/decoder_tpl.go @@ -0,0 +1,773 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "reflect" + "strconv" + "text/template" +) + +var decodeTpl map[string]*template.Template + +func init() { + decodeTpl = make(map[string]*template.Template) + + funcs := map[string]string{ + "handlerNumeric": handlerNumericTxt, + "allowTokens": allowTokensTxt, + "handleFallback": handleFallbackTxt, + "handleString": handleStringTxt, + "handleObject": handleObjectTxt, + "handleArray": handleArrayTxt, + "handleSlice": handleSliceTxt, + "handleByteSlice": handleByteSliceTxt, + "handleBool": handleBoolTxt, + "handlePtr": handlePtrTxt, + "header": headerTxt, + "ujFunc": ujFuncTxt, + "handleUnmarshaler": handleUnmarshalerTxt, + } + + tplFuncs := template.FuncMap{ + "getAllowTokens": getAllowTokens, + "getNumberSize": getNumberSize, + "getType": getType, + "handleField": handleField, + "handleFieldAddr": handleFieldAddr, + "unquoteField": unquoteField, + "getTmpVarFor": getTmpVarFor, + } + + for k, v := range funcs { + decodeTpl[k] = template.Must(template.New(k).Funcs(tplFuncs).Parse(v)) + } +} + +type handlerNumeric struct { + IC *Inception + Name string + ParseFunc string + Typ reflect.Type + TakeAddr bool +} + +var handlerNumericTxt = ` +{ + {{$ic := .IC}} + + if tok == fflib.FFTok_null { + {{if eq .TakeAddr true}} + {{.Name}} = nil + {{end}} + } else { + {{if eq .ParseFunc "ParseFloat" }} + tval, err := fflib.{{ .ParseFunc}}(fs.Output.Bytes(), {{getNumberSize .Typ}}) + {{else}} + tval, err := fflib.{{ .ParseFunc}}(fs.Output.Bytes(), 10, {{getNumberSize .Typ}}) + {{end}} + + if err != nil { + return fs.WrapErr(err) + } + {{if eq .TakeAddr true}} + ttypval := {{getType $ic .Name .Typ}}(tval) + {{.Name}} = &ttypval + {{else}} + {{.Name}} = {{getType $ic .Name .Typ}}(tval) + {{end}} + } +} +` + +type allowTokens struct { + Name string + Tokens []string +} + +var allowTokensTxt = ` +{ + if {{range $index, $element := .Tokens}}{{if ne $index 0 }}&&{{end}} tok != fflib.{{$element}}{{end}} { + return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for {{.Name}}", tok)) + } +} +` + +type handleFallback struct { + Name string + Typ reflect.Type + Kind reflect.Kind +} + +var handleFallbackTxt = ` +{ + /* Falling back. type={{printf "%v" .Typ}} kind={{printf "%v" .Kind}} */ + tbuf, err := fs.CaptureField(tok) + if err != nil { + return fs.WrapErr(err) + } + + err = json.Unmarshal(tbuf, &{{.Name}}) + if err != nil { + return fs.WrapErr(err) + } +} +` + +type handleString struct { + IC *Inception + Name string + Typ reflect.Type + TakeAddr bool + Quoted bool +} + +var handleStringTxt = ` +{ + {{$ic := .IC}} + + {{getAllowTokens .Typ.Name "FFTok_string" "FFTok_null"}} + if tok == fflib.FFTok_null { + {{if eq .TakeAddr true}} + {{.Name}} = nil + {{end}} + } else { + {{if eq .TakeAddr true}} + var tval {{getType $ic .Name .Typ}} + outBuf := fs.Output.Bytes() + {{unquoteField .Quoted}} + tval = {{getType $ic .Name .Typ}}(string(outBuf)) + {{.Name}} = &tval + {{else}} + outBuf := fs.Output.Bytes() + {{unquoteField .Quoted}} + {{.Name}} = {{getType $ic .Name .Typ}}(string(outBuf)) + {{end}} + } +} +` + +type handleObject struct { + IC *Inception + Name string + Typ reflect.Type + Ptr reflect.Kind + TakeAddr bool +} + +var handleObjectTxt = ` +{ + {{$ic := .IC}} + {{getAllowTokens .Typ.Name "FFTok_left_bracket" "FFTok_null"}} + if tok == fflib.FFTok_null { + {{.Name}} = nil + } else { + + {{if eq .TakeAddr true}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{if eq .Typ.Key.Kind .Ptr }} + var tval = make(map[*{{getType $ic .Name .Typ.Key.Elem}}]*{{getType $ic .Name .Typ.Elem.Elem}}, 0) + {{else}} + var tval = make(map[{{getType $ic .Name .Typ.Key}}]*{{getType $ic .Name .Typ.Elem.Elem}}, 0) + {{end}} + {{else}} + {{if eq .Typ.Key.Kind .Ptr }} + var tval = make(map[*{{getType $ic .Name .Typ.Key.Elem}}]{{getType $ic .Name .Typ.Elem}}, 0) + {{else}} + var tval = make(map[{{getType $ic .Name .Typ.Key}}]{{getType $ic .Name .Typ.Elem}}, 0) + {{end}} + {{end}} + {{else}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{if eq .Typ.Key.Kind .Ptr }} + {{.Name}} = make(map[*{{getType $ic .Name .Typ.Key.Elem}}]*{{getType $ic .Name .Typ.Elem.Elem}}, 0) + {{else}} + {{.Name}} = make(map[{{getType $ic .Name .Typ.Key}}]*{{getType $ic .Name .Typ.Elem.Elem}}, 0) + {{end}} + {{else}} + {{if eq .Typ.Key.Kind .Ptr }} + {{.Name}} = make(map[*{{getType $ic .Name .Typ.Key.Elem}}]{{getType $ic .Name .Typ.Elem}}, 0) + {{else}} + {{.Name}} = make(map[{{getType $ic .Name .Typ.Key}}]{{getType $ic .Name .Typ.Elem}}, 0) + {{end}} + {{end}} + {{end}} + + wantVal := true + + for { + {{$keyPtr := false}} + {{if eq .Typ.Key.Kind .Ptr }} + {{$keyPtr := true}} + var k *{{getType $ic .Name .Typ.Key.Elem}} + {{else}} + var k {{getType $ic .Name .Typ.Key}} + {{end}} + + {{$valPtr := false}} + {{$tmpVar := getTmpVarFor .Name}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{$valPtr := true}} + var {{$tmpVar}} *{{getType $ic .Name .Typ.Elem.Elem}} + {{else}} + var {{$tmpVar}} {{getType $ic .Name .Typ.Elem}} + {{end}} + + tok = fs.Scan() + if tok == fflib.FFTok_error { + goto tokerror + } + if tok == fflib.FFTok_right_bracket { + break + } + + if tok == fflib.FFTok_comma { + if wantVal == true { + // TODO(pquerna): this isn't an ideal error message, this handles + // things like [,,,] as an array value. + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) + } + continue + } else { + wantVal = true + } + + {{handleField .IC "k" .Typ.Key $keyPtr false}} + + // Expect ':' after key + tok = fs.Scan() + if tok != fflib.FFTok_colon { + return fs.WrapErr(fmt.Errorf("wanted colon token, but got token: %v", tok)) + } + + tok = fs.Scan() + {{handleField .IC $tmpVar .Typ.Elem $valPtr false}} + + {{if eq .TakeAddr true}} + tval[k] = {{$tmpVar}} + {{else}} + {{.Name}}[k] = {{$tmpVar}} + {{end}} + wantVal = false + } + + {{if eq .TakeAddr true}} + {{.Name}} = &tval + {{end}} + } +} +` + +type handleArray struct { + IC *Inception + Name string + Typ reflect.Type + Ptr reflect.Kind + UseReflectToSet bool + IsPtr bool +} + +var handleArrayTxt = ` +{ + {{$ic := .IC}} + {{getAllowTokens .Typ.Name "FFTok_left_brace" "FFTok_null"}} + {{if eq .Typ.Elem.Kind .Ptr}} + {{.Name}} = [{{.Typ.Len}}]*{{getType $ic .Name .Typ.Elem.Elem}}{} + {{else}} + {{.Name}} = [{{.Typ.Len}}]{{getType $ic .Name .Typ.Elem}}{} + {{end}} + if tok != fflib.FFTok_null { + wantVal := true + + idx := 0 + for { + {{$ptr := false}} + {{$tmpVar := getTmpVarFor .Name}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{$ptr := true}} + var {{$tmpVar}} *{{getType $ic .Name .Typ.Elem.Elem}} + {{else}} + var {{$tmpVar}} {{getType $ic .Name .Typ.Elem}} + {{end}} + + tok = fs.Scan() + if tok == fflib.FFTok_error { + goto tokerror + } + if tok == fflib.FFTok_right_brace { + break + } + + if tok == fflib.FFTok_comma { + if wantVal == true { + // TODO(pquerna): this isn't an ideal error message, this handles + // things like [,,,] as an array value. + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) + } + continue + } else { + wantVal = true + } + + {{handleField .IC $tmpVar .Typ.Elem $ptr false}} + + // Standard json.Unmarshal ignores elements out of array bounds, + // that what we do as well. + if idx < {{.Typ.Len}} { + {{.Name}}[idx] = {{$tmpVar}} + idx++ + } + + wantVal = false + } + } +} +` + +var handleSliceTxt = ` +{ + {{$ic := .IC}} + {{getAllowTokens .Typ.Name "FFTok_left_brace" "FFTok_null"}} + if tok == fflib.FFTok_null { + {{.Name}} = nil + } else { + {{if eq .Typ.Elem.Kind .Ptr }} + {{if eq .IsPtr true}} + {{.Name}} = &[]*{{getType $ic .Name .Typ.Elem.Elem}}{} + {{else}} + {{.Name}} = []*{{getType $ic .Name .Typ.Elem.Elem}}{} + {{end}} + {{else}} + {{if eq .IsPtr true}} + {{.Name}} = &[]{{getType $ic .Name .Typ.Elem}}{} + {{else}} + {{.Name}} = []{{getType $ic .Name .Typ.Elem}}{} + {{end}} + {{end}} + + wantVal := true + + for { + {{$ptr := false}} + {{$tmpVar := getTmpVarFor .Name}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{$ptr := true}} + var {{$tmpVar}} *{{getType $ic .Name .Typ.Elem.Elem}} + {{else}} + var {{$tmpVar}} {{getType $ic .Name .Typ.Elem}} + {{end}} + + tok = fs.Scan() + if tok == fflib.FFTok_error { + goto tokerror + } + if tok == fflib.FFTok_right_brace { + break + } + + if tok == fflib.FFTok_comma { + if wantVal == true { + // TODO(pquerna): this isn't an ideal error message, this handles + // things like [,,,] as an array value. + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) + } + continue + } else { + wantVal = true + } + + {{handleField .IC $tmpVar .Typ.Elem $ptr false}} + {{if eq .IsPtr true}} + *{{.Name}} = append(*{{.Name}}, {{$tmpVar}}) + {{else}} + {{.Name}} = append({{.Name}}, {{$tmpVar}}) + {{end}} + wantVal = false + } + } +} +` + +var handleByteSliceTxt = ` +{ + {{getAllowTokens .Typ.Name "FFTok_string" "FFTok_null"}} + if tok == fflib.FFTok_null { + {{.Name}} = nil + } else { + b := make([]byte, base64.StdEncoding.DecodedLen(fs.Output.Len())) + n, err := base64.StdEncoding.Decode(b, fs.Output.Bytes()) + if err != nil { + return fs.WrapErr(err) + } + {{if eq .UseReflectToSet true}} + v := reflect.ValueOf(&{{.Name}}).Elem() + v.SetBytes(b[0:n]) + {{else}} + {{.Name}} = append([]byte(), b[0:n]...) + {{end}} + } +} +` + +type handleBool struct { + Name string + Typ reflect.Type + TakeAddr bool +} + +var handleBoolTxt = ` +{ + if tok == fflib.FFTok_null { + {{if eq .TakeAddr true}} + {{.Name}} = nil + {{end}} + } else { + tmpb := fs.Output.Bytes() + + {{if eq .TakeAddr true}} + var tval bool + {{end}} + + if bytes.Compare([]byte{'t', 'r', 'u', 'e'}, tmpb) == 0 { + {{if eq .TakeAddr true}} + tval = true + {{else}} + {{.Name}} = true + {{end}} + } else if bytes.Compare([]byte{'f', 'a', 'l', 's', 'e'}, tmpb) == 0 { + {{if eq .TakeAddr true}} + tval = false + {{else}} + {{.Name}} = false + {{end}} + } else { + err = errors.New("unexpected bytes for true/false value") + return fs.WrapErr(err) + } + + {{if eq .TakeAddr true}} + {{.Name}} = &tval + {{end}} + } +} +` + +type handlePtr struct { + IC *Inception + Name string + Typ reflect.Type + Quoted bool +} + +var handlePtrTxt = ` +{ + {{$ic := .IC}} + + if tok == fflib.FFTok_null { + {{.Name}} = nil + } else { + if {{.Name}} == nil { + {{.Name}} = new({{getType $ic .Typ.Elem.Name .Typ.Elem}}) + } + + {{handleFieldAddr .IC .Name true .Typ.Elem false .Quoted}} + } +} +` + +type header struct { + IC *Inception + SI *StructInfo +} + +var headerTxt = ` +const ( + ffjt{{.SI.Name}}base = iota + ffjt{{.SI.Name}}nosuchkey + {{with $si := .SI}} + {{range $index, $field := $si.Fields}} + {{if ne $field.JsonName "-"}} + ffjt{{$si.Name}}{{$field.Name}} + {{end}} + {{end}} + {{end}} +) + +{{with $si := .SI}} + {{range $index, $field := $si.Fields}} + {{if ne $field.JsonName "-"}} +var ffjKey{{$si.Name}}{{$field.Name}} = []byte({{$field.JsonName}}) + {{end}} + {{end}} +{{end}} + +` + +type ujFunc struct { + IC *Inception + SI *StructInfo + ValidValues []string + ResetFields bool +} + +var ujFuncTxt = ` +{{$si := .SI}} +{{$ic := .IC}} + +// UnmarshalJSON umarshall json - template of ffjson +func (j *{{.SI.Name}}) UnmarshalJSON(input []byte) error { + fs := fflib.NewFFLexer(input) + return j.UnmarshalJSONFFLexer(fs, fflib.FFParse_map_start) +} + +// UnmarshalJSONFFLexer fast json unmarshall - template ffjson +func (j *{{.SI.Name}}) UnmarshalJSONFFLexer(fs *fflib.FFLexer, state fflib.FFParseState) error { + var err error + currentKey := ffjt{{.SI.Name}}base + _ = currentKey + tok := fflib.FFTok_init + wantedTok := fflib.FFTok_init + + {{if eq .ResetFields true}} + {{range $index, $field := $si.Fields}} + var ffjSet{{$si.Name}}{{$field.Name}} = false + {{end}} + {{end}} + +mainparse: + for { + tok = fs.Scan() + // println(fmt.Sprintf("debug: tok: %v state: %v", tok, state)) + if tok == fflib.FFTok_error { + goto tokerror + } + + switch state { + + case fflib.FFParse_map_start: + if tok != fflib.FFTok_left_bracket { + wantedTok = fflib.FFTok_left_bracket + goto wrongtokenerror + } + state = fflib.FFParse_want_key + continue + + case fflib.FFParse_after_value: + if tok == fflib.FFTok_comma { + state = fflib.FFParse_want_key + } else if tok == fflib.FFTok_right_bracket { + goto done + } else { + wantedTok = fflib.FFTok_comma + goto wrongtokenerror + } + + case fflib.FFParse_want_key: + // json {} ended. goto exit. woo. + if tok == fflib.FFTok_right_bracket { + goto done + } + if tok != fflib.FFTok_string { + wantedTok = fflib.FFTok_string + goto wrongtokenerror + } + + kn := fs.Output.Bytes() + if len(kn) <= 0 { + // "" case. hrm. + currentKey = ffjt{{.SI.Name}}nosuchkey + state = fflib.FFParse_want_colon + goto mainparse + } else { + switch kn[0] { + {{range $byte, $fields := $si.FieldsByFirstByte}} + case '{{$byte}}': + {{range $index, $field := $fields}} + {{if ne $index 0 }}} else if {{else}}if {{end}} bytes.Equal(ffjKey{{$si.Name}}{{$field.Name}}, kn) { + currentKey = ffjt{{$si.Name}}{{$field.Name}} + state = fflib.FFParse_want_colon + goto mainparse + {{end}} } + {{end}} + } + {{range $index, $field := $si.ReverseFields}} + if {{$field.FoldFuncName}}(ffjKey{{$si.Name}}{{$field.Name}}, kn) { + currentKey = ffjt{{$si.Name}}{{$field.Name}} + state = fflib.FFParse_want_colon + goto mainparse + } + {{end}} + currentKey = ffjt{{.SI.Name}}nosuchkey + state = fflib.FFParse_want_colon + goto mainparse + } + + case fflib.FFParse_want_colon: + if tok != fflib.FFTok_colon { + wantedTok = fflib.FFTok_colon + goto wrongtokenerror + } + state = fflib.FFParse_want_value + continue + case fflib.FFParse_want_value: + + if {{range $index, $v := .ValidValues}}{{if ne $index 0 }}||{{end}}tok == fflib.{{$v}}{{end}} { + switch currentKey { + {{range $index, $field := $si.Fields}} + case ffjt{{$si.Name}}{{$field.Name}}: + goto handle_{{$field.Name}} + {{end}} + case ffjt{{$si.Name}}nosuchkey: + err = fs.SkipField(tok) + if err != nil { + return fs.WrapErr(err) + } + state = fflib.FFParse_after_value + goto mainparse + } + } else { + goto wantedvalue + } + } + } +{{range $index, $field := $si.Fields}} +handle_{{$field.Name}}: + {{with $fieldName := $field.Name | printf "j.%s"}} + {{handleField $ic $fieldName $field.Typ $field.Pointer $field.ForceString}} + {{if eq $.ResetFields true}} + ffjSet{{$si.Name}}{{$field.Name}} = true + {{end}} + state = fflib.FFParse_after_value + goto mainparse + {{end}} +{{end}} + +wantedvalue: + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) +wrongtokenerror: + return fs.WrapErr(fmt.Errorf("ffjson: wanted token: %v, but got token: %v output=%s", wantedTok, tok, fs.Output.String())) +tokerror: + if fs.BigError != nil { + return fs.WrapErr(fs.BigError) + } + err = fs.Error.ToError() + if err != nil { + return fs.WrapErr(err) + } + panic("ffjson-generated: unreachable, please report bug.") +done: +{{if eq .ResetFields true}} +{{range $index, $field := $si.Fields}} + if !ffjSet{{$si.Name}}{{$field.Name}} { + {{with $fieldName := $field.Name | printf "j.%s"}} + {{if eq $field.Pointer true}} + {{$fieldName}} = nil + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Interface), 10) + `}} + {{$fieldName}} = nil + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Slice), 10) + `}} + {{$fieldName}} = nil + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Array), 10) + `}} + {{$fieldName}} = [{{$field.Typ.Len}}]{{getType $ic $fieldName $field.Typ.Elem}}{} + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Map), 10) + `}} + {{$fieldName}} = nil + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Bool), 10) + `}} + {{$fieldName}} = false + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.String), 10) + `}} + {{$fieldName}} = "" + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Struct), 10) + `}} + {{$fieldName}} = {{getType $ic $fieldName $field.Typ}}{} + {{else}} + {{$fieldName}} = {{getType $ic $fieldName $field.Typ}}(0) + {{end}} + {{end}} + } +{{end}} +{{end}} + return nil +} +` + +type handleUnmarshaler struct { + IC *Inception + Name string + Typ reflect.Type + Ptr reflect.Kind + TakeAddr bool + UnmarshalJSONFFLexer bool + Unmarshaler bool +} + +var handleUnmarshalerTxt = ` + {{$ic := .IC}} + + {{if eq .UnmarshalJSONFFLexer true}} + { + if tok == fflib.FFTok_null { + {{if eq .Typ.Kind .Ptr }} + {{.Name}} = nil + {{end}} + {{if eq .TakeAddr true }} + {{.Name}} = nil + {{end}} + } else { + {{if eq .Typ.Kind .Ptr }} + if {{.Name}} == nil { + {{.Name}} = new({{getType $ic .Typ.Elem.Name .Typ.Elem}}) + } + {{end}} + {{if eq .TakeAddr true }} + if {{.Name}} == nil { + {{.Name}} = new({{getType $ic .Typ.Name .Typ}}) + } + {{end}} + err = {{.Name}}.UnmarshalJSONFFLexer(fs, fflib.FFParse_want_key) + if err != nil { + return err + } + } + state = fflib.FFParse_after_value + } + {{else}} + {{if eq .Unmarshaler true}} + { + if tok == fflib.FFTok_null { + {{if eq .TakeAddr true }} + {{.Name}} = nil + {{end}} + } else { + + tbuf, err := fs.CaptureField(tok) + if err != nil { + return fs.WrapErr(err) + } + + {{if eq .TakeAddr true }} + if {{.Name}} == nil { + {{.Name}} = new({{getType $ic .Typ.Name .Typ}}) + } + {{end}} + err = {{.Name}}.UnmarshalJSON(tbuf) + if err != nil { + return fs.WrapErr(err) + } + } + state = fflib.FFParse_after_value + } + {{end}} + {{end}} +` diff --git a/inception/encoder.go b/inception/encoder.go new file mode 100644 index 0000000..3e37a28 --- /dev/null +++ b/inception/encoder.go @@ -0,0 +1,544 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "fmt" + "reflect" + + "github.com/pquerna/ffjson/shared" +) + +func typeInInception(ic *Inception, typ reflect.Type, f shared.Feature) bool { + for _, v := range ic.objs { + if v.Typ == typ { + return v.Options.HasFeature(f) + } + if typ.Kind() == reflect.Ptr { + if v.Typ == typ.Elem() { + return v.Options.HasFeature(f) + } + } + } + + return false +} + +func getOmitEmpty(ic *Inception, sf *StructField) string { + ptname := "j." + sf.Name + if sf.Pointer { + ptname = "*" + ptname + return "if true {\n" + } + switch sf.Typ.Kind() { + + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return "if len(" + ptname + ") != 0 {" + "\n" + + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64, + reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64, + reflect.Uintptr, + reflect.Float32, + reflect.Float64: + return "if " + ptname + " != 0 {" + "\n" + + case reflect.Bool: + return "if " + ptname + " != false {" + "\n" + + case reflect.Interface, reflect.Ptr: + return "if " + ptname + " != nil {" + "\n" + + default: + // TODO(pquerna): fix types + return "if true {" + "\n" + } +} + +func getMapValue(ic *Inception, name string, typ reflect.Type, ptr bool, forceString bool) string { + var out = "" + + if typ.Key().Kind() != reflect.String { + out += fmt.Sprintf("/* Falling back. type=%v kind=%v */\n", typ, typ.Kind()) + out += ic.q.Flush() + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + return out + } + + var elemKind reflect.Kind + elemKind = typ.Elem().Kind() + + switch elemKind { + case reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, + reflect.Float64, + reflect.Bool: + + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + + out += "if " + name + " == nil {" + "\n" + ic.q.Write("null") + out += ic.q.GetQueued() + ic.q.DeleteLast() + out += "} else {" + "\n" + out += ic.q.WriteFlush("{ ") + out += " for key, value := range " + name + " {" + "\n" + out += " fflib.WriteJsonString(buf, key)" + "\n" + out += " buf.WriteString(`:`)" + "\n" + out += getGetInnerValue(ic, "value", typ.Elem(), false, forceString) + out += " buf.WriteByte(',')" + "\n" + out += " }" + "\n" + out += "buf.Rewind(1)" + "\n" + out += ic.q.WriteFlush("}") + out += "}" + "\n" + + default: + out += ic.q.Flush() + out += fmt.Sprintf("/* Falling back. type=%v kind=%v */\n", typ, typ.Kind()) + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + } + return out +} + +func getGetInnerValue(ic *Inception, name string, typ reflect.Type, ptr bool, forceString bool) string { + var out = "" + + // Flush if not bool or maps + if typ.Kind() != reflect.Bool && typ.Kind() != reflect.Map && typ.Kind() != reflect.Struct { + out += ic.q.Flush() + } + + if typ.Implements(marshalerFasterType) || + reflect.PtrTo(typ).Implements(marshalerFasterType) || + typeInInception(ic, typ, shared.MustEncoder) || + typ.Implements(marshalerType) || + reflect.PtrTo(typ).Implements(marshalerType) { + + out += ic.q.Flush() + out += tplStr(encodeTpl["handleMarshaler"], handleMarshaler{ + IC: ic, + Name: name, + Typ: typ, + Ptr: reflect.Ptr, + MarshalJSONBuf: typ.Implements(marshalerFasterType) || reflect.PtrTo(typ).Implements(marshalerFasterType) || typeInInception(ic, typ, shared.MustEncoder), + Marshaler: typ.Implements(marshalerType) || reflect.PtrTo(typ).Implements(marshalerType), + }) + return out + } + + ptname := name + if ptr { + ptname = "*" + name + } + + switch typ.Kind() { + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64: + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + out += "fflib.FormatBits2(buf, uint64(" + ptname + "), 10, " + ptname + " < 0)" + "\n" + case reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64, + reflect.Uintptr: + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + out += "fflib.FormatBits2(buf, uint64(" + ptname + "), 10, false)" + "\n" + case reflect.Float32: + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + out += "fflib.AppendFloat(buf, float64(" + ptname + "), 'g', -1, 32)" + "\n" + case reflect.Float64: + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + out += "fflib.AppendFloat(buf, float64(" + ptname + "), 'g', -1, 64)" + "\n" + case reflect.Array, + reflect.Slice: + + // Arrays cannot be nil + if typ.Kind() != reflect.Array { + out += "if " + name + "!= nil {" + "\n" + } + // Array and slice values encode as JSON arrays, except that + // []byte encodes as a base64-encoded string, and a nil slice + // encodes as the null JSON object. + if typ.Kind() == reflect.Slice && typ.Elem().Kind() == reflect.Uint8 { + ic.OutputImports[`"encoding/base64"`] = true + + out += "buf.WriteString(`\"`)" + "\n" + out += `{` + "\n" + out += `enc := base64.NewEncoder(base64.StdEncoding, buf)` + "\n" + if typ.Elem().Name() != "byte" { + ic.OutputImports[`"reflect"`] = true + out += `enc.Write(reflect.Indirect(reflect.ValueOf(` + ptname + `)).Bytes())` + "\n" + + } else { + out += `enc.Write(` + ptname + `)` + "\n" + } + out += `enc.Close()` + "\n" + out += `}` + "\n" + out += "buf.WriteString(`\"`)" + "\n" + } else { + out += "buf.WriteString(`[`)" + "\n" + out += "for i, v := range " + ptname + "{" + "\n" + out += "if i != 0 {" + "\n" + out += "buf.WriteString(`,`)" + "\n" + out += "}" + "\n" + out += getGetInnerValue(ic, "v", typ.Elem(), false, false) + out += "}" + "\n" + out += "buf.WriteString(`]`)" + "\n" + } + if typ.Kind() != reflect.Array { + out += "} else {" + "\n" + out += "buf.WriteString(`null`)" + "\n" + out += "}" + "\n" + } + case reflect.String: + // Is it a json.Number? + if typ.PkgPath() == "encoding/json" && typ.Name() == "Number" { + // Fall back to json package to rely on the valid number check. + // See: https://github.com/golang/go/blob/92cd6e3af9f423ab4d8ac78f24e7fd81c31a8ce6/src/encoding/json/encode.go#L550 + out += fmt.Sprintf("/* json.Number */\n") + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + } else { + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + if forceString { + // Forcestring on strings does double-escaping of the entire value. + // We create a temporary buffer, encode to that an re-encode it. + out += "{" + "\n" + out += "tmpbuf := fflib.Buffer{}" + "\n" + out += "tmpbuf.Grow(len(" + ptname + ") + 16)" + "\n" + out += "fflib.WriteJsonString(&tmpbuf, string(" + ptname + "))" + "\n" + out += "fflib.WriteJsonString(buf, string( tmpbuf.Bytes() " + `))` + "\n" + out += "}" + "\n" + } else { + out += "fflib.WriteJsonString(buf, string(" + ptname + "))" + "\n" + } + } + case reflect.Ptr: + out += "if " + name + "!= nil {" + "\n" + switch typ.Elem().Kind() { + case reflect.Struct: + out += getGetInnerValue(ic, name, typ.Elem(), false, false) + default: + out += getGetInnerValue(ic, "*"+name, typ.Elem(), false, false) + } + out += "} else {" + "\n" + out += "buf.WriteString(`null`)" + "\n" + out += "}" + "\n" + case reflect.Bool: + out += "if " + ptname + " {" + "\n" + ic.q.Write("true") + out += ic.q.GetQueued() + out += "} else {" + "\n" + // Delete 'true' + ic.q.DeleteLast() + out += ic.q.WriteFlush("false") + out += "}" + "\n" + case reflect.Interface: + out += fmt.Sprintf("/* Interface types must use runtime reflection. type=%v kind=%v */\n", typ, typ.Kind()) + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + case reflect.Map: + out += getMapValue(ic, ptname, typ, ptr, forceString) + case reflect.Struct: + if typ.Name() == "" { + ic.q.Write("{") + ic.q.Write(" ") + out += fmt.Sprintf("/* Inline struct. type=%v kind=%v */\n", typ, typ.Kind()) + newV := reflect.Indirect(reflect.New(typ)).Interface() + fields := extractFields(newV) + + // Output all fields + for _, field := range fields { + // Adjust field name + field.Name = name + "." + field.Name + out += getField(ic, field, "") + } + + if lastConditional(fields) { + out += ic.q.Flush() + out += `buf.Rewind(1)` + "\n" + } else { + ic.q.DeleteLast() + } + out += ic.q.WriteFlush("}") + } else { + out += fmt.Sprintf("/* Struct fall back. type=%v kind=%v */\n", typ, typ.Kind()) + out += ic.q.Flush() + if ptr { + out += "err = buf.Encode(" + name + ")" + "\n" + } else { + // We send pointer to avoid copying entire struct + out += "err = buf.Encode(&" + name + ")" + "\n" + } + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + } + default: + out += fmt.Sprintf("/* Falling back. type=%v kind=%v */\n", typ, typ.Kind()) + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + } + + return out +} + +func getValue(ic *Inception, sf *StructField, prefix string) string { + closequote := false + if sf.ForceString { + switch sf.Typ.Kind() { + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64, + reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64, + reflect.Uintptr, + reflect.Float32, + reflect.Float64, + reflect.Bool: + ic.q.Write(`"`) + closequote = true + } + } + out := getGetInnerValue(ic, prefix+sf.Name, sf.Typ, sf.Pointer, sf.ForceString) + if closequote { + if sf.Pointer { + out += ic.q.WriteFlush(`"`) + } else { + ic.q.Write(`"`) + } + } + + return out +} + +func p2(v uint32) uint32 { + v-- + v |= v >> 1 + v |= v >> 2 + v |= v >> 4 + v |= v >> 8 + v |= v >> 16 + v++ + return v +} + +func getTypeSize(t reflect.Type) uint32 { + switch t.Kind() { + case reflect.String: + // TODO: consider runtime analysis. + return 32 + case reflect.Array, reflect.Map, reflect.Slice: + // TODO: consider runtime analysis. + return 4 * getTypeSize(t.Elem()) + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32: + return 8 + case reflect.Int64, + reflect.Uint64, + reflect.Uintptr: + return 16 + case reflect.Float32, + reflect.Float64: + return 16 + case reflect.Bool: + return 4 + case reflect.Ptr: + return getTypeSize(t.Elem()) + default: + return 16 + } +} + +func getTotalSize(si *StructInfo) uint32 { + rv := uint32(si.Typ.Size()) + for _, f := range si.Fields { + rv += getTypeSize(f.Typ) + } + return rv +} + +func getBufGrowSize(si *StructInfo) uint32 { + + // TOOD(pquerna): automatically calc a better grow size based on history + // of a struct. + return p2(getTotalSize(si)) +} + +func isIntish(t reflect.Type) bool { + if t.Kind() >= reflect.Int && t.Kind() <= reflect.Uintptr { + return true + } + if t.Kind() == reflect.Array || t.Kind() == reflect.Slice || t.Kind() == reflect.Ptr { + if t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Uint8 { + // base64 special case. + return false + } else { + return isIntish(t.Elem()) + } + } + return false +} + +func getField(ic *Inception, f *StructField, prefix string) string { + out := "" + if f.OmitEmpty { + out += ic.q.Flush() + if f.Pointer { + out += "if " + prefix + f.Name + " != nil {" + "\n" + } + out += getOmitEmpty(ic, f) + } + + if f.Pointer && !f.OmitEmpty { + // Pointer values encode as the value pointed to. A nil pointer encodes as the null JSON object. + out += "if " + prefix + f.Name + " != nil {" + "\n" + } + + // JsonName is already escaped and quoted. + // getInnervalue should flush + ic.q.Write(f.JsonName + ":") + // We save a copy in case we need it + t := ic.q + + out += getValue(ic, f, prefix) + ic.q.Write(",") + + if f.Pointer && !f.OmitEmpty { + out += "} else {" + "\n" + out += t.WriteFlush("null") + out += "}" + "\n" + } + + if f.OmitEmpty { + out += ic.q.Flush() + if f.Pointer { + out += "}" + "\n" + } + out += "}" + "\n" + } + return out +} + +// We check if the last field is conditional. +func lastConditional(fields []*StructField) bool { + if len(fields) > 0 { + f := fields[len(fields)-1] + return f.OmitEmpty + } + return false +} + +func CreateMarshalJSON(ic *Inception, si *StructInfo) error { + conditionalWrites := lastConditional(si.Fields) + out := "" + + out += "// MarshalJSON marshal bytes to json - template\n" + out += `func (j *` + si.Name + `) MarshalJSON() ([]byte, error) {` + "\n" + out += `var buf fflib.Buffer` + "\n" + + out += `if j == nil {` + "\n" + out += ` buf.WriteString("null")` + "\n" + out += " return buf.Bytes(), nil" + "\n" + out += `}` + "\n" + + out += `err := j.MarshalJSONBuf(&buf)` + "\n" + out += `if err != nil {` + "\n" + out += " return nil, err" + "\n" + out += `}` + "\n" + out += `return buf.Bytes(), nil` + "\n" + out += `}` + "\n" + + out += "// MarshalJSONBuf marshal buff to json - template\n" + out += `func (j *` + si.Name + `) MarshalJSONBuf(buf fflib.EncodingBuffer) (error) {` + "\n" + out += ` if j == nil {` + "\n" + out += ` buf.WriteString("null")` + "\n" + out += " return nil" + "\n" + out += ` }` + "\n" + + out += `var err error` + "\n" + out += `var obj []byte` + "\n" + out += `_ = obj` + "\n" + out += `_ = err` + "\n" + + ic.q.Write("{") + + // The extra space is inserted here. + // If nothing is written to the field this will be deleted + // instead of the last comma. + if conditionalWrites || len(si.Fields) == 0 { + ic.q.Write(" ") + } + + for _, f := range si.Fields { + out += getField(ic, f, "j.") + } + + // Handling the last comma is tricky. + // If the last field has omitempty, conditionalWrites is set. + // If something has been written, we delete the last comma, + // by backing up the buffer, otherwise it will delete a space. + if conditionalWrites { + out += ic.q.Flush() + out += `buf.Rewind(1)` + "\n" + } else { + ic.q.DeleteLast() + } + + out += ic.q.WriteFlush("}") + out += `return nil` + "\n" + out += `}` + "\n" + ic.OutputFuncs = append(ic.OutputFuncs, out) + return nil +} diff --git a/inception/encoder_tpl.go b/inception/encoder_tpl.go new file mode 100644 index 0000000..22ab529 --- /dev/null +++ b/inception/encoder_tpl.go @@ -0,0 +1,73 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "reflect" + "text/template" +) + +var encodeTpl map[string]*template.Template + +func init() { + encodeTpl = make(map[string]*template.Template) + + funcs := map[string]string{ + "handleMarshaler": handleMarshalerTxt, + } + tplFuncs := template.FuncMap{} + + for k, v := range funcs { + encodeTpl[k] = template.Must(template.New(k).Funcs(tplFuncs).Parse(v)) + } +} + +type handleMarshaler struct { + IC *Inception + Name string + Typ reflect.Type + Ptr reflect.Kind + MarshalJSONBuf bool + Marshaler bool +} + +var handleMarshalerTxt = ` + { + {{if eq .Typ.Kind .Ptr}} + if {{.Name}} == nil { + buf.WriteString("null") + } else { + {{end}} + + {{if eq .MarshalJSONBuf true}} + err = {{.Name}}.MarshalJSONBuf(buf) + if err != nil { + return err + } + {{else if eq .Marshaler true}} + obj, err = {{.Name}}.MarshalJSON() + if err != nil { + return err + } + buf.Write(obj) + {{end}} + {{if eq .Typ.Kind .Ptr}} + } + {{end}} + } +` diff --git a/inception/inception.go b/inception/inception.go new file mode 100644 index 0000000..10cb271 --- /dev/null +++ b/inception/inception.go @@ -0,0 +1,160 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "errors" + "fmt" + "github.com/pquerna/ffjson/shared" + "io/ioutil" + "os" + "reflect" + "sort" +) + +type Inception struct { + objs []*StructInfo + InputPath string + OutputPath string + PackageName string + PackagePath string + OutputImports map[string]bool + OutputFuncs []string + q ConditionalWrite + ResetFields bool +} + +func NewInception(inputPath string, packageName string, outputPath string, resetFields bool) *Inception { + return &Inception{ + objs: make([]*StructInfo, 0), + InputPath: inputPath, + OutputPath: outputPath, + PackageName: packageName, + OutputFuncs: make([]string, 0), + OutputImports: make(map[string]bool), + ResetFields: resetFields, + } +} + +func (i *Inception) AddMany(objs []shared.InceptionType) { + for _, obj := range objs { + i.Add(obj) + } +} + +func (i *Inception) Add(obj shared.InceptionType) { + i.objs = append(i.objs, NewStructInfo(obj)) + i.PackagePath = i.objs[0].Typ.PkgPath() +} + +func (i *Inception) wantUnmarshal(si *StructInfo) bool { + if si.Options.SkipDecoder { + return false + } + typ := si.Typ + umlx := typ.Implements(unmarshalFasterType) || reflect.PtrTo(typ).Implements(unmarshalFasterType) + umlstd := typ.Implements(unmarshalerType) || reflect.PtrTo(typ).Implements(unmarshalerType) + if umlstd && !umlx { + // structure has UnmarshalJSON, but not our faster version -- skip it. + return false + } + return true +} + +func (i *Inception) wantMarshal(si *StructInfo) bool { + if si.Options.SkipEncoder { + return false + } + typ := si.Typ + mlx := typ.Implements(marshalerFasterType) || reflect.PtrTo(typ).Implements(marshalerFasterType) + mlstd := typ.Implements(marshalerType) || reflect.PtrTo(typ).Implements(marshalerType) + if mlstd && !mlx { + // structure has MarshalJSON, but not our faster version -- skip it. + return false + } + return true +} + +type sortedStructs []*StructInfo + +func (p sortedStructs) Len() int { return len(p) } +func (p sortedStructs) Less(i, j int) bool { return p[i].Name < p[j].Name } +func (p sortedStructs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p sortedStructs) Sort() { sort.Sort(p) } + +func (i *Inception) generateCode() error { + // We sort the structs by name, so output if predictable. + sorted := sortedStructs(i.objs) + sorted.Sort() + + for _, si := range sorted { + if i.wantMarshal(si) { + err := CreateMarshalJSON(i, si) + if err != nil { + return err + } + } + + if i.wantUnmarshal(si) { + err := CreateUnmarshalJSON(i, si) + if err != nil { + return err + } + } + } + return nil +} + +func (i *Inception) handleError(err error) { + fmt.Fprintf(os.Stderr, "Error: %s:\n\n", err) + os.Exit(1) +} + +func (i *Inception) Execute() { + if len(os.Args) != 1 { + i.handleError(errors.New(fmt.Sprintf("Internal ffjson error: inception executable takes no args: %v", os.Args))) + return + } + + err := i.generateCode() + if err != nil { + i.handleError(err) + return + } + + data, err := RenderTemplate(i) + if err != nil { + i.handleError(err) + return + } + + stat, err := os.Stat(i.InputPath) + + if err != nil { + i.handleError(err) + return + } + + err = ioutil.WriteFile(i.OutputPath, data, stat.Mode()) + + if err != nil { + i.handleError(err) + return + } + +} diff --git a/inception/reflect.go b/inception/reflect.go new file mode 100644 index 0000000..8fb0bd5 --- /dev/null +++ b/inception/reflect.go @@ -0,0 +1,290 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + fflib "github.com/pquerna/ffjson/fflib/v1" + "github.com/pquerna/ffjson/shared" + + "bytes" + "encoding/json" + "reflect" + "unicode/utf8" +) + +type StructField struct { + Name string + JsonName string + FoldFuncName string + Typ reflect.Type + OmitEmpty bool + ForceString bool + HasMarshalJSON bool + HasUnmarshalJSON bool + Pointer bool + Tagged bool +} + +type FieldByJsonName []*StructField + +func (a FieldByJsonName) Len() int { return len(a) } +func (a FieldByJsonName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a FieldByJsonName) Less(i, j int) bool { return a[i].JsonName < a[j].JsonName } + +type StructInfo struct { + Name string + Obj interface{} + Typ reflect.Type + Fields []*StructField + Options shared.StructOptions +} + +func NewStructInfo(obj shared.InceptionType) *StructInfo { + t := reflect.TypeOf(obj.Obj) + return &StructInfo{ + Obj: obj.Obj, + Name: t.Name(), + Typ: t, + Fields: extractFields(obj.Obj), + Options: obj.Options, + } +} + +func (si *StructInfo) FieldsByFirstByte() map[string][]*StructField { + rv := make(map[string][]*StructField) + for _, f := range si.Fields { + b := string(f.JsonName[1]) + rv[b] = append(rv[b], f) + } + return rv +} + +func (si *StructInfo) ReverseFields() []*StructField { + var i int + rv := make([]*StructField, 0) + for i = len(si.Fields) - 1; i >= 0; i-- { + rv = append(rv, si.Fields[i]) + } + return rv +} + +const ( + caseMask = ^byte(0x20) // Mask to ignore case in ASCII. +) + +func foldFunc(key []byte) string { + nonLetter := false + special := false // special letter + for _, b := range key { + if b >= utf8.RuneSelf { + return "bytes.EqualFold" + } + upper := b & caseMask + if upper < 'A' || upper > 'Z' { + nonLetter = true + } else if upper == 'K' || upper == 'S' { + // See above for why these letters are special. + special = true + } + } + if special { + return "fflib.EqualFoldRight" + } + if nonLetter { + return "fflib.AsciiEqualFold" + } + return "fflib.SimpleLetterEqualFold" +} + +type MarshalerFaster interface { + MarshalJSONBuf(buf fflib.EncodingBuffer) error +} + +type UnmarshalFaster interface { + UnmarshalJSONFFLexer(l *fflib.FFLexer, state fflib.FFParseState) error +} + +var marshalerType = reflect.TypeOf(new(json.Marshaler)).Elem() +var marshalerFasterType = reflect.TypeOf(new(MarshalerFaster)).Elem() +var unmarshalerType = reflect.TypeOf(new(json.Unmarshaler)).Elem() +var unmarshalFasterType = reflect.TypeOf(new(UnmarshalFaster)).Elem() + +// extractFields returns a list of fields that JSON should recognize for the given type. +// The algorithm is breadth-first search over the set of structs to include - the top struct +// and then any reachable anonymous structs. +func extractFields(obj interface{}) []*StructField { + t := reflect.TypeOf(obj) + // Anonymous fields to explore at the current level and the next. + current := []StructField{} + next := []StructField{{Typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []*StructField + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.Typ] { + continue + } + visited[f.Typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.Typ.NumField(); i++ { + sf := f.Typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + tag := sf.Tag.Get("json") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if !isValidTag(name) { + name = "" + } + + ft := sf.Type + ptr := false + if ft.Kind() == reflect.Ptr { + ptr = true + } + + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := name != "" + if name == "" { + name = sf.Name + } + + var buf bytes.Buffer + fflib.WriteJsonString(&buf, name) + + field := &StructField{ + Name: sf.Name, + JsonName: string(buf.Bytes()), + FoldFuncName: foldFunc([]byte(name)), + Typ: ft, + HasMarshalJSON: ft.Implements(marshalerType), + HasUnmarshalJSON: ft.Implements(unmarshalerType), + OmitEmpty: opts.Contains("omitempty"), + ForceString: opts.Contains("string"), + Pointer: ptr, + Tagged: tagged, + } + + fields = append(fields, field) + + if count[f.Typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + next = append(next, StructField{ + Name: ft.Name(), + Typ: ft, + }) + } + } + } + } + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with JSON tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.JsonName + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.JsonName != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// JSON tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []*StructField) (*StructField, bool) { + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if f.Tagged { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return nil, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return nil, false + } + return fields[0], true +} diff --git a/inception/tags.go b/inception/tags.go new file mode 100644 index 0000000..ccce101 --- /dev/null +++ b/inception/tags.go @@ -0,0 +1,79 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "strings" + "unicode" +) + +// from: http://golang.org/src/pkg/encoding/json/tags.go + +// tagOptions is the string following a comma in a struct field's "json" +// tag, or the empty string. It does not include the leading comma. +type tagOptions string + +// parseTag splits a struct field's json tag into its name and +// comma-separated options. +func parseTag(tag string) (string, tagOptions) { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx], tagOptions(tag[idx+1:]) + } + return tag, tagOptions("") +} + +// Contains reports whether a comma-separated list of options +// contains a particular substr flag. substr must be surrounded by a +// string boundary or commas. +func (o tagOptions) Contains(optionName string) bool { + if len(o) == 0 { + return false + } + s := string(o) + for s != "" { + var next string + i := strings.Index(s, ",") + if i >= 0 { + s, next = s[:i], s[i+1:] + } + if s == optionName { + return true + } + s = next + } + return false +} + +func isValidTag(s string) bool { + if s == "" { + return false + } + for _, c := range s { + switch { + case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): + // Backslash and quote chars are reserved, but + // otherwise any punctuation chars are allowed + // in a tag name. + default: + if !unicode.IsLetter(c) && !unicode.IsDigit(c) { + return false + } + } + } + return true +} diff --git a/inception/template.go b/inception/template.go new file mode 100644 index 0000000..121a23d --- /dev/null +++ b/inception/template.go @@ -0,0 +1,60 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "bytes" + "go/format" + "text/template" +) + +const ffjsonTemplate = ` +// Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. +// source: {{.InputPath}} + +package {{.PackageName}} + +import ( +{{range $k, $v := .OutputImports}}{{$k}} +{{end}} +) + +{{range .OutputFuncs}} +{{.}} +{{end}} + +` + +func RenderTemplate(ic *Inception) ([]byte, error) { + t := template.Must(template.New("ffjson.go").Parse(ffjsonTemplate)) + buf := new(bytes.Buffer) + err := t.Execute(buf, ic) + if err != nil { + return nil, err + } + return format.Source(buf.Bytes()) +} + +func tplStr(t *template.Template, data interface{}) string { + buf := bytes.Buffer{} + err := t.Execute(&buf, data) + if err != nil { + panic(err) + } + return buf.String() +} diff --git a/inception/writerstack.go b/inception/writerstack.go new file mode 100644 index 0000000..1521961 --- /dev/null +++ b/inception/writerstack.go @@ -0,0 +1,65 @@ +package ffjsoninception + +import "strings" + +// ConditionalWrite is a stack containing a number of pending writes +type ConditionalWrite struct { + Queued []string +} + +// Write will add a string to be written +func (w *ConditionalWrite) Write(s string) { + w.Queued = append(w.Queued, s) +} + +// DeleteLast will delete the last added write +func (w *ConditionalWrite) DeleteLast() { + if len(w.Queued) == 0 { + return + } + w.Queued = w.Queued[:len(w.Queued)-1] +} + +// Last will return the last added write +func (w *ConditionalWrite) Last() string { + if len(w.Queued) == 0 { + return "" + } + return w.Queued[len(w.Queued)-1] +} + +// Flush will return all queued writes, and return +// "" (empty string) in nothing has been queued +// "buf.WriteByte('" + byte + "')" + '\n' if one bute has been queued. +// "buf.WriteString(`" + string + "`)" + "\n" if more than one byte has been queued. +func (w *ConditionalWrite) Flush() string { + combined := strings.Join(w.Queued, "") + if len(combined) == 0 { + return "" + } + + w.Queued = nil + if len(combined) == 1 { + return "buf.WriteByte('" + combined + "')" + "\n" + } + return "buf.WriteString(`" + combined + "`)" + "\n" +} + +func (w *ConditionalWrite) FlushTo(out string) string { + out += w.Flush() + return out +} + +// WriteFlush will add a string and return the Flush result for the queue +func (w *ConditionalWrite) WriteFlush(s string) string { + w.Write(s) + return w.Flush() +} + +// GetQueued will return the current queued content without flushing. +func (w *ConditionalWrite) GetQueued() string { + t := w.Queued + s := w.Flush() + w.Queued = t + return s +} diff --git a/shared/options.go b/shared/options.go new file mode 100644 index 0000000..d74edc1 --- /dev/null +++ b/shared/options.go @@ -0,0 +1,51 @@ +/** + * Copyright 2014 Paul Querna, Klaus Post + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package shared + +type StructOptions struct { + SkipDecoder bool + SkipEncoder bool +} + +type InceptionType struct { + Obj interface{} + Options StructOptions +} +type Feature int + +const ( + Nothing Feature = 0 + MustDecoder = 1 << 1 + MustEncoder = 1 << 2 + MustEncDec = MustDecoder | MustEncoder +) + +func (i InceptionType) HasFeature(f Feature) bool { + return i.HasFeature(f) +} + +func (s StructOptions) HasFeature(f Feature) bool { + hasNeeded := true + if f&MustDecoder != 0 && s.SkipDecoder { + hasNeeded = false + } + if f&MustEncoder != 0 && s.SkipEncoder { + hasNeeded = false + } + return hasNeeded +} diff --git a/tests/base.go b/tests/base.go new file mode 100644 index 0000000..c4a111b --- /dev/null +++ b/tests/base.go @@ -0,0 +1,18 @@ +package tff + +// Foo struct +type Foo struct { + Blah int +} + +// Record struct +type Record struct { + Timestamp int64 `json:"id,omitempty"` + OriginID uint32 + Bar Foo + Method string `json:"meth"` + ReqID string + ServerIP string + RemoteIP string + BytesSent uint64 +} diff --git a/tests/bench.cmd b/tests/bench.cmd new file mode 100644 index 0000000..e690c8c --- /dev/null +++ b/tests/bench.cmd @@ -0,0 +1,9 @@ +del ff_ffjson.go +ffjson ff.go + +go test -benchmem -bench MarshalJSON + +REM ### Bench CPU ### +rem go test -benchmem -test.run=none -bench MarshalJSONNative -cpuprofile="cpu.dat" -benchtime 10s &&go tool pprof -gif tests.test.exe cpu.dat >out.gif + + diff --git a/tests/encode_test.go b/tests/encode_test.go new file mode 100644 index 0000000..cf48f7d --- /dev/null +++ b/tests/encode_test.go @@ -0,0 +1,267 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tff + +import ( + "bytes" + "encoding/json" + "testing" +) + +func TestOmitEmpty(t *testing.T) { + var o Optionals + o.Sw = "something" + o.Mr = map[string]interface{}{} + o.Mo = map[string]interface{}{} + + got, err := json.MarshalIndent(&o, "", " ") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != optionalsExpected { + t.Errorf(" got: %s\nwant: %s\n", got, optionalsExpected) + } +} + +func TestOmitEmptyAll(t *testing.T) { + var o OmitAll + + got, err := json.MarshalIndent(&o, "", " ") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != omitAllExpected { + t.Errorf(" got: %s\nwant: %s\n", got, omitAllExpected) + } +} + +func TestNoExported(t *testing.T) { + var o NoExported + + got, err := json.MarshalIndent(&o, "", " ") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != noExportedExpected { + t.Errorf(" got: %s\nwant: %s\n", got, noExportedExpected) + } +} + +func TestOmitFirst(t *testing.T) { + var o OmitFirst + + got, err := json.MarshalIndent(&o, "", " ") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != omitFirstExpected { + t.Errorf(" got: %s\nwant: %s\n", got, omitFirstExpected) + } +} + +func TestOmitLast(t *testing.T) { + var o OmitLast + + got, err := json.MarshalIndent(&o, "", " ") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != omitLastExpected { + t.Errorf(" got: %s\nwant: %s\n", got, omitLastExpected) + } +} + +func TestStringTag(t *testing.T) { + var s StringTag + s.BoolStr = true + s.IntStr = 42 + s.StrStr = "xzbit" + got, err := json.MarshalIndent(&s, "", " ") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != stringTagExpected { + t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected) + } +} + +func TestUnsupportedValues(t *testing.T) { + for _, v := range unsupportedValues { + if _, err := json.Marshal(v); err != nil { + if _, ok := err.(*json.UnsupportedValueError); !ok { + t.Errorf("for %v, got %T want UnsupportedValueError", v, err) + } + } else { + t.Errorf("for %v, expected error", v) + } + } +} + +func TestRefValMarshal(t *testing.T) { + var s = struct { + R0 Ref + R1 *Ref + R2 RefText + R3 *RefText + V0 Val + V1 *Val + V2 ValText + V3 *ValText + }{ + R0: 12, + R1: new(Ref), + R2: 14, + R3: new(RefText), + V0: 13, + V1: new(Val), + V2: 15, + V3: new(ValText), + } + const want = `{"R0":"ref","R1":"ref","R2":"\"ref\"","R3":"\"ref\"","V0":"val","V1":"val","V2":"\"val\"","V3":"\"val\""}` + b, err := json.Marshal(&s) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if got := string(b); got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +func TestMarshalerEscaping(t *testing.T) { + var c C + want := `"\u003c\u0026\u003e"` + b, err := json.Marshal(c) + if err != nil { + t.Fatalf("Marshal(c): %v", err) + } + if got := string(b); got != want { + t.Errorf("Marshal(c) = %#q, want %#q", got, want) + } + + var ct CText + want = `"\"\u003c\u0026\u003e\""` + b, err = json.Marshal(ct) + if err != nil { + t.Fatalf("Marshal(ct): %v", err) + } + if got := string(b); got != want { + t.Errorf("Marshal(ct) = %#q, want %#q", got, want) + } +} + +func TestAnonymousNonstruct(t *testing.T) { + var i IntType = 11 + a := MyStruct{i} + const want = `{"IntType":11}` + + b, err := json.Marshal(a) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if got := string(b); got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// Issue 5245. +func TestEmbeddedBug(t *testing.T) { + v := BugB{ + BugA{"A"}, + "B", + } + b, err := json.Marshal(v) + if err != nil { + t.Fatal("Marshal:", err) + } + want := `{"S":"B"}` + got := string(b) + if got != want { + t.Fatalf("Marshal: got %s want %s", got, want) + } + // Now check that the duplicate field, S, does not appear. + x := BugX{ + A: 23, + } + b, err = json.Marshal(x) + if err != nil { + t.Fatal("Marshal:", err) + } + want = `{"A":23}` + got = string(b) + if got != want { + t.Fatalf("Marshal: got %s want %s", got, want) + } +} + +// Test that a field with a tag dominates untagged fields. +func TestTaggedFieldDominates(t *testing.T) { + v := BugY{ + BugA{"BugA"}, + BugD{"BugD"}, + } + b, err := json.Marshal(v) + if err != nil { + t.Fatal("Marshal:", err) + } + want := `{"S":"BugD"}` + got := string(b) + if got != want { + t.Fatalf("Marshal: got %s want %s", got, want) + } +} + +func TestDuplicatedFieldDisappears(t *testing.T) { + v := BugZ{ + BugA{"BugA"}, + BugC{"BugC"}, + BugY{ + BugA{"nested BugA"}, + BugD{"nested BugD"}, + }, + } + b, err := json.Marshal(v) + if err != nil { + t.Fatal("Marshal:", err) + } + want := `{}` + got := string(b) + if got != want { + t.Fatalf("Marshal: got %s want %s", got, want) + } +} + +func TestIssue6458(t *testing.T) { + type Foo struct { + M json.RawMessage + } + x := Foo{json.RawMessage(`"foo"`)} + + b, err := json.Marshal(&x) + if err != nil { + t.Fatal(err) + } + if want := `{"M":"foo"}`; string(b) != want { + t.Errorf("Marshal(&x) = %#q; want %#q", b, want) + } + + b, err = json.Marshal(x) + if err != nil { + t.Fatal(err) + } + + if want := `{"M":"foo"}`; string(b) != want { + t.Errorf("Marshal(x) = %#q; want %#q", b, want) + } +} + +func TestHTMLEscape(t *testing.T) { + var b, want bytes.Buffer + m := `{"M":"<html>foo &` + "\xe2\x80\xa8 \xe2\x80\xa9" + `</html>"}` + want.Write([]byte(`{"M":"\u003chtml\u003efoo \u0026\u2028 \u2029\u003c/html\u003e"}`)) + json.HTMLEscape(&b, []byte(m)) + if !bytes.Equal(b.Bytes(), want.Bytes()) { + t.Errorf("HTMLEscape(&b, []byte(m)) = %s; want %s", b.Bytes(), want.Bytes()) + } +} diff --git a/tests/ff.go b/tests/ff.go new file mode 100644 index 0000000..bf6423d --- /dev/null +++ b/tests/ff.go @@ -0,0 +1,2307 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tff + +import ( + "errors" + "math" + "time" +) + +// FFFoo struc... just blah +type FFFoo struct { + Blah int +} + +// FFRecord struct +type FFRecord struct { + Timestamp int64 `json:"id,omitempty"` + OriginID uint32 + Bar FFFoo + Method string `json:"meth"` + ReqID string + ServerIP string + RemoteIP string + BytesSent uint64 +} + +// TI18nName struct +// ffjson: skip +type TI18nName struct { + Ændret int64 + Aוההקלדה uint32 + Позната string +} + +// XI18nName struct +type XI18nName struct { + Ændret int64 + Aוההקלדה uint32 + Позната string +} + +type mystring string + +// TsortName struct +// ffjson: skip +type TsortName struct { + C string + B int `json:"A"` +} + +// XsortName struct +type XsortName struct { + C string + B int `json:"A"` +} + +// Tobj struct +// ffjson: skip +type Tobj struct { + X Tint +} + +// Xobj struct +type Xobj struct { + X Xint +} + +// Tduration struct +// ffjson: skip +type Tduration struct { + X time.Duration +} + +// Xduration struct +type Xduration struct { + X time.Duration +} + +// TtimePtr struct +// ffjson: skip +type TtimePtr struct { + X *time.Time +} + +// XtimePtr struct +type XtimePtr struct { + X *time.Time +} + +// Tarray struct +// ffjson: skip +type Tarray struct { + X [3]int +} + +// Xarray struct +type Xarray struct { + X [3]int +} + +// TarrayPtr struct +// ffjson: skip +type TarrayPtr struct { + X [3]*int +} + +// XarrayPtr struct +type XarrayPtr struct { + X [3]*int +} + +// Tslice struct +// ffjson: skip +type Tslice struct { + X []int +} + +//Xslice struct +type Xslice struct { + X []int +} + +// TslicePtr struct +// ffjson: skip +type TslicePtr struct { + X []*int +} + +// XslicePtr struct +type XslicePtr struct { + X []*int +} + +// TMapStringPtr struct +// ffjson: skip +type TMapStringPtr struct { + X map[string]*int +} + +// XMapStringPtr struct +type XMapStringPtr struct { + X map[string]*int +} + +// TslicePtrStruct struct +// ffjson: skip +type TslicePtrStruct struct { + X []*Xstring +} + +// XslicePtrStruct struct +type XslicePtrStruct struct { + X []*Xstring +} + +// TMapPtrStruct struct +// ffjson: skip +type TMapPtrStruct struct { + X map[string]*Xstring +} + +// XMapPtrStruct struct +type XMapPtrStruct struct { + X map[string]*Xstring +} + +// Tstring struct +// ffjson: skip +type Tstring struct { + X string +} + +// Xstring struct +type Xstring struct { + X string +} + +// Tmystring struct +// ffjson: skip +type Tmystring struct { + X mystring +} + +// Xmystring struct +type Xmystring struct { + X mystring +} + +// TmystringPtr struct +// ffjson: skip +type TmystringPtr struct { + X *mystring +} + +// XmystringPtr struct +type XmystringPtr struct { + X *mystring +} + +// TstringTagged struct +// ffjson: skip +type TstringTagged struct { + X string `json:",string"` +} + +// XstringTagged struct +type XstringTagged struct { + X string `json:",string"` +} + +// TstringTaggedPtr struct +// ffjson: skip +type TstringTaggedPtr struct { + X *string `json:",string"` +} + +// XstringTaggedPtr struct +type XstringTaggedPtr struct { + X *string `json:",string"` +} + +// TintTagged struct +// ffjson: skip +type TintTagged struct { + X int `json:",string"` +} + +//XintTagged struct +type XintTagged struct { + X int `json:",string"` +} + +// TboolTagged struct +// ffjson: skip +type TboolTagged struct { + X int `json:",string"` +} + +// XboolTagged struct +type XboolTagged struct { + X int `json:",string"` +} + +// TMapStringString struct +// ffjson: skip +type TMapStringString struct { + X map[string]string +} + +// XMapStringString struct +type XMapStringString struct { + X map[string]string +} + +// Tbool struct +// ffjson: skip +type Tbool struct { + X bool +} + +// Xbool struct +type Xbool struct { + X bool +} + +// Tint struct +// ffjson: skip +type Tint struct { + X int +} + +// Xint struct +type Xint struct { + X int +} + +// Tbyte struct +// ffjson: skip +type Tbyte struct { + X byte +} + +// Xbyte struct +type Xbyte struct { + X byte +} + +// Tint8 struct +// ffjson: skip +type Tint8 struct { + X int8 +} + +// Xint8 struct +type Xint8 struct { + X int8 +} + +// Tint16 struct +// ffjson: skip +type Tint16 struct { + X int16 +} + +// Xint16 stuct +type Xint16 struct { + X int16 +} + +// Tint32 struct +// ffjson: skip +type Tint32 struct { + X int32 +} + +// Xint32 struct +type Xint32 struct { + X int32 +} + +// Tint64 struct +// ffjson: skip +type Tint64 struct { + X int64 +} + +// Xint64 struct +type Xint64 struct { + X int64 +} + +// Tuint struct +// ffjson: skip +type Tuint struct { + X uint +} + +// Xuint struct +type Xuint struct { + X uint +} + +// Tuint8 struct +// ffjson: skip +type Tuint8 struct { + X uint8 +} + +// Xuint8 struct +type Xuint8 struct { + X uint8 +} + +// Tuint16 struct +// ffjson: skip +type Tuint16 struct { + X uint16 +} + +// Xuint16 struct +type Xuint16 struct { + X uint16 +} + +// Tuint32 struct +// ffjson: skip +type Tuint32 struct { + X uint32 +} + +// Xuint32 struct +type Xuint32 struct { + X uint32 +} + +// Tuint64 struct +// ffjson: skip +type Tuint64 struct { + X uint64 +} + +// Xuint64 struct +type Xuint64 struct { + X uint64 +} + +// Tuintptr struct +// ffjson: skip +type Tuintptr struct { + X uintptr +} + +// Xuintptr struct +type Xuintptr struct { + X uintptr +} + +// Tfloat32 struct +// ffjson: skip +type Tfloat32 struct { + X float32 +} + +// Xfloat32 struct +type Xfloat32 struct { + X float32 +} + +// Tfloat64 struct +// ffjson: skip +type Tfloat64 struct { + X float64 +} + +// Xfloat64 struct +type Xfloat64 struct { + X float64 +} + +// ATduration struct +// Arrays +// ffjson: skip +type ATduration struct { + X [3]time.Duration +} + +// AXduration struct +type AXduration struct { + X [3]time.Duration +} + +// ATbool struct +// ffjson: skip +type ATbool struct { + X [3]bool +} + +// AXbool struct +type AXbool struct { + X [3]bool +} + +// ATint struct +// ffjson: skip +type ATint struct { + X [3]int +} + +// AXint struct +type AXint struct { + X [3]int +} + +// ATbyte struct +// ffjson: skip +type ATbyte struct { + X [3]byte +} + +// AXbyte struct +type AXbyte struct { + X [3]byte +} + +// ATint8 struct +// ffjson: skip +type ATint8 struct { + X [3]int8 +} + +// AXint8 struct +type AXint8 struct { + X [3]int8 +} + +// ATint16 struct +// ffjson: skip +type ATint16 struct { + X [3]int16 +} + +// AXint16 struct +type AXint16 struct { + X [3]int16 +} + +// ATint32 struct +// ffjson: skip +type ATint32 struct { + X [3]int32 +} + +// AXint32 struct +type AXint32 struct { + X [3]int32 +} + +// ATint64 struct +// ffjson: skip +type ATint64 struct { + X [3]int64 +} + +// AXint64 struct +type AXint64 struct { + X [3]int64 +} + +// ATuint struct +// ffjson: skip +type ATuint struct { + X [3]uint +} + +// AXuint struct +type AXuint struct { + X [3]uint +} + +// ATuint8 struct +// ffjson: skip +type ATuint8 struct { + X [3]uint8 +} + +// AXuint8 struct +type AXuint8 struct { + X [3]uint8 +} + +// ATuint16 struct +// ffjson: skip +type ATuint16 struct { + X [3]uint16 +} + +// AXuint16 struct +type AXuint16 struct { + X [3]uint16 +} + +// ATuint32 struct +// ffjson: skip +type ATuint32 struct { + X [3]uint32 +} + +// AXuint32 struct +type AXuint32 struct { + X [3]uint32 +} + +// ATuint64 struct +// ffjson: skip +type ATuint64 struct { + X [3]uint64 +} + +// AXuint64 struct +type AXuint64 struct { + X [3]uint64 +} + +// ATuintptr struct +// ffjson: skip +type ATuintptr struct { + X [3]uintptr +} + +// AXuintptr struct +type AXuintptr struct { + X [3]uintptr +} + +// ATfloat32 struct +// ffjson: skip +type ATfloat32 struct { + X [3]float32 +} + +// AXfloat32 struct +type AXfloat32 struct { + X [3]float32 +} + +// ATfloat64 struct +// ffjson: skip +type ATfloat64 struct { + X [3]float64 +} + +// AXfloat64 struct +type AXfloat64 struct { + X [3]float64 +} + +// ATtime struct +// ffjson: skip +type ATtime struct { + X [3]time.Time +} + +// AXtime struct +type AXtime struct { + X [3]time.Time +} + +// STduration struct +// Slices +// ffjson: skip +type STduration struct { + X []time.Duration +} + +// SXduration struct +type SXduration struct { + X []time.Duration +} + +// STbool struct +// ffjson: skip +type STbool struct { + X []bool +} + +// SXbool struct +type SXbool struct { + X []bool +} + +// STint struct +// ffjson: skip +type STint struct { + X []int +} + +// SXint struct +type SXint struct { + X []int +} + +// STbyte struct +// ffjson: skip +type STbyte struct { + X []byte +} + +// SXbyte struct +type SXbyte struct { + X []byte +} + +// STint8 struct +// ffjson: skip +type STint8 struct { + X []int8 +} + +// SXint8 struct +type SXint8 struct { + X []int8 +} + +// STint16 struct +// ffjson: skip +type STint16 struct { + X []int16 +} + +// SXint16 struct +type SXint16 struct { + X []int16 +} + +// STint32 struct +// ffjson: skip +type STint32 struct { + X []int32 +} + +// SXint32 struct +type SXint32 struct { + X []int32 +} + +// STint64 struct +// ffjson: skip +type STint64 struct { + X []int64 +} + +// SXint64 struct +type SXint64 struct { + X []int64 +} + +// STuint struct +// ffjson: skip +type STuint struct { + X []uint +} + +// SXuint struct +type SXuint struct { + X []uint +} + +// STuint8 struct +// ffjson: skip +type STuint8 struct { + X []uint8 +} + +// SXuint8 struct +type SXuint8 struct { + X []uint8 +} + +// STuint16 struct +// ffjson: skip +type STuint16 struct { + X []uint16 +} + +// SXuint16 struct +type SXuint16 struct { + X []uint16 +} + +// STuint32 struct +// ffjson: skip +type STuint32 struct { + X []uint32 +} + +// SXuint32 struct +type SXuint32 struct { + X []uint32 +} + +// STuint64 struct +// ffjson: skip +type STuint64 struct { + X []uint64 +} + +// SXuint64 struct +type SXuint64 struct { + X []uint64 +} + +// STuintptr struct +// ffjson: skip +type STuintptr struct { + X []uintptr +} + +// SXuintptr struct +type SXuintptr struct { + X []uintptr +} + +// STfloat32 struct +// ffjson: skip +type STfloat32 struct { + X []float32 +} + +// SXfloat32 struct +type SXfloat32 struct { + X []float32 +} + +// STfloat64 struct +// ffjson: skip +type STfloat64 struct { + X []float64 +} + +// SXfloat64 struct +type SXfloat64 struct { + X []float64 +} + +// STtime struct +// ffjson: skip +type STtime struct { + X []time.Time +} + +// SXtime struct +type SXtime struct { + X []time.Time +} + +// TMapStringMapString struct +// Nested +// ffjson: skip +type TMapStringMapString struct { + X map[string]map[string]string +} + +// XMapStringMapString struct +type XMapStringMapString struct { + X map[string]map[string]string +} + +// TMapStringAString struct +// ffjson: skip +type TMapStringAString struct { + X map[string][3]string +} + +// XMapStringAString struct +type XMapStringAString struct { + X map[string][3]string +} + +// TSAAtring struct +// ffjson: skip +type TSAAtring struct { + X [2][3]string +} + +// XSAAtring struct +type XSAAtring struct { + X [2][3]string +} + +// TSAString struct +// ffjson: skip +type TSAString struct { + X [][3]string +} + +// XSAString struct +type XSAString struct { + X [][3]string +} + +// Optionals tests from golang test suite +type Optionals struct { + Sr string `json:"sr"` + So string `json:"so,omitempty"` + Sw string `json:"-"` + + Ir int `json:"omitempty"` // actually named omitempty, not an option + Io int `json:"io,omitempty"` + + Slr []string `json:"slr,random"` + Slo []string `json:"slo,omitempty"` + + Mr map[string]interface{} `json:"mr"` + Mo map[string]interface{} `json:",omitempty"` + + Fr float64 `json:"fr"` + Fo float64 `json:"fo,omitempty"` + + Br bool `json:"br"` + Bo bool `json:"bo,omitempty"` + + Ur uint `json:"ur"` + Uo uint `json:"uo,omitempty"` + + Str struct{} `json:"str"` + Sto struct{} `json:"sto,omitempty"` +} + +var unsupportedValues = []interface{}{ + math.NaN(), + math.Inf(-1), + math.Inf(1), +} + +var optionalsExpected = `{ + "sr": "", + "omitempty": 0, + "slr": null, + "mr": {}, + "fr": 0, + "br": false, + "ur": 0, + "str": {}, + "sto": {} +}` + +// StringTag struct +type StringTag struct { + BoolStr bool `json:",string"` + IntStr int64 `json:",string"` + FltStr float64 `json:",string"` + StrStr string `json:",string"` +} + +var stringTagExpected = `{ + "BoolStr": "true", + "IntStr": "42", + "FltStr": "0", + "StrStr": "\"xzbit\"" +}` + +// OmitAll struct +type OmitAll struct { + Ostr string `json:",omitempty"` + Oint int `json:",omitempty"` + Obool bool `json:",omitempty"` + Odouble float64 `json:",omitempty"` + Ointer interface{} `json:",omitempty"` + Omap map[string]interface{} `json:",omitempty"` + OstrP *string `json:",omitempty"` + OintP *int `json:",omitempty"` + // TODO: Re-enable when issue #55 is fixed. + OboolP *bool `json:",omitempty"` + OmapP *map[string]interface{} `json:",omitempty"` + Astr []string `json:",omitempty"` + Aint []int `json:",omitempty"` + Abool []bool `json:",omitempty"` + Adouble []float64 `json:",omitempty"` +} + +var omitAllExpected = `{}` + +// NoExported struct +type NoExported struct { + field1 string + field2 string + field3 string +} + +var noExportedExpected = `{}` + +// OmitFirst struct +type OmitFirst struct { + Ostr string `json:",omitempty"` + Str string +} + +var omitFirstExpected = `{ + "Str": "" +}` + +// OmitLast struct +type OmitLast struct { + Xstr string `json:",omitempty"` + Str string +} + +var omitLastExpected = `{ + "Str": "" +}` + +// byte slices are special even if they're renamed types. +type renamedByte byte +type renamedByteSlice []byte +type renamedRenamedByteSlice []renamedByte + +// ByteSliceNormal struct +type ByteSliceNormal struct { + X []byte +} + +// ByteSliceRenamed stuct +type ByteSliceRenamed struct { + X renamedByteSlice +} + +// ByteSliceDoubleRenamed struct +type ByteSliceDoubleRenamed struct { + X renamedRenamedByteSlice +} + +// Ref has Marshaler and Unmarshaler methods with pointer receiver. +type Ref int + +// MarshalJSON func +func (*Ref) MarshalJSON() ([]byte, error) { + return []byte(`"ref"`), nil +} + +// UnmarshalJSON func +func (r *Ref) UnmarshalJSON([]byte) error { + *r = 12 + return nil +} + +// Val has Marshaler methods with value receiver. +type Val int + +// MarshalJSON var +func (Val) MarshalJSON() ([]byte, error) { + return []byte(`"val"`), nil +} + +// RefText has Marshaler and Unmarshaler methods with pointer receiver. +type RefText int + +// MarshalText func +func (*RefText) MarshalText() ([]byte, error) { + return []byte(`"ref"`), nil +} + +// UnmarshalText func +func (r *RefText) UnmarshalText([]byte) error { + *r = 13 + return nil +} + +// ValText has Marshaler methods with value receiver. +type ValText int + +// MarshalText val +func (ValText) MarshalText() ([]byte, error) { + return []byte(`"val"`), nil +} + +// C implements Marshaler and returns unescaped JSON. +type C int + +// MarshalJSON func +func (C) MarshalJSON() ([]byte, error) { + return []byte(`"<&>"`), nil +} + +// CText implements Marshaler and returns unescaped text. +type CText int + +// MarshalText func +func (CText) MarshalText() ([]byte, error) { + return []byte(`"<&>"`), nil +} + +// ErrGiveError generates error +var ErrGiveError = errors.New("GiveError error") + +// GiveError always returns an ErrGiveError on Marshal/Unmarshal. +type GiveError struct{} + +// MarshalJSON func +func (r GiveError) MarshalJSON() ([]byte, error) { + return nil, ErrGiveError +} + +// UnmarshalJSON func +func (r *GiveError) UnmarshalJSON([]byte) error { + return ErrGiveError +} + +// IntType type +type IntType int + +// MyStruct struc +type MyStruct struct { + IntType +} + +// BugA struct +type BugA struct { + S string +} + +// BugB struct +type BugB struct { + BugA + S string +} + +// BugC struct +type BugC struct { + S string +} + +// BugX struct +// Legal Go: We never use the repeated embedded field (S). +type BugX struct { + A int + BugA + BugB +} + +// BugD struct +type BugD struct { // Same as BugA after tagging. + XXX string `json:"S"` +} + +// BugY struct +// BugD's tagged S field should dominate BugA's. +type BugY struct { + BugA + BugD +} + +// BugZ struct +// There are no tags here, so S should not appear. +type BugZ struct { + BugA + BugC + BugY // Contains a tagged S field through BugD; should not dominate. +} + +// FfFuzz struct +type FfFuzz struct { + A uint8 + B uint16 + C uint32 + D uint64 + + E int8 + F int16 + G int32 + H int64 + + I float32 + J float64 + + M byte + N rune + + O int + P uint + Q string + R bool + S time.Time + + Ap *uint8 + Bp *uint16 + Cp *uint32 + Dp *uint64 + + Ep *int8 + Fp *int16 + Gp *int32 + Hp *int64 + + IP *float32 + Jp *float64 + + Mp *byte + Np *rune + + Op *int + Pp *uint + Qp *string + Rp *bool + Sp *time.Time + + Aa []uint8 + Ba []uint16 + Ca []uint32 + Da []uint64 + + Ea []int8 + Fa []int16 + Ga []int32 + Ha []int64 + + Ia []float32 + Ja []float64 + + Ma []byte + Na []rune + + Oa []int + Pa []uint + Qa []string + Ra []bool + + Aap []*uint8 + Bap []*uint16 + Cap []*uint32 + Dap []*uint64 + + Eap []*int8 + Fap []*int16 + Gap []*int32 + Hap []*int64 + + Iap []*float32 + Jap []*float64 + + Map []*byte + Nap []*rune + + Oap []*int + Pap []*uint + Qap []*string + Rap []*bool +} + +// FuzzOmitEmpty struct +// ffjson: skip +type FuzzOmitEmpty struct { + A uint8 `json:",omitempty"` + B uint16 `json:",omitempty"` + C uint32 `json:",omitempty"` + D uint64 `json:",omitempty"` + + E int8 `json:",omitempty"` + F int16 `json:",omitempty"` + G int32 `json:",omitempty"` + H int64 `json:",omitempty"` + + I float32 `json:",omitempty"` + J float64 `json:",omitempty"` + + M byte `json:",omitempty"` + N rune `json:",omitempty"` + + O int `json:",omitempty"` + P uint `json:",omitempty"` + Q string `json:",omitempty"` + R bool `json:",omitempty"` + S time.Time `json:",omitempty"` + + Ap *uint8 `json:",omitempty"` + Bp *uint16 `json:",omitempty"` + Cp *uint32 `json:",omitempty"` + Dp *uint64 `json:",omitempty"` + + Ep *int8 `json:",omitempty"` + Fp *int16 `json:",omitempty"` + Gp *int32 `json:",omitempty"` + Hp *int64 `json:",omitempty"` + + IP *float32 `json:",omitempty"` + Jp *float64 `json:",omitempty"` + + Mp *byte `json:",omitempty"` + Np *rune `json:",omitempty"` + + Op *int `json:",omitempty"` + Pp *uint `json:",omitempty"` + Qp *string `json:",omitempty"` + Rp *bool `json:",omitempty"` + Sp *time.Time `json:",omitempty"` + + Aa []uint8 `json:",omitempty"` + Ba []uint16 `json:",omitempty"` + Ca []uint32 `json:",omitempty"` + Da []uint64 `json:",omitempty"` + + Ea []int8 `json:",omitempty"` + Fa []int16 `json:",omitempty"` + Ga []int32 `json:",omitempty"` + Ha []int64 `json:",omitempty"` + + Ia []float32 `json:",omitempty"` + Ja []float64 `json:",omitempty"` + + Ma []byte `json:",omitempty"` + Na []rune `json:",omitempty"` + + Oa []int `json:",omitempty"` + Pa []uint `json:",omitempty"` + Qa []string `json:",omitempty"` + Ra []bool `json:",omitempty"` + + Aap []*uint8 `json:",omitempty"` + Bap []*uint16 `json:",omitempty"` + Cap []*uint32 `json:",omitempty"` + Dap []*uint64 `json:",omitempty"` + + Eap []*int8 `json:",omitempty"` + Fap []*int16 `json:",omitempty"` + Gap []*int32 `json:",omitempty"` + Hap []*int64 `json:",omitempty"` + + Iap []*float32 `json:",omitempty"` + Jap []*float64 `json:",omitempty"` + + Map []*byte `json:",omitempty"` + Nap []*rune `json:",omitempty"` + + Oap []*int `json:",omitempty"` + Pap []*uint `json:",omitempty"` + Qap []*string `json:",omitempty"` + Rap []*bool `json:",omitempty"` +} + +// FfFuzzOmitEmpty struct +type FfFuzzOmitEmpty struct { + A uint8 `json:",omitempty"` + B uint16 `json:",omitempty"` + C uint32 `json:",omitempty"` + D uint64 `json:",omitempty"` + + E int8 `json:",omitempty"` + F int16 `json:",omitempty"` + G int32 `json:",omitempty"` + H int64 `json:",omitempty"` + + I float32 `json:",omitempty"` + J float64 `json:",omitempty"` + + M byte `json:",omitempty"` + N rune `json:",omitempty"` + + O int `json:",omitempty"` + P uint `json:",omitempty"` + Q string `json:",omitempty"` + R bool `json:",omitempty"` + S time.Time `json:",omitempty"` + + Ap *uint8 `json:",omitempty"` + Bp *uint16 `json:",omitempty"` + Cp *uint32 `json:",omitempty"` + Dp *uint64 `json:",omitempty"` + + Ep *int8 `json:",omitempty"` + Fp *int16 `json:",omitempty"` + Gp *int32 `json:",omitempty"` + Hp *int64 `json:",omitempty"` + + IP *float32 `json:",omitempty"` + Jp *float64 `json:",omitempty"` + + Mp *byte `json:",omitempty"` + Np *rune `json:",omitempty"` + + Op *int `json:",omitempty"` + Pp *uint `json:",omitempty"` + Qp *string `json:",omitempty"` + Rp *bool `json:",omitempty"` + Sp *time.Time `json:",omitempty"` + + Aa []uint8 `json:",omitempty"` + Ba []uint16 `json:",omitempty"` + Ca []uint32 `json:",omitempty"` + Da []uint64 `json:",omitempty"` + + Ea []int8 `json:",omitempty"` + Fa []int16 `json:",omitempty"` + Ga []int32 `json:",omitempty"` + Ha []int64 `json:",omitempty"` + + Ia []float32 `json:",omitempty"` + Ja []float64 `json:",omitempty"` + + Ma []byte `json:",omitempty"` + Na []rune `json:",omitempty"` + + Oa []int `json:",omitempty"` + Pa []uint `json:",omitempty"` + Qa []string `json:",omitempty"` + Ra []bool `json:",omitempty"` + + Aap []*uint8 `json:",omitempty"` + Bap []*uint16 `json:",omitempty"` + Cap []*uint32 `json:",omitempty"` + Dap []*uint64 `json:",omitempty"` + + Eap []*int8 `json:",omitempty"` + Fap []*int16 `json:",omitempty"` + Gap []*int32 `json:",omitempty"` + Hap []*int64 `json:",omitempty"` + + Iap []*float32 `json:",omitempty"` + Jap []*float64 `json:",omitempty"` + + Map []*byte `json:",omitempty"` + Nap []*rune `json:",omitempty"` + + Oap []*int `json:",omitempty"` + Pap []*uint `json:",omitempty"` + Qap []*string `json:",omitempty"` + Rap []*bool `json:",omitempty"` +} + +// FuzzString struct +// ffjson: skip +type FuzzString struct { + A uint8 `json:",string"` + B uint16 `json:",string"` + C uint32 `json:",string"` + D uint64 `json:",string"` + + E int8 `json:",string"` + F int16 `json:",string"` + G int32 `json:",string"` + H int64 `json:",string"` + + I float32 `json:",string"` + J float64 `json:",string"` + + M byte `json:",string"` + N rune `json:",string"` + + O int `json:",string"` + P uint `json:",string"` + + Q string `json:",string"` + + R bool `json:",string"` + // https://github.com/golang/go/issues/9812 + // S time.Time `json:",string"` + + Ap *uint8 `json:",string"` + Bp *uint16 `json:",string"` + Cp *uint32 `json:",string"` + Dp *uint64 `json:",string"` + + Ep *int8 `json:",string"` + Fp *int16 `json:",string"` + Gp *int32 `json:",string"` + Hp *int64 `json:",string"` + + IP *float32 `json:",string"` + Jp *float64 `json:",string"` + + Mp *byte `json:",string"` + Np *rune `json:",string"` + + Op *int `json:",string"` + Pp *uint `json:",string"` + Qp *string `json:",string"` + Rp *bool `json:",string"` + // https://github.com/golang/go/issues/9812 + // Sp *time.Time `json:",string"` +} + +// FfFuzzString struct +type FfFuzzString struct { + A uint8 `json:",string"` + B uint16 `json:",string"` + C uint32 `json:",string"` + D uint64 `json:",string"` + + E int8 `json:",string"` + F int16 `json:",string"` + G int32 `json:",string"` + H int64 `json:",string"` + + I float32 `json:",string"` + J float64 `json:",string"` + + M byte `json:",string"` + N rune `json:",string"` + + O int `json:",string"` + P uint `json:",string"` + + Q string `json:",string"` + + R bool `json:",string"` + // https://github.com/golang/go/issues/9812 + // S time.Time `json:",string"` + + Ap *uint8 `json:",string"` + Bp *uint16 `json:",string"` + Cp *uint32 `json:",string"` + Dp *uint64 `json:",string"` + + Ep *int8 `json:",string"` + Fp *int16 `json:",string"` + Gp *int32 `json:",string"` + Hp *int64 `json:",string"` + + IP *float32 `json:",string"` + Jp *float64 `json:",string"` + + Mp *byte `json:",string"` + Np *rune `json:",string"` + + Op *int `json:",string"` + Pp *uint `json:",string"` + Qp *string `json:",string"` + Rp *bool `json:",string"` + // https://github.com/golang/go/issues/9812 + // Sp *time.Time `json:",string"` +} + +// TTestMaps struct +// ffjson: skip +type TTestMaps struct { + Aa map[string]uint8 + Ba map[string]uint16 + Ca map[string]uint32 + Da map[string]uint64 + + Ea map[string]int8 + Fa map[string]int16 + Ga map[string]int32 + Ha map[string]int64 + + Ia map[string]float32 + Ja map[string]float64 + + Ma map[string]byte + Na map[string]rune + + Oa map[string]int + Pa map[string]uint + Qa map[string]string + Ra map[string]bool + + AaP map[string]*uint8 + BaP map[string]*uint16 + CaP map[string]*uint32 + DaP map[string]*uint64 + + EaP map[string]*int8 + FaP map[string]*int16 + GaP map[string]*int32 + HaP map[string]*int64 + + IaP map[string]*float32 + JaP map[string]*float64 + + MaP map[string]*byte + NaP map[string]*rune + + OaP map[string]*int + PaP map[string]*uint + QaP map[string]*string + RaP map[string]*bool +} + +// XTestMaps struct +type XTestMaps struct { + TTestMaps +} + +// NoEncoder struct +// ffjson: noencoder +type NoEncoder struct { + C string + B int `json:"A"` +} + +// NoDecoder struct +// ffjson: nodecoder +type NoDecoder struct { + C string + B int `json:"A"` +} + +// TEmbeddedStructures struct +// ffjson: skip +type TEmbeddedStructures struct { + X []interface{} + Y struct { + X int + } + Z []struct { + X int + } + U map[string]struct { + X int + } + V []map[string]struct { + X int + } + W [5]map[string]struct { + X int + } + Q [][]string +} + +// XEmbeddedStructures struct +type XEmbeddedStructures struct { + X []interface{} + Y struct { + X int + } + Z []struct { + X int + } + U map[string]struct { + X int + } + V []map[string]struct { + X int + } + W [5]map[string]struct { + X int + } + Q [][]string +} + +// TRenameTypes struct +// ffjson: skip +// Side-effect of this test is also to verify that Encoder/Decoder skipping works. +type TRenameTypes struct { + X struct { + X int + } `json:"X-renamed"` + Y NoEncoder `json:"Y-renamed"` + Z string `json:"Z-renamed"` + U *NoDecoder `json:"U-renamed"` +} + +// XRenameTypes struct +type XRenameTypes struct { + X struct { + X int + } `json:"X-renamed"` + Y NoEncoder `json:"Y-renamed"` + Z string `json:"Z-renamed"` + U *NoDecoder `json:"U-renamed"` +} + +// ReTypedA type +type ReTypedA uint8 + +// ReTypedB type +type ReTypedB uint16 + +// ReTypedC type +type ReTypedC uint32 + +// ReTypedD type +type ReTypedD uint64 + +// ReTypedE type +type ReTypedE int8 + +// ReTypedF type +type ReTypedF int16 + +// ReTypedG type +type ReTypedG int32 + +// ReTypedH type +type ReTypedH int64 + +// ReTypedI type +type ReTypedI float32 + +// ReTypedJ type +type ReTypedJ float64 + +// ReTypedM type +type ReTypedM byte + +// ReTypedN type +type ReTypedN rune + +// ReTypedO type +type ReTypedO int + +// ReTypedP type +type ReTypedP uint + +// ReTypedQ type +type ReTypedQ string + +// ReTypedR type +type ReTypedR bool + +// ReTypedS type +type ReTypedS time.Time + +// ReTypedAp type +type ReTypedAp *uint8 + +// ReTypedBp type +type ReTypedBp *uint16 + +// ReTypedCp type +type ReTypedCp *uint32 + +// ReTypedDp type +type ReTypedDp *uint64 + +// ReTypedEp type +type ReTypedEp *int8 + +// ReTypedFp type +type ReTypedFp *int16 + +// ReTypedGp type +type ReTypedGp *int32 + +// ReTypedHp type +type ReTypedHp *int64 + +// ReTypedIP type +type ReTypedIP *float32 + +// ReTypedJp type +type ReTypedJp *float64 + +// ReTypedMp type +type ReTypedMp *byte + +// ReTypedNp type +type ReTypedNp *rune + +// ReTypedOp type +type ReTypedOp *int + +// ReTypedPp type +type ReTypedPp *uint + +// ReTypedQp type +type ReTypedQp *string + +// ReTypedRp type +type ReTypedRp *bool + +// ReTypedSp type +type ReTypedSp *time.Time + +// ReTypedAa type +type ReTypedAa []uint8 + +// ReTypedBa type +type ReTypedBa []uint16 + +// ReTypedCa type +type ReTypedCa []uint32 + +// ReTypedDa type +type ReTypedDa []uint64 + +// ReTypedEa type +type ReTypedEa []int8 + +// ReTypedFa type +type ReTypedFa []int16 + +// ReTypedGa type +type ReTypedGa []int32 + +// ReTypedHa type +type ReTypedHa []int64 + +// ReTypedIa type +type ReTypedIa []float32 + +// ReTypedJa type +type ReTypedJa []float64 + +// ReTypedMa type +type ReTypedMa []byte + +// ReTypedNa type +type ReTypedNa []rune + +// ReTypedOa type +type ReTypedOa []int + +// ReTypedPa type +type ReTypedPa []uint + +// ReTypedQa type +type ReTypedQa []string + +// ReTypedRa type +type ReTypedRa []bool + +// ReTypedAap type +type ReTypedAap []*uint8 + +// ReTypedBap type +type ReTypedBap []*uint16 + +// ReTypedCap type +type ReTypedCap []*uint32 + +// ReTypedDap type +type ReTypedDap []*uint64 + +// ReTypedEap type +type ReTypedEap []*int8 + +// ReTypedFap type +type ReTypedFap []*int16 + +// ReTypedGap type +type ReTypedGap []*int32 + +// ReTypedHap type +type ReTypedHap []*int64 + +// ReTypedIap type +type ReTypedIap []*float32 + +// ReTypedJap type +type ReTypedJap []*float64 + +// ReTypedMap type +type ReTypedMap []*byte + +// ReTypedNap type +type ReTypedNap []*rune + +// ReTypedOap type +type ReTypedOap []*int + +// ReTypedPap type +type ReTypedPap []*uint + +// ReTypedQap type +type ReTypedQap []*string + +// ReTypedRap type +type ReTypedRap []*bool + +// ReTypedXa type +type ReTypedXa NoDecoder + +// ReTypedXb type +type ReTypedXb NoEncoder + +// ReTypedXc type +type ReTypedXc *NoDecoder + +// ReTypedXd type +type ReTypedXd *NoEncoder + +// ReReTypedA type +type ReReTypedA ReTypedA + +// ReReTypedS type +type ReReTypedS ReTypedS + +// ReReTypedAp type +type ReReTypedAp ReTypedAp + +// ReReTypedSp type +type ReReTypedSp ReTypedSp + +// ReReTypedAa type +type ReReTypedAa ReTypedAa + +// ReReTypedAap type +type ReReTypedAap ReTypedAap + +// ReReTypedXa type +type ReReTypedXa ReTypedXa + +// ReReTypedXb type +type ReReTypedXb ReTypedXb + +// ReReTypedXc type +type ReReTypedXc ReTypedXc + +// ReReTypedXd type +type ReReTypedXd ReTypedXd + +// RePReTypedA type +type RePReTypedA *ReTypedA + +// ReSReTypedS type +type ReSReTypedS []ReTypedS + +// ReAReTypedAp type +type ReAReTypedAp [4]ReTypedAp + +// TReTyped struct +// ffjson: ignore +type TReTyped struct { + A ReTypedA + B ReTypedB + C ReTypedC + D ReTypedD + + E ReTypedE + F ReTypedF + G ReTypedG + H ReTypedH + + I ReTypedI + J ReTypedJ + + M ReTypedM + N ReTypedN + + O ReTypedO + P ReTypedP + Q ReTypedQ + R ReTypedR + S ReTypedS + + Ap ReTypedAp + Bp ReTypedBp + Cp ReTypedCp + Dp ReTypedDp + + Ep ReTypedEp + Fp ReTypedFp + Gp ReTypedGp + Hp ReTypedHp + + IP ReTypedIP + Jp ReTypedJp + + Mp ReTypedMp + Np ReTypedNp + + Op ReTypedOp + Pp ReTypedPp + Qp ReTypedQp + Rp ReTypedRp + // FIXME: https://github.com/pquerna/ffjson/issues/108 + //Sp ReTypedSp + + // Bug in encoding/json: Bug in encoding/json: json: cannot unmarshal string into Go value of type tff.ReTypedAa + //Aa ReTypedAa + Ba ReTypedBa + Ca ReTypedCa + Da ReTypedDa + + Ea ReTypedEa + Fa ReTypedFa + Ga ReTypedGa + Ha ReTypedHa + + Ia ReTypedIa + Ja ReTypedJa + + // Bug in encoding/json: json: cannot unmarshal string into Go value of type tff.ReTypedMa + // Ma ReTypedMa + Na ReTypedNa + + Oa ReTypedOa + Pa ReTypedPa + Qa ReTypedQa + Ra ReTypedRa + + Aap ReTypedAap + Bap ReTypedBap + Cap ReTypedCap + Dap ReTypedDap + + Eap ReTypedEap + Fap ReTypedFap + Gap ReTypedGap + Hap ReTypedHap + + Iap ReTypedIap + Jap ReTypedJap + + Map ReTypedMap + Nap ReTypedNap + + Oap ReTypedOap + Pap ReTypedPap + Qap ReTypedQap + Rap ReTypedRap + + Xa ReTypedXa + Xb ReTypedXb + + Rra ReReTypedA + Rrs ReReTypedS + Rrap ReReTypedAp + // FIXME: https://github.com/pquerna/ffjson/issues/108 + // Rrsp ReReTypedSp + // Rrxc ReReTypedXc + // Rrxd ReReTypedXd + + // Bug in encoding/json: json: json: cannot unmarshal string into Go value of type tff.ReReTypedAa + // Rraa ReReTypedAa + Rraap ReReTypedAap + Rrxa ReReTypedXa + Rrxb ReReTypedXb + + Rpra RePReTypedA + Rsrs ReSReTypedS +} + +// XReTyped struct +type XReTyped struct { + A ReTypedA + B ReTypedB + C ReTypedC + D ReTypedD + + E ReTypedE + F ReTypedF + G ReTypedG + H ReTypedH + + I ReTypedI + J ReTypedJ + + M ReTypedM + N ReTypedN + + O ReTypedO + P ReTypedP + Q ReTypedQ + R ReTypedR + S ReTypedS + + Ap ReTypedAp + Bp ReTypedBp + Cp ReTypedCp + Dp ReTypedDp + + Ep ReTypedEp + Fp ReTypedFp + Gp ReTypedGp + Hp ReTypedHp + + IP ReTypedIP + Jp ReTypedJp + + Mp ReTypedMp + Np ReTypedNp + + Op ReTypedOp + Pp ReTypedPp + Qp ReTypedQp + Rp ReTypedRp + // FIXME: https://github.com/pquerna/ffjson/issues/108 + //Sp ReTypedSp + + // Bug in encoding/json: Bug in encoding/json: json: cannot unmarshal string into Go value of type tff.ReTypedAa + // Aa ReTypedAa + Ba ReTypedBa + Ca ReTypedCa + Da ReTypedDa + + Ea ReTypedEa + Fa ReTypedFa + Ga ReTypedGa + Ha ReTypedHa + + Ia ReTypedIa + Ja ReTypedJa + + // Bug in encoding/json: Bug in encoding/json: json: cannot unmarshal string into Go value of type tff.ReTypedMa + //Ma ReTypedMa + Na ReTypedNa + + Oa ReTypedOa + Pa ReTypedPa + Qa ReTypedQa + Ra ReTypedRa + + Aap ReTypedAap + Bap ReTypedBap + Cap ReTypedCap + Dap ReTypedDap + + Eap ReTypedEap + Fap ReTypedFap + Gap ReTypedGap + Hap ReTypedHap + + Iap ReTypedIap + Jap ReTypedJap + + Map ReTypedMap + Nap ReTypedNap + + Oap ReTypedOap + Pap ReTypedPap + Qap ReTypedQap + Rap ReTypedRap + + Xa ReTypedXa + Xb ReTypedXb + + Rra ReReTypedA + Rrs ReReTypedS + Rrap ReReTypedAp + // FIXME: https://github.com/pquerna/ffjson/issues/108 + // Rrsp ReReTypedSp + // Rrxc ReReTypedXc + // Rrxd ReReTypedXd + + // Bug in encoding/json: json: json: cannot unmarshal string into Go value of type tff.ReReTypedAa + // Rraa ReReTypedAa + Rraap ReReTypedAap + Rrxa ReReTypedXa + Rrxb ReReTypedXb + + Rpra RePReTypedA + Rsrs ReSReTypedS +} + +// TInlineStructs struct +// ffjson: skip +type TInlineStructs struct { + B struct { + A uint8 + B uint16 + C uint32 + D uint64 + + E int8 + F int16 + G int32 + H int64 + + I float32 + J float64 + + M byte + N rune + + O int + P uint + Q string + R bool + S time.Time + + Ap *uint8 + Bp *uint16 + Cp *uint32 + Dp *uint64 + + Ep *int8 + Fp *int16 + Gp *int32 + Hp *int64 + + IP *float32 + Jp *float64 + + Mp *byte + Np *rune + + Op *int + Pp *uint + Qp *string + Rp *bool + Sp *time.Time + + Aa []uint8 + Ba []uint16 + Ca []uint32 + Da []uint64 + + Ea []int8 + Fa []int16 + Ga []int32 + Ha []int64 + + Ia []float32 + Ja []float64 + + Ma []byte + Na []rune + + Oa []int + Pa []uint + Qa []string + Ra []bool + + Aap []*uint8 + Bap []*uint16 + Cap []*uint32 + Dap []*uint64 + + Eap []*int8 + Fap []*int16 + Gap []*int32 + Hap []*int64 + + Iap []*float32 + Jap []*float64 + + Map []*byte + Nap []*rune + + Oap []*int + Pap []*uint + Qap []*string + Rap []*bool + } + PtStr *struct { + X int + } + InceptionStr struct { + Y []struct { + X *int + } + } +} + +// XInlineStructs struct +type XInlineStructs struct { + B struct { + A uint8 + B uint16 + C uint32 + D uint64 + + E int8 + F int16 + G int32 + H int64 + + I float32 + J float64 + + M byte + N rune + + O int + P uint + Q string + R bool + S time.Time + + Ap *uint8 + Bp *uint16 + Cp *uint32 + Dp *uint64 + + Ep *int8 + Fp *int16 + Gp *int32 + Hp *int64 + + IP *float32 + Jp *float64 + + Mp *byte + Np *rune + + Op *int + Pp *uint + Qp *string + Rp *bool + Sp *time.Time + + Aa []uint8 + Ba []uint16 + Ca []uint32 + Da []uint64 + + Ea []int8 + Fa []int16 + Ga []int32 + Ha []int64 + + Ia []float32 + Ja []float64 + + Ma []byte + Na []rune + + Oa []int + Pa []uint + Qa []string + Ra []bool + + Aap []*uint8 + Bap []*uint16 + Cap []*uint32 + Dap []*uint64 + + Eap []*int8 + Fap []*int16 + Gap []*int32 + Hap []*int64 + + Iap []*float32 + Jap []*float64 + + Map []*byte + Nap []*rune + + Oap []*int + Pap []*uint + Qap []*string + Rap []*bool + } + PtStr *struct { + X int + } + InceptionStr struct { + Y []struct { + X *int + } + } +} + +// TDominantField struct +// ffjson: skip +type TDominantField struct { + X *int `json:"Name,omitempty"` + Y *int `json:"Name,omitempty"` + Other string + Name *int `json",omitempty"` + A *struct{ X int } `json:"Name,omitempty"` +} + +// XDominantField struct +type XDominantField struct { + X *int `json:"Name,omitempty"` + Y *int `json:"Name,omitempty"` + Other string + Name *int `json",omitempty"` + A *struct{ X int } `json:"Name,omitempty"` +} diff --git a/tests/ff_float_test.go b/tests/ff_float_test.go new file mode 100644 index 0000000..8f740a1 --- /dev/null +++ b/tests/ff_float_test.go @@ -0,0 +1,103 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tff + +import ( + "testing" +) + +// Test data from https://github.com/akheron/jansson/tree/master/test/suites/valid +// jansson, Copyright (c) 2009-2014 Petri Lehtinen <petri@digip.org> +// (MIT Licensed) + +func TestFloatRealCapitalENegativeExponent(t *testing.T) { + testExpectedXValBare(t, + 0.01, + `1E-2`, + &Xfloat64{}) +} + +func TestFloatRealCapitalEPositiveExponent(t *testing.T) { + testExpectedXValBare(t, + 100.0, + `1E+2`, + &Xfloat64{}) +} + +func TestFloatRealCapital(t *testing.T) { + testExpectedXValBare(t, + 1e22, + `1E22`, + &Xfloat64{}) +} + +func TestFloatRealExponent(t *testing.T) { + testExpectedXValBare(t, + 1.2299999999999999e47, + `123e45`, + &Xfloat64{}) +} + +func TestFloatRealFractionExponent(t *testing.T) { + testExpectedXValBare(t, + 1.23456e80, + `123.456e78`, + &Xfloat64{}) +} + +func TestFloatRealNegativeExponent(t *testing.T) { + testExpectedXValBare(t, + 0.01, + `1e-2`, + &Xfloat64{}) +} + +func TestFloatRealPositiveExponent(t *testing.T) { + testExpectedXValBare(t, + 100.0, + `1e2`, + &Xfloat64{}) +} + +func TestFloatRealSubnormalNumber(t *testing.T) { + testExpectedXValBare(t, + 1.8011670033376514e-308, + `1.8011670033376514e-308`, + &Xfloat64{}) +} + +func TestFloatRealUnderflow(t *testing.T) { + testExpectedXValBare(t, + 0.0, + `123e-10000000`, + &Xfloat64{}) +} + +func TestFloatNull(t *testing.T) { + testExpectedXValBare(t, + 0.0, + `null`, + &Xfloat64{}) +} + +func TestFloatInt(t *testing.T) { + testExpectedXValBare(t, + 1.0, + `1`, + &Xfloat64{}) +} diff --git a/tests/ff_invalid_test.go b/tests/ff_invalid_test.go new file mode 100644 index 0000000..94b2397 --- /dev/null +++ b/tests/ff_invalid_test.go @@ -0,0 +1,190 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tff + +import ( + fflib "github.com/pquerna/ffjson/fflib/v1" + + _ "encoding/json" + "testing" +) + +// Test data from https://github.com/akheron/jansson/tree/master/test/suites/invalid +// jansson, Copyright (c) 2009-2014 Petri Lehtinen <petri@digip.org> +// (MIT Licensed) + +func TestInvalidApostrophe(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `'`, + &Xstring{}) +} + +func TestInvalidASCIIUnicodeIdentifier(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `aå`, + &Xstring{}) +} + +func TestInvalidBraceComma(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{,}`, + &Xstring{}) +} + +func TestInvalidBracketComma(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `[,]`, + &Xarray{}) +} + +func TestInvalidBracketValueComma(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `[1,`, + &Xarray{}) +} + +func TestInvalidEmptyValue(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + ``, + &Xarray{}) +} + +func TestInvalidGarbageAfterNewline(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + "[1,2,3]\nfoo", + &Xarray{}) +} + +func TestInvalidGarbageAtEnd(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + "[1,2,3]foo", + &Xarray{}) +} + +func TestInvalidIntStartingWithZero(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + "012", + &Xint64{}) +} + +func TestInvalidEscape(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `"\a <-- invalid escape"`, + &Xstring{}) +} + +func TestInvalidIdentifier(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `troo`, + &Xbool{}) +} + +func TestInvalidNegativeInt(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `-123foo`, + &Xint{}) +} + +func TestInvalidNegativeFloat(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `-124.123foo`, + &Xfloat64{}) +} + +func TestInvalidSecondSurrogate(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `"\uD888\u3210 (first surrogate and invalid second surrogate)"`, + &Xstring{}) +} + +func TestInvalidLoneOpenBrace(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{`, + &Xstring{}) +} + +func TestInvalidLoneOpenBracket(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `[`, + &Xarray{}) +} + +func TestInvalidLoneCloseBrace(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `}`, + &Xstring{}) +} + +func TestInvalidHighBytes(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + string('\xFF'), + &Xstring{}) +} + +func TestInvalidLoneCloseBracket(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `]`, + &Xarray{}) +} + +func TestInvalidMinusSignWithoutNumber(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `-`, + &Xint{}) +} + +func TestInvalidNullByte(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + "\u0000", + &Xstring{}) +} + +func TestInvalidNullByteInString(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + "\"\u0000 <- null byte\"", + &Xstring{}) +} + +func TestInvalidFloatGarbageAfterE(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `1ea`, + &Xfloat64{}) +} diff --git a/tests/ff_obj_test.go b/tests/ff_obj_test.go new file mode 100644 index 0000000..7ba81f7 --- /dev/null +++ b/tests/ff_obj_test.go @@ -0,0 +1,106 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tff + +import ( + fflib "github.com/pquerna/ffjson/fflib/v1" + + "testing" +) + +// Test data from https://github.com/akheron/jansson/tree/master/test/suites +// jansson, Copyright (c) 2009-2014 Petri Lehtinen <petri@digip.org> +// (MIT Licensed) + +func TestInvalidBareKey(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{X:"foo"}`, + &Xobj{}) +} + +func TestInvalidNoValue(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{"X":}`, + &Xobj{}) +} + +func TestInvalidTrailingComma(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{"X":"foo",}`, + &Xobj{}) +} + +func TestInvalidRougeComma(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{,}`, + &Xobj{}) +} + +func TestInvalidRougeColon(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{:}`, + &Xobj{}) +} + +func TestInvalidMissingColon(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{"X""foo"}`, + &Xobj{}) + testExpectedError(t, + &fflib.LexerError{}, + `{"X" "foo"}`, + &Xobj{}) + testExpectedError(t, + &fflib.LexerError{}, + `{"X","foo"}`, + &Xobj{}) +} + +func TestInvalidUnmatchedBrace(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `[`, + &Xobj{}) +} + +func TestInvalidUnmatchedBracket(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{`, + &Xobj{}) +} + +func TestInvalidExpectedObjGotArray(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `[]`, + &Xobj{}) +} + +func TestInvalidUnterminatedValue(t *testing.T) { + testExpectedError(t, + &fflib.LexerError{}, + `{"X": "foo`, + &Xobj{}) +} diff --git a/tests/ff_string_test.go b/tests/ff_string_test.go new file mode 100644 index 0000000..75bb5cd --- /dev/null +++ b/tests/ff_string_test.go @@ -0,0 +1,151 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tff + +import ( + "testing" + "strings" + "runtime" +) + +// Test data from https://github.com/akheron/jansson/tree/master/test/suites/valid +// jansson, Copyright (c) 2009-2014 Petri Lehtinen <petri@digip.org> +// (MIT Licensed) + +func TestString(t *testing.T) { + testType(t, &Tstring{}, &Xstring{}) + testType(t, &Tmystring{}, &Xmystring{}) + testType(t, &TmystringPtr{}, &XmystringPtr{}) +} + +func TestMapStringString(t *testing.T) { + m := map[string]string{"陫ʋsş\")珷<ºɖgȏ哙ȍ": "2ħ籦ö嗏ʑ>季"} + testCycle(t, &TMapStringString{X: m}, &XMapStringString{X: m}) +} + +func TestMapStringStringLong(t *testing.T) { + m := map[string]string{"ɥ³ƞsɁ8^ʥǔTĪȸŹă": "ɩÅ議Ǹ轺@)蓳嗘TʡȂ", "丯Ƙ枛牐ɺ皚|": "\\p[", "ȉ": "ģ毋Ó6dz娝嘚", "ʒUɦOŖ": "斎AO6ĴC浔Ű壝ž", "/C龷ȪÆl殛瓷雼浢Ü礽绅": "D¡", "Lɋ聻鎥ʟ<$洅ɹ7\\弌Þ帺萸Do©": "A", "yǠ/淹\\韲翁&ʢsɜ": "`诫z徃鷢6ȥ啕禗Ǐ2啗塧ȱ蓿彭聡A", "瓧嫭塓烀罁胾^拜": "ǒɿʒ刽ʼn掏1ſ盷褎weLJ", "姥呄鐊唊飙Ş-U圴÷a/ɔ}摁(": "瓘ǓvjĜ蛶78Ȋ²@H", "IJ斬³;": "鯿r", "勽Ƙq/Ź u衲": "ŭDz鯰硰{舁", "枊a8衍`Ĩɘ.蘯6ċV夸eɑeʤ脽ě": "6/ʕVŚ(ĿȊ甞谐颋DžSǡƏS$+", "1ØœȠƬQg鄠": "军g>郵[+扴ȨŮ+朷Ǝ膯lj", "礶惇¸t颟.鵫ǚ灄鸫rʤî萨z": "", "ȶ网棊ʢ=wǕɳɷ9Ì": "'WKw(ğ儴Ůĺ}潷ʒ胵輓Ɔ", "}ȧ外ĺ稥氹Ç|¶鎚¡ Ɠ(嘒ėf倐": "窮秳ķ蟒苾h^", "?瞲Ť倱<įXŋ朘瑥A徙": "nh0åȂ町恰nj揠8lj黳鈫ʕ禒", "丩ŽoǠŻʘY賃ɪ鐊": "ľǎɳ,ǿ飏騀呣ǎ", "ȇe媹Hǝ呮}臷Ľð»ųKĵ": "踪鄌eÞȦY籎顒ǥŴ唼Ģ猇õǶț", "偐ę腬瓷碑=ɉ鎷卩蝾H韹寬娬ï瓼猀2": "ǰ溟ɴ扵閝ȝ鐵儣廡ɑ龫`劳", "ʮ馜ü": "", "șƶ4ĩĉş蝿ɖȃ賲鐅臬dH巧": "_瀹鞎sn芞QÄȻȊ+?", "E@Ȗs«ö": "蚛隖<ǶĬ4y£軶ǃ*ʙ嫙&蒒5靇C'", "忄*齧獚敆Ȏ": "螩B", "圠=l畣潁谯耨V6&]鴍Ɋ恧ȭ%ƎÜ": "涽託仭w-檮", "ʌ鴜": "琔n宂¬轚9Ȏ瀮昃2Ō¾\\", "ƅTG": "ǺƶȤ^}穠C]躢|)黰eȪ嵛4$%Q", "ǹ_Áȉ彂Ŵ廷s": "", "t莭琽§ć\\ ïì": "", "擓ƖHVe熼'FD剂讼ɓȌʟni酛": "/ɸɎ R§耶FfBls3!", "狞夌碕ʂɭ": "Ƽ@hDrȮO励鹗塢", "ʁgɸ=ǤÆ": "?讦ĭÐ", "陫ʋsş\")珷<ºɖgȏ哙ȍ": "2ħ籦ö嗏ʑ>季", "": "昕Ĭ", "Ⱦdz@ùƸʋŀ": "ǐƲE'iþŹʣy豎@ɀ羭,铻OŤǢʭ", ">犵殇ŕ-Ɂ圯W:ĸ輦唊#v铿ʩȂ4": "屡ʁ", "1Rƥ贫d飼$俊跾|@?鷅bȻN": "H炮掊°nʮ閼咎櫸eʔŊƞ究:ho", "ƻ悖ȩ0Ƹ[": "Ndǂ>5姣>懔%熷谟þ蛯ɰ", "ŵw^Ü郀叚Fi皬择": ":5塋訩塶\"=y钡n)İ笓", "'容": "誒j剐", "猤痈C*ĕ": "鴈o_鹈ɹ坼É/pȿŘ阌"} + testCycle(t, &TMapStringString{X: m}, &XMapStringString{X: m}) +} + +func TestStringEscapedControlCharacter(t *testing.T) { + testExpectedXVal(t, + "\x12 escaped control character", + `\u0012 escaped control character`, + &Xstring{}) +} + +func TestStringOneByteUTF8(t *testing.T) { + testExpectedXVal(t, + ", one-byte UTF-8", + `\u002c one-byte UTF-8`, + &Xstring{}) +} + +func TestStringUtf8Escape(t *testing.T) { + testExpectedXVal(t, + "2ħ籦ö嗏ʑ>嫀", + `2ħ籦ö嗏ʑ\u003e嫀`, + &Xstring{}) +} + +func TestStringTwoByteUTF8(t *testing.T) { + testExpectedXVal(t, + "ģ two-byte UTF-8", + `\u0123 two-byte UTF-8`, + &Xstring{}) +} + +func TestStringThreeByteUTF8(t *testing.T) { + testExpectedXVal(t, + "ࠡ three-byte UTF-8", + `\u0821 three-byte UTF-8`, + &Xstring{}) +} + +func TestStringEsccapes(t *testing.T) { + testExpectedXVal(t, + `"\`+"\b\f\n\r\t", + `\"\\\b\f\n\r\t`, + &Xstring{}) + + testExpectedXVal(t, + `/`, + `\/`, + &Xstring{}) +} + +func TestStringSomeUTF8(t *testing.T) { + testExpectedXVal(t, + `€þıœəßð some utf-8 ĸʒ×ŋµåäö𝄞`, + `€þıœəßð some utf-8 ĸʒ×ŋµåäö𝄞`, + &Xstring{}) +} + +func TestBytesInString(t *testing.T) { + testExpectedXVal(t, + string('\xff')+` <- xFF byte`, + string('\xff')+` <- xFF byte`, + &Xstring{}) +} + +func TestString4ByteSurrogate(t *testing.T) { + testExpectedXVal(t, + "𝄞 surrogate, four-byte UTF-8", + `\uD834\uDD1E surrogate, four-byte UTF-8`, + &Xstring{}) +} + +func TestStringNull(t *testing.T) { + testExpectedXValBare(t, + "foobar", + `null`, + &Xstring{X: "foobar"}) +} + +func TestStringQuoted(t *testing.T) { + ver := runtime.Version() + if strings.Contains(ver, "go1.3") || strings.Contains(ver, "go1.2") { + t.Skipf("Test requires go v1.4 or later, this is %s", ver) + } + + testStrQuoted(t, "\x12 escaped control character") + testStrQuoted(t, `\u0012 escaped control character`) + testStrQuoted(t, ", one-byte UTF-8") + testStrQuoted(t, `\u002c one-byte UTF-8`) + testStrQuoted(t, "2ħ籦ö嗏ʑ>嫀") + testStrQuoted(t, `2ħ籦ö嗏ʑ\u003e嫀`) + testStrQuoted(t, "ģ two-byte UTF-8") + testStrQuoted(t, `\u0123 two-byte UTF-8`) + testStrQuoted(t, "ࠡ three-byte UTF-8") + testStrQuoted(t, `\u0821 three-byte UTF-8`) + testStrQuoted(t, `"\`+"\b\f\n\r\t") + testStrQuoted(t, "𝄞 surrogate, four-byte UTF-8") + testStrQuoted(t, string('\xff')+` <- xFF byte`) + testStrQuoted(t, `€þıœəßð some utf-8 ĸʒ×ŋµåäö𝄞`) + testStrQuoted(t, `\/`) + testStrQuoted(t, `/`) + testStrQuoted(t, `\"\\\b\f\n\r\t`) + testStrQuoted(t, `\uD834\uDD1E surrogate, four-byte UTF-8`) + testStrQuoted(t, `null`) +} + +func testStrQuoted(t *testing.T, str string) { + testCycle(t, &TstringTagged{X: str}, &XstringTagged{X: str}) + testCycle(t, &TstringTaggedPtr{X: &str}, &XstringTaggedPtr{X: &str}) +}
\ No newline at end of file diff --git a/tests/ff_test.go b/tests/ff_test.go new file mode 100644 index 0000000..c17f2db --- /dev/null +++ b/tests/ff_test.go @@ -0,0 +1,862 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tff + +import ( + "github.com/pquerna/ffjson/ffjson" + fflib "github.com/pquerna/ffjson/fflib/v1" + "github.com/stretchr/testify/require" + + "bytes" + "encoding/gob" + "encoding/json" + "fmt" + "os" + "reflect" + "testing" + "time" +) + +// If this is enabled testSameMarshal and testCycle will output failures to files +// for easy debugging. +var outputFileOnError = false + +func newLogRecord() *Record { + return &Record{ + OriginID: 11, + Method: "POST", + } +} + +func newLogFFRecord() *FFRecord { + return &FFRecord{ + OriginID: 11, + Method: "POST", + } +} + +func BenchmarkMarshalJSON(b *testing.B) { + record := newLogRecord() + + buf, err := json.Marshal(&record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := json.Marshal(&record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + } +} + +func BenchmarkMarshalJSONNative(b *testing.B) { + record := newLogFFRecord() + + buf, err := json.Marshal(record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := ffjson.MarshalFast(record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + } +} + +func BenchmarkMarshalJSONNativePool(b *testing.B) { + record := newLogFFRecord() + + buf, err := json.Marshal(&record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + bytes, err := ffjson.MarshalFast(record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + ffjson.Pool(bytes) + } +} + +type NopWriter struct{} + +func (*NopWriter) Write(buf []byte) (int, error) { + return len(buf), nil +} + +func BenchmarkMarshalJSONNativeReuse(b *testing.B) { + record := newLogFFRecord() + + buf, err := json.Marshal(&record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + b.SetBytes(int64(len(buf))) + + enc := ffjson.NewEncoder(&NopWriter{}) + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := enc.Encode(record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + } +} + +func BenchmarkSimpleUnmarshal(b *testing.B) { + record := newLogFFRecord() + buf := []byte(`{"id": 123213, "OriginID": 22, "meth": "GET"}`) + err := record.UnmarshalJSON(buf) + if err != nil { + b.Fatalf("UnmarshalJSON: %v", err) + } + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := record.UnmarshalJSON(buf) + if err != nil { + b.Fatalf("UnmarshalJSON: %v", err) + } + } +} + +func BenchmarkSXimpleUnmarshalNative(b *testing.B) { + record := newLogRecord() + buf := []byte(`{"id": 123213, "OriginID": 22, "meth": "GET"}`) + err := json.Unmarshal(buf, record) + if err != nil { + b.Fatalf("json.Unmarshal: %v", err) + } + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := ffjson.UnmarshalFast(buf, record) + if err != nil { + b.Fatalf("json.Unmarshal: %v", err) + } + } +} + +func TestMarshalFaster(t *testing.T) { + record := newLogFFRecord() + _, err := ffjson.MarshalFast(record) + require.NoError(t, err) + + r2 := newLogRecord() + _, err = ffjson.MarshalFast(r2) + require.Error(t, err, "Record should not support MarshalFast") + _, err = ffjson.Marshal(r2) + require.NoError(t, err) +} + +func TestMarshalEncoder(t *testing.T) { + record := newLogFFRecord() + out := bytes.Buffer{} + enc := ffjson.NewEncoder(&out) + err := enc.Encode(record) + require.NoError(t, err) + require.NotEqual(t, 0, out.Len(), "encoded buffer size should not be 0") + + out.Reset() + err = enc.EncodeFast(record) + require.NoError(t, err) + require.NotEqual(t, 0, out.Len(), "encoded buffer size should not be 0") +} + +func TestMarshalEncoderError(t *testing.T) { + out := NopWriter{} + enc := ffjson.NewEncoder(&out) + v := GiveError{} + err := enc.Encode(v) + require.Error(t, err, "excpected error from encoder") + err = enc.Encode(newLogFFRecord()) + require.NoError(t, err, "error did not clear as expected.") + + err = enc.EncodeFast(newLogRecord()) + require.Error(t, err, "excpected error from encoder on type that isn't fast") +} + +func TestUnmarshalFaster(t *testing.T) { + buf := []byte(`{"id": 123213, "OriginID": 22, "meth": "GET"}`) + record := newLogFFRecord() + err := ffjson.UnmarshalFast(buf, record) + require.NoError(t, err) + + r2 := newLogRecord() + err = ffjson.UnmarshalFast(buf, r2) + require.Error(t, err, "Record should not support UnmarshalFast") + err = ffjson.Unmarshal(buf, r2) + require.NoError(t, err) +} + +func TestSimpleUnmarshal(t *testing.T) { + record := newLogFFRecord() + + err := record.UnmarshalJSON([]byte(`{"id": 123213, "OriginID": 22, "meth": "GET"}`)) + if err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + + if record.Timestamp != 123213 { + t.Fatalf("record.Timestamp: expected: 0 got: %v", record.Timestamp) + } + + if record.OriginID != 22 { + t.Fatalf("record.OriginID: expected: 22 got: %v", record.OriginID) + } + + if record.Method != "GET" { + t.Fatalf("record.Method: expected: GET got: %v", record.Method) + } +} + +type marshalerFaster interface { + MarshalJSONBuf(buf fflib.EncodingBuffer) error +} + +type unmarshalFaster interface { + UnmarshalJSONFFLexer(l *fflib.FFLexer, state fflib.FFParseState) error +} + +// emptyInterface creates a new instance of the object sent +// It the returned interface is writable and contains the zero value. +func emptyInterface(a interface{}) interface{} { + aval := reflect.ValueOf(a) + indirect := reflect.Indirect(aval) + newIndirect := reflect.New(indirect.Type()) + + return newIndirect.Interface() +} +func testType(t *testing.T, base interface{}, ff interface{}) { + require.Implements(t, (*json.Marshaler)(nil), ff) + require.Implements(t, (*json.Unmarshaler)(nil), ff) + require.Implements(t, (*marshalerFaster)(nil), ff) + require.Implements(t, (*unmarshalFaster)(nil), ff) + + if _, ok := base.(unmarshalFaster); ok { + require.FailNow(t, "base should not have a UnmarshalJSONFFLexer") + } + + if _, ok := base.(marshalerFaster); ok { + require.FailNow(t, "base should not have a MarshalJSONBuf") + } + + testSameMarshal(t, base, ff) + testCycle(t, base, ff) +} + +func testSameMarshal(t *testing.T, base interface{}, ff interface{}) { + bufbase, err := json.MarshalIndent(base, " ", " ") + require.NoError(t, err, "base[%T] failed to Marshal", base) + + bufff, err := json.MarshalIndent(ff, " ", " ") + if err != nil { + msg := fmt.Sprintf("golang output:\n%s\n", string(bufbase)) + mf, ok := ff.(json.Marshaler) + var raw []byte + if ok { + var err2 error + raw, err2 = mf.MarshalJSON() + msg += fmt.Sprintf("Raw output:\n%s\nErros:%v", string(raw), err2) + } + if outputFileOnError { + typeName := reflect.Indirect(reflect.ValueOf(base)).Type().String() + file, err := os.Create(fmt.Sprintf("fail-%s-marshal-go.json", typeName)) + if err == nil { + file.Write(bufbase) + file.Close() + } + if len(raw) != 0 { + file, err = os.Create(fmt.Sprintf("fail-%s-marshal-ffjson-raw.json", typeName)) + if err == nil { + file.Write(raw) + file.Close() + } + } + } + require.NoError(t, err, "ff[%T] failed to Marshal:%s", ff, msg) + } + + if outputFileOnError { + if string(bufbase) != string(bufff) { + typeName := reflect.Indirect(reflect.ValueOf(base)).Type().String() + file, err := os.Create(fmt.Sprintf("fail-%s-marshal-base.json", typeName)) + if err == nil { + file.Write(bufbase) + file.Close() + } + file, err = os.Create(fmt.Sprintf("fail-%s-marshal-ffjson.json", typeName)) + if err == nil { + file.Write(bufff) + file.Close() + } + } + } + + require.Equal(t, string(bufbase), string(bufff), "json.Marshal of base[%T] != ff[%T]", base, ff) +} + +func testCycle(t *testing.T, base interface{}, ff interface{}) { + setXValue(t, base) + + buf, err := json.MarshalIndent(base, " ", " ") + require.NoError(t, err, "base[%T] failed to Marshal", base) + + ffDst := emptyInterface(ff) + baseDst := emptyInterface(base) + + err = json.Unmarshal(buf, ffDst) + errGo := json.Unmarshal(buf, baseDst) + if outputFileOnError && err != nil { + typeName := reflect.Indirect(reflect.ValueOf(base)).Type().String() + file, err := os.Create(fmt.Sprintf("fail-%s-unmarshal-decoder-input.json", typeName)) + if err == nil { + file.Write(buf) + file.Close() + } + if errGo == nil { + file, err := os.Create(fmt.Sprintf("fail-%s-unmarshal-decoder-output-base.txt", typeName)) + if err == nil { + fmt.Fprintf(file, "%#v", baseDst) + file.Close() + } + } + } + require.Nil(t, err, "json.Unmarshal of encoded ff[%T],\nErrors golang:%v,\nffjson:%v", ff, errGo, err) + require.Nil(t, errGo, "json.Unmarshal of encoded ff[%T],\nerrors golang:%v,\nffjson:%v", base, errGo, err) + + require.EqualValues(t, baseDst, ffDst, "json.Unmarshal of base[%T] into ff[%T]", base, ff) +} + +func testExpectedX(t *testing.T, expected interface{}, base interface{}, ff interface{}) { + buf, err := json.Marshal(base) + require.NoError(t, err, "base[%T] failed to Marshal", base) + + err = json.Unmarshal(buf, ff) + require.NoError(t, err, "ff[%T] failed to Unmarshal", ff) + + require.Equal(t, expected, getXValue(ff), "json.Unmarshal of base[%T] into ff[%T]", base, ff) +} + +func testExpectedXValBare(t *testing.T, expected interface{}, xval string, ff interface{}) { + buf := []byte(`{"X":` + xval + `}`) + err := json.Unmarshal(buf, ff) + require.NoError(t, err, "ff[%T] failed to Unmarshal", ff) + + require.Equal(t, expected, getXValue(ff), "json.Unmarshal of %T into ff[%T]", xval, ff) +} + +func testExpectedXVal(t *testing.T, expected interface{}, xval string, ff interface{}) { + testExpectedXValBare(t, expected, `"`+xval+`"`, ff) +} + +func testExpectedError(t *testing.T, expected error, xval string, ff json.Unmarshaler) { + buf := []byte(`{"X":` + xval + `}`) + err := ff.UnmarshalJSON(buf) + require.Errorf(t, err, "ff[%T] failed to Unmarshal", ff) + require.IsType(t, expected, err) +} + +func setXValue(t *testing.T, thing interface{}) { + v := reflect.ValueOf(thing) + v = reflect.Indirect(v) + f := v.FieldByName("X") + switch f.Kind() { + case reflect.Bool: + f.SetBool(true) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + f.SetInt(-42) + case reflect.Uint, reflect.Uintptr, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + f.SetUint(42) + case reflect.Float32, reflect.Float64: + f.SetFloat(3.141592653) + case reflect.String: + f.SetString("hello world") + } +} + +func getXValue(thing interface{}) interface{} { + v := reflect.ValueOf(thing) + v = reflect.Indirect(v) + f := v.FieldByName("X") + switch f.Kind() { + case reflect.Bool: + return f.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return f.Int() + case reflect.Uint, reflect.Uintptr, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return f.Uint() + case reflect.Float32, reflect.Float64: + return f.Float() + case reflect.String: + return f.String() + } + + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + enc.Encode(f) + return buf.String() +} + +func TestArray(t *testing.T) { + testType(t, &Tarray{X: [3]int{}}, &Xarray{X: [3]int{}}) + testCycle(t, &Tarray{X: [3]int{42, -42, 44}}, &Xarray{X: [3]int{}}) + + x := Xarray{X: [3]int{222}} + buf := []byte(`{"X": null}`) + err := json.Unmarshal(buf, &x) + require.NoError(t, err, "Unmarshal of null into array.") + var eq = [3]int{} + require.Equal(t, x.X, eq) +} + +func TestArrayPtr(t *testing.T) { + testType(t, &TarrayPtr{X: [3]*int{}}, &XarrayPtr{X: [3]*int{}}) + v := 33 + testCycle(t, &TarrayPtr{X: [3]*int{&v}}, &XarrayPtr{X: [3]*int{}}) +} + +func TestSlice(t *testing.T) { + testType(t, &Tslice{X: []int{}}, &Xslice{X: []int{}}) + testCycle(t, &Tslice{X: []int{42, -42, 44}}, &Xslice{X: []int{}}) + + x := Xslice{X: []int{222}} + buf := []byte(`{"X": null}`) + err := json.Unmarshal(buf, &x) + require.NoError(t, err, "Unmarshal of null into slice.") + var eq []int + require.Equal(t, x.X, eq) +} + +func TestSlicePtr(t *testing.T) { + testType(t, &TslicePtr{X: []*int{}}, &XslicePtr{X: []*int{}}) + v := 33 + testCycle(t, &TslicePtr{X: []*int{&v}}, &XslicePtr{X: []*int{}}) +} + +func TestSlicePtrNils(t *testing.T) { + v1 := 3 + v2 := 4 + testType(t, &TslicePtr{X: []*int{nil, &v1, nil, &v2}}, &XslicePtr{X: []*int{nil, &v1, nil, &v2}}) +} + +func TestMapPtrNils(t *testing.T) { + v1 := 3 + v2 := 4 + testType(t, &TMapStringPtr{X: map[string]*int{"a": nil, "b": &v1, "c": nil, "d": &v2}}, &XMapStringPtr{X: map[string]*int{"a": nil, "b": &v1, "c": nil, "d": &v2}}) +} + +func TestSlicePtrStructNils(t *testing.T) { + v1 := "v1" + v2 := "v2" + testType(t, &TslicePtrStruct{X: []*Xstring{nil, &Xstring{v1}, nil, &Xstring{v2}}}, &XslicePtrStruct{X: []*Xstring{nil, &Xstring{v1}, nil, &Xstring{v2}}}) +} + +func TestMapPtrStructNils(t *testing.T) { + v1 := "v1" + v2 := "v2" + testType(t, &TMapPtrStruct{X: map[string]*Xstring{"a": nil, "b": &Xstring{v1}, "c": nil, "d": &Xstring{v2}}}, &XMapPtrStruct{X: map[string]*Xstring{"a": nil, "b": &Xstring{v1}, "c": nil, "d": &Xstring{v2}}}) +} + +func TestTimeDuration(t *testing.T) { + testType(t, &Tduration{}, &Xduration{}) +} + +func TestI18nName(t *testing.T) { + testType(t, &TI18nName{}, &XI18nName{}) +} + +func TestTimeTimePtr(t *testing.T) { + tm := time.Date(2014, 12, 13, 15, 16, 17, 18, time.UTC) + testType(t, &TtimePtr{X: &tm}, &XtimePtr{X: &tm}) +} + +func TestTimeNullTimePtr(t *testing.T) { + testType(t, &TtimePtr{}, &XtimePtr{}) +} + +func TestBool(t *testing.T) { + testType(t, &Tbool{}, &Xbool{}) + testExpectedXValBare(t, + true, + `null`, + &Xbool{X: true}) +} + +func TestInt(t *testing.T) { + testType(t, &Tint{}, &Xint{}) +} + +func TestByte(t *testing.T) { + testType(t, &Tbyte{}, &Xbyte{}) +} + +func TestInt8(t *testing.T) { + testType(t, &Tint8{}, &Xint8{}) +} + +func TestInt16(t *testing.T) { + testType(t, &Tint16{}, &Xint16{}) +} + +func TestInt32(t *testing.T) { + testType(t, &Tint32{}, &Xint32{}) +} + +func TestInt64(t *testing.T) { + testType(t, &Tint64{}, &Xint64{}) +} + +func TestUint(t *testing.T) { + testType(t, &Tuint{}, &Xuint{}) +} + +func TestUint8(t *testing.T) { + testType(t, &Tuint8{}, &Xuint8{}) +} + +func TestUint16(t *testing.T) { + testType(t, &Tuint16{}, &Xuint16{}) +} + +func TestUint32(t *testing.T) { + testType(t, &Tuint32{}, &Xuint32{}) +} + +func TestUint64(t *testing.T) { + testType(t, &Tuint64{}, &Xuint64{}) +} + +func TestUintptr(t *testing.T) { + testType(t, &Tuintptr{}, &Xuintptr{}) +} + +func TestFloat32(t *testing.T) { + testType(t, &Tfloat32{}, &Xfloat32{}) +} + +func TestFloat64(t *testing.T) { + testType(t, &Tfloat64{}, &Xfloat64{}) +} + +func TestForceStringTagged(t *testing.T) { + // testSameMarshal is used instead of testType because + // the string tag is a one way effect, Unmarshaling doesn't + // work because the receiving type must be a string. + testSameMarshal(t, &TstringTagged{}, &XstringTagged{}) + testSameMarshal(t, &TintTagged{}, &XintTagged{}) + testSameMarshal(t, &TboolTagged{}, &XboolTagged{}) +} + +func TestForceStringTaggedEscape(t *testing.T) { + testSameMarshal(t, &TstringTagged{X: `"`}, &XstringTagged{X: `"`}) +} + +func TestForceStringTaggedDecoder(t *testing.T) { + testCycle(t, &TstringTagged{}, &XstringTagged{}) + testCycle(t, &TintTagged{}, &XintTagged{}) + testCycle(t, &TboolTagged{}, &XboolTagged{}) +} + +func TestSortSame(t *testing.T) { + testSameMarshal(t, &TsortName{C: "foo", B: 12}, &XsortName{C: "foo", B: 12}) +} + +func TestEncodeRenamedByteSlice(t *testing.T) { + expect := `{"X":"YWJj"}` + + s := ByteSliceNormal{X: []byte("abc")} + result, err := s.MarshalJSON() + require.NoError(t, err) + require.Equal(t, string(result), expect) + + r := ByteSliceRenamed{X: renamedByteSlice("abc")} + result, err = r.MarshalJSON() + require.NoError(t, err) + require.Equal(t, string(result), expect) + + rr := ByteSliceDoubleRenamed{X: renamedRenamedByteSlice("abc")} + result, err = rr.MarshalJSON() + require.NoError(t, err) + require.Equal(t, string(result), expect) +} + +// Test arrays +func TestArrayBool(t *testing.T) { + testType(t, &ATbool{}, &AXbool{}) +} + +func TestArrayInt(t *testing.T) { + testType(t, &ATint{}, &AXint{}) +} + +func TestArrayByte(t *testing.T) { + testType(t, &ATbyte{}, &AXbyte{}) +} + +func TestArrayInt8(t *testing.T) { + testType(t, &ATint8{}, &AXint8{}) +} + +func TestArrayInt16(t *testing.T) { + testType(t, &ATint16{}, &AXint16{}) +} + +func TestArrayInt32(t *testing.T) { + testType(t, &ATint32{}, &AXint32{}) +} + +func TestArrayInt64(t *testing.T) { + testType(t, &ATint64{}, &AXint64{}) +} + +func TestArrayUint(t *testing.T) { + testType(t, &ATuint{}, &AXuint{}) +} + +func TestArrayUint8(t *testing.T) { + testType(t, &ATuint8{}, &AXuint8{}) +} + +func TestArrayUint16(t *testing.T) { + testType(t, &ATuint16{}, &AXuint16{}) +} + +func TestArrayUint32(t *testing.T) { + testType(t, &ATuint32{}, &AXuint32{}) +} + +func TestArrayUint64(t *testing.T) { + testType(t, &ATuint64{}, &AXuint64{}) +} + +func TestArrayUintptr(t *testing.T) { + testType(t, &ATuintptr{}, &AXuintptr{}) +} + +func TestArrayFloat32(t *testing.T) { + testType(t, &ATfloat32{}, &AXfloat32{}) +} + +func TestArrayFloat64(t *testing.T) { + testType(t, &ATfloat64{}, &AXfloat64{}) +} + +func TestArrayTime(t *testing.T) { + testType(t, &ATtime{}, &AXtime{}) +} + +// Test slices +func TestSliceBool(t *testing.T) { + testType(t, &STbool{}, &SXbool{}) +} + +func TestSliceInt(t *testing.T) { + testType(t, &STint{}, &SXint{}) +} + +func TestSliceByte(t *testing.T) { + testType(t, &STbyte{}, &SXbyte{}) +} + +func TestSliceInt8(t *testing.T) { + testType(t, &STint8{}, &SXint8{}) +} + +func TestSliceInt16(t *testing.T) { + testType(t, &STint16{}, &SXint16{}) +} + +func TestSliceInt32(t *testing.T) { + testType(t, &STint32{}, &SXint32{}) +} + +func TestSliceInt64(t *testing.T) { + testType(t, &STint64{}, &SXint64{}) +} + +func TestSliceUint(t *testing.T) { + testType(t, &STuint{}, &SXuint{}) +} + +func TestSliceUint8(t *testing.T) { + testType(t, &STuint8{}, &SXuint8{}) +} + +func TestSliceUint16(t *testing.T) { + testType(t, &STuint16{}, &SXuint16{}) +} + +func TestSliceUint32(t *testing.T) { + testType(t, &STuint32{}, &SXuint32{}) +} + +func TestSliceUint64(t *testing.T) { + testType(t, &STuint64{}, &SXuint64{}) +} + +func TestSliceUintptr(t *testing.T) { + testType(t, &STuintptr{}, &SXuintptr{}) +} + +func TestSliceFloat32(t *testing.T) { + testType(t, &STfloat32{}, &SXfloat32{}) +} + +func TestSliceFloat64(t *testing.T) { + testType(t, &STfloat64{}, &SXfloat64{}) +} + +func TestSliceTime(t *testing.T) { + testType(t, &STtime{}, &SXtime{}) +} + +func TestTMapStringMapString(t *testing.T) { + testType(t, &TMapStringMapString{}, &XMapStringMapString{}) +} + +func TestTMapStringAString(t *testing.T) { + testType(t, &TMapStringAString{}, &XMapStringAString{}) +} + +func TestTSAAtring(t *testing.T) { + testType(t, &TSAAtring{}, &XSAAtring{}) +} + +func TestTSAString(t *testing.T) { + testType(t, &TSAString{}, &XSAString{}) +} + +func TestNoDecoder(t *testing.T) { + var test interface{} = &NoDecoder{} + if _, ok := test.(unmarshalFaster); ok { + require.FailNow(t, "NoDecoder should not have a UnmarshalJSONFFLexer") + } +} + +func TestNoEncoder(t *testing.T) { + var test interface{} = &NoEncoder{} + if _, ok := test.(marshalerFaster); ok { + require.FailNow(t, "NoEncoder should not have a MarshalJSONBuf") + } +} + +func TestCaseSensitiveUnmarshalSimple(t *testing.T) { + base := Tint{} + ff := Xint{} + + err := json.Unmarshal([]byte(`{"x": 123213}`), &base) + if err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + + err = json.Unmarshal([]byte(`{"x": 123213}`), &ff) + if err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + require.EqualValues(t, base, ff, "json.Unmarshal of Record with mixed case JSON") +} + +func TestEmbedded(t *testing.T) { + a := TEmbeddedStructures{} + a.X = make([]interface{}, 0) + a.X = append(a.X, "testString") + a.Y.X = 73 + a.Z = make([]struct{ X int }, 2) + a.Z[0].X = 12 + a.Z[1].X = 34 + a.U = make(map[string]struct{ X int }) + a.U["sample"] = struct{ X int }{X: 56} + a.U["value"] = struct{ X int }{X: 78} + a.V = make([]map[string]struct{ X int }, 3) + for i := range a.V { + a.V[i] = make(map[string]struct{ X int }) + a.V[i]["sample"] = struct{ X int }{X: i * 3} + } + for i := range a.W { + a.W[i] = make(map[string]struct{ X int }) + a.W[i]["sample"] = struct{ X int }{X: i * 3} + a.W[i]["value"] = struct{ X int }{X: i * 5} + } + a.Q = make([][]string, 3) + for i := range a.Q { + a.Q[i] = make([]string, 1) + a.Q[i][0] = fmt.Sprintf("thestring #%d", i) + } + + b := XEmbeddedStructures{} + b.X = make([]interface{}, 0) + b.X = append(b.X, "testString") + b.Y.X = 73 + b.Z = make([]struct{ X int }, 2) + b.Z[0].X = 12 + b.Z[1].X = 34 + b.U = make(map[string]struct{ X int }) + b.U["sample"] = struct{ X int }{X: 56} + b.U["value"] = struct{ X int }{X: 78} + b.V = make([]map[string]struct{ X int }, 3) + for i := range b.V { + b.V[i] = make(map[string]struct{ X int }) + b.V[i]["sample"] = struct{ X int }{X: i * 3} + } + for i := range b.W { + b.W[i] = make(map[string]struct{ X int }) + b.W[i]["sample"] = struct{ X int }{X: i * 3} + b.W[i]["value"] = struct{ X int }{X: i * 5} + } + b.Q = make([][]string, 3) + for i := range a.Q { + b.Q[i] = make([]string, 1) + b.Q[i][0] = fmt.Sprintf("thestring #%d", i) + } + testSameMarshal(t, &a, &b) + testCycle(t, &a, &b) +} + +func TestRenameTypes(t *testing.T) { + testType(t, &TRenameTypes{}, &XRenameTypes{}) +} + +func TestInlineStructs(t *testing.T) { + a := TInlineStructs{} + b := XInlineStructs{} + testSameMarshal(t, &a, &b) + testCycle(t, &a, &b) +} + +// This tests that we behave the same way as encoding/json. +// That means that if there is more than one field that has the same name +// set via the json tag ALL fields with this name are dropped. +func TestDominantField(t *testing.T) { + i := 43 + testType(t, &TDominantField{Y: &i}, &XDominantField{Y: &i}) +} diff --git a/tests/fuzz_test.go b/tests/fuzz_test.go new file mode 100644 index 0000000..fe94274 --- /dev/null +++ b/tests/fuzz_test.go @@ -0,0 +1,717 @@ +/** + * Copyright 2014 Paul Querna, Klaus Post + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tff + +import ( + "encoding/json" + "math/rand" + "runtime" + "strings" + "testing" + "time" + + fuzz "github.com/google/gofuzz" + "github.com/stretchr/testify/require" +) + +type Fuzz struct { + A uint8 + B uint16 + C uint32 + D uint64 + + E int8 + F int16 + G int32 + H int64 + + I float32 + J float64 + + M byte + N rune + + O int + P uint + Q string + R bool + S time.Time + + Ap *uint8 + Bp *uint16 + Cp *uint32 + Dp *uint64 + + Ep *int8 + Fp *int16 + Gp *int32 + Hp *int64 + + IP *float32 + Jp *float64 + + Mp *byte + Np *rune + + Op *int + Pp *uint + Qp *string + Rp *bool + Sp *time.Time + + Aa []uint8 + Ba []uint16 + Ca []uint32 + Da []uint64 + + Ea []int8 + Fa []int16 + Ga []int32 + Ha []int64 + + Ia []float32 + Ja []float64 + + Ma []byte + Na []rune + + Oa []int + Pa []uint + Qa []string + Ra []bool + + Aap []*uint8 + Bap []*uint16 + Cap []*uint32 + Dap []*uint64 + + Eap []*int8 + Fap []*int16 + Gap []*int32 + Hap []*int64 + + Iap []*float32 + Jap []*float64 + + Map []*byte + Nap []*rune + + Oap []*int + Pap []*uint + Qap []*string + Rap []*bool +} + +// Return a time no later than 5000 years from unix datum. +// JSON cannot handle dates after year 9999. +func fuzzTime(t *time.Time, c fuzz.Continue) { + sec := c.Rand.Int63() + nsec := c.Rand.Int63() + // No more than 5000 years in the future + sec %= 5000 * 365 * 24 * 60 * 60 + *t = time.Unix(sec, nsec) +} + +func fuzzTimeSlice(t *[]time.Time, c fuzz.Continue) { + var i uint64 + rv := make([]time.Time, 0) + count := c.RandUint64() % 50 + for i = 0; i < count; i++ { + var tmp time.Time + fuzzTime(&tmp, c) + rv = append(rv, tmp) + } + *t = rv +} + +// Test 1000 iterations +func TestFuzzCycle(t *testing.T) { + f := fuzz.New() + f.NumElements(0, 50) + f.NilChance(0.1) + f.Funcs(fuzzTime) + + rFF := FfFuzz{} + r := Fuzz{} + for i := 0; i < 1000; i++ { + if i > 0 { + f.RandSource(rand.New(rand.NewSource(int64(i * 324221)))) + f.Fuzz(&r) + + // TODO: remove these after we marshal 0.00000012 to 1.2e-7. + r.I = 0 + r.J = 0 + r.IP = nil + r.Jp = nil + r.Ia = []float32{0} + r.Ja = []float64{0} + r.Iap = nil + r.Jap = nil + } + rFF.A = r.A + rFF.B = r.B + rFF.C = r.C + rFF.D = r.D + rFF.E = r.E + rFF.F = r.F + rFF.G = r.G + rFF.H = r.H + rFF.I = r.I + rFF.J = r.J + rFF.M = r.M + rFF.N = r.N + rFF.O = r.O + rFF.P = r.P + rFF.Q = r.Q + rFF.R = r.R + rFF.S = r.S + + rFF.Ap = r.Ap + rFF.Bp = r.Bp + rFF.Cp = r.Cp + rFF.Dp = r.Dp + rFF.Ep = r.Ep + rFF.Fp = r.Fp + rFF.Gp = r.Gp + rFF.Hp = r.Hp + rFF.IP = r.IP + rFF.Jp = r.Jp + rFF.Mp = r.Mp + rFF.Np = r.Np + rFF.Op = r.Op + rFF.Pp = r.Pp + rFF.Qp = r.Qp + rFF.Rp = r.Rp + rFF.Sp = r.Sp + + rFF.Aa = r.Aa + rFF.Ba = r.Ba + rFF.Ca = r.Ca + rFF.Da = r.Da + rFF.Ea = r.Ea + rFF.Fa = r.Fa + rFF.Ga = r.Ga + rFF.Ha = r.Ha + rFF.Ia = r.Ia + rFF.Ja = r.Ja + rFF.Ma = r.Ma + rFF.Na = r.Na + rFF.Oa = r.Oa + rFF.Pa = r.Pa + rFF.Qa = r.Qa + rFF.Ra = r.Ra + + rFF.Aap = r.Aap + rFF.Bap = r.Bap + rFF.Cap = r.Cap + rFF.Dap = r.Dap + rFF.Eap = r.Eap + rFF.Fap = r.Fap + rFF.Gap = r.Gap + rFF.Hap = r.Hap + rFF.Iap = r.Iap + rFF.Jap = r.Jap + rFF.Map = r.Map + rFF.Nap = r.Nap + rFF.Oap = r.Oap + rFF.Pap = r.Pap + rFF.Qap = r.Qap + rFF.Rap = r.Rap + testSameMarshal(t, &r, &rFF) + testCycle(t, &r, &rFF) + } +} + +// Test 1000 iterations +func TestFuzzOmitCycle(t *testing.T) { + f := fuzz.New() + f.NumElements(0, 10) + f.NilChance(0.5) + f.Funcs(fuzzTime) + + rFF := FfFuzzOmitEmpty{} + r := FuzzOmitEmpty{} + for i := 0; i <= 1000; i++ { + if i > 0 { + f.RandSource(rand.New(rand.NewSource(int64(i * 324221)))) + f.Fuzz(&r) + + // TODO: remove these after we marshal 0.00000012 to 1.2e-7. + r.J = 0 + r.Jp = nil + r.Ja = []float64{0} + r.Jap = nil + } + rFF.A = r.A + rFF.B = r.B + rFF.C = r.C + rFF.D = r.D + rFF.E = r.E + rFF.F = r.F + rFF.G = r.G + rFF.H = r.H + rFF.I = r.I + rFF.J = r.J + rFF.M = r.M + rFF.N = r.N + rFF.O = r.O + rFF.P = r.P + rFF.Q = r.Q + rFF.R = r.R + rFF.S = r.S + + rFF.Ap = r.Ap + rFF.Bp = r.Bp + rFF.Cp = r.Cp + rFF.Dp = r.Dp + rFF.Ep = r.Ep + rFF.Fp = r.Fp + rFF.Gp = r.Gp + rFF.Hp = r.Hp + rFF.IP = r.IP + rFF.Jp = r.Jp + rFF.Mp = r.Mp + rFF.Np = r.Np + rFF.Op = r.Op + rFF.Pp = r.Pp + rFF.Qp = r.Qp + rFF.Rp = r.Rp + rFF.Sp = r.Sp + + rFF.Aa = r.Aa + rFF.Ba = r.Ba + rFF.Ca = r.Ca + rFF.Da = r.Da + rFF.Ea = r.Ea + rFF.Fa = r.Fa + rFF.Ga = r.Ga + rFF.Ha = r.Ha + rFF.Ia = r.Ia + rFF.Ja = r.Ja + rFF.Ma = r.Ma + rFF.Na = r.Na + rFF.Oa = r.Oa + rFF.Pa = r.Pa + rFF.Qa = r.Qa + rFF.Ra = r.Ra + + rFF.Aap = r.Aap + rFF.Bap = r.Bap + rFF.Cap = r.Cap + rFF.Dap = r.Dap + rFF.Eap = r.Eap + rFF.Fap = r.Fap + rFF.Gap = r.Gap + rFF.Hap = r.Hap + rFF.Iap = r.Iap + rFF.Jap = r.Jap + rFF.Map = r.Map + rFF.Nap = r.Nap + rFF.Oap = r.Oap + rFF.Pap = r.Pap + rFF.Qap = r.Qap + rFF.Rap = r.Rap + testSameMarshal(t, &r, &rFF) + testCycle(t, &r, &rFF) + } +} + +// Test 1000 iterations +func TestFuzzStringCycle(t *testing.T) { + ver := runtime.Version() + if strings.Contains(ver, "go1.3") || strings.Contains(ver, "go1.2") { + t.Skipf("Test requires go v1.4 or later, this is %s", ver) + } + f := fuzz.New() + f.NumElements(0, 50) + f.NilChance(0.1) + f.Funcs(fuzzTime) + + rFF := FfFuzzString{} + r := FuzzString{} + for i := 0; i < 1000; i++ { + if i > 0 { + f.RandSource(rand.New(rand.NewSource(int64(i * 324221)))) + f.Fuzz(&r) + } + rFF.A = r.A + rFF.B = r.B + rFF.C = r.C + rFF.D = r.D + rFF.E = r.E + rFF.F = r.F + rFF.G = r.G + rFF.H = r.H + rFF.I = r.I + rFF.J = r.J + rFF.M = r.M + rFF.N = r.N + rFF.O = r.O + rFF.P = r.P + rFF.Q = r.Q + rFF.R = r.R + + // https://github.com/golang/go/issues/9812 + // rFF.S = r.S + + rFF.Ap = r.Ap + rFF.Bp = r.Bp + rFF.Cp = r.Cp + rFF.Dp = r.Dp + rFF.Ep = r.Ep + rFF.Fp = r.Fp + rFF.Gp = r.Gp + rFF.Hp = r.Hp + rFF.IP = r.IP + rFF.Jp = r.Jp + rFF.Mp = r.Mp + rFF.Np = r.Np + rFF.Op = r.Op + rFF.Pp = r.Pp + rFF.Qp = r.Qp + rFF.Rp = r.Rp + // https://github.com/golang/go/issues/9812 + // rFF.Sp = r.Sp + + // The "string" option signals that a field is stored as JSON inside a JSON-encoded string. It applies only to fields of string, floating point, or integer types. This extra level of encoding is sometimes used when communicating with JavaScript programs. + // Therefore tests on byte arrays are removed, since the golang decoder chokes on them. + testSameMarshal(t, &r, &rFF) + + // Test for https://github.com/pquerna/ffjson/issues/80 + // testCycle(t, &r, &rFF) + } +} + +// Fuzz test for 1000 iterations +func testTypeFuzz(t *testing.T, base interface{}, ff interface{}) { + testTypeFuzzN(t, base, ff, 1000) +} + +// Fuzz test for N iterations +func testTypeFuzzN(t *testing.T, base interface{}, ff interface{}, n int) { + require.Implements(t, (*json.Marshaler)(nil), ff) + require.Implements(t, (*json.Unmarshaler)(nil), ff) + require.Implements(t, (*marshalerFaster)(nil), ff) + require.Implements(t, (*unmarshalFaster)(nil), ff) + + if _, ok := base.(unmarshalFaster); ok { + require.FailNow(t, "base should not have a UnmarshalJSONFFLexer") + } + + if _, ok := base.(marshalerFaster); ok { + require.FailNow(t, "base should not have a MarshalJSONBuf") + } + + f := fuzz.New() + f.NumElements(0, 1+n/40) + f.NilChance(0.2) + f.Funcs(fuzzTime, fuzzTimeSlice) + for i := 0; i < n; i++ { + f.RandSource(rand.New(rand.NewSource(int64(i * 5275)))) + f.Fuzz(base) + f.RandSource(rand.New(rand.NewSource(int64(i * 5275)))) + f.Fuzz(ff) + + testSameMarshal(t, base, ff) + testCycle(t, base, ff) + } +} + +func TestFuzzArray(t *testing.T) { + testTypeFuzz(t, &Tarray{X: [3]int{}}, &Xarray{X: [3]int{}}) +} + +func TestFuzzArrayPtr(t *testing.T) { + testTypeFuzz(t, &TarrayPtr{X: [3]*int{}}, &XarrayPtr{X: [3]*int{}}) +} + +func TestFuzzSlice(t *testing.T) { + testTypeFuzz(t, &Tslice{X: []int{}}, &Xslice{X: []int{}}) +} + +func TestFuzzSlicePtr(t *testing.T) { + testTypeFuzz(t, &TslicePtr{X: []*int{}}, &XslicePtr{X: []*int{}}) +} + +func TestFuzzTimeDuration(t *testing.T) { + testTypeFuzz(t, &Tduration{}, &Xduration{}) +} + +func TestFuzzBool(t *testing.T) { + testTypeFuzz(t, &Tbool{}, &Xbool{}) +} + +func TestFuzzInt(t *testing.T) { + testTypeFuzz(t, &Tint{}, &Xint{}) +} + +func TestFuzzByte(t *testing.T) { + testTypeFuzz(t, &Tbyte{}, &Xbyte{}) +} + +func TestFuzzInt8(t *testing.T) { + testTypeFuzz(t, &Tint8{}, &Xint8{}) +} + +func TestFuzzInt16(t *testing.T) { + testTypeFuzz(t, &Tint16{}, &Xint16{}) +} + +func TestFuzzInt32(t *testing.T) { + testTypeFuzz(t, &Tint32{}, &Xint32{}) +} + +func TestFuzzInt64(t *testing.T) { + testTypeFuzz(t, &Tint64{}, &Xint64{}) +} + +func TestFuzzUint(t *testing.T) { + testTypeFuzz(t, &Tuint{}, &Xuint{}) +} + +func TestFuzzUint8(t *testing.T) { + testTypeFuzz(t, &Tuint8{}, &Xuint8{}) +} + +func TestFuzzUint16(t *testing.T) { + testTypeFuzz(t, &Tuint16{}, &Xuint16{}) +} + +func TestFuzzUint32(t *testing.T) { + testTypeFuzz(t, &Tuint32{}, &Xuint32{}) +} + +func TestFuzzUint64(t *testing.T) { + testTypeFuzz(t, &Tuint64{}, &Xuint64{}) +} + +func TestFuzzUintptr(t *testing.T) { + testTypeFuzz(t, &Tuintptr{}, &Xuintptr{}) +} + +func TestFuzzFloat32(t *testing.T) { + testTypeFuzz(t, &Tfloat32{}, &Xfloat32{}) +} + +func TestFuzzFloat64(t *testing.T) { + testTypeFuzz(t, &Tfloat64{}, &Xfloat64{}) +} + +func TestFuzzString(t *testing.T) { + testTypeFuzz(t, &Tstring{}, &Xstring{}) + testTypeFuzz(t, &Tmystring{}, &Xmystring{}) + testTypeFuzz(t, &TmystringPtr{}, &XmystringPtr{}) +} + +func TestFuzzArrayTimeDuration(t *testing.T) { + testTypeFuzz(t, &ATduration{}, &AXduration{}) +} + +func TestFuzzArrayBool(t *testing.T) { + testTypeFuzz(t, &ATbool{}, &AXbool{}) +} + +func TestFuzzArrayInt(t *testing.T) { + testTypeFuzz(t, &ATint{}, &AXint{}) +} + +func TestFuzzArrayByte(t *testing.T) { + testTypeFuzz(t, &ATbyte{}, &AXbyte{}) +} + +func TestFuzzArrayInt8(t *testing.T) { + testTypeFuzz(t, &ATint8{}, &AXint8{}) +} + +func TestFuzzArrayInt16(t *testing.T) { + testTypeFuzz(t, &ATint16{}, &AXint16{}) +} + +func TestFuzzArrayInt32(t *testing.T) { + testTypeFuzz(t, &ATint32{}, &AXint32{}) +} + +func TestFuzzArrayInt64(t *testing.T) { + testTypeFuzz(t, &ATint64{}, &AXint64{}) +} + +func TestFuzzArrayUint(t *testing.T) { + testTypeFuzz(t, &ATuint{}, &AXuint{}) +} + +func TestFuzzArrayUint8(t *testing.T) { + testTypeFuzz(t, &ATuint8{}, &AXuint8{}) +} + +func TestFuzzArrayUint16(t *testing.T) { + testTypeFuzz(t, &ATuint16{}, &AXuint16{}) +} + +func TestFuzzArrayUint32(t *testing.T) { + testTypeFuzz(t, &ATuint32{}, &AXuint32{}) +} + +func TestFuzzArrayUint64(t *testing.T) { + testTypeFuzz(t, &ATuint64{}, &AXuint64{}) +} + +func TestFuzzArrayUintptr(t *testing.T) { + testTypeFuzz(t, &ATuintptr{}, &AXuintptr{}) +} + +func TestFuzzArrayFloat32(t *testing.T) { + testTypeFuzz(t, &ATfloat32{}, &AXfloat32{}) +} + +func TestFuzzArrayFloat64(t *testing.T) { + testTypeFuzz(t, &ATfloat64{}, &AXfloat64{}) +} + +func TestFuzzArrayTime(t *testing.T) { + testTypeFuzz(t, &ATtime{}, &AXtime{}) +} + +func TestFuzzSliceTimeDuration(t *testing.T) { + testTypeFuzz(t, &STduration{}, &SXduration{}) +} + +func TestFuzzSliceBool(t *testing.T) { + testTypeFuzz(t, &STbool{}, &SXbool{}) +} + +func TestFuzzSliceInt(t *testing.T) { + testTypeFuzz(t, &STint{}, &SXint{}) +} + +func TestFuzzSliceByte(t *testing.T) { + testTypeFuzz(t, &STbyte{}, &SXbyte{}) +} + +func TestFuzzSliceInt8(t *testing.T) { + testTypeFuzz(t, &STint8{}, &SXint8{}) +} + +func TestFuzzSliceInt16(t *testing.T) { + testTypeFuzz(t, &STint16{}, &SXint16{}) +} + +func TestFuzzSliceInt32(t *testing.T) { + testTypeFuzz(t, &STint32{}, &SXint32{}) +} + +func TestFuzzSliceInt64(t *testing.T) { + testTypeFuzz(t, &STint64{}, &SXint64{}) +} + +func TestFuzzSliceUint(t *testing.T) { + testTypeFuzz(t, &STuint{}, &SXuint{}) +} + +func TestFuzzSliceUint8(t *testing.T) { + testTypeFuzz(t, &STuint8{}, &SXuint8{}) +} + +func TestFuzzSliceUint16(t *testing.T) { + testTypeFuzz(t, &STuint16{}, &SXuint16{}) +} + +func TestFuzzSliceUint32(t *testing.T) { + testTypeFuzz(t, &STuint32{}, &SXuint32{}) +} + +func TestFuzzSliceUint64(t *testing.T) { + testTypeFuzz(t, &STuint64{}, &SXuint64{}) +} + +func TestFuzzSliceUintptr(t *testing.T) { + testTypeFuzz(t, &STuintptr{}, &SXuintptr{}) +} + +func TestFuzzSliceFloat32(t *testing.T) { + testTypeFuzz(t, &STfloat32{}, &SXfloat32{}) +} + +func TestFuzzSliceFloat64(t *testing.T) { + testTypeFuzz(t, &STfloat64{}, &SXfloat64{}) +} + +func TestFuzzSliceTime(t *testing.T) { + testTypeFuzz(t, &STtime{}, &SXtime{}) +} + +func TestFuzzI18nName(t *testing.T) { + testTypeFuzz(t, &TI18nName{}, &XI18nName{}) +} + +func TestFuzzInlineStructs(t *testing.T) { + testTypeFuzzN(t, &TInlineStructs{}, &XInlineStructs{}, 100) +} + +func TestFuzzTMapStringMapString(t *testing.T) { + testType(t, &TMapStringMapString{}, &XMapStringMapString{}) +} + +func TestFuzzTMapStringAString(t *testing.T) { + testType(t, &TMapStringAString{}, &XMapStringAString{}) +} + +func TestFuzzTSAAtring(t *testing.T) { + testType(t, &TSAAtring{}, &XSAAtring{}) +} + +func TestFuzzTSAString(t *testing.T) { + testType(t, &TSAString{}, &XSAString{}) +} + +// This contains maps. +// Since map order is random, we can expect the encoding order to be random +// Therefore we cannot use binary compare. +func TestFuzzMapToType(t *testing.T) { + base := &TTestMaps{} + ff := &XTestMaps{} + f := fuzz.New() + f.NumElements(0, 50) + f.NilChance(0.1) + f.Funcs(fuzzTime) + for i := 0; i < 100; i++ { + f.RandSource(rand.New(rand.NewSource(int64(i * 5275)))) + f.Fuzz(base) + ff = &XTestMaps{*base} + + bufbase, err := json.Marshal(base) + require.NoError(t, err, "base[%T] failed to Marshal", base) + + bufff, err := json.Marshal(ff) + require.NoError(t, err, "ff[%T] failed to Marshal", ff) + + var baseD map[string]interface{} + var ffD map[string]interface{} + + err = json.Unmarshal(bufbase, &baseD) + require.NoError(t, err, "ff[%T] failed to Unmarshal", base) + + err = json.Unmarshal(bufff, &ffD) + require.NoError(t, err, "ff[%T] failed to Unmarshal", ff) + + require.Equal(t, baseD, ffD, "Inspected struct difference of base[%T] != ff[%T]", base, ff) + } +} + +func TestFuzzReType(t *testing.T) { + testTypeFuzzN(t, &TReTyped{}, &XReTyped{}, 100) +} diff --git a/tests/go.stripe/base/customer.go b/tests/go.stripe/base/customer.go new file mode 100644 index 0000000..de6f71e --- /dev/null +++ b/tests/go.stripe/base/customer.go @@ -0,0 +1,199 @@ +package stripe + +import ( + "time" +) + +// Customer encapsulates details about a Customer registered in Stripe. +// +// see https://stripe.com/docs/api#customer_object +type Customer struct { + ID string `json:"id"` + Desc string `json:"description,omitempty"` + Email string `json:"email,omitempty"` + Created int64 `json:"created"` + Balance int64 `json:"account_balance"` + Delinquent bool `json:"delinquent"` + Cards CardData `json:"cards,omitempty"` + Discount *Discount `json:"discount,omitempty"` + Subscription *Subscription `json:"subscription,omitempty"` + Livemode bool `json:"livemode"` + DefaultCard string `json:"default_card"` +} + +// CardData detaiks about cards +type CardData struct { + Object string `json:"object"` + Count int `json:"count"` + URL string `json:"url"` + Data []*Card `json:"data"` +} + +// Credit Card Types accepted by the Stripe API. +const ( + AmericanExpress = "American Express" + DinersClub = "Diners Club" + Discover = "Discover" + JCB = "JCB" + MasterCard = "MasterCard" + Visa = "Visa" + UnknownCard = "Unknown" +) + +// Card represents details about a Credit Card entered into Stripe. +type Card struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Type string `json:"type"` + ExpMonth int `json:"exp_month"` + ExpYear int `json:"exp_year"` + Last4 string `json:"last4"` + Fingerprint string `json:"fingerprint"` + Country string `json:"country,omitempty"` + AddrUess1 string `json:"address_line1,omitempty"` + Address2 string `json:"address_line2,omitempty"` + AddressCountry string `json:"address_country,omitempty"` + AddressState string `json:"address_state,omitempty"` + AddressZip string `json:"address_zip,omitempty"` + AddressCity string `json:"address_city"` + AddressLine1Check string `json:"address_line1_check,omitempty"` + AddressZipCheck string `json:"address_zip_check,omitempty"` + CVCCheck string `json:"cvc_check,omitempty"` +} + +// Discount represents the actual application of a coupon to a particular +// customer. +// +// see https://stripe.com/docs/api#discount_object +type Discount struct { + ID string `json:"id"` + Customer string `json:"customer"` + Start int64 `json:"start"` + End int64 `json:"end"` + Coupon *Coupon `json:"coupon"` +} + +// Coupon represents percent-off discount you might want to apply to a customer. +// +// see https://stripe.com/docs/api#coupon_object +type Coupon struct { + ID string `json:"id"` + Duration string `json:"duration"` + PercentOff int `json:"percent_off"` + DurationInMonths int `json:"duration_in_months,omitempty"` + MaxRedemptions int `json:"max_redemptions,omitempty"` + RedeemBy int64 `json:"redeem_by,omitempty"` + TimesRedeemed int `json:"times_redeemed,omitempty"` + Livemode bool `json:"livemode"` +} + +// Subscription Statuses +const ( + SubscriptionTrialing = "trialing" + SubscriptionActive = "active" + SubscriptionPastDue = "past_due" + SubscriptionCanceled = "canceled" + SubscriptionUnpaid = "unpaid" +) + +// Subscription represents a recurring charge a customer's card. +// +// see https://stripe.com/docs/api#subscription_object +type Subscription struct { + Customer string `json:"customer"` + Status string `json:"status"` + Plan *Plan `json:"plan"` + Start int64 `json:"start"` + EndedAt int64 `json:"ended_at"` + CurrentPeriodStart int64 `json:"current_period_start"` + CurrentPeriodEnd int64 `json:"current_period_end"` + TrialStart int64 `json:"trial_start"` + TrialEnd int64 `json:"trial_end"` + CanceledAt int64 `json:"canceled_at"` + CancelAtPeriodEnd bool `json:"cancel_at_period_end"` + Quantity int64 `json:"quantity"` +} + +// Plan holds details about pricing information for different products and +// feature levels on your site. For example, you might have a $10/month plan +// for basic features and a different $20/month plan for premium features. +// +// see https://stripe.com/docs/api#plan_object +type Plan struct { + ID string `json:"id"` + Name string `json:"name"` + Amount int64 `json:"amount"` + Interval string `json:"interval"` + IntervalCount int `json:"interval_count"` + Currency string `json:"currency"` + TrialPeriodDays int `json:"trial_period_days"` + Livemode bool `json:"livemode"` +} + +// NewCustomer creates a new customer +func NewCustomer() *Customer { + + return &Customer{ + ID: "hooN5ne7ug", + Desc: "A very nice customer.", + Email: "customer@example.com", + Created: time.Now().UnixNano(), + Balance: 10, + Delinquent: false, + Cards: CardData{ + Object: "A92F4CFE-8B6B-4176-873E-887AC0D120EB", + Count: 1, + URL: "https://stripe.example.com/card/A92F4CFE-8B6B-4176-873E-887AC0D120EB", + Data: []*Card{ + &Card{ + Name: "John Smith", + ID: "7526EC97-A0B6-47B2-AAE5-17443626A116", + Fingerprint: "4242424242424242", + ExpYear: time.Now().Year() + 1, + ExpMonth: 1, + }, + }, + }, + Discount: &Discount{ + ID: "Ee9ieZ8zie", + Customer: "hooN5ne7ug", + Start: time.Now().UnixNano(), + End: time.Now().UnixNano(), + Coupon: &Coupon{ + ID: "ieQuo5Aiph", + Duration: "2m", + PercentOff: 10, + DurationInMonths: 2, + MaxRedemptions: 1, + RedeemBy: time.Now().UnixNano(), + TimesRedeemed: 1, + Livemode: true, + }, + }, + Subscription: &Subscription{ + Customer: "hooN5ne7ug", + Status: SubscriptionActive, + Plan: &Plan{ + ID: "gaiyeLua5u", + Name: "Great Plan (TM)", + Amount: 10, + Interval: "monthly", + IntervalCount: 3, + Currency: "USD", + TrialPeriodDays: 15, + Livemode: true, + }, + Start: time.Now().UnixNano(), + EndedAt: 0, + CurrentPeriodStart: time.Now().UnixNano(), + CurrentPeriodEnd: time.Now().UnixNano(), + TrialStart: time.Now().UnixNano(), + TrialEnd: time.Now().UnixNano(), + CanceledAt: 0, + CancelAtPeriodEnd: false, + Quantity: 2, + }, + Livemode: true, + DefaultCard: "7526EC97-A0B6-47B2-AAE5-17443626A116", + } +} diff --git a/tests/go.stripe/ff/customer.go b/tests/go.stripe/ff/customer.go new file mode 100644 index 0000000..fff5a22 --- /dev/null +++ b/tests/go.stripe/ff/customer.go @@ -0,0 +1,199 @@ +package stripe + +import ( + "time" +) + +// Customer encapsulates details about a Customer registered in Stripe. +// +// see https://stripe.com/docs/api#customer_object +type Customer struct { + ID string `json:"id"` + Desc string `json:"description,omitempty"` + Email string `json:"email,omitempty"` + Created int64 `json:"created"` + Balance int64 `json:"account_balance"` + Delinquent bool `json:"delinquent"` + Cards CardData `json:"cards,omitempty"` + Discount *Discount `json:"discount,omitempty"` + Subscription *Subscription `json:"subscription,omitempty"` + Livemode bool `json:"livemode"` + DefaultCard string `json:"default_card"` +} + +// CardData struct +type CardData struct { + Object string `json:"object"` + Count int `json:"count"` + URL string `json:"url"` + Data []*Card `json:"data"` +} + +// Credit Card Types accepted by the Stripe API. +const ( + AmericanExpress = "American Express" + DinersClub = "Diners Club" + Discover = "Discover" + JCB = "JCB" + MasterCard = "MasterCard" + Visa = "Visa" + UnknownCard = "Unknown" +) + +// Card represents details about a Credit Card entered into Stripe. +type Card struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Type string `json:"type"` + ExpMonth int `json:"exp_month"` + ExpYear int `json:"exp_year"` + Last4 string `json:"last4"` + Fingerprint string `json:"fingerprint"` + Country string `json:"country,omitempty"` + AddrUess1 string `json:"address_line1,omitempty"` + Address2 string `json:"address_line2,omitempty"` + AddressCountry string `json:"address_country,omitempty"` + AddressState string `json:"address_state,omitempty"` + AddressZip string `json:"address_zip,omitempty"` + AddressCity string `json:"address_city"` + AddressLine1Check string `json:"address_line1_check,omitempty"` + AddressZipCheck string `json:"address_zip_check,omitempty"` + CVCCheck string `json:"cvc_check,omitempty"` +} + +// Discount represents the actual application of a coupon to a particular +// customer. +// +// see https://stripe.com/docs/api#discount_object +type Discount struct { + ID string `json:"id"` + Customer string `json:"customer"` + Start int64 `json:"start"` + End int64 `json:"end"` + Coupon *Coupon `json:"coupon"` +} + +// Coupon represents percent-off discount you might want to apply to a customer. +// +// see https://stripe.com/docs/api#coupon_object +type Coupon struct { + ID string `json:"id"` + Duration string `json:"duration"` + PercentOff int `json:"percent_off"` + DurationInMonths int `json:"duration_in_months,omitempty"` + MaxRedemptions int `json:"max_redemptions,omitempty"` + RedeemBy int64 `json:"redeem_by,omitempty"` + TimesRedeemed int `json:"times_redeemed,omitempty"` + Livemode bool `json:"livemode"` +} + +// Subscription Statuses +const ( + SubscriptionTrialing = "trialing" + SubscriptionActive = "active" + SubscriptionPastDue = "past_due" + SubscriptionCanceled = "canceled" + SubscriptionUnpaid = "unpaid" +) + +// Subscription represents a recurring charge a customer's card. +// +// see https://stripe.com/docs/api#subscription_object +type Subscription struct { + Customer string `json:"customer"` + Status string `json:"status"` + Plan *Plan `json:"plan"` + Start int64 `json:"start"` + EndedAt int64 `json:"ended_at"` + CurrentPeriodStart int64 `json:"current_period_start"` + CurrentPeriodEnd int64 `json:"current_period_end"` + TrialStart int64 `json:"trial_start"` + TrialEnd int64 `json:"trial_end"` + CanceledAt int64 `json:"canceled_at"` + CancelAtPeriodEnd bool `json:"cancel_at_period_end"` + Quantity int64 `json:"quantity"` +} + +// Plan holds details about pricing information for different products and +// feature levels on your site. For example, you might have a $10/month plan +// for basic features and a different $20/month plan for premium features. +// +// see https://stripe.com/docs/api#plan_object +type Plan struct { + ID string `json:"id"` + Name string `json:"name"` + Amount int64 `json:"amount"` + Interval string `json:"interval"` + IntervalCount int `json:"interval_count"` + Currency string `json:"currency"` + TrialPeriodDays int `json:"trial_period_days"` + Livemode bool `json:"livemode"` +} + +// NewCustomer creates a customer +func NewCustomer() *Customer { + + return &Customer{ + ID: "hooN5ne7ug", + Desc: "A very nice customer.", + Email: "customer@example.com", + Created: time.Now().UnixNano(), + Balance: 10, + Delinquent: false, + Cards: CardData{ + Object: "A92F4CFE-8B6B-4176-873E-887AC0D120EB", + Count: 1, + URL: "https://stripe.example.com/card/A92F4CFE-8B6B-4176-873E-887AC0D120EB", + Data: []*Card{ + &Card{ + Name: "John Smith", + ID: "7526EC97-A0B6-47B2-AAE5-17443626A116", + Fingerprint: "4242424242424242", + ExpYear: time.Now().Year() + 1, + ExpMonth: 1, + }, + }, + }, + Discount: &Discount{ + ID: "Ee9ieZ8zie", + Customer: "hooN5ne7ug", + Start: time.Now().UnixNano(), + End: time.Now().UnixNano(), + Coupon: &Coupon{ + ID: "ieQuo5Aiph", + Duration: "2m", + PercentOff: 10, + DurationInMonths: 2, + MaxRedemptions: 1, + RedeemBy: time.Now().UnixNano(), + TimesRedeemed: 1, + Livemode: true, + }, + }, + Subscription: &Subscription{ + Customer: "hooN5ne7ug", + Status: SubscriptionActive, + Plan: &Plan{ + ID: "gaiyeLua5u", + Name: "Great Plan (TM)", + Amount: 10, + Interval: "monthly", + IntervalCount: 3, + Currency: "USD", + TrialPeriodDays: 15, + Livemode: true, + }, + Start: time.Now().UnixNano(), + EndedAt: 0, + CurrentPeriodStart: time.Now().UnixNano(), + CurrentPeriodEnd: time.Now().UnixNano(), + TrialStart: time.Now().UnixNano(), + TrialEnd: time.Now().UnixNano(), + CanceledAt: 0, + CancelAtPeriodEnd: false, + Quantity: 2, + }, + Livemode: true, + DefaultCard: "7526EC97-A0B6-47B2-AAE5-17443626A116", + } +} diff --git a/tests/go.stripe/stripe_test.go b/tests/go.stripe/stripe_test.go new file mode 100644 index 0000000..eea77b8 --- /dev/null +++ b/tests/go.stripe/stripe_test.go @@ -0,0 +1,118 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package goser + +import ( + "encoding/json" + base "github.com/pquerna/ffjson/tests/go.stripe/base" + ff "github.com/pquerna/ffjson/tests/go.stripe/ff" + "testing" +) + +func TestRoundTrip(t *testing.T) { + var customerTripped ff.Customer + customer := ff.NewCustomer() + + buf1, err := json.Marshal(&customer) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + err = json.Unmarshal(buf1, &customerTripped) + if err != nil { + print(string(buf1)) + t.Fatalf("Unmarshal: %v", err) + } +} + +func BenchmarkMarshalJSON(b *testing.B) { + cust := base.NewCustomer() + + buf, err := json.Marshal(&cust) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := json.Marshal(&cust) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + } +} + +func BenchmarkFFMarshalJSON(b *testing.B) { + cust := ff.NewCustomer() + + buf, err := cust.MarshalJSON() + if err != nil { + b.Fatalf("Marshal: %v", err) + } + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := cust.MarshalJSON() + if err != nil { + b.Fatalf("Marshal: %v", err) + } + } +} + +type fatalF interface { + Fatalf(format string, args ...interface{}) +} + +func getBaseData(b fatalF) []byte { + cust := base.NewCustomer() + buf, err := json.MarshalIndent(&cust, "", " ") + if err != nil { + b.Fatalf("Marshal: %v", err) + } + return buf +} + +func BenchmarkUnmarshalJSON(b *testing.B) { + rec := base.Customer{} + buf := getBaseData(b) + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := json.Unmarshal(buf, &rec) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + } +} + +func BenchmarkFFUnmarshalJSON(b *testing.B) { + rec := ff.Customer{} + buf := getBaseData(b) + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := rec.UnmarshalJSON(buf) + if err != nil { + b.Fatalf("UnmarshalJSON: %v", err) + } + } +} diff --git a/tests/goser/base/goser.go b/tests/goser/base/goser.go new file mode 100644 index 0000000..afc165a --- /dev/null +++ b/tests/goser/base/goser.go @@ -0,0 +1,208 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package goser + +import ( + "io" + "net" + "time" +) + +// CacheStatus of goser +type CacheStatus int32 + +const ( + // CACHESTATUSUNKNOWN unknown cache status + CACHESTATUSUNKNOWN CacheStatus = 0 + // CACHESTATUSMISS miss cache status + CACHESTATUSMISS CacheStatus = 1 + // CACHESTATUSEXPIRED exipred cache status + CACHESTATUSEXPIRED CacheStatus = 2 + // CACHESTATUSHIT hit cache status + CACHESTATUSHIT CacheStatus = 3 +) + +// HTTPProtocol of goser +type HTTPProtocol int32 + +const ( + // HTTPPROTOCOLUNKNOWN http protocol unknown + HTTPPROTOCOLUNKNOWN HTTPProtocol = 0 + // HTTPPROTOCOL10 http protocol 10 + HTTPPROTOCOL10 HTTPProtocol = 1 + // HTTPPROTOCOL11 http protocol 11 + HTTPPROTOCOL11 HTTPProtocol = 2 +) + +// HTTPMethod of goser +type HTTPMethod int32 + +const ( + // HTTPMETHODUNKNOWN unknown http method + HTTPMETHODUNKNOWN HTTPMethod = 0 + // HTTPMETHODGET get http method + HTTPMETHODGET HTTPMethod = 1 + // HTTPMETHODPOST post http method + HTTPMETHODPOST HTTPMethod = 2 + // HTTPMETHODDELETE delete http method + HTTPMETHODDELETE HTTPMethod = 3 + // HTTPMETHODPUT put http method + HTTPMETHODPUT HTTPMethod = 4 + // HTTPMETHODHEAD head http method + HTTPMETHODHEAD HTTPMethod = 5 + // HTTPMETHODPURGE purge http method + HTTPMETHODPURGE HTTPMethod = 6 + // HTTPMETHODOPTIONS options http method + HTTPMETHODOPTIONS HTTPMethod = 7 + // HTTPMETHODPROPFIND propfind http method + HTTPMETHODPROPFIND HTTPMethod = 8 + // HTTPMETHODMKCOL mkcol http method + HTTPMETHODMKCOL HTTPMethod = 9 + // HTTPMETHODPATCH patch http method + HTTPMETHODPATCH HTTPMethod = 10 +) + +// OriginProtocol type +type OriginProtocol int32 + +const ( + // ORIGINPROTOCOLUNKNOWN origin protocol unknown + ORIGINPROTOCOLUNKNOWN OriginProtocol = 0 + // ORIGINPROTOCOLHTTP origin protocol http + ORIGINPROTOCOLHTTP OriginProtocol = 1 + // ORIGINPROTOCOLHTTPS origin protocol https + ORIGINPROTOCOLHTTPS OriginProtocol = 2 +) + +// HTTP struct type +type HTTP struct { + Protocol HTTPProtocol `json:"protocol"` + Status uint32 `json:"status"` + HostStatus uint32 `json:"hostStatus"` + UpStatus uint32 `json:"upStatus"` + Method HTTPMethod `json:"method"` + ContentType string `json:"contentType"` + UserAgent string `json:"userAgent"` + Referer string `json:"referer"` + RequestURI string `json:"requestURI"` + Unrecognized []byte `json:"-"` +} + +// Origin struct +type Origin struct { + IP IP `json:"ip"` + Port uint32 `json:"port"` + Hostname string `json:"hostname"` + Protocol OriginProtocol `json:"protocol"` +} + +// ZonePlan type +type ZonePlan int32 + +const ( + // ZONEPLANUNKNOWN unknwon zone plan + ZONEPLANUNKNOWN ZonePlan = 0 + // ZONEPLANFREE free zone plan + ZONEPLANFREE ZonePlan = 1 + // ZONEPLANPRO pro zone plan + ZONEPLANPRO ZonePlan = 2 + // ZONEPLANBIZ biz zone plan + ZONEPLANBIZ ZonePlan = 3 + // ZONEPLANENT ent zone plan + ZONEPLANENT ZonePlan = 4 +) + +// Country type +type Country int32 + +const ( + // COUNTRYUNKNOWN unknwon country + COUNTRYUNKNOWN Country = 0 + // COUNTRYUS us country + COUNTRYUS Country = 238 +) + +// Log struct +type Log struct { + Timestamp int64 `json:"timestamp"` + ZoneID uint32 `json:"zoneId"` + ZonePlan ZonePlan `json:"zonePlan"` + HTTP HTTP `json:"http"` + Origin Origin `json:"origin"` + Country Country `json:"country"` + CacheStatus CacheStatus `json:"cacheStatus"` + ServerIP IP `json:"serverIp"` + ServerName string `json:"serverName"` + RemoteIP IP `json:"remoteIp"` + BytesDlv uint64 `json:"bytesDlv"` + RayID string `json:"rayId"` + Unrecognized []byte `json:"-"` +} + +// IP type +type IP net.IP + +// MarshalJSON function +func (ip IP) MarshalJSON() ([]byte, error) { + return []byte("\"" + net.IP(ip).String() + "\""), nil +} + +// UnmarshalJSON function +func (ip *IP) UnmarshalJSON(data []byte) error { + if len(data) < 2 { + return io.ErrShortBuffer + } + *ip = IP(net.ParseIP(string(data[1 : len(data)-1])).To4()) + return nil +} + +const userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.146 Safari/537.36" + +// NewLog creates a new log +func NewLog(record *Log) { + record.Timestamp = time.Now().UnixNano() + record.ZoneID = 123456 + record.ZonePlan = ZONEPLANFREE + + record.HTTP = HTTP{ + Protocol: HTTPPROTOCOL11, + Status: 200, + HostStatus: 503, + UpStatus: 520, + Method: HTTPMETHODGET, + ContentType: "text/html", + UserAgent: userAgent, + Referer: "https://www.cloudflare.com/", + RequestURI: "/cdn-cgi/trace", + } + + record.Origin = Origin{ + IP: IP(net.IPv4(1, 2, 3, 4).To4()), + Port: 8080, + Hostname: "www.example.com", + Protocol: ORIGINPROTOCOLHTTPS, + } + + record.Country = COUNTRYUS + record.CacheStatus = CACHESTATUSHIT + record.ServerIP = IP(net.IPv4(192, 168, 1, 1).To4()) + record.ServerName = "metal.cloudflare.com" + record.RemoteIP = IP(net.IPv4(10, 1, 2, 3).To4()) + record.BytesDlv = 123456 + record.RayID = "10c73629cce30078-LAX" +} diff --git a/tests/goser/ff/goser.go b/tests/goser/ff/goser.go new file mode 100644 index 0000000..d3e731d --- /dev/null +++ b/tests/goser/ff/goser.go @@ -0,0 +1,218 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package goser + +import ( + fflib "github.com/pquerna/ffjson/fflib/v1" + + "io" + "net" + "time" +) + +// CacheStatus of goser +type CacheStatus int32 + +const ( + // CACHESTATUSUNKNOWN unknown cache status + CACHESTATUSUNKNOWN CacheStatus = 0 + // CACHESTATUSMISS miss cache status + CACHESTATUSMISS CacheStatus = 1 + // CACHESTATUSEXPIRED exipred cache status + CACHESTATUSEXPIRED CacheStatus = 2 + // CACHESTATUSHIT hit cache status + CACHESTATUSHIT CacheStatus = 3 +) + +// HTTPProtocol of goser +type HTTPProtocol int32 + +const ( + // HTTPPROTOCOLUNKNOWN http protocol unknown + HTTPPROTOCOLUNKNOWN HTTPProtocol = 0 + // HTTPPROTOCOL10 http protocol 10 + HTTPPROTOCOL10 HTTPProtocol = 1 + // HTTPPROTOCOL11 http protocol 11 + HTTPPROTOCOL11 HTTPProtocol = 2 +) + +// HTTPMethod of goser +type HTTPMethod int32 + +const ( + // HTTPMETHODUNKNOWN unknown http method + HTTPMETHODUNKNOWN HTTPMethod = 0 + // HTTPMETHODGET get http method + HTTPMETHODGET HTTPMethod = 1 + // HTTPMETHODPOST post http method + HTTPMETHODPOST HTTPMethod = 2 + // HTTPMETHODDELETE delete http method + HTTPMETHODDELETE HTTPMethod = 3 + // HTTPMETHODPUT put http method + HTTPMETHODPUT HTTPMethod = 4 + // HTTPMETHODHEAD head http method + HTTPMETHODHEAD HTTPMethod = 5 + // HTTPMETHODPURGE purge http method + HTTPMETHODPURGE HTTPMethod = 6 + // HTTPMETHODOPTIONS options http method + HTTPMETHODOPTIONS HTTPMethod = 7 + // HTTPMETHODPROPFIND propfind http method + HTTPMETHODPROPFIND HTTPMethod = 8 + // HTTPMETHODMKCOL mkcol http method + HTTPMETHODMKCOL HTTPMethod = 9 + // HTTPMETHODPATCH patch http method + HTTPMETHODPATCH HTTPMethod = 10 +) + +// OriginProtocol type +type OriginProtocol int32 + +const ( + // ORIGINPROTOCOLUNKNOWN origin protocol unknown + ORIGINPROTOCOLUNKNOWN OriginProtocol = 0 + // ORIGINPROTOCOLHTTP origin protocol http + ORIGINPROTOCOLHTTP OriginProtocol = 1 + // ORIGINPROTOCOLHTTPS origin protocol https + ORIGINPROTOCOLHTTPS OriginProtocol = 2 +) + +// HTTP struct type +type HTTP struct { + Protocol HTTPProtocol `json:"protocol"` + Status uint32 `json:"status"` + HostStatus uint32 `json:"hostStatus"` + UpStatus uint32 `json:"upStatus"` + Method HTTPMethod `json:"method"` + ContentType string `json:"contentType"` + UserAgent string `json:"userAgent"` + Referer string `json:"referer"` + RequestURI string `json:"requestURI"` + Unrecognized []byte `json:"-"` +} + +// Origin struct +type Origin struct { + IP IP `json:"ip"` + Port uint32 `json:"port"` + Hostname string `json:"hostname"` + Protocol OriginProtocol `json:"protocol"` +} + +// ZonePlan type +type ZonePlan int32 + +const ( + // ZONEPLANUNKNOWN unknwon zone plan + ZONEPLANUNKNOWN ZonePlan = 0 + // ZONEPLANFREE free zone plan + ZONEPLANFREE ZonePlan = 1 + // ZONEPLANPRO pro zone plan + ZONEPLANPRO ZonePlan = 2 + // ZONEPLANBIZ biz zone plan + ZONEPLANBIZ ZonePlan = 3 + // ZONEPLANENT ent zone plan + ZONEPLANENT ZonePlan = 4 +) + +// Country type +type Country int32 + +const ( + // COUNTRYUNKNOWN unknwon country + COUNTRYUNKNOWN Country = 0 + // COUNTRYUS us country + COUNTRYUS Country = 238 +) + +// Log struct +type Log struct { + Timestamp int64 `json:"timestamp"` + ZoneID uint32 `json:"zoneId"` + ZonePlan ZonePlan `json:"zonePlan"` + HTTP HTTP `json:"http"` + Origin Origin `json:"origin"` + Country Country `json:"country"` + CacheStatus CacheStatus `json:"cacheStatus"` + ServerIP IP `json:"serverIp"` + ServerName string `json:"serverName"` + RemoteIP IP `json:"remoteIp"` + BytesDlv uint64 `json:"bytesDlv"` + RayID string `json:"rayId"` + Unrecognized []byte `json:"-"` +} + +// IP type +type IP net.IP + +// MarshalJSON set ip to json +func (ip IP) MarshalJSON() ([]byte, error) { + return []byte("\"" + net.IP(ip).String() + "\""), nil +} + +// MarshalJSONBuf set ip to json with buf +func (ip IP) MarshalJSONBuf(buf fflib.EncodingBuffer) error { + buf.WriteByte('"') + buf.WriteString(net.IP(ip).String()) + buf.WriteByte('"') + return nil +} + +// UnmarshalJSON umarshall json to ip +func (ip *IP) UnmarshalJSON(data []byte) error { + if len(data) < 2 { + return io.ErrShortBuffer + } + *ip = IP(net.ParseIP(string(data[1 : len(data)-1])).To4()) + return nil +} + +const userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.146 Safari/537.36" + +// NewLog creates a new log +func NewLog(record *Log) { + record.Timestamp = time.Now().UnixNano() + record.ZoneID = 123456 + record.ZonePlan = ZONEPLANFREE + + record.HTTP = HTTP{ + Protocol: HTTPPROTOCOL11, + Status: 200, + HostStatus: 503, + UpStatus: 520, + Method: HTTPMETHODGET, + ContentType: "text/html", + UserAgent: userAgent, + Referer: "https://www.cloudflare.com/", + RequestURI: "/cdn-cgi/trace", + } + + record.Origin = Origin{ + IP: IP(net.IPv4(1, 2, 3, 4).To4()), + Port: 8080, + Hostname: "www.example.com", + Protocol: ORIGINPROTOCOLHTTPS, + } + + record.Country = COUNTRYUS + record.CacheStatus = CACHESTATUSHIT + record.ServerIP = IP(net.IPv4(192, 168, 1, 1).To4()) + record.ServerName = "metal.cloudflare.com" + record.RemoteIP = IP(net.IPv4(10, 1, 2, 3).To4()) + record.BytesDlv = 123456 + record.RayID = "10c73629cce30078-LAX" +} diff --git a/tests/goser/goser_test.go b/tests/goser/goser_test.go new file mode 100644 index 0000000..a8414de --- /dev/null +++ b/tests/goser/goser_test.go @@ -0,0 +1,146 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package goser + +import ( + "encoding/json" + "fmt" + base "github.com/pquerna/ffjson/tests/goser/base" + ff "github.com/pquerna/ffjson/tests/goser/ff" + "reflect" + "testing" +) + +func TestRoundTrip(t *testing.T) { + var record ff.Log + var recordTripped ff.Log + ff.NewLog(&record) + + buf1, err := json.Marshal(&record) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + err = json.Unmarshal(buf1, &recordTripped) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + good := reflect.DeepEqual(record, recordTripped) + if !good { + t.Fatalf("Expected: %v\n Got: %v", record, recordTripped) + } +} + +func BenchmarkMarshalJSON(b *testing.B) { + var record base.Log + base.NewLog(&record) + + buf, err := json.Marshal(&record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := json.Marshal(&record) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + } +} + +func BenchmarkFFMarshalJSON(b *testing.B) { + var record ff.Log + ff.NewLog(&record) + + buf, err := record.MarshalJSON() + if err != nil { + b.Fatalf("Marshal: %v", err) + } + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := record.MarshalJSON() + if err != nil { + b.Fatalf("Marshal: %v", err) + } + } +} + +type fatalF interface { + Fatalf(format string, args ...interface{}) +} + +func getBaseData(b fatalF) []byte { + var record base.Log + base.NewLog(&record) + buf, err := json.MarshalIndent(&record, "", " ") + if err != nil { + b.Fatalf("Marshal: %v", err) + } + return buf +} + +func BenchmarkUnmarshalJSON(b *testing.B) { + rec := base.Log{} + buf := getBaseData(b) + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := json.Unmarshal(buf, &rec) + if err != nil { + b.Fatalf("Marshal: %v", err) + } + } +} + +func BenchmarkFFUnmarshalJSON(b *testing.B) { + rec := ff.Log{} + buf := getBaseData(b) + b.SetBytes(int64(len(buf))) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := rec.UnmarshalJSON(buf) + if err != nil { + b.Fatalf("UnmarshalJSON: %v", err) + } + } +} + +func TestUnmarshal(t *testing.T) { + rec := ff.Log{} + buf := getBaseData(t) + + err := rec.UnmarshalJSON(buf) + if err != nil { + t.Fatalf("Unmarshal: %v from %s", err, string(buf)) + } + + rec2 := base.Log{} + json.Unmarshal(buf, &rec2) + + a := fmt.Sprintf("%v", rec) + b := fmt.Sprintf("%v", rec2) + if a != b { + t.Fatalf("Expected: %v\n Got: %v\n from: %s", rec2, rec, string(buf)) + } +} diff --git a/tests/number/ff/number.go b/tests/number/ff/number.go new file mode 100644 index 0000000..e219716 --- /dev/null +++ b/tests/number/ff/number.go @@ -0,0 +1,34 @@ +/** + * Copyright 2016 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ff + +import ( + "encoding/json" +) + +// Number struct +type Number struct { + Int json.Number + Float json.Number +} + +// NewNumber creates a new number +func NewNumber(e *Number) { + e.Int = "1" + e.Float = "3.14" +} diff --git a/tests/number/number_test.go b/tests/number/number_test.go new file mode 100644 index 0000000..f76ff4c --- /dev/null +++ b/tests/number/number_test.go @@ -0,0 +1,70 @@ +/** + * Copyright 2016 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package types + +import ( + "encoding/json" + "reflect" + "testing" + + ff "github.com/pquerna/ffjson/tests/number/ff" +) + +func TestRoundTrip(t *testing.T) { + var record ff.Number + var recordTripped ff.Number + ff.NewNumber(&record) + + buf1, err := json.Marshal(&record) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + err = json.Unmarshal(buf1, &recordTripped) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + good := reflect.DeepEqual(record, recordTripped) + if !good { + t.Fatalf("Expected: %v\n Got: %v", record, recordTripped) + } +} + +func TestUnmarshalEmpty(t *testing.T) { + record := ff.Number{} + err := record.UnmarshalJSON([]byte(`{}`)) + if err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } +} + +const ( + numberJSON = `{ + "Int": 1, + "Float": 3.14 +}` +) + +func TestUnmarshalFull(t *testing.T) { + record := ff.Number{} + err := record.UnmarshalJSON([]byte(numberJSON)) + if err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } +} diff --git a/tests/t.cmd b/tests/t.cmd new file mode 100644 index 0000000..2aec0cb --- /dev/null +++ b/tests/t.cmd @@ -0,0 +1,12 @@ +go install github.com/pquerna/ffjson +del ff_ffjson.go +del goser\ff\goser_ffjson.go +del go.stripe\ff\customer_ffjson.go +del types\ff\everything_ffjson.go + +go test -v github.com/pquerna/ffjson/fflib/v1 github.com/pquerna/ffjson/generator github.com/pquerna/ffjson/inception && ffjson ff.go && go test -v +ffjson goser/ff/goser.go && go test github.com/pquerna/ffjson/tests/goser +ffjson go.stripe/ff/customer.go && go test github.com/pquerna/ffjson/tests/go.stripe +ffjson types/ff/everything.go && go test github.com/pquerna/ffjson/tests/types + + diff --git a/tests/t.sh b/tests/t.sh new file mode 100755 index 0000000..53f0a40 --- /dev/null +++ b/tests/t.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +make -C .. +ffjson ff.go + +# +# https://twitter.com/jpetazzo/status/446476354930757632/photo/1 +# + +go test -benchmem -bench MarshalJSON +go test -benchmem -bench MarshalJSONNative -cpuprofile="prof.dat" -benchtime 10s +go tool pprof -gif tests.test prof.dat >out.gif diff --git a/tests/types/ff/everything.go b/tests/types/ff/everything.go new file mode 100644 index 0000000..2c85838 --- /dev/null +++ b/tests/types/ff/everything.go @@ -0,0 +1,184 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ff + +import ( + "regexp" + "runtime" + "strconv" + + "github.com/foo/vendored" +) + +// ExpectedSomethingValue maybe expects something of value +var ExpectedSomethingValue int8 + +// GoLangVersionPre16 indicates if golang before 1.6 +var GoLangVersionPre16 bool + +func init() { + // since go1.6 reflect package changed behaivour: + // + // -------- + // https://tip.golang.org/doc/go1.6 + // + // The reflect package has resolved a long-standing incompatibility between + // the gc and gccgo toolchains regarding embedded unexported struct types + // containing exported fields. Code that walks data structures using + // reflection, especially to implement serialization in the spirit of the + // encoding/json and encoding/xml packages, may need to be updated. + // + // The problem arises when using reflection to walk through an embedded + // unexported struct-typed field into an exported field of that struct. In + // this case, reflect had incorrectly reported the embedded field as exported, + // by returning an empty Field.PkgPath. Now it correctly reports the field as + // unexported but ignores that fact when evaluating access to exported fields + // contained within the struct. + // + // Updating: Typically, code that previously walked over structs and used + // + // f.PkgPath != "" + // to exclude inaccessible fields should now use + // + // f.PkgPath != "" && !f.Anonymous + // For example, see the changes to the implementations of encoding/json and + // encoding/xml. + // + // -------- + // + // I didn't find better option to get Go's version rather then parsing + // runtime.Version(). Godoc say that Version() can return multiple things: + // + // Version returns the Go tree's version string. It is either the commit + // hash and date at the time of the build or, when possible, a release tag + // like "go1.3". + // + // So, I'll assumes that if Version() returns not a release tag, running + // version is younger then 1.5. Patches welcome :-) + + versionRegexp := regexp.MustCompile("^go[0-9]+\\.([0-9]+)") + if res := versionRegexp.FindStringSubmatch(runtime.Version()); len(res) > 1 { + if i, _ := strconv.Atoi(res[1]); i < 6 { + // pre go1.6 + GoLangVersionPre16 = true + ExpectedSomethingValue = 99 + } + } +} + +// SweetInterface is a sweet interface +type SweetInterface interface { + Cats() int +} + +// Cats they allways fallback on their legs +type Cats struct { + FieldOnCats int +} + +// Cats initialize a cat +func (c *Cats) Cats() int { + return 42 +} + +// Embed structure +type Embed struct { + SuperBool bool +} + +// Everything a bit of everything... take care what yy-ou which for +type Everything struct { + Embed + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Uintptr uintptr + Float32 float32 + Float64 float64 + Array [2]int + Slice []int + SlicePointer *[]string + Map map[string]int + String string + StringPointer *string + Int64Pointer *int64 + FooStruct *Foo + MySweetInterface SweetInterface + MapMap map[string]map[string]string + MapArraySlice map[string][3][]int + nonexported +} + +type nonexported struct { + Something int8 +} + +// Foo a foo's structure (it's a bar !?!) +type Foo struct { + Bar int + Baz vendored.Foo +} + +// NewEverything kind of renew the world +func NewEverything(e *Everything) { + e.SuperBool = true + e.Bool = true + e.Int = 1 + e.Int8 = 2 + e.Int16 = 3 + e.Int32 = -4 + e.Int64 = 2 ^ 59 + e.Uint = 100 + e.Uint8 = 101 + e.Uint16 = 102 + e.Uint64 = 103 + e.Uintptr = 104 + e.Float32 = 3.14 + e.Float64 = 3.15 + e.Array = [2]int{11, 12} + e.Slice = []int{1, 2, 3} + e.SlicePointer = &[]string{"a", "b"} + e.Map = map[string]int{ + "foo": 1, + "bar": 2, + } + e.String = "snowman->☃" + e.FooStruct = &Foo{Bar: 1, Baz: vendored.Foo{A: "a", B: 1}} + e.Something = ExpectedSomethingValue + e.MySweetInterface = &Cats{} + e.MapMap = map[string]map[string]string{ + "a": map[string]string{"b": "2", "c": "3", "d": "4"}, + "e": map[string]string{}, + "f": map[string]string{"g": "9"}, + } + e.MapArraySlice = map[string][3][]int{ + "a": [3][]int{ + 0: []int{1, 2, 3}, + 1: []int{}, + 2: []int{4}, + }, + } +} diff --git a/tests/types/types_test.go b/tests/types/types_test.go new file mode 100644 index 0000000..6f1106d --- /dev/null +++ b/tests/types/types_test.go @@ -0,0 +1,188 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package types + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + ff "github.com/pquerna/ffjson/tests/types/ff" +) + +func TestRoundTrip(t *testing.T) { + var record ff.Everything + var recordTripped ff.Everything + ff.NewEverything(&record) + + buf1, err := json.Marshal(&record) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + recordTripped.MySweetInterface = &ff.Cats{} + err = json.Unmarshal(buf1, &recordTripped) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + good := reflect.DeepEqual(record.FooStruct, recordTripped.FooStruct) + if !good { + t.Fatalf("Expected: %v\n Got: %v", *record.FooStruct, *recordTripped.FooStruct) + } + + record.FooStruct = nil + recordTripped.FooStruct = nil + + good = reflect.DeepEqual(record, recordTripped) + if !good { + t.Fatalf("Expected: %v\n Got: %v", record, recordTripped) + } + + if recordTripped.SuperBool != true { + t.Fatal("Embeded struct didn't Unmarshal") + } + + if recordTripped.Something != ff.ExpectedSomethingValue { + t.Fatal("Embeded nonexported-struct didn't Unmarshal") + } +} + +func TestUnmarshalEmpty(t *testing.T) { + record := ff.Everything{} + err := record.UnmarshalJSON([]byte(`{}`)) + if err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } +} + +const ( + everythingJSON = `{ + "Bool": true, + "Int": 1, + "Int8": 2, + "Int16": 3, + "Int32": -4, + "Int64": 57, + "Uint": 100, + "Uint8": 101, + "Uint16": 102, + "Uint32": 0, + "Uint64": 103, + "Uintptr": 104, + "Float32": 3.14, + "Float64": 3.15, + "Array": [ + 1, + 2, + 3 + ], + "Map": { + "bar": 2, + "foo": 1 + }, + "String": "snowman☃\uD801\uDC37", + "StringPointer": null, + "Int64Pointer": null, + "FooStruct": { + "Bar": 1 + }, + "Something": 99 +}` +) + +func TestUnmarshalFull(t *testing.T) { + record := ff.Everything{} + // TODO(pquerna): add unicode snowman + // TODO(pquerna): handle Bar subtype + err := record.UnmarshalJSON([]byte(everythingJSON)) + if err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + + expect := "snowman☃𐐷" + if record.String != expect { + t.Fatalf("record.String decoding problem, expected: %v got: %v", expect, record.String) + } + + if record.Something != ff.ExpectedSomethingValue { + t.Fatalf("record.Something decoding problem, expected: %d got: %v", + ff.ExpectedSomethingValue, record.Something) + } +} + +func TestUnmarshalNullPointer(t *testing.T) { + record := ff.Everything{} + err := record.UnmarshalJSON([]byte(`{"FooStruct": null,"Something":99}`)) + if err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + if record.FooStruct != nil { + t.Fatalf("record.Something decoding problem, expected: nil got: %v", record.FooStruct) + } +} + +func TestUnmarshalToReusedObject(t *testing.T) { + JSONParts := []string{ + `"Bool":true`, + `"Int":1`, + `"Int8": 2`, + `"Int16": 3`, + `"Int32": -4`, + `"Int64": 57`, + `"Uint": 100`, + `"Uint8": 101`, + `"Uint16": 102`, + `"Uint32": 50`, + `"Uint64": 103`, + `"Uintptr": 104`, + `"Float32": 3.14`, + `"Float64": 3.15`, + `"Array": [1,2,3]`, + `"Map": {"bar": 2,"foo": 1}`, + `"String": "snowman☃\uD801\uDC37"`, + `"StringPointer": "pointed snowman☃\uD801\uDC37"`, + `"Int64Pointer": 44`, + `"FooStruct": {"Bar": 1}`, + `"MapMap": {"a0": {"b0":"foo"}, "a1":{"a2":"bar"}}`, + `"MapArraySlice": {"foo":[[1,2,3],[4,5,6],[7]], "bar": [[1,2,3,4],[5,6,7]]}`, + `"Something": 99`, + } + + JSONWhole := "{" + strings.Join(JSONParts, ",") + "}" + var record ff.Everything + if err := record.UnmarshalJSON([]byte(JSONWhole)); err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + + for _, part := range JSONParts { + reuseRecord := record + if err := reuseRecord.UnmarshalJSON([]byte("{" + part + "}")); err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + var emptyRecord ff.Everything + if err := emptyRecord.UnmarshalJSON([]byte("{" + part + "}")); err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + + if !reflect.DeepEqual(reuseRecord, emptyRecord) { + t.Errorf("%#v should be equal to %#v", reuseRecord, emptyRecord) + } + } +} |