upgrade buntdb and dependencies

This commit is contained in:
Shivaram Lingamneni 2020-11-08 17:55:22 -05:00
parent 025f062a43
commit 008416e4dd
21 changed files with 985 additions and 1566 deletions

2
go.mod

@ -16,7 +16,7 @@ require (
github.com/oragono/confusables v0.0.0-20190624102032-fe1cf31a24b0
github.com/oragono/go-ident v0.0.0-20200511222032-830550b1d775
github.com/stretchr/testify v1.4.0 // indirect
github.com/tidwall/buntdb v1.1.2
github.com/tidwall/buntdb v1.1.4
github.com/toorop/go-dkim v0.0.0-20200526084421-76378ae5207e
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f // indirect

10
go.sum

@ -42,18 +42,28 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY=
github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/buntdb v1.1.2 h1:noCrqQXL9EKMtcdwJcmuVKSEjqu1ua99RHHgbLTEHRo=
github.com/tidwall/buntdb v1.1.2/go.mod h1:xAzi36Hir4FarpSHyfuZ6JzPJdjRZ8QlLZSntE2mqlI=
github.com/tidwall/buntdb v1.1.4 h1:W7y9+2dM3GOswU0t3pz6+BcwZXjj/tVOhPcO6EHufME=
github.com/tidwall/buntdb v1.1.4/go.mod h1:06+/n7EFf6uUaIG5r9xZcExYN3H0Lnc+g/Kqx0fZFkI=
github.com/tidwall/gjson v1.3.4 h1:On5waDnyKKk3SWE4EthbjjirAWXp43xx5cKCUZY1eZw=
github.com/tidwall/gjson v1.3.4/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e h1:+NL1GDIUOKxVfbp2KoJQD9cTQ6dyP2co9q4yzmT9FZo=
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 h1:BsKSRhu0TDB6Snq8SutN9KQHc6vqHEXJTcAFwyGNius=
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
github.com/toorop/go-dkim v0.0.0-20200526084421-76378ae5207e h1:uZTp+hhFm+PCH0t0Px5oE+QYlVTwVJ+XKNQr7ct4Q7w=

@ -1,202 +1,18 @@
Copyright (c) 2020 Josh Baker
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -1,107 +1,5 @@
BTree implementation for Go
===========================
# B-tree for Go
![Travis CI Build Status](https://api.travis-ci.org/tidwall/btree.svg?branch=master)
[![GoDoc](https://godoc.org/github.com/tidwall/btree?status.svg)](https://godoc.org/github.com/tidwall/btree)
This package provides an in-memory B-Tree implementation for Go, useful as
an ordered, mutable data structure.
This is a fork of the wonderful [google/btree](https://github.com/google/btree) package. It's has all the same great features and adds a few more.
- Descend* functions for iterating backwards.
- Iteration performance boost.
- User defined context.
User defined context
--------------------
This is a great new feature that allows for entering the same item into multiple B-trees, and each B-tree have a different ordering formula.
For example:
```go
package main
import (
"fmt"
"github.com/tidwall/btree"
)
type Item struct {
Key, Val string
}
func (i1 *Item) Less(item btree.Item, ctx interface{}) bool {
i2 := item.(*Item)
switch tag := ctx.(type) {
case string:
if tag == "vals" {
if i1.Val < i2.Val {
return true
} else if i1.Val > i2.Val {
return false
}
// Both vals are equal so we should fall though
// and let the key comparison take over.
}
}
return i1.Key < i2.Key
}
func main() {
// Create a tree for keys and a tree for values.
// The "keys" tree will be sorted on the Keys field.
// The "values" tree will be sorted on the Values field.
keys := btree.New(16, "keys")
vals := btree.New(16, "vals")
// Create some items.
users := []*Item{
&Item{Key: "user:1", Val: "Jane"},
&Item{Key: "user:2", Val: "Andy"},
&Item{Key: "user:3", Val: "Steve"},
&Item{Key: "user:4", Val: "Andrea"},
&Item{Key: "user:5", Val: "Janet"},
&Item{Key: "user:6", Val: "Andy"},
}
// Insert each user into both trees
for _, user := range users {
keys.ReplaceOrInsert(user)
vals.ReplaceOrInsert(user)
}
// Iterate over each user in the key tree
keys.Ascend(func(item btree.Item) bool {
kvi := item.(*Item)
fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
return true
})
fmt.Printf("\n")
// Iterate over each user in the val tree
vals.Ascend(func(item btree.Item) bool {
kvi := item.(*Item)
fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
return true
})
}
// Should see the results
/*
user:1 Jane
user:2 Andy
user:3 Steve
user:4 Andrea
user:5 Janet
user:6 Andy
user:4 Andrea
user:2 Andy
user:6 Andy
user:1 Jane
user:3 Steve
*/
```

File diff suppressed because it is too large Load Diff

@ -1 +1,4 @@
language: go
go:
- 1.15.x

@ -576,7 +576,7 @@ var config buntdb.Config
if err := db.ReadConfig(&config); err != nil{
log.Fatal(err)
}
if err := db.WriteConfig(config); err != nil{
if err := db.SetConfig(config); err != nil{
log.Fatal(err)
}
```

@ -1,7 +1,7 @@
// Package buntdb implements a low-level in-memory key/value store in pure Go.
// It persists to disk, is ACID compliant, and uses locking for multiple
// readers and a single writer. Bunt is ideal for projects that need
// a dependable database, and favor speed over data size.
// readers and a single writer. Bunt is ideal for projects that need a
// dependable database, and favor speed over data size.
package buntdb
import (
@ -122,10 +122,11 @@ type Config struct {
// has been expired.
OnExpired func(keys []string)
// OnExpiredSync will be called inside the same transaction that is performing
// the deletion of expired items. If OnExpired is present then this callback
// will not be called. If this callback is present, then the deletion of the
// timeed-out item is the explicit responsibility of this callback.
// OnExpiredSync will be called inside the same transaction that is
// performing the deletion of expired items. If OnExpired is present then
// this callback will not be called. If this callback is present, then the
// deletion of the timeed-out item is the explicit responsibility of this
// callback.
OnExpiredSync func(key, value string, tx *Tx) error
}
@ -142,8 +143,8 @@ const btreeDegrees = 64
func Open(path string) (*DB, error) {
db := &DB{}
// initialize trees and indexes
db.keys = btree.New(btreeDegrees, nil)
db.exps = btree.New(btreeDegrees, &exctx{db})
db.keys = btree.New(lessCtx(nil))
db.exps = btree.New(lessCtx(&exctx{db}))
db.idxs = make(map[string]*index)
// initialize default configuration
db.config = Config{
@ -204,7 +205,7 @@ func (db *DB) Save(wr io.Writer) error {
// use a buffered writer and flush every 4MB
var buf []byte
// iterated through every item in the database and write to the buffer
db.keys.Ascend(func(item btree.Item) bool {
btreeAscend(db.keys, func(item interface{}) bool {
dbi := item.(*dbItem)
buf = dbi.writeSetTo(buf)
if len(buf) > 1024*1024*4 {
@ -285,7 +286,7 @@ func (idx *index) clearCopy() *index {
}
// initialize with empty trees
if nidx.less != nil {
nidx.btr = btree.New(btreeDegrees, nidx)
nidx.btr = btree.New(lessCtx(nidx))
}
if nidx.rect != nil {
nidx.rtr = rtree.New(nidx)
@ -297,20 +298,20 @@ func (idx *index) clearCopy() *index {
func (idx *index) rebuild() {
// initialize trees
if idx.less != nil {
idx.btr = btree.New(btreeDegrees, idx)
idx.btr = btree.New(lessCtx(idx))
}
if idx.rect != nil {
idx.rtr = rtree.New(idx)
}
// iterate through all keys and fill the index
idx.db.keys.Ascend(func(item btree.Item) bool {
btreeAscend(idx.db.keys, func(item interface{}) bool {
dbi := item.(*dbItem)
if !idx.match(dbi.key) {
// does not match the pattern, conintue
// does not match the pattern, continue
return true
}
if idx.less != nil {
idx.btr.ReplaceOrInsert(dbi)
idx.btr.Set(dbi)
}
if idx.rect != nil {
idx.rtr.Insert(dbi)
@ -456,7 +457,7 @@ func (db *DB) SetConfig(config Config) error {
// will be replaced with the new one, and return the previous item.
func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
var pdbi *dbItem
prev := db.keys.ReplaceOrInsert(item)
prev := db.keys.Set(item)
if prev != nil {
// A previous item was removed from the keys tree. Let's
// fully delete this item from all indexes.
@ -479,7 +480,7 @@ func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
if item.opts != nil && item.opts.ex {
// The new item has eviction options. Add it to the
// expires tree
db.exps.ReplaceOrInsert(item)
db.exps.Set(item)
}
for _, idx := range db.idxs {
if !idx.match(item.key) {
@ -487,7 +488,7 @@ func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
}
if idx.btr != nil {
// Add new item to btree index.
idx.btr.ReplaceOrInsert(item)
idx.btr.Set(item)
}
if idx.rtr != nil {
// Add new item to rtree index.
@ -557,9 +558,9 @@ func (db *DB) backgroundManager() {
}
}
// produce a list of expired items that need removing
db.exps.AscendLessThan(&dbItem{
btreeAscendLessThan(db.exps, &dbItem{
opts: &dbItemOpts{ex: true, exat: time.Now()},
}, func(item btree.Item) bool {
}, func(item interface{}) bool {
expired = append(expired, item.(*dbItem))
return true
})
@ -674,8 +675,8 @@ func (db *DB) Shrink() error {
}
done = true
var n int
db.keys.AscendGreaterOrEqual(&dbItem{key: pivot},
func(item btree.Item) bool {
btreeAscendGreaterOrEqual(db.keys, &dbItem{key: pivot},
func(item interface{}) bool {
dbi := item.(*dbItem)
// 1000 items or 64MB buffer
if n > 1000 || len(buf) > 64*1024*1024 {
@ -849,7 +850,7 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
if len(parts) == 0 {
continue
}
if (parts[0][0] == 's' || parts[0][1] == 'S') &&
if (parts[0][0] == 's' || parts[0][0] == 'S') &&
(parts[0][1] == 'e' || parts[0][1] == 'E') &&
(parts[0][2] == 't' || parts[0][2] == 'T') {
// SET
@ -879,7 +880,7 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
} else {
db.insertIntoDatabase(&dbItem{key: parts[1], val: parts[2]})
}
} else if (parts[0][0] == 'd' || parts[0][1] == 'D') &&
} else if (parts[0][0] == 'd' || parts[0][0] == 'D') &&
(parts[0][1] == 'e' || parts[0][1] == 'E') &&
(parts[0][2] == 'l' || parts[0][2] == 'L') {
// DEL
@ -887,10 +888,10 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
return ErrInvalid
}
db.deleteFromDatabase(&dbItem{key: parts[1]})
} else if (parts[0][0] == 'f' || parts[0][1] == 'F') &&
} else if (parts[0][0] == 'f' || parts[0][0] == 'F') &&
strings.ToLower(parts[0]) == "flushdb" {
db.keys = btree.New(btreeDegrees, nil)
db.exps = btree.New(btreeDegrees, &exctx{db})
db.keys = btree.New(lessCtx(nil))
db.exps = btree.New(lessCtx(&exctx{db}))
db.idxs = make(map[string]*index)
} else {
return ErrInvalid
@ -1025,8 +1026,8 @@ func (tx *Tx) DeleteAll() error {
}
// now reset the live database trees
tx.db.keys = btree.New(btreeDegrees, nil)
tx.db.exps = btree.New(btreeDegrees, &exctx{tx.db})
tx.db.keys = btree.New(lessCtx(nil))
tx.db.exps = btree.New(lessCtx(&exctx{tx.db}))
tx.db.idxs = make(map[string]*index)
// finally re-create the indexes
@ -1264,8 +1265,7 @@ func (dbi *dbItem) expiresAt() time.Time {
// to note that the ctx parameter is used to help with determine which
// formula to use on an item. Each b-tree should use a different ctx when
// sharing the same item.
func (dbi *dbItem) Less(item btree.Item, ctx interface{}) bool {
dbi2 := item.(*dbItem)
func (dbi *dbItem) Less(dbi2 *dbItem, ctx interface{}) bool {
switch ctx := ctx.(type) {
case *exctx:
// The expires b-tree formula
@ -1295,6 +1295,12 @@ func (dbi *dbItem) Less(item btree.Item, ctx interface{}) bool {
return dbi.key < dbi2.key
}
func lessCtx(ctx interface{}) func(a, b interface{}) bool {
return func(a, b interface{}) bool {
return a.(*dbItem).Less(b.(*dbItem), ctx)
}
}
// Rect converts a string to a rectangle.
// An invalid rectangle will cause a panic.
func (dbi *dbItem) Rect(ctx interface{}) (min, max []float64) {
@ -1498,7 +1504,7 @@ func (tx *Tx) scan(desc, gt, lt bool, index, start, stop string,
return ErrTxClosed
}
// wrap a btree specific iterator around the user-defined iterator.
iter := func(item btree.Item) bool {
iter := func(item interface{}) bool {
dbi := item.(*dbItem)
return iterator(dbi.key, dbi.val)
}
@ -1542,26 +1548,26 @@ func (tx *Tx) scan(desc, gt, lt bool, index, start, stop string,
if desc {
if gt {
if lt {
tr.DescendRange(itemA, itemB, iter)
btreeDescendRange(tr, itemA, itemB, iter)
} else {
tr.DescendGreaterThan(itemA, iter)
btreeDescendGreaterThan(tr, itemA, iter)
}
} else if lt {
tr.DescendLessOrEqual(itemA, iter)
btreeDescendLessOrEqual(tr, itemA, iter)
} else {
tr.Descend(iter)
btreeDescend(tr, iter)
}
} else {
if gt {
if lt {
tr.AscendRange(itemA, itemB, iter)
btreeAscendRange(tr, itemA, itemB, iter)
} else {
tr.AscendGreaterOrEqual(itemA, iter)
btreeAscendGreaterOrEqual(tr, itemA, iter)
}
} else if lt {
tr.AscendLessThan(itemA, iter)
btreeAscendLessThan(tr, itemA, iter)
} else {
tr.Ascend(iter)
btreeAscend(tr, iter)
}
}
return nil
@ -2014,7 +2020,8 @@ func (tx *Tx) createIndex(name string, pattern string,
if tx.wc.rbkeys == nil {
// store the index in the rollback map.
if _, ok := tx.wc.rollbackIndexes[name]; !ok {
// we use nil to indicate that the index should be removed upon rollback.
// we use nil to indicate that the index should be removed upon
// rollback.
tx.wc.rollbackIndexes[name] = nil
}
}
@ -2044,8 +2051,8 @@ func (tx *Tx) DropIndex(name string) error {
if tx.wc.rbkeys == nil {
// store the index in the rollback map.
if _, ok := tx.wc.rollbackIndexes[name]; !ok {
// we use a non-nil copy of the index without the data to indicate that the
// index should be rebuilt upon rollback.
// we use a non-nil copy of the index without the data to indicate
// that the index should be rebuilt upon rollback.
tx.wc.rollbackIndexes[name] = idx.clearCopy()
}
}
@ -2181,3 +2188,67 @@ func IndexJSONCaseSensitive(path string) func(a, b string) bool {
func Desc(less func(a, b string) bool) func(a, b string) bool {
return func(a, b string) bool { return less(b, a) }
}
//// Wrappers around btree Ascend/Descend
func bLT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(a, b) }
func bGT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(b, a) }
// func bLTE(tr *btree.BTree, a, b interface{}) bool { return !tr.Less(b, a) }
// func bGTE(tr *btree.BTree, a, b interface{}) bool { return !tr.Less(a, b) }
// Ascend
func btreeAscend(tr *btree.BTree, iter func(item interface{}) bool) {
tr.Ascend(nil, iter)
}
func btreeAscendLessThan(tr *btree.BTree, pivot interface{},
iter func(item interface{}) bool,
) {
tr.Ascend(nil, func(item interface{}) bool {
return bLT(tr, item, pivot) && iter(item)
})
}
func btreeAscendGreaterOrEqual(tr *btree.BTree, pivot interface{},
iter func(item interface{}) bool,
) {
tr.Ascend(pivot, iter)
}
func btreeAscendRange(tr *btree.BTree, greaterOrEqual, lessThan interface{},
iter func(item interface{}) bool,
) {
tr.Ascend(greaterOrEqual, func(item interface{}) bool {
return bLT(tr, item, lessThan) && iter(item)
})
}
// Descend
func btreeDescend(tr *btree.BTree, iter func(item interface{}) bool) {
tr.Descend(nil, iter)
}
func btreeDescendGreaterThan(tr *btree.BTree, pivot interface{},
iter func(item interface{}) bool,
) {
tr.Descend(nil, func(item interface{}) bool {
return bGT(tr, item, pivot) && iter(item)
})
}
func btreeDescendRange(tr *btree.BTree, lessOrEqual, greaterThan interface{},
iter func(item interface{}) bool,
) {
tr.Descend(lessOrEqual, func(item interface{}) bool {
return bGT(tr, item, greaterThan) && iter(item)
})
}
func btreeDescendLessOrEqual(tr *btree.BTree, pivot interface{},
iter func(item interface{}) bool,
) {
tr.Descend(pivot, iter)
}

@ -1,12 +1,12 @@
module github.com/tidwall/buntdb
go 1.13
go 1.15
require (
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274
github.com/tidwall/gjson v1.3.4
github.com/tidwall/btree v0.2.2
github.com/tidwall/gjson v1.6.1
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb
github.com/tidwall/match v1.0.1
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect
)

@ -1,14 +1,14 @@
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/gjson v1.3.4 h1:On5waDnyKKk3SWE4EthbjjirAWXp43xx5cKCUZY1eZw=
github.com/tidwall/gjson v1.3.4/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY=
github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e h1:+NL1GDIUOKxVfbp2KoJQD9cTQ6dyP2co9q4yzmT9FZo=
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 h1:BsKSRhu0TDB6Snq8SutN9KQHc6vqHEXJTcAFwyGNius=
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=

@ -193,11 +193,15 @@ we'll get `children` array and reverse the order:
"children|@reverse|0" >> "Jack"
```
There are currently three built-in modifiers:
There are currently the following built-in modifiers:
- `@reverse`: Reverse an array or the members of an object.
- `@ugly`: Remove all whitespace from a json document.
- `@pretty`: Make the json document more human readable.
- `@this`: Returns the current element. It can be used to retrieve the root element.
- `@valid`: Ensure the json document is valid.
- `@flatten`: Flattens an array.
- `@join`: Joins multiple objects into a single object.
### Modifier arguments

@ -77,6 +77,14 @@ Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`.
fav\.movie "Deer Hunter"
```
You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in source code.
```go
res := gjson.Get(json, "fav\\.movie") // must escape the slash
res := gjson.Get(json, `fav\.movie`) // no need to escape the slash
```
### Arrays
The `#` character allows for digging into JSON Arrays.
@ -181,11 +189,15 @@ children.@reverse ["Jack","Alex","Sara"]
children.@reverse.0 "Jack"
```
There are currently three built-in modifiers:
There are currently the following built-in modifiers:
- `@reverse`: Reverse an array or the members of an object.
- `@ugly`: Remove all whitespace from JSON.
- `@pretty`: Make the JSON more human readable.
- `@this`: Returns the current element. It can be used to retrieve the root element.
- `@valid`: Ensure the json document is valid.
- `@flatten`: Flattens an array.
- `@join`: Joins multiple objects into a single object.
#### Modifier arguments

@ -2,17 +2,14 @@
package gjson
import (
"encoding/base64"
"encoding/json"
"errors"
"reflect"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"unicode/utf16"
"unicode/utf8"
"unsafe"
"github.com/tidwall/match"
"github.com/tidwall/pretty"
@ -464,11 +461,13 @@ func ParseBytes(json []byte) Result {
}
func squash(json string) string {
// expects that the lead character is a '[' or '{' or '('
// expects that the lead character is a '[' or '{' or '(' or '"'
// squash the value, ignoring all nested arrays and objects.
// the first '[' or '{' or '(', has already been read
depth := 1
for i := 1; i < len(json); i++ {
var i, depth int
if json[0] != '"' {
i, depth = 1, 1
}
for ; i < len(json); i++ {
if json[i] >= '"' && json[i] <= '}' {
switch json[i] {
case '"':
@ -495,6 +494,9 @@ func squash(json string) string {
break
}
}
if depth == 0 {
return json[:i+1]
}
case '{', '[', '(':
depth++
case '}', ']', ')':
@ -1984,7 +1986,7 @@ func runeit(json string) rune {
}
// unescape unescapes a string
func unescape(json string) string { //, error) {
func unescape(json string) string {
var str = make([]byte, 0, len(json))
for i := 0; i < len(json); i++ {
switch {
@ -2194,145 +2196,6 @@ func GetManyBytes(json []byte, path ...string) []Result {
return res
}
var fieldsmu sync.RWMutex
var fields = make(map[string]map[string]int)
func assign(jsval Result, goval reflect.Value) {
if jsval.Type == Null {
return
}
switch goval.Kind() {
default:
case reflect.Ptr:
if !goval.IsNil() {
newval := reflect.New(goval.Elem().Type())
assign(jsval, newval.Elem())
goval.Elem().Set(newval.Elem())
} else {
newval := reflect.New(goval.Type().Elem())
assign(jsval, newval.Elem())
goval.Set(newval)
}
case reflect.Struct:
fieldsmu.RLock()
sf := fields[goval.Type().String()]
fieldsmu.RUnlock()
if sf == nil {
fieldsmu.Lock()
sf = make(map[string]int)
for i := 0; i < goval.Type().NumField(); i++ {
f := goval.Type().Field(i)
tag := strings.Split(f.Tag.Get("json"), ",")[0]
if tag != "-" {
if tag != "" {
sf[tag] = i
sf[f.Name] = i
} else {
sf[f.Name] = i
}
}
}
fields[goval.Type().String()] = sf
fieldsmu.Unlock()
}
jsval.ForEach(func(key, value Result) bool {
if idx, ok := sf[key.Str]; ok {
f := goval.Field(idx)
if f.CanSet() {
assign(value, f)
}
}
return true
})
case reflect.Slice:
if goval.Type().Elem().Kind() == reflect.Uint8 &&
jsval.Type == String {
data, _ := base64.StdEncoding.DecodeString(jsval.String())
goval.Set(reflect.ValueOf(data))
} else {
jsvals := jsval.Array()
slice := reflect.MakeSlice(goval.Type(), len(jsvals), len(jsvals))
for i := 0; i < len(jsvals); i++ {
assign(jsvals[i], slice.Index(i))
}
goval.Set(slice)
}
case reflect.Array:
i, n := 0, goval.Len()
jsval.ForEach(func(_, value Result) bool {
if i == n {
return false
}
assign(value, goval.Index(i))
i++
return true
})
case reflect.Map:
if goval.Type().Key().Kind() == reflect.String &&
goval.Type().Elem().Kind() == reflect.Interface {
goval.Set(reflect.ValueOf(jsval.Value()))
}
case reflect.Interface:
goval.Set(reflect.ValueOf(jsval.Value()))
case reflect.Bool:
goval.SetBool(jsval.Bool())
case reflect.Float32, reflect.Float64:
goval.SetFloat(jsval.Float())
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
reflect.Int64:
goval.SetInt(jsval.Int())
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32,
reflect.Uint64:
goval.SetUint(jsval.Uint())
case reflect.String:
goval.SetString(jsval.String())
}
if len(goval.Type().PkgPath()) > 0 {
v := goval.Addr()
if v.Type().NumMethod() > 0 {
if u, ok := v.Interface().(json.Unmarshaler); ok {
u.UnmarshalJSON([]byte(jsval.Raw))
}
}
}
}
var validate uintptr = 1
// UnmarshalValidationEnabled provides the option to disable JSON validation
// during the Unmarshal routine. Validation is enabled by default.
//
// Deprecated: Use encoder/json.Unmarshal instead
func UnmarshalValidationEnabled(enabled bool) {
if enabled {
atomic.StoreUintptr(&validate, 1)
} else {
atomic.StoreUintptr(&validate, 0)
}
}
// Unmarshal loads the JSON data into the value pointed to by v.
//
// This function works almost identically to json.Unmarshal except that
// gjson.Unmarshal will automatically attempt to convert JSON values to any Go
// type. For example, the JSON string "100" or the JSON number 100 can be
// equally assigned to Go string, int, byte, uint64, etc. This rule applies to
// all types.
//
// Deprecated: Use encoder/json.Unmarshal instead
func Unmarshal(data []byte, v interface{}) error {
if atomic.LoadUintptr(&validate) == 1 {
_, ok := validpayload(data, 0)
if !ok {
return errors.New("invalid json")
}
}
if v := reflect.ValueOf(v); v.Kind() == reflect.Ptr {
assign(ParseBytes(data), v)
}
return nil
}
func validpayload(data []byte, i int) (outi int, ok bool) {
for ; i < len(data); i++ {
switch data[i] {
@ -2713,7 +2576,7 @@ func execModifier(json, path string) (pathOut, res string, ok bool) {
case '{', '[', '"':
res := Parse(pathOut)
if res.Exists() {
_, args = parseSquash(pathOut, 0)
args = squash(pathOut)
pathOut = pathOut[len(args):]
parsedArgs = true
}
@ -2734,6 +2597,15 @@ func execModifier(json, path string) (pathOut, res string, ok bool) {
return pathOut, res, false
}
// unwrap removes the '[]' or '{}' characters around json
func unwrap(json string) string {
json = trim(json)
if len(json) >= 2 && json[0] == '[' || json[0] == '{' {
json = json[1 : len(json)-1]
}
return json
}
// DisableModifiers will disable the modifier syntax
var DisableModifiers = false
@ -2741,6 +2613,10 @@ var modifiers = map[string]func(json, arg string) string{
"pretty": modPretty,
"ugly": modUgly,
"reverse": modReverse,
"this": modThis,
"flatten": modFlatten,
"join": modJoin,
"valid": modValid,
}
// AddModifier binds a custom modifier command to the GJSON syntax.
@ -2778,6 +2654,11 @@ func modPretty(json, arg string) string {
return bytesString(pretty.Pretty(stringBytes(json)))
}
// @this returns the current element. Can be used to retrieve the root element.
func modThis(json, arg string) string {
return json
}
// @ugly modifier removes all whitespace.
func modUgly(json, arg string) string {
return bytesString(pretty.Ugly(stringBytes(json)))
@ -2824,3 +2705,194 @@ func modReverse(json, arg string) string {
}
return json
}
// @flatten an array with child arrays.
// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,[6,7]]
// The {"deep":true} arg can be provide for deep flattening.
// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,6,7]
// The original json is returned when the json is not an array.
func modFlatten(json, arg string) string {
res := Parse(json)
if !res.IsArray() {
return json
}
var deep bool
if arg != "" {
Parse(arg).ForEach(func(key, value Result) bool {
if key.String() == "deep" {
deep = value.Bool()
}
return true
})
}
var out []byte
out = append(out, '[')
var idx int
res.ForEach(func(_, value Result) bool {
if idx > 0 {
out = append(out, ',')
}
if value.IsArray() {
if deep {
out = append(out, unwrap(modFlatten(value.Raw, arg))...)
} else {
out = append(out, unwrap(value.Raw)...)
}
} else {
out = append(out, value.Raw...)
}
idx++
return true
})
out = append(out, ']')
return bytesString(out)
}
// @join multiple objects into a single object.
// [{"first":"Tom"},{"last":"Smith"}] -> {"first","Tom","last":"Smith"}
// The arg can be "true" to specify that duplicate keys should be preserved.
// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":37,"age":41}
// Without preserved keys:
// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":41}
// The original json is returned when the json is not an object.
func modJoin(json, arg string) string {
res := Parse(json)
if !res.IsArray() {
return json
}
var preserve bool
if arg != "" {
Parse(arg).ForEach(func(key, value Result) bool {
if key.String() == "preserve" {
preserve = value.Bool()
}
return true
})
}
var out []byte
out = append(out, '{')
if preserve {
// Preserve duplicate keys.
var idx int
res.ForEach(func(_, value Result) bool {
if !value.IsObject() {
return true
}
if idx > 0 {
out = append(out, ',')
}
out = append(out, unwrap(value.Raw)...)
idx++
return true
})
} else {
// Deduplicate keys and generate an object with stable ordering.
var keys []Result
kvals := make(map[string]Result)
res.ForEach(func(_, value Result) bool {
if !value.IsObject() {
return true
}
value.ForEach(func(key, value Result) bool {
k := key.String()
if _, ok := kvals[k]; !ok {
keys = append(keys, key)
}
kvals[k] = value
return true
})
return true
})
for i := 0; i < len(keys); i++ {
if i > 0 {
out = append(out, ',')
}
out = append(out, keys[i].Raw...)
out = append(out, ':')
out = append(out, kvals[keys[i].String()].Raw...)
}
}
out = append(out, '}')
return bytesString(out)
}
// @valid ensures that the json is valid before moving on. An empty string is
// returned when the json is not valid, otherwise it returns the original json.
func modValid(json, arg string) string {
if !Valid(json) {
return ""
}
return json
}
// getBytes casts the input json bytes to a string and safely returns the
// results as uniquely allocated data. This operation is intended to minimize
// copies and allocations for the large json string->[]byte.
func getBytes(json []byte, path string) Result {
var result Result
if json != nil {
// unsafe cast to string
result = Get(*(*string)(unsafe.Pointer(&json)), path)
// safely get the string headers
rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw))
strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str))
// create byte slice headers
rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len}
strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len}
if strh.Data == 0 {
// str is nil
if rawh.Data == 0 {
// raw is nil
result.Raw = ""
} else {
// raw has data, safely copy the slice header to a string
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
}
result.Str = ""
} else if rawh.Data == 0 {
// raw is nil
result.Raw = ""
// str has data, safely copy the slice header to a string
result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
} else if strh.Data >= rawh.Data &&
int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len {
// Str is a substring of Raw.
start := int(strh.Data - rawh.Data)
// safely copy the raw slice header
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
// substring the raw
result.Str = result.Raw[start : start+strh.Len]
} else {
// safely copy both the raw and str slice headers to strings
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
}
}
return result
}
// fillIndex finds the position of Raw data and assigns it to the Index field
// of the resulting value. If the position cannot be found then Index zero is
// used instead.
func fillIndex(json string, c *parseContext) {
if len(c.value.Raw) > 0 && !c.calcd {
jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json))
rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw)))
c.value.Index = int(rhdr.Data - jhdr.Data)
if c.value.Index < 0 || c.value.Index >= len(json) {
c.value.Index = 0
}
}
}
func stringBytes(s string) []byte {
return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
Data: (*reflect.StringHeader)(unsafe.Pointer(&s)).Data,
Len: len(s),
Cap: len(s),
}))
}
func bytesString(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}

@ -1,18 +0,0 @@
//+build appengine js
package gjson
func getBytes(json []byte, path string) Result {
return Get(string(json), path)
}
func fillIndex(json string, c *parseContext) {
// noop. Use zero for the Index value.
}
func stringBytes(s string) []byte {
return []byte(s)
}
func bytesString(b []byte) string {
return string(b)
}

@ -1,81 +0,0 @@
//+build !appengine
//+build !js
package gjson
import (
"reflect"
"unsafe"
)
// getBytes casts the input json bytes to a string and safely returns the
// results as uniquely allocated data. This operation is intended to minimize
// copies and allocations for the large json string->[]byte.
func getBytes(json []byte, path string) Result {
var result Result
if json != nil {
// unsafe cast to string
result = Get(*(*string)(unsafe.Pointer(&json)), path)
// safely get the string headers
rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw))
strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str))
// create byte slice headers
rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len}
strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len}
if strh.Data == 0 {
// str is nil
if rawh.Data == 0 {
// raw is nil
result.Raw = ""
} else {
// raw has data, safely copy the slice header to a string
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
}
result.Str = ""
} else if rawh.Data == 0 {
// raw is nil
result.Raw = ""
// str has data, safely copy the slice header to a string
result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
} else if strh.Data >= rawh.Data &&
int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len {
// Str is a substring of Raw.
start := int(strh.Data - rawh.Data)
// safely copy the raw slice header
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
// substring the raw
result.Str = result.Raw[start : start+strh.Len]
} else {
// safely copy both the raw and str slice headers to strings
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
}
}
return result
}
// fillIndex finds the position of Raw data and assigns it to the Index field
// of the resulting value. If the position cannot be found then Index zero is
// used instead.
func fillIndex(json string, c *parseContext) {
if len(c.value.Raw) > 0 && !c.calcd {
jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json))
rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw)))
c.value.Index = int(rhdr.Data - jhdr.Data)
if c.value.Index < 0 || c.value.Index >= len(json) {
c.value.Index = 0
}
}
}
func stringBytes(s string) []byte {
return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
Data: (*reflect.StringHeader)(unsafe.Pointer(&s)).Data,
Len: len(s),
Cap: len(s),
}))
}
func bytesString(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}

@ -4,5 +4,5 @@ go 1.12
require (
github.com/tidwall/match v1.0.1
github.com/tidwall/pretty v1.0.0
github.com/tidwall/pretty v1.0.2
)

@ -1,4 +1,4 @@
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

@ -1,7 +1,7 @@
# Pretty
[![Build Status](https://img.shields.io/travis/tidwall/pretty.svg?style=flat-square)](https://travis-ci.org/tidwall/prettty)
[![Coverage Status](https://img.shields.io/badge/coverage-100%25-brightgreen.svg?style=flat-square)](http://gocover.io/github.com/tidwall/pretty)
[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://godoc.org/github.com/tidwall/pretty)
[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty)
Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads.

@ -318,21 +318,25 @@ func hexp(p byte) byte {
}
// TerminalStyle is for terminals
var TerminalStyle = &Style{
Key: [2]string{"\x1B[94m", "\x1B[0m"},
String: [2]string{"\x1B[92m", "\x1B[0m"},
Number: [2]string{"\x1B[93m", "\x1B[0m"},
True: [2]string{"\x1B[96m", "\x1B[0m"},
False: [2]string{"\x1B[96m", "\x1B[0m"},
Null: [2]string{"\x1B[91m", "\x1B[0m"},
Append: func(dst []byte, c byte) []byte {
if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
dst = append(dst, "\\u00"...)
dst = append(dst, hexp((c>>4)&0xF))
return append(dst, hexp((c)&0xF))
}
return append(dst, c)
},
var TerminalStyle *Style
func init() {
TerminalStyle = &Style{
Key: [2]string{"\x1B[94m", "\x1B[0m"},
String: [2]string{"\x1B[92m", "\x1B[0m"},
Number: [2]string{"\x1B[93m", "\x1B[0m"},
True: [2]string{"\x1B[96m", "\x1B[0m"},
False: [2]string{"\x1B[96m", "\x1B[0m"},
Null: [2]string{"\x1B[91m", "\x1B[0m"},
Append: func(dst []byte, c byte) []byte {
if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
dst = append(dst, "\\u00"...)
dst = append(dst, hexp((c>>4)&0xF))
return append(dst, hexp((c)&0xF))
}
return append(dst, c)
},
}
}
// Color will colorize the json. The style parma is used for customizing

@ -183,6 +183,7 @@ type treeItem struct {
item interface{}
}
//go:nocheckptr
func (item *treeItem) unsafeNode() *treeNode {
return (*treeNode)(unsafe.Pointer(item))
}

10
vendor/modules.txt vendored

@ -37,20 +37,20 @@ github.com/oragono/confusables
github.com/oragono/go-ident
# github.com/stretchr/testify v1.4.0
## explicit
# github.com/tidwall/btree v0.0.0-20191029221954-400434d76274
# github.com/tidwall/btree v0.2.2
github.com/tidwall/btree
# github.com/tidwall/buntdb v1.1.2
# github.com/tidwall/buntdb v1.1.4
## explicit
github.com/tidwall/buntdb
# github.com/tidwall/gjson v1.3.4
# github.com/tidwall/gjson v1.6.1
github.com/tidwall/gjson
# github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb
github.com/tidwall/grect
# github.com/tidwall/match v1.0.1
github.com/tidwall/match
# github.com/tidwall/pretty v1.0.0
# github.com/tidwall/pretty v1.0.2
github.com/tidwall/pretty
# github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e
# github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8
github.com/tidwall/rtree
github.com/tidwall/rtree/base
# github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563