Skip to content

Commit 52c177c

Browse files
aschmahmannlidelmarten-seemannguseggertschomatis
authored
feat: go-libp2p 0.16, UnixFS autosharding and go-datastore with contexts (#8563)
* plumb through go-datastore context changes * update go-libp2p to v0.16.0 * use LIBP2P_TCP_REUSEPORT instead of IPFS_REUSEPORT * use relay config * making deprecation notice match the go-ipfs-config key * docs(config): circuit relay v2 * docs(config): fix links and headers * feat(config): Internal.Libp2pForceReachability This switches to config that supports setting and reading Internal.Libp2pForceReachability OptionalString flag * use configuration option for static relays * chore: go-ipfs-config v0.18.0 https://github.com/ipfs/go-ipfs-config/releases/tag/v0.18.0 * feat: circuit v1 migration prompt when Swarm.EnableRelayHop is set (#8559) * exit when Swarm.EnableRelayHop is set * docs: Experimental.ShardingEnabled migration This ensures existing users of global sharding experiment get notified that the flag no longer works + that autosharding happens automatically. For people who NEED to keep the old behavior (eg. have no time to migrate today) there is a note about restoring it with `UnixFSShardingSizeThreshold`. * chore: add dag-jose code to the cid command output * add support for setting automatic unixfs sharding threshold from the config * test: have tests use low cutoff for sharding to mimic old behavior * test: change error message to match the current error * test: Add automatic sharding/unsharding tests (#8547) * test: refactored naming in the sharding sharness tests to make more sense * ci: set interop test executor to convenience image for Go1.16 + Node * ci: use interop master Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: Marten Seemann <martenseemann@gmail.com> Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: Gus Eggert <gus@gus.dev> Co-authored-by: Lucas Molas <schomatis@gmail.com>
1 parent c00065c commit 52c177c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+798
-792
lines changed

‎.circleci/main.yml

+5-4
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ executors:
5656
environment:
5757
<<: *default_environment
5858
NO_SANDBOX: true
59-
IPFS_REUSEPORT: false
59+
LIBP2P_TCP_REUSEPORT: false
6060
LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
6161
E2E_IPFSD_TYPE: go
6262
dockerizer:
@@ -216,7 +216,8 @@ jobs:
216216
- bin/ipfs
217217
- *store_gomod
218218
interop:
219-
executor: node
219+
docker:
220+
- image: cimg/go:1.16-node
220221
parallelism: 4
221222
steps:
222223
- *make_out_dirs
@@ -227,7 +228,7 @@ jobs:
227228
command: |
228229
npm init -y
229230
npm install ipfs@^0.59.1
230-
npm install ipfs-interop@^7.0.3
231+
npm install ipfs/interop#master
231232
npm install mocha-circleci-reporter@0.0.3
232233
working_directory: ~/ipfs/go-ipfs/interop
233234
- run:
@@ -238,7 +239,7 @@ jobs:
238239
npx ipfs-interop -- -t node -f $(sed -n -e "s|^require('\(.*\)')$|test/\1|p" node_modules/ipfs-interop/test/node.js | circleci tests split) -- --reporter mocha-circleci-reporter
239240
working_directory: ~/ipfs/go-ipfs/interop
240241
environment:
241-
IPFS_REUSEPORT: false
242+
LIBP2P_TCP_REUSEPORT: false
242243
LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
243244
IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs
244245
- store_test_results:

‎Rules.mk

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ include mk/golang.mk
2222
ifeq ($(TEST_NO_FUSE),1)
2323
GOTAGS += nofuse
2424
endif
25-
export IPFS_REUSEPORT=false
25+
export LIBP2P_TCP_REUSEPORT=false
2626

2727
# -------------------- #
2828
# sub-files #

‎blocks/blockstoreutil/remove.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88

99
cid "github.com/ipfs/go-cid"
1010
bs "github.com/ipfs/go-ipfs-blockstore"
11-
"github.com/ipfs/go-ipfs-pinner"
11+
pin "github.com/ipfs/go-ipfs-pinner"
1212
)
1313

1414
// RemovedBlock is used to represent the result of removing a block.
@@ -40,15 +40,15 @@ func RmBlocks(ctx context.Context, blocks bs.GCBlockstore, pins pin.Pinner, cids
4040
go func() {
4141
defer close(out)
4242

43-
unlocker := blocks.GCLock()
44-
defer unlocker.Unlock()
43+
unlocker := blocks.GCLock(ctx)
44+
defer unlocker.Unlock(ctx)
4545

4646
stillOkay := FilterPinned(ctx, pins, out, cids)
4747

4848
for _, c := range stillOkay {
4949
// Kept for backwards compatibility. We may want to
5050
// remove this sometime in the future.
51-
has, err := blocks.Has(c)
51+
has, err := blocks.Has(ctx, c)
5252
if err != nil {
5353
out <- &RemovedBlock{Hash: c.String(), Error: err.Error()}
5454
continue
@@ -58,7 +58,7 @@ func RmBlocks(ctx context.Context, blocks bs.GCBlockstore, pins pin.Pinner, cids
5858
continue
5959
}
6060

61-
err = blocks.DeleteBlock(c)
61+
err = blocks.DeleteBlock(ctx, c)
6262
if err != nil {
6363
out <- &RemovedBlock{Hash: c.String(), Error: err.Error()}
6464
} else if !opts.Quiet {

‎core/commands/dag/import.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
4242
// This is especially important for use cases like dagger:
4343
// ipfs dag import $( ... | ipfs-dagger --stdout=carfifos )
4444
//
45-
unlocker := node.Blockstore.PinLock()
46-
defer unlocker.Unlock()
45+
unlocker := node.Blockstore.PinLock(req.Context)
46+
defer unlocker.Unlock(req.Context)
4747

4848
doPinRoots, _ := req.Options[pinRootsOptionName].(bool)
4949

@@ -87,7 +87,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
8787

8888
ret := RootMeta{Cid: c}
8989

90-
if block, err := node.Blockstore.Get(c); err != nil {
90+
if block, err := node.Blockstore.Get(req.Context, c); err != nil {
9191
ret.PinErrorMsg = err.Error()
9292
} else if nd, err := ipld.Decode(block); err != nil {
9393
ret.PinErrorMsg = err.Error()

‎core/commands/dht.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ var provideRefDhtCmd = &cmds.Command{
273273
return err
274274
}
275275

276-
has, err := nd.Blockstore.Has(c)
276+
has, err := nd.Blockstore.Has(req.Context, c)
277277
if err != nil {
278278
return err
279279
}

‎core/commands/filestore.go

+11-10
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
package commands
22

33
import (
4+
"context"
45
"fmt"
56
"io"
67
"os"
78

89
filestore "github.com/ipfs/go-filestore"
10+
cmds "github.com/ipfs/go-ipfs-cmds"
911
core "github.com/ipfs/go-ipfs/core"
1012
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
1113
e "github.com/ipfs/go-ipfs/core/commands/e"
1214

1315
"github.com/ipfs/go-cid"
14-
"github.com/ipfs/go-ipfs-cmds"
1516
)
1617

1718
var FileStoreCmd = &cmds.Command{
@@ -56,17 +57,17 @@ The output is:
5657
}
5758
args := req.Arguments
5859
if len(args) > 0 {
59-
return listByArgs(res, fs, args)
60+
return listByArgs(req.Context, res, fs, args)
6061
}
6162

6263
fileOrder, _ := req.Options[fileOrderOptionName].(bool)
63-
next, err := filestore.ListAll(fs, fileOrder)
64+
next, err := filestore.ListAll(req.Context, fs, fileOrder)
6465
if err != nil {
6566
return err
6667
}
6768

6869
for {
69-
r := next()
70+
r := next(req.Context)
7071
if r == nil {
7172
break
7273
}
@@ -133,17 +134,17 @@ For ERROR entries the error will also be printed to stderr.
133134
}
134135
args := req.Arguments
135136
if len(args) > 0 {
136-
return listByArgs(res, fs, args)
137+
return listByArgs(req.Context, res, fs, args)
137138
}
138139

139140
fileOrder, _ := req.Options[fileOrderOptionName].(bool)
140-
next, err := filestore.VerifyAll(fs, fileOrder)
141+
next, err := filestore.VerifyAll(req.Context, fs, fileOrder)
141142
if err != nil {
142143
return err
143144
}
144145

145146
for {
146-
r := next()
147+
r := next(req.Context)
147148
if r == nil {
148149
break
149150
}
@@ -206,7 +207,7 @@ var dupsFileStore = &cmds.Command{
206207
}
207208

208209
for cid := range ch {
209-
have, err := fs.MainBlockstore().Has(cid)
210+
have, err := fs.MainBlockstore().Has(req.Context, cid)
210211
if err != nil {
211212
return res.Emit(&RefWrapper{Err: err.Error()})
212213
}
@@ -235,7 +236,7 @@ func getFilestore(env cmds.Environment) (*core.IpfsNode, *filestore.Filestore, e
235236
return n, fs, err
236237
}
237238

238-
func listByArgs(res cmds.ResponseEmitter, fs *filestore.Filestore, args []string) error {
239+
func listByArgs(ctx context.Context, res cmds.ResponseEmitter, fs *filestore.Filestore, args []string) error {
239240
for _, arg := range args {
240241
c, err := cid.Decode(arg)
241242
if err != nil {
@@ -248,7 +249,7 @@ func listByArgs(res cmds.ResponseEmitter, fs *filestore.Filestore, args []string
248249
}
249250
continue
250251
}
251-
r := filestore.Verify(fs, c)
252+
r := filestore.Verify(ctx, fs, c)
252253
if err := res.Emit(r); err != nil {
253254
return err
254255
}

‎core/commands/pin/remotepin.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ NOTE: a comma-separated notation is supported in CLI for convenience:
174174
return err
175175
}
176176

177-
isInBlockstore, err := node.Blockstore.Has(rp.Cid())
177+
isInBlockstore, err := node.Blockstore.Has(req.Context, rp.Cid())
178178
if err != nil {
179179
return err
180180
}

‎core/commands/repo.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Ci
239239
defer wg.Done()
240240

241241
for k := range keys {
242-
_, err := bs.Get(k)
242+
_, err := bs.Get(ctx, k)
243243
if err != nil {
244244
select {
245245
case results <- fmt.Sprintf("block %s was corrupt (%s)", k, err):

‎core/coreapi/block.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
4646
}
4747

4848
if settings.Pin {
49-
defer api.blockstore.PinLock().Unlock()
49+
defer api.blockstore.PinLock(ctx).Unlock(ctx)
5050
}
5151

52-
err = api.blocks.AddBlock(b)
52+
err = api.blocks.AddBlock(ctx, b)
5353
if err != nil {
5454
return nil, err
5555
}

‎core/coreapi/dag.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"context"
55

66
cid "github.com/ipfs/go-cid"
7-
"github.com/ipfs/go-ipfs-pinner"
7+
pin "github.com/ipfs/go-ipfs-pinner"
88
ipld "github.com/ipfs/go-ipld-format"
99
dag "github.com/ipfs/go-merkledag"
1010
)
@@ -18,7 +18,7 @@ type dagAPI struct {
1818
type pinningAdder CoreAPI
1919

2020
func (adder *pinningAdder) Add(ctx context.Context, nd ipld.Node) error {
21-
defer adder.blockstore.PinLock().Unlock()
21+
defer adder.blockstore.PinLock(ctx).Unlock(ctx)
2222

2323
if err := adder.dag.Add(ctx, nd); err != nil {
2424
return err
@@ -30,7 +30,7 @@ func (adder *pinningAdder) Add(ctx context.Context, nd ipld.Node) error {
3030
}
3131

3232
func (adder *pinningAdder) AddMany(ctx context.Context, nds []ipld.Node) error {
33-
defer adder.blockstore.PinLock().Unlock()
33+
defer adder.blockstore.PinLock(ctx).Unlock(ctx)
3434

3535
if err := adder.dag.AddMany(ctx, nds); err != nil {
3636
return err

‎core/coreapi/dht.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ func (api *DhtAPI) Provide(ctx context.Context, path path.Path, opts ...caopts.D
7676

7777
c := rp.Cid()
7878

79-
has, err := api.blockstore.Has(c)
79+
has, err := api.blockstore.Has(ctx, c)
8080
if err != nil {
8181
return err
8282
}

‎core/coreapi/object.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
"io/ioutil"
1313

1414
cid "github.com/ipfs/go-cid"
15-
"github.com/ipfs/go-ipfs-pinner"
15+
pin "github.com/ipfs/go-ipfs-pinner"
1616
ipld "github.com/ipfs/go-ipld-format"
1717
dag "github.com/ipfs/go-merkledag"
1818
"github.com/ipfs/go-merkledag/dagutils"
@@ -110,7 +110,7 @@ func (api *ObjectAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Obj
110110
}
111111

112112
if options.Pin {
113-
defer api.blockstore.PinLock().Unlock()
113+
defer api.blockstore.PinLock(ctx).Unlock(ctx)
114114
}
115115

116116
err = api.dag.Add(ctx, dagnode)

‎core/coreapi/pin.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp
2727
return err
2828
}
2929

30-
defer api.blockstore.PinLock().Unlock()
30+
defer api.blockstore.PinLock(ctx).Unlock(ctx)
3131

3232
err = api.pinning.Pin(ctx, dagNode, settings.Recursive)
3333
if err != nil {
@@ -89,7 +89,7 @@ func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOpti
8989

9090
// Note: after unpin the pin sets are flushed to the blockstore, so we need
9191
// to take a lock to prevent a concurrent garbage collection
92-
defer api.blockstore.PinLock().Unlock()
92+
defer api.blockstore.PinLock(ctx).Unlock(ctx)
9393

9494
if err = api.pinning.Unpin(ctx, rp.Cid(), settings.Recursive); err != nil {
9595
return err
@@ -114,7 +114,7 @@ func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opt
114114
return err
115115
}
116116

117-
defer api.blockstore.PinLock().Unlock()
117+
defer api.blockstore.PinLock(ctx).Unlock(ctx)
118118

119119
err = api.pinning.Update(ctx, fp.Cid(), tp.Cid(), settings.Unpin)
120120
if err != nil {

‎core/coreapi/test/path_test.go

+5-6
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
"github.com/ipld/go-ipld-prime"
1515
)
1616

17-
1817
func TestPathUnixFSHAMTPartial(t *testing.T) {
1918
ctx, cancel := context.WithCancel(context.Background())
2019
defer cancel()
@@ -27,16 +26,16 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
2726
a := apis[0]
2827

2928
// Setting this after instantiating the swarm so that it's not clobbered by loading the go-ipfs config
30-
prevVal := uio.UseHAMTSharding
31-
uio.UseHAMTSharding = true
29+
prevVal := uio.HAMTShardingSize
30+
uio.HAMTShardingSize = 1
3231
defer func() {
33-
uio.UseHAMTSharding = prevVal
32+
uio.HAMTShardingSize = prevVal
3433
}()
3534

3635
// Create and add a sharded directory
3736
dir := make(map[string]files.Node)
3837
// Make sure we have at least two levels of sharding
39-
for i := 0; i < uio.DefaultShardWidth + 1; i++ {
38+
for i := 0; i < uio.DefaultShardWidth+1; i++ {
4039
dir[strconv.Itoa(i)] = files.NewBytesFile([]byte(strconv.Itoa(i)))
4140
}
4241

@@ -67,7 +66,7 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
6766
for k := range dir {
6867
// The node will go out to the (non-existent) network looking for the missing block. Make sure we're erroring
6968
// because we exceeded the timeout on our query
70-
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second * 1)
69+
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second*1)
7170
_, err := a.ResolveNode(timeoutCtx, path.Join(r, k))
7271
if err != nil {
7372
if timeoutCtx.Err() == nil {

‎core/coreapi/unixfs.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
111111
DAGService: dserv,
112112
syncFn: func() error {
113113
ds := api.repo.Datastore()
114-
if err := ds.Sync(bstore.BlockPrefix); err != nil {
114+
if err := ds.Sync(ctx, bstore.BlockPrefix); err != nil {
115115
return err
116116
}
117-
return ds.Sync(filestore.FilestorePrefix)
117+
return ds.Sync(ctx, filestore.FilestorePrefix)
118118
},
119119
}
120120
}
@@ -164,7 +164,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
164164
fileAdder.SetMfsRoot(mr)
165165
}
166166

167-
nd, err := fileAdder.AddAllAndPin(files)
167+
nd, err := fileAdder.AddAllAndPin(ctx, files)
168168
if err != nil {
169169
return nil, err
170170
}

‎core/corehttp/metrics_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ func TestPeersTotal(t *testing.T) {
2121
hosts := make([]*bhost.BasicHost, 4)
2222
for i := 0; i < 4; i++ {
2323
var err error
24-
hosts[i], err = bhost.NewHost(ctx, swarmt.GenSwarm(t, ctx), nil)
24+
hosts[i], err = bhost.NewHost(swarmt.GenSwarm(t), nil)
2525
if err != nil {
2626
t.Fatal(err)
2727
}

‎core/corerepo/gc.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ func ConditionalGC(ctx context.Context, node *core.IpfsNode, offset uint64) erro
205205
}
206206

207207
func (gc *GC) maybeGC(ctx context.Context, offset uint64) error {
208-
storage, err := gc.Repo.GetStorageUsage()
208+
storage, err := gc.Repo.GetStorageUsage(ctx)
209209
if err != nil {
210210
return err
211211
}

‎core/corerepo/stat.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ func RepoSize(ctx context.Context, n *core.IpfsNode) (SizeStat, error) {
7171
return SizeStat{}, err
7272
}
7373

74-
usage, err := r.GetStorageUsage()
74+
usage, err := r.GetStorageUsage(ctx)
7575
if err != nil {
7676
return SizeStat{}, err
7777
}

0 commit comments

Comments
 (0)