Skip to content

Commit 1e5ce93

Browse files
hacdiashacdias
committed
fix: use %-encoded headers in most compatible way
1 parent 5e9bad9 commit 1e5ce93

File tree

14 files changed

+233
-49
lines changed

14 files changed

+233
-49
lines changed

‎client/rpc/api.go

Lines changed: 53 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
package rpc
22

33
import (
4+
"context"
5+
"encoding/json"
46
"errors"
57
"fmt"
68
"net/http"
79
"os"
810
"path/filepath"
911
"strings"
12+
"sync"
13+
"time"
1014

15+
"github.com/blang/semver/v4"
1116
iface "github.com/ipfs/boxo/coreiface"
1217
caopts "github.com/ipfs/boxo/coreiface/options"
1318
"github.com/ipfs/boxo/ipld/merkledag"
1419
"github.com/ipfs/go-cid"
1520
legacy "github.com/ipfs/go-ipld-legacy"
21+
ipfs "github.com/ipfs/kubo"
1622
dagpb "github.com/ipld/go-codec-dagpb"
1723
_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
1824
"github.com/ipld/go-ipld-prime/node/basicnode"
@@ -42,6 +48,8 @@ type HttpApi struct {
4248
Headers http.Header
4349
applyGlobal func(*requestBuilder)
4450
ipldDecoder *legacy.Decoder
51+
versionMu sync.Mutex
52+
version *semver.Version
4553
}
4654

4755
// NewLocalApi tries to construct new HttpApi instance communicating with local
@@ -151,6 +159,7 @@ func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) {
151159
api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
152160
return fmt.Errorf("unexpected redirect")
153161
}
162+
154163
return api, nil
155164
}
156165

@@ -160,14 +169,19 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error)
160169
return nil, err
161170
}
162171

163-
subApi := *api
164-
subApi.applyGlobal = func(req *requestBuilder) {
165-
if options.Offline {
166-
req.Option("offline", options.Offline)
167-
}
172+
subApi := &HttpApi{
173+
url: api.url,
174+
httpcli: api.httpcli,
175+
Headers: api.Headers,
176+
applyGlobal: func(req *requestBuilder) {
177+
if options.Offline {
178+
req.Option("offline", options.Offline)
179+
}
180+
},
181+
ipldDecoder: api.ipldDecoder,
168182
}
169183

170-
return &subApi, nil
184+
return subApi, nil
171185
}
172186

173187
func (api *HttpApi) Request(command string, args ...string) RequestBuilder {
@@ -228,3 +242,36 @@ func (api *HttpApi) PubSub() iface.PubSubAPI {
228242
func (api *HttpApi) Routing() iface.RoutingAPI {
229243
return (*RoutingAPI)(api)
230244
}
245+
246+
func (api *HttpApi) loadRemoteVersion() (*semver.Version, error) {
247+
api.versionMu.Lock()
248+
defer api.versionMu.Unlock()
249+
250+
if api.version == nil {
251+
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30))
252+
defer cancel()
253+
254+
resp, err := api.Request("version").Send(ctx)
255+
if err != nil {
256+
return nil, err
257+
}
258+
if resp.Error != nil {
259+
return nil, resp.Error
260+
}
261+
defer resp.Close()
262+
var out ipfs.VersionInfo
263+
dec := json.NewDecoder(resp.Output)
264+
if err := dec.Decode(&out); err != nil {
265+
return nil, err
266+
}
267+
268+
remoteVersion, err := semver.New(out.Version)
269+
if err != nil {
270+
return nil, err
271+
}
272+
273+
api.version = remoteVersion
274+
}
275+
276+
return api.version, nil
277+
}

‎client/rpc/requestbuilder.go

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"strconv"
99
"strings"
1010

11+
"github.com/blang/semver/v4"
1112
"github.com/ipfs/boxo/files"
1213
)
1314

@@ -23,13 +24,18 @@ type RequestBuilder interface {
2324
Exec(ctx context.Context, res interface{}) error
2425
}
2526

27+
// encodedAbsolutePathVersion is the version from which the absolute path header in
28+
// multipart requests is %-encoded. Before this version, its sent raw.
29+
var encodedAbsolutePathVersion = semver.MustParse("0.23.0-dev")
30+
2631
// requestBuilder is an IPFS commands request builder.
2732
type requestBuilder struct {
28-
command string
29-
args []string
30-
opts map[string]string
31-
headers map[string]string
32-
body io.Reader
33+
command string
34+
args []string
35+
opts map[string]string
36+
headers map[string]string
37+
body io.Reader
38+
buildError error
3339

3440
shell *HttpApi
3541
}
@@ -60,7 +66,18 @@ func (r *requestBuilder) Body(body io.Reader) RequestBuilder {
6066
func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder {
6167
pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil)
6268
d := files.NewMapDirectory(map[string]files.Node{"": pr})
63-
r.body = files.NewMultiFileReader(d, false)
69+
70+
version, err := r.shell.loadRemoteVersion()
71+
if err != nil {
72+
// Unfortunately, we cannot return an error here. Changing this API is also
73+
// not the best since we would otherwise have an inconsistent RequestBuilder.
74+
// We save the error and return it when calling [requestBuilder.Send].
75+
r.buildError = err
76+
return r
77+
}
78+
79+
useEncodedAbsPaths := version.LT(encodedAbsolutePathVersion)
80+
r.body = files.NewMultiFileReader(d, false, useEncodedAbsPaths)
6481

6582
return r
6683
}
@@ -97,6 +114,10 @@ func (r *requestBuilder) Header(name, value string) RequestBuilder {
97114

98115
// Send sends the request and return the response.
99116
func (r *requestBuilder) Send(ctx context.Context) (*Response, error) {
117+
if r.buildError != nil {
118+
return nil, r.buildError
119+
}
120+
100121
r.shell.applyGlobal(r)
101122

102123
req := NewRequest(ctx, r.shell.url, r.command, r.args...)

‎client/rpc/unixfs.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,13 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
6262
}
6363

6464
d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side
65-
req.Body(files.NewMultiFileReader(d, false))
65+
66+
version, err := api.core().loadRemoteVersion()
67+
if err != nil {
68+
return nil, err
69+
}
70+
useEncodedAbsPaths := version.LT(encodedAbsolutePathVersion)
71+
req.Body(files.NewMultiFileReader(d, false, useEncodedAbsPaths))
6672

6773
var out addEvent
6874
resp, err := req.Send(ctx)

‎cmd/ipfs/main.go

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,27 @@
22
package main
33

44
import (
5+
"bytes"
56
"context"
7+
"encoding/json"
68
"errors"
79
"fmt"
10+
"io"
811
"net"
912
"net/http"
1013
"os"
1114
"runtime/pprof"
1215
"strings"
1316
"time"
1417

18+
"github.com/blang/semver/v4"
1519
"github.com/google/uuid"
1620
u "github.com/ipfs/boxo/util"
1721
cmds "github.com/ipfs/go-ipfs-cmds"
1822
"github.com/ipfs/go-ipfs-cmds/cli"
1923
cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
2024
logging "github.com/ipfs/go-log"
25+
ipfs "github.com/ipfs/kubo"
2126
"github.com/ipfs/kubo/cmd/ipfs/util"
2227
oldcmds "github.com/ipfs/kubo/commands"
2328
"github.com/ipfs/kubo/core"
@@ -224,6 +229,10 @@ func apiAddrOption(req *cmds.Request) (ma.Multiaddr, error) {
224229
return ma.NewMultiaddr(apiAddrStr)
225230
}
226231

232+
// encodedAbsolutePathVersion is the version from which the absolute path header in
233+
// multipart requests is %-encoded. Before this version, its sent raw.
234+
var encodedAbsolutePathVersion = semver.MustParse("0.23.0-dev")
235+
227236
func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) {
228237
exe := tracingWrappedExecutor{cmds.NewExecutor(req.Root)}
229238
cctx := env.(*oldcmds.Context)
@@ -315,9 +324,18 @@ func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) {
315324
default:
316325
return nil, fmt.Errorf("unsupported API address: %s", apiAddr)
317326
}
318-
opts = append(opts, cmdhttp.ClientWithHTTPClient(&http.Client{
327+
328+
httpClient := &http.Client{
319329
Transport: otelhttp.NewTransport(tpt),
320-
}))
330+
}
331+
opts = append(opts, cmdhttp.ClientWithHTTPClient(httpClient))
332+
333+
// Fetch remove version, as some feature compatibility might change depending on it.
334+
remoteVersion, err := getRemoteVersion(tracingWrappedExecutor{cmdhttp.NewClient(host, opts...)})
335+
if err != nil {
336+
return nil, err
337+
}
338+
opts = append(opts, cmdhttp.ClientWithRawAbsPath(remoteVersion.LT(encodedAbsolutePathVersion)))
321339

322340
return tracingWrappedExecutor{cmdhttp.NewClient(host, opts...)}, nil
323341
}
@@ -417,3 +435,40 @@ func resolveAddr(ctx context.Context, addr ma.Multiaddr) (ma.Multiaddr, error) {
417435

418436
return addrs[0], nil
419437
}
438+
439+
type nopWriter struct {
440+
io.Writer
441+
}
442+
443+
func (nw nopWriter) Close() error {
444+
return nil
445+
}
446+
447+
func getRemoteVersion(exe cmds.Executor) (*semver.Version, error) {
448+
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30))
449+
defer cancel()
450+
451+
req, err := cmds.NewRequest(ctx, []string{"version"}, nil, nil, nil, Root)
452+
if err != nil {
453+
return nil, err
454+
}
455+
456+
var buf bytes.Buffer
457+
re, err := cmds.NewWriterResponseEmitter(nopWriter{&buf}, req)
458+
if err != nil {
459+
return nil, err
460+
}
461+
462+
err = exe.Execute(req, re, nil)
463+
if err != nil {
464+
return nil, err
465+
}
466+
467+
var out ipfs.VersionInfo
468+
dec := json.NewDecoder(&buf)
469+
if err := dec.Decode(&out); err != nil {
470+
return nil, err
471+
}
472+
473+
return semver.New(out.Version)
474+
}

‎docs/changelogs/v0.23.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
- [🔦 Highlights](#-highlights)
99
- [Mplex deprecation](#mplex-deprecation)
1010
- [Gateway: meaningful CAR responses on Not Found errors](#gateway-meaningful-car-responses-on-not-found-errors)
11+
- [Binary characters in file names: no longer works with old clients and new Kubo servers](#binary-characters-in-file-names-no-longer-works-with-old-clients-and-new-kubo-servers)
1112
- [📝 Changelog](#-changelog)
1213
- [👨‍👩‍👧‍👦 Contributors](#-contributors)
1314

@@ -43,6 +44,21 @@ the path does not exist, a CAR will be sent with a root of `bafkqaaa` (empty CID
4344
This CAR will contain all blocks necessary to validate that the path does not
4445
exist without having to trust the gateway.
4546

47+
#### Binary characters in file names: no longer works with old clients and new Kubo servers
48+
49+
In this version, we updated Kubo to support Go 1.20+. In Go 1.20, a regression
50+
regarding multipart headers was [introduced](https://github.com/golang/go/issues/60674).
51+
This only affects `ipfs add` when a file name has binary characters in its name.
52+
As a consequence, we had to update the encoding of the file name headers. This is
53+
the compatibility table:
54+
55+
| | New Client | Old Client |
56+
|------------|------------|-------------|
57+
| New Server | ✅ | 🟡* |
58+
| Old Server | ✅ | ✅ |
59+
60+
*Old clients can only send Unicode file paths to the server.
61+
4662
### 📝 Changelog
4763

4864
### 👨‍👩‍👧‍👦 Contributors

‎docs/examples/kubo-as-a-library/go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ go 1.20
77
replace github.com/ipfs/kubo => ./../../..
88

99
require (
10-
github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3
10+
github.com/ipfs/boxo v0.12.0
1111
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
1212
github.com/libp2p/go-libp2p v0.29.2
1313
github.com/multiformats/go-multiaddr v0.10.1
@@ -91,7 +91,7 @@ require (
9191
github.com/ipfs/go-unixfsnode v1.7.1 // indirect
9292
github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 // indirect
9393
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
94-
github.com/ipld/go-ipld-prime v0.20.0 // indirect
94+
github.com/ipld/go-ipld-prime v0.21.0 // indirect
9595
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
9696
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
9797
github.com/jbenet/goprocess v0.1.4 // indirect

‎docs/examples/kubo-as-a-library/go.sum

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn
159159
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
160160
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
161161
github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
162-
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
162+
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
163163
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
164164
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
165165
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -301,8 +301,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
301301
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
302302
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
303303
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
304-
github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3 h1:oiMqmivloEHtlCkS+8rCq1Pkz/rf6m3sC45lL4cnwBA=
305-
github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w=
304+
github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ=
305+
github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg=
306306
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
307307
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
308308
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -398,8 +398,8 @@ github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6
398398
github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
399399
github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
400400
github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
401-
github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
402-
github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
401+
github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
402+
github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
403403
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
404404
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
405405
github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
@@ -737,7 +737,7 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI
737737
github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
738738
github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
739739
github.com/warpfork/go-testmark v0.9.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
740-
github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U=
740+
github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
741741
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
742742
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
743743
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=

0 commit comments

Comments
 (0)