Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 6eeaca4

Browse files
lidellidel
authored
fix(pubsub): multibase in pubsub http rpc (#3922)
This PR aims to restore interop with go-ipfs by applying the same changes as in ipfs/kubo#8183 TLDR is that we clean up and unify the API. BREAKING CHANGE: We had to make breaking changes to `pubsub` commands sent over HTTP RPC to fix data corruption caused by topic names and payload bytes that included `\n`. More details in ipfs/kubo#7939 and ipfs/kubo#8183
1 parent 33f1034 commit 6eeaca4

File tree

23 files changed

+159
-82
lines changed

23 files changed

+159
-82
lines changed

‎.github/workflows/test.yml

+6-6
Original file line numberDiff line numberDiff line change
@@ -331,9 +331,9 @@ jobs:
331331
- name: ipfs browser exchange files
332332
repo: https://github.com./ipfs-examples/js-ipfs-browser-exchange-files.git
333333
deps: ipfs-core@$PWD/packages/ipfs-core/dist,ipfs@$PWD/packages/ipfs/dist,ipfs-core-types@$PWD/packages/ipfs-core-types/dist,ipfs-http-client@$PWD/packages/ipfs-http-client/dist
334-
- name: ipfs browser ipns publish
335-
repo: https://github.com./ipfs-examples/js-ipfs-browser-ipns-publish.git
336-
deps: ipfs-core@$PWD/packages/ipfs-core/dist,ipfs-http-client@$PWD/packages/ipfs-http-client/dist
334+
#- name: ipfs browser ipns publish TODO: re-enable after example bumped to go-ipfs 0.11 and ipfs-http-client from https://github.com./ipfs/js-ipfs/pull/3922
335+
# repo: https://github.com./ipfs-examples/js-ipfs-browser-ipns-publish.git
336+
# deps: ipfs-core@$PWD/packages/ipfs-core/dist,ipfs-http-client@$PWD/packages/ipfs-http-client/dist
337337
- name: ipfs browser mfs
338338
repo: https://github.com./ipfs-examples/js-ipfs-browser-mfs.git
339339
deps: ipfs-core@$PWD/packages/ipfs-core/dist
@@ -373,9 +373,9 @@ jobs:
373373
- name: ipfs custom libp2p
374374
repo: https://github.com./ipfs-examples/js-ipfs-custom-libp2p.git
375375
deps: ipfs-core@$PWD/packages/ipfs-core/dist
376-
- name: ipfs-http-client browser pubsub
377-
repo: https://github.com./ipfs-examples/js-ipfs-http-client-browser-pubsub.git
378-
deps: ipfs-http-client@$PWD/packages/ipfs-http-client/dist,ipfs@$PWD/packages/ipfs/dist
376+
#- name: ipfs-http-client browser pubsub TODO: re-enable after example bumped to go-ipfs 0.11 and ipfs-http-client from https://github.com./ipfs/js-ipfs/pull/3922
377+
# repo: https://github.com./ipfs-examples/js-ipfs-http-client-browser-pubsub.git
378+
# deps: ipfs-http-client@$PWD/packages/ipfs-http-client/dist,ipfs@$PWD/packages/ipfs/dist
379379
- name: ipfs-http-client bundle webpack
380380
repo: https://github.com./ipfs-examples/js-ipfs-http-client-bundle-webpack.git
381381
deps: ipfs-http-client@$PWD/packages/ipfs-http-client/dist,ipfs@$PWD/packages/ipfs/dist

‎packages/interface-ipfs-core/src/add.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -463,9 +463,9 @@ export function testAdd (factory, options) {
463463
sharding: true
464464
},
465465
config: {
466-
// enable sharding for go
467-
Experimental: {
468-
ShardingEnabled: true
466+
// enable sharding for go with automatic threshold dropped to the minimum so it shards everything
467+
Internal: {
468+
UnixFSShardingSizeThreshold: '1B'
469469
}
470470
}
471471
}

‎packages/interface-ipfs-core/src/files/cp.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -358,9 +358,9 @@ export function testCp (factory, options) {
358358
sharding: true
359359
},
360360
config: {
361-
// enable sharding for go
362-
Experimental: {
363-
ShardingEnabled: true
361+
// enable sharding for go with automatic threshold dropped to the minimum so it shards everything
362+
Internal: {
363+
UnixFSShardingSizeThreshold: '1B'
364364
}
365365
}
366366
}

‎packages/interface-ipfs-core/src/files/ls.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,9 @@ export function testLs (factory, options) {
174174
sharding: true
175175
},
176176
config: {
177-
// enable sharding for go
178-
Experimental: {
179-
ShardingEnabled: true
177+
// enable sharding for go with automatic threshold dropped to the minimum so it shards everything
178+
Internal: {
179+
UnixFSShardingSizeThreshold: '1B'
180180
}
181181
}
182182
}

‎packages/interface-ipfs-core/src/files/mkdir.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -235,9 +235,9 @@ export function testMkdir (factory, options) {
235235
sharding: true
236236
},
237237
config: {
238-
// enable sharding for go
239-
Experimental: {
240-
ShardingEnabled: true
238+
// enable sharding for go with automatic threshold dropped to the minimum so it shards everything
239+
Internal: {
240+
UnixFSShardingSizeThreshold: '1B'
241241
}
242242
}
243243
}

‎packages/interface-ipfs-core/src/files/mv.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -120,9 +120,9 @@ export function testMv (factory, options) {
120120
sharding: true
121121
},
122122
config: {
123-
// enable sharding for go
124-
Experimental: {
125-
ShardingEnabled: true
123+
// enable sharding for go with automatic threshold dropped to the minimum so it shards everything
124+
Internal: {
125+
UnixFSShardingSizeThreshold: '1B'
126126
}
127127
}
128128
}

‎packages/interface-ipfs-core/src/files/read.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,9 @@ export function testRead (factory, options) {
124124
sharding: true
125125
},
126126
config: {
127-
// enable sharding for go
128-
Experimental: {
129-
ShardingEnabled: true
127+
// enable sharding for go with automatic threshold dropped to the minimum so it shards everything
128+
Internal: {
129+
UnixFSShardingSizeThreshold: '1B'
130130
}
131131
}
132132
}

‎packages/interface-ipfs-core/src/files/rm.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -143,9 +143,9 @@ export function testRm (factory, options) {
143143
sharding: true
144144
},
145145
config: {
146-
// enable sharding for go
147-
Experimental: {
148-
ShardingEnabled: true
146+
// enable sharding for go with automatic threshold dropped to the minimum so it shards everything
147+
Internal: {
148+
UnixFSShardingSizeThreshold: '1B'
149149
}
150150
}
151151
}

‎packages/interface-ipfs-core/src/files/stat.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -382,9 +382,9 @@ export function testStat (factory, options) {
382382
sharding: true
383383
},
384384
config: {
385-
// enable sharding for go
386-
Experimental: {
387-
ShardingEnabled: true
385+
// enable sharding for go with automatic threshold dropped to the minimum so it shards everything
386+
Internal: {
387+
UnixFSShardingSizeThreshold: '1B'
388388
}
389389
}
390390
}

‎packages/interface-ipfs-core/src/files/write.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -661,9 +661,9 @@ export function testWrite (factory, options) {
661661
sharding: true
662662
},
663663
config: {
664-
// enable sharding for go
665-
Experimental: {
666-
ShardingEnabled: true
664+
// enable sharding for go with automatic threshold dropped to the minimum so it shards everything
665+
Internal: {
666+
UnixFSShardingSizeThreshold: '1B'
667667
}
668668
}
669669
}

‎packages/ipfs-cli/test/pubsub.spec.js

+11-11
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ describe('pubsub', () => {
6767
timeout: undefined
6868
}
6969

70-
it('should list toic peers', async () => {
70+
it('should list topic peers', async () => {
7171
const subName = 'sub-name'
7272
const peer = 'peer-id'
7373

@@ -79,7 +79,7 @@ describe('pubsub', () => {
7979
expect(out).to.equal(`${peer}\n`)
8080
})
8181

82-
it('should list toic peers with a timeout', async () => {
82+
it('should list topic peers with a timeout', async () => {
8383
const subName = 'sub-name'
8484
const peer = 'peer-id'
8585

@@ -101,19 +101,19 @@ describe('pubsub', () => {
101101
}
102102

103103
it('should publish message', async () => {
104-
const subName = 'sub-name'
105-
const data = 'data'
104+
const subName = 'sub-name-1'
105+
const data = 'data\r\nfirst\nZażółć gęślą jaźń😇'
106106

107-
await cli(`pubsub pub ${subName} ${data}`, { ipfs })
107+
await cli(`pubsub pub ${subName} "${data}"`, { ipfs })
108108

109109
expect(ipfs.pubsub.publish.calledWith(subName, uint8ArrayFromString(data), defaultOptions)).to.be.true()
110110
})
111111

112112
it('should publish message with timeout', async () => {
113-
const subName = 'sub-name'
114-
const data = 'data'
113+
const subName = 'sub-name-2'
114+
const data = 'data\r\nsecond\nZażółć gęślą jaźń😇'
115115

116-
await cli(`pubsub pub ${subName} ${data} --timeout=1s`, { ipfs })
116+
await cli(`pubsub pub ${subName} "${data}" --timeout=1s`, { ipfs })
117117

118118
expect(ipfs.pubsub.publish.calledWith(subName, uint8ArrayFromString(data), {
119119
...defaultOptions,
@@ -128,17 +128,17 @@ describe('pubsub', () => {
128128
}
129129

130130
it('should subscribe', async () => {
131-
const subName = 'sub-name'
131+
const subName = 'sub\nname'
132132

133-
await cli(`pubsub sub ${subName}`, { ipfs })
133+
await cli(`pubsub sub "${subName}"`, { ipfs })
134134

135135
expect(ipfs.pubsub.subscribe.calledWith(subName, sinon.match.func, defaultOptions)).to.be.true()
136136
})
137137

138138
it('should subscribe with a timeout', async () => {
139139
const subName = 'sub-name'
140140

141-
await cli(`pubsub sub ${subName} --timeout=1s`, { ipfs })
141+
await cli(`pubsub sub "${subName}" --timeout=1s`, { ipfs })
142142

143143
expect(ipfs.pubsub.subscribe.calledWith(subName, sinon.match.func, {
144144
...defaultOptions,

‎packages/ipfs-core/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@
136136
"@types/rimraf": "^3.0.1",
137137
"aegir": "^36.0.1",
138138
"delay": "^5.0.0",
139-
"go-ipfs": "0.10.0",
139+
"go-ipfs": "0.11.0",
140140
"interface-blockstore-tests": "^2.0.1",
141141
"interface-ipfs-core": "^0.152.2",
142142
"ipfsd-ctl": "^10.0.4",

‎packages/ipfs-http-client/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@
7777
"devDependencies": {
7878
"aegir": "^36.0.1",
7979
"delay": "^5.0.0",
80-
"go-ipfs": "0.10.0",
80+
"go-ipfs": "0.11.0",
8181
"ipfsd-ctl": "^10.0.4",
8282
"it-all": "^1.0.4",
8383
"it-first": "^1.0.4",

‎packages/ipfs-http-client/src/files/rm.js

+10-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { configure } from '../lib/configure.js'
22
import { toUrlSearchParams } from '../lib/to-url-search-params.js'
3+
import HTTP from 'ipfs-utils/src/http.js'
34

45
/**
56
* @typedef {import('../types').HTTPClientExtraOptions} HTTPClientExtraOptions
@@ -20,7 +21,15 @@ export const createRm = configure(api => {
2021
headers: options.headers
2122
})
2223

23-
await res.text()
24+
const body = await res.text()
25+
// we don't expect text body to be ever present
26+
// (if so, it means an error such as https://github.com./ipfs/go-ipfs/issues/8606)
27+
if (body !== '') {
28+
/** @type {Error} */
29+
const error = new HTTP.HTTPError(res)
30+
error.message = body
31+
throw error
32+
}
2433
}
2534
return rm
2635
})

‎packages/ipfs-http-client/src/lib/http-rpc-wire-format.js

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
2+
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
3+
import { base64url } from 'multiformats/bases/base64'
4+
5+
/* HTTP RPC:
6+
* - wraps binary data in multibase. base64url is used to avoid issues
7+
* when a binary data is passed as search param in URL.
8+
* Historical context: https://github.com./ipfs/go-ipfs/issues/7939
9+
* Multibase wrapping introduced in: https://github.com./ipfs/go-ipfs/pull/8183
10+
*/
11+
12+
/**
13+
* @param {Array<string>} strings
14+
* @returns {Array<string>} strings
15+
*/
16+
const rpcArrayToTextArray = strings => {
17+
if (Array.isArray(strings)) {
18+
return strings.map(rpcToText)
19+
}
20+
return strings
21+
}
22+
23+
/**
24+
* @param {string} mb
25+
* @returns {string}
26+
*/
27+
const rpcToText = mb => uint8ArrayToString(rpcToBytes(mb))
28+
29+
/**
30+
* @param {string} mb
31+
* @returns {Uint8Array}
32+
*/
33+
const rpcToBytes = mb => base64url.decode(mb)
34+
35+
/**
36+
* @param {string} text
37+
* @returns {string}
38+
*/
39+
const textToUrlSafeRpc = text => base64url.encode(uint8ArrayFromString(text))
40+
41+
export { rpcArrayToTextArray, rpcToText, rpcToBytes, textToUrlSafeRpc }

‎packages/ipfs-http-client/src/pubsub/ls.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { configure } from '../lib/configure.js'
22
import { toUrlSearchParams } from '../lib/to-url-search-params.js'
3+
import { rpcArrayToTextArray } from '../lib/http-rpc-wire-format.js'
34

45
/**
56
* @typedef {import('../types').HTTPClientExtraOptions} HTTPClientExtraOptions
@@ -17,7 +18,7 @@ export const createLs = configure(api => {
1718
headers: options.headers
1819
})).json()
1920

20-
return Strings || []
21+
return rpcArrayToTextArray(Strings) || []
2122
}
2223
return ls
2324
})

‎packages/ipfs-http-client/src/pubsub/peers.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { configure } from '../lib/configure.js'
22
import { toUrlSearchParams } from '../lib/to-url-search-params.js'
3+
import { textToUrlSafeRpc } from '../lib/http-rpc-wire-format.js'
34

45
/**
56
* @typedef {import('../types').HTTPClientExtraOptions} HTTPClientExtraOptions
@@ -14,7 +15,7 @@ export const createPeers = configure(api => {
1415
const res = await api.post('pubsub/peers', {
1516
signal: options.signal,
1617
searchParams: toUrlSearchParams({
17-
arg: topic,
18+
arg: textToUrlSafeRpc(topic),
1819
...options
1920
}),
2021
headers: options.headers

‎packages/ipfs-http-client/src/pubsub/publish.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { configure } from '../lib/configure.js'
22
import { toUrlSearchParams } from '../lib/to-url-search-params.js'
33
import { multipartRequest } from 'ipfs-core-utils/multipart-request'
44
import { abortSignal } from '../lib/abort-signal.js'
5+
import { textToUrlSafeRpc } from '../lib/http-rpc-wire-format.js'
56
import { AbortController } from 'native-abort-controller'
67

78
/**
@@ -15,7 +16,7 @@ export const createPublish = configure(api => {
1516
*/
1617
async function publish (topic, data, options = {}) {
1718
const searchParams = toUrlSearchParams({
18-
arg: topic,
19+
arg: textToUrlSafeRpc(topic),
1920
...options
2021
})
2122

‎packages/ipfs-http-client/src/pubsub/subscribe.js

+6-7
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
2-
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
31
import debug from 'debug'
42
import { configure } from '../lib/configure.js'
53
import { toUrlSearchParams } from '../lib/to-url-search-params.js'
4+
import { textToUrlSafeRpc, rpcArrayToTextArray, rpcToBytes } from '../lib/http-rpc-wire-format.js'
65
const log = debug('ipfs-http-client:pubsub:subscribe')
76

87
/**
@@ -43,7 +42,7 @@ export const createSubscribe = (options, subsTracker) => {
4342
api.post('pubsub/sub', {
4443
signal: options.signal,
4544
searchParams: toUrlSearchParams({
46-
arg: topic,
45+
arg: textToUrlSafeRpc(topic),
4746
...options
4847
}),
4948
headers: options.headers
@@ -95,10 +94,10 @@ async function readMessages (response, { onMessage, onEnd, onError }) {
9594
}
9695

9796
onMessage({
98-
from: uint8ArrayToString(uint8ArrayFromString(msg.from, 'base64pad'), 'base58btc'),
99-
data: uint8ArrayFromString(msg.data, 'base64pad'),
100-
seqno: uint8ArrayFromString(msg.seqno, 'base64pad'),
101-
topicIDs: msg.topicIDs
97+
from: msg.from,
98+
data: rpcToBytes(msg.data),
99+
seqno: rpcToBytes(msg.seqno),
100+
topicIDs: rpcArrayToTextArray(msg.topicIDs)
102101
})
103102
} catch (/** @type {any} */ err) {
104103
err.message = `Failed to parse pubsub message: ${err.message}`

‎packages/ipfs-http-client/test/utils/factory.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ const commonOptions = {
1616

1717
const commonOverrides = {
1818
go: {
19-
ipfsBin: isNode ? path() : undefined
19+
ipfsBin: isNode ? (process.env.IPFS_GO_EXEC || path()) : undefined
2020
}
2121
}
2222

0 commit comments

Comments
 (0)