Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 860165c

Browse files
dignifiedquiredaviddias
authored andcommitted
feat: DHT integration PART I
1 parent 92f8ef9 commit 860165c

File tree

17 files changed

+428
-96
lines changed

17 files changed

+428
-96
lines changed

README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,8 @@ const node = new IPFS({
232232
EXPERIMENTAL: { // enable experimental features
233233
pubsub: true,
234234
sharding: true, // enable dir sharding
235-
wrtcLinuxWindows: true // use unstable wrtc module on Linux or Windows with Node.js
235+
wrtcLinuxWindows: true // use unstable wrtc module on Linux or Windows with Node.js,
236+
dht: true // enable KadDHT, currently not interopable with go-ipfs
236237
},
237238
config: { // overload the default config
238239
Addresses: {

package.json

+20-20
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131
"test:unit:node:http": "TEST=http npm run test:unit:node",
3232
"test:unit:node:cli": "TEST=cli npm run test:unit:node",
3333
"test:unit:browser": "gulp test:browser --dom",
34-
"test:interop": "mocha -t 60000 test/interop",
34+
"test:interop": "npm run test:interop:node",
3535
"test:interop:node": "mocha -t 60000 test/interop/node.js",
36-
"test:interop:browser": "mocha test/interop/browser.js",
36+
"test:interop:browser": "mocha -t 60000 test/interop/browser.js",
3737
"test:benchmark": "echo \"Error: no benchmarks yet\" && exit 1",
3838
"test:benchmark:node": "echo \"Error: no benchmarks yet\" && exit 1",
3939
"test:benchmark:node:core": "echo \"Error: no benchmarks yet\" && exit 1",
@@ -63,23 +63,23 @@
6363
},
6464
"homepage": "https://github.com./ipfs/js-ipfs#readme",
6565
"devDependencies": {
66-
"aegir": "^11.0.1",
66+
"aegir": "^11.0.2",
6767
"buffer-loader": "0.0.1",
6868
"chai": "^3.5.0",
6969
"delay": "^2.0.0",
7070
"detect-node": "^2.0.3",
71-
"dir-compare": "^1.3.0",
71+
"dir-compare": "^1.4.0",
7272
"dirty-chai": "^1.2.2",
73-
"eslint-plugin-react": "^6.10.3",
73+
"eslint-plugin-react": "^7.0.1",
7474
"execa": "^0.6.3",
7575
"expose-loader": "^0.7.3",
76-
"form-data": "^2.1.2",
76+
"form-data": "^2.1.4",
7777
"gulp": "^3.9.1",
78-
"interface-ipfs-core": "~0.27.0",
78+
"interface-ipfs-core": "~0.27.2",
7979
"ipfsd-ctl": "~0.20.0",
8080
"left-pad": "^1.1.3",
8181
"lodash": "^4.17.4",
82-
"mocha": "^3.2.0",
82+
"mocha": "^3.4.1",
8383
"ncp": "^2.0.0",
8484
"nexpect": "^0.5.0",
8585
"pre-commit": "^1.2.2",
@@ -91,17 +91,17 @@
9191
"transform-loader": "^0.2.4"
9292
},
9393
"dependencies": {
94-
"async": "^2.3.0",
95-
"bl": "^1.2.0",
94+
"async": "^2.4.0",
95+
"bl": "^1.2.1",
9696
"boom": "^4.3.1",
9797
"cids": "^0.5.0",
98-
"debug": "^2.6.3",
98+
"debug": "^2.6.8",
9999
"fsm-event": "^2.1.0",
100100
"glob": "^7.1.1",
101101
"hapi": "^16.1.1",
102102
"hapi-set-header": "^1.0.2",
103103
"hoek": "^4.1.1",
104-
"ipfs-api": "^14.0.0",
104+
"ipfs-api": "^14.0.1",
105105
"ipfs-bitswap": "~0.13.0",
106106
"ipfs-block": "~0.6.0",
107107
"ipfs-block-service": "~0.9.0",
@@ -111,10 +111,10 @@
111111
"ipfs-unixfs-engine": "~0.19.1",
112112
"ipld-resolver": "~0.11.0",
113113
"isstream": "^0.1.2",
114-
"joi": "^10.4.1",
114+
"joi": "^10.5.0",
115115
"libp2p-floodsub": "~0.9.4",
116-
"libp2p-ipfs-browser": "~0.23.0",
117-
"libp2p-ipfs-nodejs": "~0.23.0",
116+
"libp2p-ipfs-browser": "~0.24.1",
117+
"libp2p-ipfs-nodejs": "~0.25.2",
118118
"lodash.flatmap": "^4.5.0",
119119
"lodash.get": "^4.4.2",
120120
"lodash.has": "^4.5.2",
@@ -132,21 +132,21 @@
132132
"peer-info": "~0.9.2",
133133
"promisify-es6": "^1.0.2",
134134
"pull-file": "^1.0.0",
135-
"pull-paramap": "^1.2.1",
136-
"pull-pushable": "^2.0.1",
135+
"pull-paramap": "^1.2.2",
136+
"pull-pushable": "^2.1.1",
137137
"pull-sort": "^1.0.0",
138-
"pull-stream": "^3.5.0",
138+
"pull-stream": "^3.6.0",
139139
"pull-stream-to-stream": "^1.3.4",
140140
"pull-zip": "^2.0.1",
141141
"read-pkg-up": "^2.0.0",
142142
"readable-stream": "1.1.14",
143143
"safe-buffer": "^5.0.1",
144144
"stream-to-pull-stream": "^1.7.2",
145-
"tar-stream": "^1.5.2",
145+
"tar-stream": "^1.5.4",
146146
"temp": "^0.8.3",
147147
"through2": "^2.0.3",
148148
"update-notifier": "^2.1.0",
149-
"yargs": "7.0.2"
149+
"yargs": "8.0.1"
150150
},
151151
"contributors": [
152152
"Andrew de Andrade <[email protected]>",

src/core/components/dht.js

+163
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
'use strict'
2+
3+
const promisify = require('promisify-es6')
4+
const every = require('async/every')
5+
const PeerId = require('peer-id')
6+
const CID = require('cids')
7+
const each = require('async/each')
8+
// const bsplit = require('buffer-split')
9+
10+
module.exports = (self) => {
11+
return {
12+
/**
13+
* Given a key, query the DHT for its best value.
14+
*
15+
* @param {Buffer} key
16+
* @param {function(Error)} [callback]
17+
* @returns {Promise|void}
18+
*/
19+
get: promisify((key, options, callback) => {
20+
if (!Buffer.isBuffer(key)) {
21+
return callback(new Error('Not valid key'))
22+
}
23+
24+
if (typeof options === 'function') {
25+
callback = options
26+
options = {}
27+
}
28+
29+
self._libp2pNode.dht.get(key, options.timeout, callback)
30+
}),
31+
32+
/**
33+
* Write a key/value pair to the DHT.
34+
*
35+
* Given a key of the form /foo/bar and a value of any
36+
* form, this will write that value to the DHT with
37+
* that key.
38+
*
39+
* @param {Buffer} key
40+
* @param {Buffer} value
41+
* @param {function(Error)} [callback]
42+
* @returns {Promise|void}
43+
*/
44+
put: promisify((key, value, callback) => {
45+
if (!Buffer.isBuffer(key)) {
46+
return callback(new Error('Not valid key'))
47+
}
48+
49+
self._libp2pNode.dht.put(key, value, callback)
50+
}),
51+
52+
/**
53+
* Find peers in the DHT that can provide a specific value, given a key.
54+
*
55+
* @param {CID} key - They key to find providers for.
56+
* @param {function(Error, Array<PeerInfo>)} [callback]
57+
* @returns {Promise<PeerInfo>|void}
58+
*/
59+
findprovs: promisify((key, callback) => {
60+
if (typeof key === 'string') {
61+
key = new CID(key)
62+
}
63+
64+
self._libp2pNode.contentRouting.findProviders(key, callback)
65+
}),
66+
67+
/**
68+
* Query the DHT for all multiaddresses associated with a `PeerId`.
69+
*
70+
* @param {PeerId} peer - The id of the peer to search for.
71+
* @param {function(Error, Array<Multiaddr>)} [callback]
72+
* @returns {Promise<Array<Multiaddr>>|void}
73+
*/
74+
findpeer: promisify((peer, callback) => {
75+
if (typeof peer === 'string') {
76+
peer = PeerId.createFromB58String(peer)
77+
}
78+
79+
self._libp2pNode.peerRouting.findPeer(peer, (err, info) => {
80+
if (err) {
81+
return callback(err)
82+
}
83+
84+
// convert to go-ipfs return value, we need to revisit
85+
// this. For now will just conform.
86+
const goResult = [
87+
{
88+
Responses: [{
89+
ID: info.id.toB58String(),
90+
Addresses: info.multiaddrs.toArray().map((a) => a.toString())
91+
}]
92+
}
93+
]
94+
95+
callback(null, goResult)
96+
})
97+
}),
98+
99+
/**
100+
* Announce to the network that we are providing given values.
101+
*
102+
* @param {CID|Array<CID>} keys - The keys that should be announced.
103+
* @param {Object} [options={}]
104+
* @param {bool} [options.recursive=false] - Provide not only the given object but also all objects linked from it.
105+
* @param {function(Error)} [callback]
106+
* @returns {Promise|void}
107+
*/
108+
provide: promisify((keys, options, callback) => {
109+
if (!Array.isArray(keys)) {
110+
keys = [keys]
111+
}
112+
if (typeof options === 'function') {
113+
callback = options
114+
options = {}
115+
}
116+
117+
// ensure blocks are actually local
118+
every(keys, (key, cb) => {
119+
self._repo.blockstore.has(key, cb)
120+
}, (err, has) => {
121+
if (err) {
122+
return callback(err)
123+
}
124+
/* TODO reconsider this. go-ipfs provides anyway
125+
if (!has) {
126+
return callback(new Error('Not all blocks exist locally, can not provide'))
127+
}
128+
*/
129+
130+
if (options.recursive) {
131+
// TODO: Implement recursive providing
132+
} else {
133+
each(keys, (cid, cb) => {
134+
self._libp2pNode.contentRouting.provide(cid, cb)
135+
}, callback)
136+
}
137+
})
138+
}),
139+
140+
/**
141+
* Find the closest peers to a given `PeerId`, by querying the DHT.
142+
*
143+
* @param {PeerId} peer - The `PeerId` to run the query agains.
144+
* @param {function(Error, Array<PeerId>)} [callback]
145+
* @returns {Promise<Array<PeerId>>|void}
146+
*/
147+
query: promisify((peerId, callback) => {
148+
if (typeof peerId === 'string') {
149+
peerId = PeerId.createFromB58String(peerId)
150+
}
151+
152+
// TODO expose this method in peerRouting
153+
self._libp2pNode._dht.getClosestPeers(peerId.toBytes(), (err, peerIds) => {
154+
if (err) {
155+
return callback(err)
156+
}
157+
callback(null, peerIds.map((id) => {
158+
return { ID: id.toB58String() }
159+
}))
160+
})
161+
})
162+
}
163+
}

src/core/components/id.js

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ module.exports = function id (self) {
1515
addresses: self._peerInfo.multiaddrs
1616
.toArray()
1717
.map((ma) => ma.toString())
18+
.filter((ma) => ma.indexOf('ipfs') >= 0)
1819
.sort(),
1920
agentVersion: 'js-ipfs',
2021
protocolVersion: '9000'

src/core/components/index.js

+1
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,4 @@ exports.ping = require('./ping')
1919
exports.files = require('./files')
2020
exports.bitswap = require('./bitswap')
2121
exports.pubsub = require('./pubsub')
22+
exports.dht = require('./dht')

src/core/components/libp2p.js

+13-11
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,23 @@ module.exports = function libp2p (self) {
1717
const options = {
1818
mdns: get(config, 'Discovery.MDNS.Enabled'),
1919
webRTCStar: get(config, 'Discovery.webRTCStar.Enabled'),
20-
bootstrap: get(config, 'Bootstrap')
20+
bootstrap: get(config, 'Bootstrap'),
21+
dht: self._options.EXPERIMENTAL.dht
2122
}
2223

2324
self._libp2pNode = new Node(self._peerInfo, self._peerInfoBook, options)
2425

26+
self._libp2pNode.on('peer:discovery', (peerInfo) => {
27+
if (self.isOnline()) {
28+
self._peerInfoBook.put(peerInfo)
29+
self._libp2pNode.dial(peerInfo, () => {})
30+
}
31+
})
32+
33+
self._libp2pNode.on('peer:connect', (peerInfo) => {
34+
self._peerInfoBook.put(peerInfo)
35+
})
36+
2537
self._libp2pNode.start((err) => {
2638
if (err) {
2739
return callback(err)
@@ -31,16 +43,6 @@ module.exports = function libp2p (self) {
3143
console.log('Swarm listening on', ma.toString())
3244
})
3345

34-
self._libp2pNode.on('peer:discovery', (peerInfo) => {
35-
if (self.isOnline()) {
36-
self._peerInfoBook.put(peerInfo)
37-
self._libp2pNode.dial(peerInfo, () => {})
38-
}
39-
})
40-
self._libp2pNode.on('peer:connect', (peerInfo) => {
41-
self._peerInfoBook.put(peerInfo)
42-
})
43-
4446
callback()
4547
})
4648
}

src/core/index.js

+5
Original file line numberDiff line numberDiff line change
@@ -93,13 +93,18 @@ class IPFS extends EventEmitter {
9393
this.bitswap = components.bitswap(this)
9494
this.ping = components.ping(this)
9595
this.pubsub = components.pubsub(this)
96+
this.dht = components.dht(this)
9697

9798
if (this._options.EXPERIMENTAL.pubsub) {
9899
this.log('EXPERIMENTAL pubsub is enabled')
99100
}
100101
if (this._options.EXPERIMENTAL.sharding) {
101102
this.log('EXPERIMENTAL sharding is enabled')
102103
}
104+
if (this._options.EXPERIMENTAL.dht) {
105+
this.log('EXPERIMENTAL Kademlia DHT is enabled')
106+
}
107+
103108
this.state = require('./state')(this)
104109

105110
boot(this)

test/core/interface/dht.js

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/* eslint-env mocha */
2+
3+
'use strict'
4+
5+
/*
6+
const test = require('interface-ipfs-core')
7+
const IPFSFactory = require('../../utils/ipfs-factory-instance')
8+
9+
let factory
10+
11+
const common = {
12+
setup: function (callback) {
13+
factory = new IPFSFactory()
14+
callback(null, factory)
15+
},
16+
teardown: function (callback) {
17+
factory.dismantle(callback)
18+
}
19+
}
20+
21+
test.dht(common)
22+
*/

test/core/interface/interface.spec.js

+1
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,6 @@ describe('interface-ipfs-core tests', () => {
1414
if (isNode) {
1515
require('./swarm')
1616
require('./pubsub')
17+
require('./dht')
1718
}
1819
})

0 commit comments

Comments
 (0)