diff --git a/examples/sub-module/bundles-size-KBs.csv b/examples/sub-module/bundles-size-KBs.csv index 8101c63cd..3bc5bce32 100644 --- a/examples/sub-module/bundles-size-KBs.csv +++ b/examples/sub-module/bundles-size-KBs.csv @@ -1,28 +1,28 @@ name, bundled (KBs), minified (KBs) -IPFS, 1685.41, 661.64 -add, 968.11, 344.13 -bitswap, 684.80, 232.13 -block, 724.77, 250.23 -bootstrap, 685.64, 232.39 -cat, 725.05, 250.33 -commands, 683.89, 231.76 -config, 686.87, 233.10 -dht, 688.51, 233.41 -diag, 684.97, 232.18 -files, 1120.38, 404.30 -get, 907.74, 318.35 -id, 684.31, 231.95 -key, 684.59, 232.03 -log, 685.40, 232.38 -ls, 684.00, 231.80 -mount, 684.18, 231.86 -name, 684.63, 232.06 -object, 923.66, 340.57 -pin, 685.51, 232.39 -ping, 684.59, 231.90 -pubsub, 740.40, 249.54 -refs, 684.39, 231.94 -repo, 684.56, 232.02 -swarm, 1324.18, 527.03 -update, 684.45, 231.96 -version, 684.21, 231.88 +IPFS, 1412.20, 573.44 +add, 591.17, 198.23 +bitswap, 590.14, 197.96 +block, 630.50, 216.31 +bootstrap, 590.97, 198.22 +cat, 630.78, 216.41 +commands, 589.22, 197.59 +config, 592.21, 198.93 +dht, 593.86, 199.24 +diag, 590.31, 198.00 +files, 669.07, 235.88 +get, 661.57, 233.16 +id, 589.65, 197.78 +key, 589.93, 197.86 +log, 590.74, 198.20 +ls, 589.35, 197.63 +mount, 589.53, 197.69 +name, 589.97, 197.88 +object, 833.17, 307.73 +pin, 590.86, 198.22 +ping, 589.94, 197.73 +pubsub, 595.31, 199.76 +refs, 589.74, 197.77 +repo, 589.91, 197.85 +swarm, 1239.42, 498.59 +update, 589.79, 197.79 +version, 589.55, 197.71 diff --git a/examples/sub-module/test-modules-size.sh b/examples/sub-module/test-modules-size.sh old mode 100644 new mode 100755 index ef9955438..0b330d6a2 --- a/examples/sub-module/test-modules-size.sh +++ b/examples/sub-module/test-modules-size.sh @@ -6,7 +6,7 @@ modules=($(ls modules/)) echo "name, bundled (KBs), minified (KBs)" # Full IPFS module -webpack --display none --config webpack.confg.js complete-module.js complete-bundle.js +webpack --display none --config webpack.config.js complete-module.js complete-bundle.js babili complete-bundle.js -o complete-bundle-minified.js ipfsBundleSize=($(wc -c < complete-bundle.js | awk '{b=$1/1024; printf "%.2f\n", b}' | sed 's/,/./g')) diff --git a/package.json b/package.json index 7a4577355..97be67c51 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "gulp": "^3.9.1", "interface-ipfs-core": "~0.32.1", "hapi": "^16.6.2", - "ipfsd-ctl": "~0.23.0", + "ipfsd-ctl": "~0.24.0", "pre-commit": "^1.2.2", "socket.io": "^2.0.3", "socket.io-client": "^2.0.3", diff --git a/src/files/add.js b/src/files/add.js index cfd9a279e..324a49416 100644 --- a/src/files/add.js +++ b/src/files/add.js @@ -2,8 +2,8 @@ const isStream = require('is-stream') const promisify = require('promisify-es6') -const DAGNodeStream = require('../utils/dagnode-stream') const ProgressStream = require('../utils/progress-stream') +const converter = require('../utils/converter') module.exports = (send) => { return promisify((files, opts, callback) => { @@ -44,9 +44,8 @@ module.exports = (send) => { const request = { path: 'add', files: files, qs: qs, progress: opts.progress } - // Transform the response stream to DAGNode values - const transform = (res, callback) => DAGNodeStream - .streamToValue(send, ProgressStream.fromStream(opts.progress, res), callback) - send.andTransform(request, transform, callback) + send.andTransform(request, (response, cb) => { + converter(ProgressStream.fromStream(opts.progress, response), cb) + }, callback) }) } diff --git a/src/util/fs-add.js b/src/util/fs-add.js index 0803b36a9..e305069fd 100644 --- a/src/util/fs-add.js +++ b/src/util/fs-add.js @@ -2,7 +2,7 @@ const isNode = require('detect-node') const promisify = require('promisify-es6') -const DAGNodeStream = require('../utils/dagnode-stream') +const converter = require('../utils/converter') const moduleConfig = require('../utils/module-config') module.exports = (arg) => { @@ -31,14 +31,7 @@ module.exports = (arg) => { return callback(new Error('"path" must be a string')) } - const request = { - path: 'add', - qs: opts, - files: path - } - - // Transform the response stream to DAGNode values - const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback) - send.andTransform(request, transform, callback) + const request = { path: 'add', files: path, qs: opts } + send.andTransform(request, converter, callback) }) } diff --git a/src/util/url-add.js b/src/util/url-add.js index 3ecc691d6..a9889e64f 100644 --- a/src/util/url-add.js +++ b/src/util/url-add.js @@ -4,7 +4,7 @@ const promisify = require('promisify-es6') const once = require('once') const parseUrl = require('url').parse const request = require('../utils/request') -const DAGNodeStream = require('../utils/dagnode-stream') +const converter = require('../utils/converter') const moduleConfig = require('../utils/module-config') module.exports = (arg) => { @@ -25,12 +25,12 @@ module.exports = (arg) => { opts = {} } + callback = once(callback) + if (!validUrl(url)) { return callback(new Error('"url" param must be an http(s) url')) } - callback = once(callback) - requestWithRedirect(url, opts, send, callback) }) } @@ -52,14 +52,9 @@ const requestWithRedirect = (url, opts, send, callback) => { } requestWithRedirect(redirection, opts, send, callback) } else { - const params = { - path: 'add', - qs: opts, - files: res - } - // Transform the response stream to DAGNode values - const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback) - send.andTransform(params, transform, callback) + const request = { path: 'add', files: res, qs: opts } + + send.andTransform(request, converter, callback) } }).end() } diff --git a/src/utils/dagnode-stream.js b/src/utils/converter.js similarity index 53% rename from src/utils/dagnode-stream.js rename to src/utils/converter.js index 15198649c..b372e0d09 100644 --- a/src/utils/dagnode-stream.js +++ b/src/utils/converter.js @@ -3,18 +3,18 @@ const pump = require('pump') const TransformStream = require('readable-stream').Transform const streamToValue = require('./stream-to-value') -const getDagNode = require('./get-dagnode') /* Transforms a stream of {Name, Hash} objects to include size of the DAG object. - Usage: inputStream.pipe(DAGNodeStream({ send: send })) + Usage: inputStream.pipe(new Converter()) Input object format: { Name: '/path/to/file/foo.txt', Hash: 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP' + Size: '20' } Output object format: @@ -24,38 +24,34 @@ const getDagNode = require('./get-dagnode') size: 20 } */ -class DAGNodeStream extends TransformStream { +class ConverterStream extends TransformStream { constructor (options) { const opts = Object.assign(options || {}, { objectMode: true }) super(opts) - this._send = opts.send } - static streamToValue (send, inputStream, callback) { - const outputStream = pump(inputStream, new DAGNodeStream({ send: send }), (err) => { - if (err) { - callback(err) - } + _transform (obj, enc, callback) { + this.push({ + path: obj.Name, + hash: obj.Hash, + size: parseInt(obj.Size, 10) }) - streamToValue(outputStream, callback) + + callback(null) } +} - _transform (obj, enc, callback) { - getDagNode(this._send, obj.Hash, (err, node) => { +function converter (inputStream, callback) { + const outputStream = pump( + inputStream, + new ConverterStream(), + (err) => { if (err) { - return callback(err) - } - - const result = { - path: obj.Name, - hash: obj.Hash, - size: node.size + callback(err) } - - this.push(result) - callback(null) }) - } + + streamToValue(outputStream, callback) } -module.exports = DAGNodeStream +module.exports = converter diff --git a/src/utils/get-dagnode.js b/src/utils/get-dagnode.js deleted file mode 100644 index 1cfd1dcd7..000000000 --- a/src/utils/get-dagnode.js +++ /dev/null @@ -1,54 +0,0 @@ -'use strict' - -const DAGNode = require('ipld-dag-pb').DAGNode -const parallel = require('async/parallel') -const CID = require('cids') -const streamToValue = require('./stream-to-value') - -module.exports = function (send, hash, callback) { - let cid - - try { - cid = new CID(hash) - } catch (err) { - return callback(err) - } - - // Retrieve the object and its data in parallel, then produce a DAGNode - // instance using this information. - parallel([ - (done) => { - send({ - path: 'object/get', - args: cid.toBaseEncodedString() - }, done) - }, - (done) => { - // WORKAROUND: request the object's data separately, since raw bits in JSON - // are interpreted as UTF-8 and corrupt the data. - // See https://github.com/ipfs/go-ipfs/issues/1582 for more details. - send({ - path: 'object/data', - args: cid.toBaseEncodedString() - }, done) - } - ], (err, res) => { - if (err) { - return callback(err) - } - - var object = res[0] - var stream = res[1] - - if (Buffer.isBuffer(stream)) { - DAGNode.create(stream, object.Links, callback) - } else { - streamToValue(stream, (err, data) => { - if (err) { - return callback(err) - } - DAGNode.create(data, object.Links, callback) - }) - } - }) -} diff --git a/src/utils/stream-to-value.js b/src/utils/stream-to-value.js index 69213f776..fa7068d46 100644 --- a/src/utils/stream-to-value.js +++ b/src/utils/stream-to-value.js @@ -6,9 +6,9 @@ const concat = require('concat-stream') /* Concatenate a stream to a single value. */ -function streamToValue (res, callback) { +function streamToValue (response, callback) { pump( - res, + response, concat((data) => callback(null, data)), (err) => { if (err) {