From e6f09159e3f0e84318aabcb8f972894064f5d579 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 11 Dec 2019 16:25:47 +0000 Subject: [PATCH 1/2] refactor: convert dag API to async/await --- src/cli/commands/dag/resolve.js | 3 +- src/core/components/dag.js | 170 ----------------------------- src/core/components/dag/get.js | 34 ++++++ src/core/components/dag/put.js | 71 ++++++++++++ src/core/components/dag/resolve.js | 15 +++ src/core/components/dag/tree.js | 15 +++ src/core/components/dag/utils.js | 48 ++++++++ src/core/components/index.js | 7 +- src/core/components/init.js | 12 +- src/core/components/start.js | 8 +- src/core/components/stop.js | 8 +- src/http/api/resources/dag.js | 2 +- 12 files changed, 216 insertions(+), 177 deletions(-) delete mode 100644 src/core/components/dag.js create mode 100644 src/core/components/dag/get.js create mode 100644 src/core/components/dag/put.js create mode 100644 src/core/components/dag/resolve.js create mode 100644 src/core/components/dag/tree.js create mode 100644 src/core/components/dag/utils.js diff --git a/src/cli/commands/dag/resolve.js b/src/cli/commands/dag/resolve.js index bba7886034..7a9907f427 100644 --- a/src/cli/commands/dag/resolve.js +++ b/src/cli/commands/dag/resolve.js @@ -19,10 +19,9 @@ module.exports = { const options = {} try { - const result = await ipfs.dag.resolve(ref, options) let lastCid - for (const res of result) { + for await (const res of ipfs.dag.resolve(ref, options)) { if (CID.isCID(res.value)) { lastCid = res.value } diff --git a/src/core/components/dag.js b/src/core/components/dag.js deleted file mode 100644 index fd704e8139..0000000000 --- a/src/core/components/dag.js +++ /dev/null @@ -1,170 +0,0 @@ -'use strict' - -const callbackify = require('callbackify') -const CID = require('cids') -const all = require('async-iterator-all') -const errCode = require('err-code') -const multicodec = require('multicodec') - -function parseArgs (cid, path, options) { - options = options || {} - - // Allow options in path position - if (path !== undefined && typeof path !== 'string') { - options = path - path = undefined - } - - if (typeof cid === 'string') { - if (cid.startsWith('/ipfs/')) { - cid = cid.substring(6) - } - - const split = cid.split('/') - - try { - cid = new CID(split[0]) - } catch (err) { - throw errCode(err, 'ERR_INVALID_CID') - } - - split.shift() - - if (split.length > 0) { - path = split.join('/') - } else { - path = path || '/' - } - } else if (Buffer.isBuffer(cid)) { - try { - cid = new CID(cid) - } catch (err) { - throw errCode(err, 'ERR_INVALID_CID') - } - } - - return [ - cid, - path, - options - ] -} - -module.exports = function dag (self) { - return { - put: callbackify.variadic(async (dagNode, options) => { - options = options || {} - - if (options.cid && (options.format || options.hashAlg)) { - throw new Error('Can\'t put dag node. Please provide either `cid` OR `format` and `hashAlg` options.') - } else if (((options.format && !options.hashAlg) || (!options.format && options.hashAlg))) { - throw new Error('Can\'t put dag node. Please provide `format` AND `hashAlg` options.') - } - - const optionDefaults = { - format: multicodec.DAG_CBOR, - hashAlg: multicodec.SHA2_256 - } - - // The IPLD expects the format and hashAlg as constants - if (options.format && typeof options.format === 'string') { - const constantName = options.format.toUpperCase().replace(/-/g, '_') - options.format = multicodec[constantName] - } - if (options.hashAlg && typeof options.hashAlg === 'string') { - const constantName = options.hashAlg.toUpperCase().replace(/-/g, '_') - options.hashAlg = multicodec[constantName] - } - - options = options.cid ? options : Object.assign({}, optionDefaults, options) - - // js-ipld defaults to verion 1 CIDs. Hence set version 0 explicitly for - // dag-pb nodes - if (options.version === undefined) { - if (options.format === multicodec.DAG_PB && options.hashAlg === multicodec.SHA2_256) { - options.version = 0 - } else { - options.version = 1 - } - } - - let release - - if (options.pin) { - release = await self._gcLock.readLock() - } - - try { - const cid = await self._ipld.put(dagNode, options.format, { - hashAlg: options.hashAlg, - cidVersion: options.version - }) - - if (options.pin) { - await self.pin.add(cid, { - lock: false - }) - } - - if (options.preload !== false) { - self._preload(cid) - } - - return cid - } finally { - if (release) { - release() - } - } - }), - - get: callbackify.variadic(async (cid, path, options) => { - [cid, path, options] = parseArgs(cid, path, options) - - if (options.preload !== false) { - self._preload(cid) - } - - if (path == null || path === '/') { - const value = await self._ipld.get(cid) - - return { - value, - remainderPath: '' - } - } else { - let result - - for await (const entry of self._ipld.resolve(cid, path)) { - if (options.localResolve) { - return entry - } - - result = entry - } - - return result - } - }), - - tree: callbackify.variadic(async (cid, path, options) => { // eslint-disable-line require-await - [cid, path, options] = parseArgs(cid, path, options) - - if (options.preload !== false) { - self._preload(cid) - } - - return all(self._ipld.tree(cid, path, options)) - }), - - resolve: callbackify.variadic(async (cid, path, options) => { // eslint-disable-line require-await - [cid, path, options] = parseArgs(cid, path, options) - - if (options.preload !== false) { - self._preload(cid) - } - - return all(self._ipld.resolve(cid, path)) - }) - } -} diff --git a/src/core/components/dag/get.js b/src/core/components/dag/get.js new file mode 100644 index 0000000000..11c17152bc --- /dev/null +++ b/src/core/components/dag/get.js @@ -0,0 +1,34 @@ +'use strict' + +const { parseArgs } = require('./utils') + +module.exports = ({ ipld, preload }) => { + return async function get (cid, path, options) { + [cid, path, options] = parseArgs(cid, path, options) + + if (options.preload !== false) { + preload(cid) + } + + if (path == null || path === '/') { + const value = await ipld.get(cid) + + return { + value, + remainderPath: '' + } + } else { + let result + + for await (const entry of ipld.resolve(cid, path)) { + if (options.localResolve) { + return entry + } + + result = entry + } + + return result + } + } +} diff --git a/src/core/components/dag/put.js b/src/core/components/dag/put.js new file mode 100644 index 0000000000..7ab406dd1a --- /dev/null +++ b/src/core/components/dag/put.js @@ -0,0 +1,71 @@ +'use strict' + +const multicodec = require('multicodec') + +module.exports = ({ ipld, pin, gcLock, preload }) => { + return async function put (dagNode, options) { + options = options || {} + + if (options.cid && (options.format || options.hashAlg)) { + throw new Error('Can\'t put dag node. Please provide either `cid` OR `format` and `hashAlg` options.') + } else if (((options.format && !options.hashAlg) || (!options.format && options.hashAlg))) { + throw new Error('Can\'t put dag node. Please provide `format` AND `hashAlg` options.') + } + + const optionDefaults = { + format: multicodec.DAG_CBOR, + hashAlg: multicodec.SHA2_256 + } + + // The IPLD expects the format and hashAlg as constants + if (options.format && typeof options.format === 'string') { + const constantName = options.format.toUpperCase().replace(/-/g, '_') + options.format = multicodec[constantName] + } + if (options.hashAlg && typeof options.hashAlg === 'string') { + const constantName = options.hashAlg.toUpperCase().replace(/-/g, '_') + options.hashAlg = multicodec[constantName] + } + + options = options.cid ? options : Object.assign({}, optionDefaults, options) + + // js-ipld defaults to verion 1 CIDs. Hence set version 0 explicitly for + // dag-pb nodes + if (options.version === undefined) { + if (options.format === multicodec.DAG_PB && options.hashAlg === multicodec.SHA2_256) { + options.version = 0 + } else { + options.version = 1 + } + } + + let release + + if (options.pin) { + release = await gcLock.readLock() + } + + try { + const cid = await ipld.put(dagNode, options.format, { + hashAlg: options.hashAlg, + cidVersion: options.version + }) + + if (options.pin) { + await pin.add(cid, { + lock: false + }) + } + + if (options.preload !== false) { + preload(cid) + } + + return cid + } finally { + if (release) { + release() + } + } + } +} diff --git a/src/core/components/dag/resolve.js b/src/core/components/dag/resolve.js new file mode 100644 index 0000000000..e95e5b526f --- /dev/null +++ b/src/core/components/dag/resolve.js @@ -0,0 +1,15 @@ +'use strict' + +const { parseArgs } = require('./utils') + +module.exports = ({ ipld, preload }) => { + return async function * resolve (cid, path, options) { // eslint-disable-line require-await + [cid, path, options] = parseArgs(cid, path, options) + + if (options.preload !== false) { + preload(cid) + } + + yield * ipld.resolve(cid, path) + } +} diff --git a/src/core/components/dag/tree.js b/src/core/components/dag/tree.js new file mode 100644 index 0000000000..07d2d03e65 --- /dev/null +++ b/src/core/components/dag/tree.js @@ -0,0 +1,15 @@ +'use strict' + +const { parseArgs } = require('./utils') + +module.exports = ({ ipld, preload }) => { + return async function * tree (cid, path, options) { // eslint-disable-line require-await + [cid, path, options] = parseArgs(cid, path, options) + + if (options.preload !== false) { + preload(cid) + } + + yield * ipld.tree(cid, path, options) + } +} diff --git a/src/core/components/dag/utils.js b/src/core/components/dag/utils.js new file mode 100644 index 0000000000..810b0e2f9a --- /dev/null +++ b/src/core/components/dag/utils.js @@ -0,0 +1,48 @@ +'use strict' + +const CID = require('cids') +const errCode = require('err-code') + +exports.parseArgs = (cid, path, options) => { + options = options || {} + + // Allow options in path position + if (path !== undefined && typeof path !== 'string') { + options = path + path = undefined + } + + if (typeof cid === 'string') { + if (cid.startsWith('/ipfs/')) { + cid = cid.substring(6) + } + + const split = cid.split('/') + + try { + cid = new CID(split[0]) + } catch (err) { + throw errCode(err, 'ERR_INVALID_CID') + } + + split.shift() + + if (split.length > 0) { + path = split.join('/') + } else { + path = path || '/' + } + } else if (Buffer.isBuffer(cid)) { + try { + cid = new CID(cid) + } catch (err) { + throw errCode(err, 'ERR_INVALID_CID') + } + } + + return [ + cid, + path, + options + ] +} diff --git a/src/core/components/index.js b/src/core/components/index.js index bd1cdd2add..96598727c8 100644 --- a/src/core/components/index.js +++ b/src/core/components/index.js @@ -8,6 +8,12 @@ exports.block = { stat: require('./block/stat') } exports.config = require('./config') +exports.dag = { + get: require('./dag/get'), + put: require('./dag/put'), + resolve: require('./dag/resolve'), + tree: require('./dag/tree') +} exports.init = require('./init') exports.pin = { add: require('./pin/add'), @@ -18,7 +24,6 @@ exports.start = require('./start') exports.stop = require('./stop') exports.legacy = { // TODO: these will be removed as the new API is completed - dag: require('./dag'), libp2p: require('./libp2p'), object: require('./object') } diff --git a/src/core/components/init.js b/src/core/components/init.js index f409ad62f3..2dda0eba04 100644 --- a/src/core/components/init.js +++ b/src/core/components/init.js @@ -98,7 +98,11 @@ module.exports = ({ // Make sure GC lock is specific to repo, for tests where there are // multiple instances of IPFS const gcLock = mortice(repo.path, { singleProcess: constructorOptions.repoOwner }) - const dag = Commands.legacy.dag({ _ipld: ipld, _preload: preload }) + const dag = { + get: Commands.dag.get({ ipld, preload }), + resolve: Commands.dag.resolve({ ipld, preload }), + tree: Commands.dag.tree({ ipld, preload }) + } const object = Commands.legacy.object({ _ipld: ipld, _preload: preload, dag, _gcLock: gcLock }) const pinManager = new PinManager(repo, dag) @@ -110,6 +114,9 @@ module.exports = ({ rm: Commands.pin.rm({ pinManager, gcLock, object }) } + // FIXME: resolve this circular dependency + dag.put = Commands.dag.put({ ipld, pin, gcLock, preload }) + const add = Commands.add({ ipld, dag, preload, pin, gcLock, constructorOptions }) if (!isInitialized && !options.emptyRepo) { @@ -132,6 +139,7 @@ module.exports = ({ apiManager, constructorOptions, blockService, + dag, gcLock, initOptions: options, ipld, @@ -275,6 +283,7 @@ function createApi ({ apiManager, constructorOptions, blockService, + dag, gcLock, initOptions, ipld, @@ -310,6 +319,7 @@ function createApi ({ stat: Commands.block.stat({ blockService, preload }) }, config: Commands.config({ repo }), + dag, init: () => { throw new AlreadyInitializedError() }, pin, start diff --git a/src/core/components/start.js b/src/core/components/start.js index 147493a824..e7bd337f13 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -107,13 +107,19 @@ function createApi ({ print, repo }) { - const dag = Commands.legacy.dag({ _ipld: ipld, _preload: preload }) + const dag = { + get: Commands.dag.get({ ipld, preload }), + resolve: Commands.dag.resolve({ ipld, preload }), + tree: Commands.dag.tree({ ipld, preload }) + } const object = Commands.legacy.object({ _ipld: ipld, _preload: preload, dag, _gcLock: gcLock }) const pin = { add: Commands.pin.add({ pinManager, gcLock, dag, object }), ls: Commands.pin.ls({ pinManager, object }), rm: Commands.pin.rm({ pinManager, gcLock, object }) } + // FIXME: resolve this circular dependency + dag.put = Commands.dag.put({ ipld, pin, gcLock, preload }) const add = Commands.add({ ipld, dag, preload, pin, gcLock, constructorOptions }) const stop = Commands.stop({ diff --git a/src/core/components/stop.js b/src/core/components/stop.js index 64cac5d3bc..dcec2f00b5 100644 --- a/src/core/components/stop.js +++ b/src/core/components/stop.js @@ -76,13 +76,19 @@ function createApi ({ print, repo }) { - const dag = Commands.legacy.dag({ _ipld: ipld, _preload: preload }) + const dag = { + get: Commands.dag.get({ ipld, preload }), + resolve: Commands.dag.resolve({ ipld, preload }), + tree: Commands.dag.tree({ ipld, preload }) + } const object = Commands.legacy.object({ _ipld: ipld, _preload: preload, dag, _gcLock: gcLock }) const pin = { add: Commands.pin.add({ pinManager, gcLock, dag, object }), ls: Commands.pin.ls({ pinManager, object }), rm: Commands.pin.rm({ pinManager, gcLock, object }) } + // FIXME: resolve this circular dependency + dag.put = Commands.dag.put({ ipld, pin, gcLock, preload }) const add = Commands.add({ ipld, dag, preload, pin, gcLock, constructorOptions }) const start = Commands.start({ diff --git a/src/http/api/resources/dag.js b/src/http/api/resources/dag.js index 436382bc38..f3dffc1b4f 100644 --- a/src/http/api/resources/dag.js +++ b/src/http/api/resources/dag.js @@ -248,7 +248,7 @@ exports.resolve = { let lastRemainderPath = path if (path) { - const result = ipfs._ipld.resolve(lastCid, path) + const result = ipfs.dag.resolve(lastCid, path) while (true) { const resolveResult = (await result.next()).value if (!CID.isCID(resolveResult.value)) { From 380b8f56035aa792638f8917ab3b3055456588e0 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 16 Dec 2019 21:24:44 +0000 Subject: [PATCH 2/2] refactor: pull out repeated name to codec conversion code --- src/core/components/dag/put.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/components/dag/put.js b/src/core/components/dag/put.js index 7ab406dd1a..301c87ba8c 100644 --- a/src/core/components/dag/put.js +++ b/src/core/components/dag/put.js @@ -1,6 +1,7 @@ 'use strict' const multicodec = require('multicodec') +const nameToCodec = name => multicodec[name.toUpperCase().replace(/-/g, '_')] module.exports = ({ ipld, pin, gcLock, preload }) => { return async function put (dagNode, options) { @@ -19,12 +20,10 @@ module.exports = ({ ipld, pin, gcLock, preload }) => { // The IPLD expects the format and hashAlg as constants if (options.format && typeof options.format === 'string') { - const constantName = options.format.toUpperCase().replace(/-/g, '_') - options.format = multicodec[constantName] + options.format = nameToCodec(options.format) } if (options.hashAlg && typeof options.hashAlg === 'string') { - const constantName = options.hashAlg.toUpperCase().replace(/-/g, '_') - options.hashAlg = multicodec[constantName] + options.hashAlg = nameToCodec(options.hashAlg) } options = options.cid ? options : Object.assign({}, optionDefaults, options)