docs: Tutorial - How to customize the IPFS Repo

* docs: add example to show how to customize ipfs repo

* feat: allow repo options to be passed in the ipfs constructor

* docs: update repo creation in custom repo example

* docs: update custom repo example

remove ability to add repoOptions in favor of just supplying a repo

chore: remove outdated options

* docs: fix incorrect description in custom repo example

automatically stop the node in the custom repo example

docs: update custom repo example

* docs: clarify custom-repo docs

chore: bump custom repo example ipfs-repo version

* docs: add a step to have users check their local repo

fix: resolve bugs in the custom s3 lock

Author: Jacob Heun

examples/README.md

@@ -21,6 +21,7 @@ Let us know if you find any issue or if you want to contribute and add a new tut
 - [js-ipfs in the browser with a `<script>` tag](./browser-script-tag)
 - [js-ipfs in electron](./run-in-electron)
 - [Using streams to add a directory of files to ipfs](./browser-add-readable-stream)
+- [Customizing the ipfs repository](./custom-ipfs-repo)
 
 ## Understanding the IPFS Stack
 

examples/custom-ipfs-repo/README.md

@@ -0,0 +1,32 @@
+# Customizing the IPFS Repo
+
+This example shows you how to customize your repository, including where your data is stored and how the repo locking is managed. Customizing your repo makes it easier to extend IPFS for your particular needs. You may want to customize your repo if:
+
+* If you want to store data somewhere that’s not on your local disk, like S3, a Redis instance, a different machine on your local network, or in your own database system, like MongoDB or Postgres, you might use a custom datastore.
+* If you have multiple browser windows or workers sharing the same IPFS storage, you might want to use a custom lock to coordinate between them. (Locking is currently only used to ensure a single IPFS instance can access a repo at a time. This check is done on `repo.open()`. A more complex lock, coupled with a custom datastore, could allow for safe writes to a single datastore from multiple IPFS nodes.)
+
+You can find full details on customization in the [IPFS Repo Docs](https://github.com./ipfs/js-ipfs-repo#setup).
+
+## Run this example
+
+```
+> npm install
+> npm start
+```
+
+## Other Options
+
+### Custom `storageBackends`
+This example leverages [datastore-fs](https://github.com./ipfs/js-datastore-fs) to store all data in the IPFS Repo. You can customize each of the 4 `storageBackends` to meet the needs of your project. For an example on how to manage your entire IPFS REPO on S3, you can see the [datastore-s3 example](https://github.com./ipfs/js-datastore-s3/tree/master/examples/full-s3-repo).
+
+### Custom Repo Lock
+This example uses one of the locks that comes with IPFS Repo. If you would like to control how locking happens, such as with a centralized S3 IPFS Repo, you can pass in your own custom lock. See [custom-lock.js](./custom-lock.js) for an example of a custom lock that can be used for [datastore-s3](https://github.com./ipfs/js-datastore-s3). This is also being used in the [full S3 example](https://github.com./ipfs/js-datastore-s3/tree/master/examples/full-s3-repo).
+
+```js
+const S3Lock = require('./custom-lock')
+
+const repo = new Repo('/tmp/.ipfs', {
+  ...
+  lock: new S3Lock(s3DatastoreInstance)
+})
+```

examples/custom-ipfs-repo/custom-lock.js

@@ -0,0 +1,99 @@
+'use strict'
+
+const PATH = require('path')
+
+/**
+ * Uses an object in an S3 bucket as a lock to signal that an IPFS repo is in use.
+ * When the object exists, the repo is in use. You would normally use this to make
+ * sure multiple IPFS nodes don’t use the same S3 bucket as a datastore at the same time.
+ */
+class S3Lock {
+  constructor (s3Datastore) {
+    this.s3 = s3Datastore
+  }
+
+  /**
+   * Returns the location of the lock file given the path it should be located at
+   *
+   * @private
+   * @param {string} dir
+   * @returns {string}
+   */
+  getLockfilePath (dir) {
+    return PATH.join(dir, 'repo.lock')
+  }
+
+  /**
+   * Creates the lock. This can be overriden to customize where the lock should be created
+   *
+   * @param {string} dir
+   * @param {function(Error, LockCloser)} callback
+   * @returns {void}
+   */
+  lock (dir, callback) {
+    const lockPath = this.getLockfilePath(dir)
+
+    this.locked(dir, (err, alreadyLocked) => {
+      if (err || alreadyLocked) {
+        return callback(new Error('The repo is already locked'))
+      }
+
+      // There's no lock yet, create one
+      this.s3.put(lockPath, Buffer.from(''), (err, data) => {
+        if (err) {
+          return callback(err, null)
+        }
+
+        callback(null, this.getCloser(lockPath))
+      })
+    })
+  }
+
+  /**
+   * Returns a LockCloser, which has a `close` method for removing the lock located at `lockPath`
+   *
+   * @param {string} lockPath
+   * @returns {LockCloser}
+   */
+  getCloser (lockPath) {
+    return {
+      /**
+       * Removes the lock. This can be overriden to customize how the lock is removed. This
+       * is important for removing any created locks.
+       *
+       * @param {function(Error)} callback
+       * @returns {void}
+       */
+      close: (callback) => {
+        this.s3.delete(lockPath, (err) => {
+          if (err && err.statusCode !== 404) {
+            return callback(err)
+          }
+
+          callback(null)
+        })
+      }
+    }
+  }
+
+  /**
+   * Calls back on whether or not a lock exists. Override this method to customize how the check is made.
+   *
+   * @param {string} dir
+   * @param {function(Error, boolean)} callback
+   * @returns {void}
+   */
+  locked (dir, callback) {
+    this.s3.get(this.getLockfilePath(dir), (err, data) => {
+      if (err && err.message.match(/not found/)) {
+        return callback(null, false)
+      } else if (err) {
+        return callback(err)
+      }
+
+      callback(null, true)
+    })
+  }
+}
+
+module.exports = S3Lock

examples/custom-ipfs-repo/index.js

@@ -0,0 +1,107 @@
+'use strict'
+
+const IPFS = require('ipfs')
+const Repo = require('ipfs-repo')
+const fsLock = require('ipfs-repo/src/lock')
+
+// Create our custom options
+const customRepositoryOptions = {
+
+  /**
+   * IPFS nodes store different information in separate storageBackends, or datastores.
+   * Each storage backend can use the same type of datastore or a different one — you
+   * could store your keys in a levelDB database while everything else is in files,
+   * for example. (See https://github.com./ipfs/interface-datastore for more about datastores.)
+   */
+  storageBackends: {
+    root: require('datastore-fs'), // version and config data will be saved here
+    blocks: require('datastore-fs'),
+    keys: require('datastore-fs'),
+    datastore: require('datastore-fs')
+  },
+
+  /**
+   * Storage Backend Options will get passed into the instantiation of their counterpart
+   * in `storageBackends`. If you create a custom datastore, this is where you can pass in
+   * custom constructor arguments. You can see an S3 datastore example at:
+   * https://github.com./ipfs/js-datastore-s3/tree/master/examples/full-s3-repo
+   *
+   * NOTE: The following options are being overriden for demonstration purposes only.
+   * In most instances you can simply use the default options, by not passing in any
+   * overrides, which is recommended if you have no need to override.
+   */
+  storageBackendOptions: {
+    root: {
+      extension: '.ipfsroot', // Defaults to ''. Used by datastore-fs; Appended to all files
+      errorIfExists: false, // Used by datastore-fs; If the datastore exists, don't throw an error
+      createIfMissing: true // Used by datastore-fs; If the datastore doesn't exist yet, create it
+    },
+    blocks: {
+      sharding: false, // Used by IPFSRepo Blockstore to determine sharding; Ignored by datastore-fs
+      extension: '.ipfsblock', // Defaults to '.data'.
+      errorIfExists: false,
+      createIfMissing: true
+    },
+    keys: {
+      extension: '.ipfskey', // No extension by default
+      errorIfExists: false,
+      createIfMissing: true
+    },
+    datastore: {
+      extension: '.ipfsds', // No extension by default
+      errorIfExists: false,
+      createIfMissing: true
+    }
+  },
+
+  /**
+   * A custom lock can be added here. Or the build in Repo `fs` or `memory` locks can be used.
+   * See https://github.com./ipfs/js-ipfs-repo for more details on setting the lock.
+   */
+  lock: fsLock
+}
+
+// Initialize our IPFS node with the custom repo options
+const node = new IPFS({
+  repo: new Repo('/tmp/custom-repo/.ipfs', customRepositoryOptions)
+})
+
+// Test the new repo by adding and fetching some data
+node.on('ready', () => {
+  console.log('Ready')
+  node.version()
+    .then((version) => {
+      console.log('Version:', version.version)
+    })
+    // Once we have the version, let's add a file to IPFS
+    .then(() => {
+      return node.files.add({
+        path: 'test-data.txt',
+        content: Buffer.from('We are using a customized repo!')
+      })
+    })
+    // Log out the added files metadata and cat the file from IPFS
+    .then((filesAdded) => {
+      console.log('\nAdded file:', filesAdded[0].path, filesAdded[0].hash)
+      return node.files.cat(filesAdded[0].hash)
+    })
+    // Print out the files contents to console
+    .then((data) => {
+      console.log('\nFetched file content:')
+      process.stdout.write(data)
+    })
+    // Log out the error, if there is one
+    .catch((err) => {
+      console.log('File Processing Error:', err)
+    })
+    // After everything is done, shut the node down
+    // We don't need to worry about catching errors here
+    .then(() => {
+      console.log('\n\nStopping the node')
+      return node.stop()
+    })
+    // Let users know where they can inspect the repo
+    .then(() => {
+      console.log('Check "/tmp/custom-repo/.ipfs" to see what your customized repository looks like on disk.')
+    })
+})

examples/custom-ipfs-repo/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "custom-ipfs-repo",
+  "version": "0.1.0",
+  "description": "Customizing your ipfs repo",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node index.js"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "datastore-fs": "~0.4.2",
+    "ipfs": "file:../../",
+    "ipfs-repo": "^0.20.0"
+  }
+}