feat: support UnixFSv1.5 metadata

* refactor: pass mode and mtime in headers

* refactor: refactor clunky test

* feat: store mtime as timespec

Author: achingbrain

package.json

@@ -27,12 +27,12 @@
   },
   "dependencies": {
     "@hapi/content": "^4.1.0",
-    "it-multipart": "~0.0.2"
+    "it-multipart": "^1.0.1"
   },
   "devDependencies": {
     "aegir": "^20.0.0",
     "chai": "^4.2.0",
-    "ipfs-http-client": "^35.1.0",
+    "ipfs-http-client": "^40.2.0",
     "request": "^2.88.0"
   },
   "engines": {

src/parser.js

@@ -12,22 +12,16 @@ const isDirectory = (mediatype) => mediatype === multipartFormdataType || mediat
 const parseDisposition = (disposition) => {
   const details = {}
   details.type = disposition.split(';')[0]
-  if (details.type === 'file' || details.type === 'form-data') {
-    const namePattern = / filename="(.[^"]+)"/
-    const matches = disposition.match(namePattern)
-    details.name = matches ? matches[1] : ''
-  }
-
-  return details
-}
 
-const parseHeader = (header) => {
-  const type = Content.type(header['content-type'])
-  const disposition = parseDisposition(header['content-disposition'])
+  if (details.type === 'file' || details.type === 'form-data') {
+    const filenamePattern = / filename="(.[^"]+)"/
+    const filenameMatches = disposition.match(filenamePattern)
+    details.filename = filenameMatches ? filenameMatches[1] : ''
 
-  const details = type
-  details.name = decodeURIComponent(disposition.name)
-  details.type = disposition.type
+    const namePattern = / name="(.[^"]+)"/
+    const nameMatches = disposition.match(namePattern)
+    details.name = nameMatches ? nameMatches[1] : ''
+  }
 
   return details
 }
@@ -50,49 +44,92 @@ const ignore = async (stream) => {
   }
 }
 
-async function * parser (stream, options) {
-  for await (const part of multipart(stream, options.boundary)) {
-    const partHeader = parseHeader(part.headers)
+async function * parseEntry (stream, options) {
+  for await (const part of stream) {
+    if (!part.headers['content-type']) {
+      throw new Error('No content-type in multipart part')
+    }
 
-    if (isDirectory(partHeader.mime)) {
-      yield {
-        type: 'directory',
-        name: partHeader.name
-      }
+    const type = Content.type(part.headers['content-type'])
 
-      await ignore(part.body)
+    if (type.boundary) {
+      // recursively parse nested multiparts
+      yield * parser(part.body, {
+        ...options,
+        boundary: type.boundary
+      })
 
       continue
     }
 
-    if (partHeader.mime === applicationSymlink) {
-      const target = await collect(part.body)
+    if (!part.headers['content-disposition']) {
+      throw new Error('No content disposition in multipart part')
+    }
 
-      yield {
-        type: 'symlink',
-        name: partHeader.name,
-        target: target.toString('utf8')
+    const entry = {}
+
+    if (part.headers.mtime) {
+      entry.mtime = {
+        secs: parseInt(part.headers.mtime, 10)
       }
 
-      continue
+      if (part.headers['mtime-nsecs']) {
+        entry.mtime.nsecs = parseInt(part.headers['mtime-nsecs'], 10)
+      }
     }
 
-    if (partHeader.boundary) {
-      // recursively parse nested multiparts
-      for await (const entry of parser(part, {
-        ...options,
-        boundary: partHeader.boundary
-      })) {
-        yield entry
+    if (part.headers.mode) {
+      entry.mode = parseInt(part.headers.mode, 8)
+    }
+
+    if (isDirectory(type.mime)) {
+      entry.type = 'directory'
+    } else if (type.mime === applicationSymlink) {
+      entry.type = 'symlink'
+    } else {
+      entry.type = 'file'
+    }
+
+    const disposition = parseDisposition(part.headers['content-disposition'])
+
+    entry.name = decodeURIComponent(disposition.filename)
+    entry.body = part.body
+
+    yield entry
+  }
+}
+
+async function * parser (stream, options) {
+  for await (const entry of parseEntry(multipart(stream, options.boundary), options)) {
+    if (entry.type === 'directory') {
+      yield {
+        type: 'directory',
+        name: entry.name,
+        mtime: entry.mtime,
+        mode: entry.mode
       }
 
-      continue
+      await ignore(entry.body)
+    }
+
+    if (entry.type === 'symlink') {
+      yield {
+        type: 'symlink',
+        name: entry.name,
+        target: (await collect(entry.body)).toString('utf8'),
+        mtime: entry.mtime,
+        mode: entry.mode
+      }
     }
 
-    yield {
-      type: 'file',
-      name: partHeader.name,
-      content: part.body
+    if (entry.type === 'file') {
+      yield {
+        type: 'file',
+        name: entry.name,
+        content: entry.body,
+        mtime: entry.mtime,
+        mode: entry.mode
+      }
     }
   }
 }

test/parser.spec.js

@@ -14,7 +14,7 @@ const os = require('os')
 
 const isWindows = os.platform() === 'win32'
 
-const readDir = (path, prefix, output = []) => {
+const readDir = (path, prefix, includeMetadata, output = []) => {
   const entries = fs.readdirSync(path)
 
   entries.forEach(entry => {
@@ -23,21 +23,25 @@ const readDir = (path, prefix, output = []) => {
     const type = fs.statSync(entryPath)
 
     if (type.isDirectory()) {
-      readDir(entryPath, `${prefix}/${entry}`, output)
+      readDir(entryPath, `${prefix}/${entry}`, includeMetadata, output)
+
+      output.push({
+        path: `${prefix}/${entry}`,
+        mtime: includeMetadata ? new Date(type.mtimeMs) : undefined,
+        mode: includeMetadata ? type.mode : undefined
+      })
     }
 
     if (type.isFile()) {
       output.push({
         path: `${prefix}/${entry}`,
-        content: fs.createReadStream(entryPath)
+        content: fs.createReadStream(entryPath),
+        mtime: includeMetadata ? new Date(type.mtimeMs) : undefined,
+        mode: includeMetadata ? type.mode : undefined
       })
     }
   })
 
-  output.push({
-    path: prefix
-  })
-
   return output
 }
 
@@ -75,6 +79,8 @@ describe('parser', () => {
   describe('single file', () => {
     const filePath = path.resolve(__dirname, 'fixtures/config')
     const fileContent = fs.readFileSync(filePath, 'utf8')
+    const fileMtime = parseInt(Date.now() / 1000)
+    const fileMode = parseInt('0777', 8)
 
     before(() => {
       handler = async (req) => {
@@ -84,7 +90,7 @@ describe('parser', () => {
 
         for await (const entry of parser(req)) {
           if (entry.type === 'file') {
-            const file = { name: entry.name, content: '' }
+            const file = { ...entry, content: '' }
 
             for await (const data of entry.content) {
               file.content += data.toString()
@@ -95,13 +101,12 @@ describe('parser', () => {
         }
 
         expect(files.length).to.equal(1)
-        expect(files[0].name).to.equal('config')
-        expect(files[0].content).to.equal(fileContent)
+        expect(JSON.parse(files[0].content)).to.deep.equal(JSON.parse(fileContent))
       }
     })
 
     it('parses ctl.config.replace correctly', async () => {
-      await ctl.config.replace(filePath)
+      await ctl.config.replace(JSON.parse(fileContent))
     })
 
     it('parses regular multipart requests correctly', (done) => {
@@ -111,6 +116,22 @@ describe('parser', () => {
 
       request.post({ url: `http://localhost:${PORT}`, formData: formData }, (err) => done(err))
     })
+
+    it('parses multipart requests with metadata correctly', (done) => {
+      const formData = {
+        file: {
+          value: fileContent,
+          options: {
+            header: {
+              mtime: fileMtime,
+              mode: fileMode
+            }
+          }
+        }
+      }
+
+      request.post({ url: `http://localhost:${PORT}`, formData }, (err) => done(err))
+    })
   })
 
   describe('directory', () => {
@@ -123,15 +144,15 @@ describe('parser', () => {
         expect(req.headers['content-type']).to.be.a('string')
 
         for await (const entry of parser(req)) {
-          if (entry.type === 'file') {
-            const file = { name: entry.name, content: '' }
+          const file = { ...entry, content: '' }
 
+          if (entry.content) {
             for await (const data of entry.content) {
               file.content += data.toString()
             }
-
-            files.push(file)
           }
+
+          files.push(file)
         }
       }
     })
@@ -149,12 +170,37 @@ describe('parser', () => {
         return
       }
 
-      expect(files.length).to.equal(5)
-      expect(files[0].name).to.equal('fixtures/config')
-      expect(files[1].name).to.equal('fixtures/folderlink/deepfile')
-      expect(files[2].name).to.equal('fixtures/link')
-      expect(files[3].name).to.equal('fixtures/otherfile')
-      expect(files[4].name).to.equal('fixtures/subfolder/deepfile')
+      expect(files).to.have.lengthOf(contents.length)
+
+      for (let i = 0; i < contents.length; i++) {
+        expect(files[i].name).to.equal(contents[i].path)
+        expect(files[i].mode).to.be.undefined
+        expect(files[i].mtime).to.be.undefined
+      }
+    })
+
+    it('parses ctl.add with metadata correctly', async () => {
+      const contents = readDir(dirPath, 'fixtures', true)
+
+      await ctl.add(contents, { recursive: true, followSymlinks: false })
+
+      if (isWindows) {
+        return
+      }
+
+      expect(files).to.have.lengthOf(contents.length)
+
+      for (let i = 0; i < contents.length; i++) {
+        const msecs = contents[i].mtime.getTime()
+        const secs = Math.floor(msecs / 1000)
+
+        expect(files[i].name).to.equal(contents[i].path)
+        expect(files[i].mode).to.equal(contents[i].mode)
+        expect(files[i].mtime).to.deep.equal({
+          secs,
+          nsecs: (msecs - (secs * 1000)) * 1000
+        })
+      }
     })
   })