refactor for clarity, fix dist-url, add env var dist-url functionality

PR-URL: #711
Reviewed-By: Ben Noordhuis <[email protected]>

Author: rvagg

.gitignore

@@ -1,2 +1,3 @@
 gyp/test
 node_modules
+test/.node-gyp

lib/configure.js

@@ -145,8 +145,8 @@ function configure (gyp, argv, callback) {
         log.verbose('get node dir', 'no --target version specified, falling back to host node version: %s', release.version)
       }
 
-      // make sure we have a valid version
       if (!release.semver) {
+        // could not parse the version string with semver
         return callback(new Error('Invalid version number: ' + release.version))
       }
 
@@ -304,9 +304,8 @@ function configure (gyp, argv, callback) {
     var addon_gypi = path.resolve(__dirname, '..', 'addon.gypi')
     var common_gypi = path.resolve(nodeDir, 'include/node/common.gypi')
     fs.stat(common_gypi, function (err, stat) {
-      if (err || !stat.isFile()) {
+      if (err)
         common_gypi = path.resolve(nodeDir, 'common.gypi')
-      }
 
       var output_dir = 'build'
       if (win) {

lib/install.js

@@ -45,8 +45,8 @@ function install (gyp, argv, callback) {
   // Determine which node dev files version we are installing
   log.verbose('install', 'input version string %j', release.version)
 
-  // parse the version to normalize and ensure it's valid
   if (!release.semver) {
+    // could not parse the version string with semver
     return callback(new Error('Invalid version number: ' + release.version))
   }
 
@@ -150,7 +150,7 @@ function install (gyp, argv, callback) {
   }
 
   function getContentSha(res, callback) {
-    var shasum = crypto.createHash(release.checksumAlgo)
+    var shasum = crypto.createHash('sha256')
     res.on('data', function (chunk) {
       shasum.update(chunk)
     }).on('end', function () {
@@ -307,8 +307,8 @@ function install (gyp, argv, callback) {
       }
 
       function downloadShasums(done) {
-        log.verbose('check download content checksum, need to download `' + release.shasumsFile + '`...')
-        var shasumsPath = path.resolve(devDir, release.shasumsFile)
+        log.verbose('check download content checksum, need to download `SHASUMS256.txt`...')
+        var shasumsPath = path.resolve(devDir, 'SHASUMS256.txt')
 
         log.verbose('checksum url', release.shasumsUrl)
         var req = download(release.shasumsUrl)

lib/process-release.js

@@ -1,48 +1,75 @@
 var semver = require('semver')
-var url = require('url')
-var path = require('path')
+  , url = require('url')
+  , path = require('path')
 
-var bitsre = /(x86|x64)/
-var bitsreV3 = /(x86|ia32|x64)/ // io.js v3.x.x shipped with "ia32" but should
-                                // have been "x86"
+  , bitsre = /\/win-(x86|x64)\//
+  , bitsreV3 = /\/win-(x86|ia32|x64)\// // io.js v3.x.x shipped with "ia32" but should
+                                        // have been "x86"
 
 // Captures all the logic required to determine download URLs, local directory and 
-// file names and whether we need to use SHA-1 or 2. Inputs come from command-line
-// switches (--target), `process.version` and `process.release` where it exists.
+// file names. Inputs come from command-line switches (--target, --dist-url),
+// `process.version` and `process.release` where it exists.
 function processRelease (argv, gyp, defaultVersion, defaultRelease) {
   var version = argv[0] || gyp.opts.target || defaultVersion
+    , isDefaultVersion = version === defaultVersion
+    , versionSemver = semver.parse(version)
+    , overrideDistUrl = gyp.opts['dist-url'] || gyp.opts.disturl
+    , isIojs
+    , name
+    , distBaseUrl
+    , baseUrl
+    , libUrl32
+    , libUrl64
 
-  // parse the version to normalize and ensure it's valid
-  var versionSemver = semver.parse(version)
   if (!versionSemver) {
+    // not a valid semver string, nothing we can do
     return { version: version }
   }
   // flatten version into String
   version = versionSemver.version
 
-  var overrideDistUrl = gyp.opts['dist-url'] || gyp.opts.disturl
-  var iojs = !overrideDistUrl && !defaultRelease && semver.satisfies(version, '>=1.0.0 <4.0.0')
-  var name = (defaultRelease && defaultRelease.name.replace(/\./g, '')) || (iojs ? 'iojs' : 'node') // io.js -> iojs
-  var defaultDirUrl = (overrideDistUrl || iojs ? 'https://iojs.org/download/release' : 'https://nodejs.org/dist') + '/v' + version + '/'
-  var baseUrl
-  var libUrl32
-  var libUrl64
+  // can't use process.release if we're using --target=x.y.z
+  if (!isDefaultVersion)
+    defaultRelease = null
+
+  if (defaultRelease) {
+    // v3 onward, has process.release
+    name = defaultRelease.name.replace(/io\.js/, 'iojs') // remove the '.' for directory naming purposes
+    isIojs = name === 'iojs'
+  } else {
+    // old node or alternative --target=
+    isIojs = semver.satisfies(version, '>=1.0.0 <4.0.0')
+    name = isIojs ? 'iojs' : 'node'
+  }
+
+  // check for the nvm.sh standard mirror env variables
+  if (!overrideDistUrl) {
+    if (isIojs && process.env.NVM_IOJS_ORG_MIRROR)
+      overrideDistUrl = process.env.NVM_IOJS_ORG_MIRROR
+    else if (process.env.NVM_NODEJS_ORG_MIRROR)
+      overrideDistUrl = process.env.NVM_NODEJS_ORG_MIRROR
+  }
+
+
+  if (overrideDistUrl)
+    distBaseUrl = overrideDistUrl.replace(/\/+$/, '')
+  else
+    distBaseUrl = isIojs ? 'https://iojs.org/download/release' : 'https://nodejs.org/dist'
+  distBaseUrl += '/v' + version + '/'
 
   // new style, based on process.release so we have a lot of the data we need
   if (defaultRelease && defaultRelease.headersUrl && !overrideDistUrl) {
     baseUrl = url.resolve(defaultRelease.headersUrl, './')
-    libUrl32 = resolveLibUrl(name, defaultRelease.libUrl || baseUrl || defaultDirUrl, 'x86', version)
-    libUrl64 = resolveLibUrl(name, defaultRelease.libUrl || baseUrl || defaultDirUrl, 'x64', version)
+    libUrl32 = resolveLibUrl(name, defaultRelease.libUrl || baseUrl || distBaseUrl, 'x86', version)
+    libUrl64 = resolveLibUrl(name, defaultRelease.libUrl || baseUrl || distBaseUrl, 'x64', version)
 
     return {
       version: version,
       semver: versionSemver,
       name: name,
       baseUrl: baseUrl,
       tarballUrl: defaultRelease.headersUrl,
-      shasumsFile: 'SHASUMS256.txt',
       shasumsUrl: url.resolve(baseUrl, 'SHASUMS256.txt'),
-      checksumAlgo: 'sha256',
       versionDir: (name !== 'node' ? name + '-' : '') + version,
       libUrl32: libUrl32,
       libUrl64: libUrl64,
@@ -52,25 +79,24 @@ function processRelease (argv, gyp, defaultVersion, defaultRelease) {
   }
 
   // older versions without process.release are captured here and we have to make
-  // a lot of assumptions
-
-  // distributions starting with 0.10.0 contain sha256 checksums
-  var checksumAlgo = semver.gte(version, '0.10.0') ? 'sha256' : 'sha1'
-  var shasumsFile = (checksumAlgo === 'sha256') ? 'SHASUMS256.txt' : 'SHASUMS.txt'
+  // a lot of assumptions, additionally if you --target=x.y.z then we can't use the
+  // current process.release
 
-  baseUrl = defaultDirUrl
+  baseUrl = distBaseUrl
   libUrl32 = resolveLibUrl(name, baseUrl, 'x86', version)
   libUrl64 = resolveLibUrl(name, baseUrl, 'x64', version)
+  // making the bold assumption that anything with a version number >3.0.0 will
+  // have a *-headers.tar.gz file in its dist location, even some frankenstein
+  // custom version
+  tarballUrl = url.resolve(baseUrl, name + '-v' + version + (semver.satisfies(version, '>=3') ? '-headers' : '') + '.tar.gz')
 
   return {
     version: version,
     semver: versionSemver,
     name: name,
     baseUrl: baseUrl,
-    tarballUrl: baseUrl + name + '-v' + version + '.tar.gz',
-    shasumsFile: shasumsFile,
-    shasumsUrl: baseUrl + shasumsFile,
-    checksumAlgo: checksumAlgo,
+    tarballUrl: tarballUrl,
+    shasumsUrl: url.resolve(baseUrl, 'SHASUMS256.txt'),
     versionDir: (name !== 'node' ? name + '-' : '') + version,
     libUrl32: libUrl32,
     libUrl64: libUrl64,
@@ -85,19 +111,19 @@ function normalizePath (p) {
 
 function resolveLibUrl (name, defaultUrl, arch, version) {
   var base = url.resolve(defaultUrl, './')
-  var isV3 = semver.satisfies(version, '^3')
-  var hasLibUrl = bitsre.test(defaultUrl) || (isV3 && bitsreV3.test(defaultUrl))
+    , isV3 = semver.satisfies(version, '^3')
+    , hasLibUrl = bitsre.test(defaultUrl) || (isV3 && bitsreV3.test(defaultUrl))
 
   if (!hasLibUrl) {
     // let's assume it's a baseUrl then
     if (semver.gte(version, '1.0.0'))
       return url.resolve(base, 'win-' + arch  +'/' + name + '.lib')
     // prior to [email protected] 32-bit node.lib lives in /, 64-bit lives in /x64/
-    return url.resolve(base, (arch == 'x64' ? 'x64/' : '') + name + '.lib')
+    return url.resolve(base, (arch === 'x64' ? 'x64/' : '') + name + '.lib')
   }
 
   // else we have a proper url to a .lib, just make sure it's the right arch
-  return defaultUrl.replace(isV3 ? bitsreV3 : bitsre, arch)
+  return defaultUrl.replace(isV3 ? bitsreV3 : bitsre, '/win-' + arch + '/')
 }
 
 module.exports = processRelease

test/docker.sh

@@ -5,41 +5,50 @@
 test_node_versions="0.8.28 0.10.40 0.12.7"
 test_iojs_versions="1.8.4 2.4.0 3.3.0"
 
+__dirname="$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+dot_node_gyp=${__dirname}/.node-gyp/
+
 # borrows from https://github.com./rvagg/dnt/
 
 # Simple setup function for a container:
 #  setup_container(image id, base image, commands to run to set up)
 setup_container() {
-  local ID=$1
-  local BASE=$2
-  local RUN=$3
+  local container_id="$1"
+  local base_container="$2"
+  local run_cmd="$3"
 
   # Does this image exist? If yes, ignore
-  docker inspect "$ID" &> /dev/null
+  docker inspect "$container_id" &> /dev/null
   if [[ $? -eq 0 ]]; then
-    echo "Found existing container [$ID]"
+    echo "Found existing container [$container_id]"
   else
     # No such image, so make it
-    echo "Did not find container [$ID], creating..."
-    docker run -i $BASE /bin/bash -c "$RUN"
+    echo "Did not find container [$container_id], creating..."
+    docker run -i $base_container /bin/bash -c "$run_cmd"
     sleep 2
-    docker commit $(docker ps -l -q) $ID
+    docker commit $(docker ps -l -q) $container_id
   fi
 }
 
 # Run tests inside each of the versioned containers, copy cwd into npm's copy of node-gyp
 # so it'll be invoked by npm when a compile is needed
 #  run_tests(version, test-commands)
 run_tests() {
-  local VERSION=$1
-  local RUN=$2
+  local version="$1"
+  local run_cmd="$2"
 
-  RUN="rsync -aAXx --delete --exclude .git --exclude build /node-gyp-src/ /usr/lib/node_modules/npm/node_modules/node-gyp/;
-    /bin/su -s /bin/bash node-gyp -c 'cd && ${RUN}'"
+  run_cmd="rsync -aAXx --delete --exclude .git --exclude build /node-gyp-src/ /usr/lib/node_modules/npm/node_modules/node-gyp/;
+    /bin/su -s /bin/bash node-gyp -c 'cd && ${run_cmd}'"
 
-  docker run --rm -v ~/.npm/:/dnt/.npm/ -v $(pwd):/node-gyp-src/:ro -i node-gyp-test/${VERSION} /bin/bash -c "$RUN"
-}
+  rm -rf $dot_node_gyp
 
+  docker run \
+    --rm -i \
+    -v ~/.npm/:/node-gyp/.npm/ \
+    -v ${dot_node_gyp}:/node-gyp/.node-gyp/ \
+    -v $(pwd):/node-gyp-src/:ro \
+    node-gyp-test/${version} /bin/bash -c "${run_cmd}"
+}
 
 # A base image with build tools and a user account
 setup_container "node-gyp-test/base" "ubuntu:14.04" "
@@ -74,13 +83,61 @@ for v in $test_iojs_versions; do
   "
 done
 
-
 # Run the tests for all of the test images we've created,
 # we should see node-gyp doing its download, configure and run thing
 # _NOTE: bignum doesn't compile on 0.8 currently so it'll fail for that version only_
 for v in $test_node_versions $test_iojs_versions; do
   run_tests $v "
-    cd node-buffertools && npm install --loglevel=info && npm test && cd &&
-    cd node-bignum && npm install --loglevel=info && npm test
+    cd node-buffertools && npm install --loglevel=info && npm test && cd
   "
-done
+  # removed for now, too noisy: cd node-bignum && npm install --loglevel=info && npm test
+done
+
+# Test use of --target=x.y.z to compile against alternate versions
+test_download_node_version() {
+  local run_with_ver="$1"
+  local expected_dir="$2"
+  local expected_ver="$3"
+  run_tests $run_with_ver "cd node-buffertools && npm install --loglevel=info --target=${expected_ver}"
+  local node_ver=$(cat "${dot_node_gyp}${expected_dir}/node_version.h" | grep '#define NODE_\w*_VERSION [0-9]*$')
+  node_ver=$(echo $node_ver | sed 's/#define NODE_[A-Z]*_VERSION //g' | sed 's/ /./g')
+  if [ "X$(echo $node_ver)" != "X${expected_ver}" ]; then
+    echo "Did not download v${expected_ver} using --target, instead got: $(echo $node_ver)"
+    exit 1
+  fi
+  echo "Verified correct download of [v${node_ver}]"
+}
+
+test_download_node_version "0.12.7" "0.10.30/src" "0.10.30"
+test_download_node_version "3.3.0" "iojs-1.8.4/src" "1.8.4"
+# should download the headers file
+test_download_node_version "3.3.0" "iojs-3.2.0/include/node" "3.2.0"
+
+# TODO: test --dist-url by starting up a localhost server and serving up tarballs
+
+# testing --dist-url, using simple-proxy.js to make localhost work as a distribution
+# point for tarballs
+# we can test whether it uses the proxy because after 2 connections the proxy will
+# die and therefore should not be running at the end of the test, `nc` can tell us this
+run_tests "3.3.0" "
+  (node /node-gyp-src/test/simple-proxy.js 8080 /foobar/ https://iojs.org/dist/ &) &&
+  cd node-buffertools &&
+  /node-gyp-src/bin/node-gyp.js --loglevel=info --dist-url=http://localhost:8080/foobar/ rebuild &&
+  nc -z localhost 8080 && echo -e \"\\n\\n\\033[31mFAILED TO USE LOCAL PROXY\\033[39m\\n\\n\"
+"
+
+run_tests "3.3.0" "
+  (node /node-gyp-src/test/simple-proxy.js 8080 /doobar/ https://iojs.org/dist/ &) &&
+  cd node-buffertools &&
+  NVM_IOJS_ORG_MIRROR=http://localhost:8080/doobar/ /node-gyp-src/bin/node-gyp.js --loglevel=info rebuild &&
+  nc -z localhost 8080 && echo -e \"\\n\\n\\033[31mFAILED TO USE LOCAL PROXY\\033[39m\\n\\n\"
+"
+
+run_tests "0.12.7" "
+  (node /node-gyp-src/test/simple-proxy.js 8080 /boombar/ https://nodejs.org/dist/ &) &&
+  cd node-buffertools &&
+  NVM_NODEJS_ORG_MIRROR=http://localhost:8080/boombar/ /node-gyp-src/bin/node-gyp.js --loglevel=info rebuild &&
+  nc -z localhost 8080 && echo -e \"\\n\\n\\033[31mFAILED TO USE LOCAL PROXY\\033[39m\\n\\n\"
+"
+
+rm -rf $dot_node_gyp

test/simple-proxy.js

@@ -0,0 +1,24 @@
+var http = require('http')
+  , https = require('https')
+  , server = http.createServer(handler)
+  , port = +process.argv[2]
+  , prefix = process.argv[3]
+  , upstream = process.argv[4]
+  , calls = 0
+
+server.listen(port)
+
+function handler (req, res) {
+  if (req.url.indexOf(prefix) != 0)
+    throw new Error('request url [' + req.url + '] does not start with [' + prefix + ']')
+
+  var upstreamUrl = upstream + req.url.substring(prefix.length)
+  console.log(req.url + ' -> ' + upstreamUrl)
+  https.get(upstreamUrl, function (ures) {
+    ures.on('end', function () {
+      if (++calls == 2)
+        server.close()
+    })
+    ures.pipe(res)
+  })
+}