[load][m]: rename load to loadUrl and have new load function to load from disk.

Author: rufuspollock

README.md

@@ -18,15 +18,46 @@ The following assume you've required datapackage as follows:
 var datapackage = require('datapackage-read');
 ```
 
-## Loading and normalizing a datapackage.json
+## load
 
 ```
-datapackage.load(urlToDataPackage, callback)
+datapackage.load(path, callback)
 ```
 
-Load a datapackage.json from a URL and normalize it as per the normalize function.
+* `path` {String} The path to the datapackage.json file
 
-`urlToDataPackage` can be either a url to actual datapackage.json or to a base directory.
+Load a datapackage.json from a file and normalize it as per the normalize
+function. Will search for README at REAME.md in same directory as
+datapackage.json.
+
+## loadUrl
+
+```
+datapackage.loadUrl(urlToDataPackage, callback)
+```
+
+* `urlToDataPackage` {String} url to datapackage.json or directory in which
+  datapackage.json contained (also handle provision of github urls e.g.
+  https://github.com/datasets/gold-prices)
+
+Load a datapackage.json from a URL, related files (e.g. README.md) and
+normalize it
+
+## normalize
+
+```
+datapackage.normalize(datapackage, url)
+```
+
+* `datapackage`: datapackage.json object (already parsed from JSON)
+* `url`: [optional] url to datapackage.json or the root directory in which it
+  is contained online. Used to help auto-generate things homepage 
+
+Normalize a DataPackage DataPackage.json in various ways, for example:
+
+* Setting a description if missing (from readme)
+* Setting a url for resources
+* Guessing a homepage
 
 # Tests
 

datapackage-read.js

@@ -1,13 +1,39 @@
 var fs = require('fs')
   , url = require('url')
+  , path = require('path')
   , marked = require('marked')
   , request = require('request')
   ;
 
-// Load a datapackage.json from a URL, related files (e.g. README.md) and normalize it
-//
-// Normalize is as per `normalize`
-exports.load = function(datapackage_url, cb) {
+exports.load = function(path_, cb) {
+  // TODO: support just passing a directory
+  var dpjsonPath = path_;
+  var base = path.dirname(dpjsonPath);
+  fs.readFile(dpjsonPath, function(error, body) {
+    if (error) {
+      cb(error);
+      return;
+    }
+    try {
+      var datapackage = JSON.parse(body);
+    } catch(e) {
+      cb({message: 'datapackage.json is invalid JSON. Details: ' + e.message});
+      return;
+    }
+
+    // now dig up and use README if it exists
+    var readmePath = path.join(base, 'README.md');
+    fs.readFile(readmePath, 'utf8', function(err, body) {
+      if (!err) {
+        datapackage['readme'] = body.replace(/\r\n/g, '\n');
+      }
+      datapackage = exports.normalize(datapackage, base);
+      cb(null, datapackage); 
+    });
+  });
+}
+
+exports.loadUrl = function(datapackage_url, cb) {
   var datapackage_url = exports.normalizeDataPackageUrl(datapackage_url);
   var base = datapackage_url.replace(/datapackage.json$/g, '');
   request(datapackage_url, function(error, response, body) {
@@ -44,7 +70,7 @@ exports.load = function(datapackage_url, cb) {
 // hash (keyed by Data Package names)
 //
 // @return: via the callback
-exports.loadMany = function(urls, callback) {
+exports.loadManyUrls = function(urls, callback) {
   var output = {}
     , count = urls.length
     ;
@@ -55,7 +81,7 @@ exports.loadMany = function(urls, callback) {
     }
   }
   urls.forEach(function(url) {
-    exports.load(url, function(err, dpjson) {
+    exports.loadUrl(url, function(err, dpjson) {
       if (err) {
         console.error(url, err)
       } else {
@@ -78,10 +104,6 @@ exports.normalizeDataPackageUrl = function(url) {
   return url;
 };
 
-// Normalize a DataPackage DataPackage.json in various ways
-// 
-// @param datapackage: datapackage object (already parsed from JSON)
-// @param url: [optional] url to datapackage.json or the root directory in which it is contained
 exports.normalize = function(datapackage, url_) {
   var base = url_ ? url_.replace(/datapackage.json$/g, '') : '';
   // ensure certain fields exist
@@ -98,13 +120,17 @@ exports.normalize = function(datapackage, url_) {
     datapackage.readme = datapackage.description;
   }
 
-  datapackage.readme_html = marked(datapackage.readme);
+  datapackage.readmeHtml = marked(datapackage.readme);
+
+  if (!datapackage.resources) {
+    datapackage.resources = [];
+  }
 
   datapackage.resources.forEach(function(info) {
-    if (!info.url && info.path) {
+    if (!info.url && info.path && base) {
       info.url = base + info.path;
     }
-    if (!info.name) {
+    if (!info.name && info.url) {
       info.name = _nameFromUrl(info.url);
     }
     // upgrade for change in JTS spec - https://github.com/dataprotocols/dataprotocols/issues/60

test/all.js

@@ -54,7 +54,23 @@ describe('normalize', function() {
 
 describe('load', function() {
   it('works in basic case', function(done) {
-    tools.load(sourceUrl, function(err, dpout) {
+    tools.load('test/data/dp1/datapackage.json', function(error, data) {
+      assert(!error)
+      assert.equal(data.readme, 'The README.\n');
+      done();
+    });
+  });
+  it('reports error with bad path', function(done) {
+    tools.load('test/data/dp1/does-not-exist.json', function(error, data) {
+      assert(error)
+      done();
+    });
+  });
+});
+
+describe('loadUrl', function() {
+  it('works in basic case', function(done) {
+    tools.loadUrl(sourceUrl, function(err, dpout) {
       assert(err === null);
       assert.equal(dpout.resources[0].url, sourceUrlBase + 'data/data.csv');
       assert(dpout.readme.length > 50);
@@ -66,15 +82,15 @@ describe('load', function() {
   it('works with 404', function(done) {
     this.timeout(4000);
     var badUrl = 'https://raw.github.com/datasets/gold-prices/master/xyz.txt';
-    tools.load(badUrl, function(err, dpout) {
+    tools.loadUrl(badUrl, function(err, dpout) {
       assert(err!=null);
       done();
     });
   });
 
   it('works with bad data', function(done) {
     var csvurl = 'https://raw.github.com/datasets/gold-prices/master/README.md'; 
-    dpout = tools.load(csvurl, function(err, dpout) {
+    dpout = tools.loadUrl(csvurl, function(err, dpout) {
       // disable
       // as we now add datapackage.json to url this gives 404 rather than bad JSON
       // assert.equal(err.message.indexOf('datapackage.json is invalid JSON'), 0);
@@ -83,10 +99,10 @@ describe('load', function() {
   });
 });
 
-describe('loadMany', function() {
+describe('loadManyUrls', function() {
   it('works', function(done) {
     gdpUrl = 'https://github.com/datasets/gdp';
-    tools.loadMany([sourceUrl, gdpUrl], function(err, dpout) {
+    tools.loadManyUrls([sourceUrl, gdpUrl], function(err, dpout) {
       assert.equal(Object.keys(dpout).length, 2);
       assert.equal(dpout['gdp'].homepage, gdpUrl);
       done();

test/data/dp1/README.md

@@ -0,0 +1 @@
+The README.

test/data/dp1/datapackage.json

@@ -0,0 +1,3 @@
+{
+  "name": "abc"
+}