nodejs: add support for storing URLs and introduce fileReader (#73)

* support node url storage

* add tests and refactor node storage

* add fingerprint for reactnative

* make compatible with older node versions + catch JSON.parse exception

* don't use FileStorage by default

* fix rebase conflict

* add option to set custom file reader

.gitignore

@@ -2,3 +2,4 @@ node_modules
 demos/reactnative/.expo
 lib.es5
 dist
+.DS_Store

lib/browser/storage.js

@@ -22,17 +22,23 @@ try {
 
 export const canStoreURLs = hasStorage;
 
-export function setItem(key, value) {
-  if (!hasStorage) return;
-  return localStorage.setItem(key, value);
-}
+class LocalStorage {
+  setItem(key, value, cb) {
+    if (!hasStorage) return cb();
+    cb(null, localStorage.setItem(key, value));
+  }
 
-export function getItem(key) {
-  if (!hasStorage) return;
-  return localStorage.getItem(key);
+  getItem(key, cb) {
+    if (!hasStorage) return cb();
+    cb(null, localStorage.getItem(key));
+  }
+
+  removeItem(key, cb) {
+    if (!hasStorage) return cb();
+    cb(null, localStorage.removeItem(key));
+  }
 }
 
-export function removeItem(key) {
-  if (!hasStorage) return;
-  return localStorage.removeItem(key);
+export function getStorage() {
+  return new LocalStorage();
 }

lib/fingerprint.js

@@ -1,10 +1,16 @@
+import isReactNative from "./node/isReactNative";
+
 /**
  * Generate a fingerprint for a file which will be used the store the endpoint
  *
  * @param {File} file
  * @return {String}
  */
 export default function fingerprint(file, options) {
+  if (isReactNative) {
+    return reactNativeFingerprint(file, options);
+  }
+
   return [
     "tus",
     file.name,
@@ -14,3 +20,28 @@ export default function fingerprint(file, options) {
     options.endpoint
   ].join("-");
 }
+
+function reactNativeFingerprint(file, options) {
+  let exifHash = file.exif ? hashCode(JSON.stringify(file.exif)) : "noexif";
+  return [
+    "tus",
+    file.name || "noname",
+    file.size || "nosize",
+    exifHash,
+    options.endpoint
+  ].join("/");
+}
+
+function hashCode(str) {
+  // from https://stackoverflow.com/a/8831937/151666
+  var hash = 0;
+  if (str.length === 0) {
+    return hash;
+  }
+  for (var i = 0; i < str.length; i++) {
+    var char = str.charCodeAt(i);
+    hash = ((hash << 5) - hash) + char;
+    hash = hash & hash; // Convert to 32bit integer
+  }
+  return hash;
+}

lib/index.js

@@ -1,30 +1,32 @@
 /* global window */
 import Upload from "./upload";
-import {canStoreURLs} from "./node/storage";
+import * as storage from "./node/storage";
 
 const {defaultOptions} = Upload;
-let isSupported;
+
+const moduleExport = {
+  Upload,
+  canStoreURLs: storage.canStoreURLs,
+  defaultOptions
+};
 
 if (typeof window !== "undefined") {
   // Browser environment using XMLHttpRequest
   const {XMLHttpRequest, Blob} = window;
 
-  isSupported = (
+  moduleExport.isSupported = (
     XMLHttpRequest &&
     Blob &&
     typeof Blob.prototype.slice === "function"
   );
 } else {
   // Node.js environment using http module
-  isSupported = true;
+  moduleExport.isSupported = true;
+  // make FileStorage module available as it will not be set by default.
+  moduleExport.FileStorage = storage.FileStorage;
 }
 
 // The usage of the commonjs exporting syntax instead of the new ECMAScript
 // one is actually inteded and prevents weird behaviour if we are trying to
 // import this module in another module using Babel.
-module.exports = {
-  Upload,
-  isSupported,
-  canStoreURLs,
-  defaultOptions
-};
+module.exports = moduleExport;

lib/node/isReactNative.js

@@ -0,0 +1,3 @@
+const isReactNative = false;
+
+export default isReactNative;

lib/node/storage.js

@@ -1,15 +1,111 @@
 /* eslint no-unused-vars: 0 */
+import { readFile, writeFile } from "fs";
+import * as lockfile from "proper-lockfile";
 
-export const canStoreURLs = false;
-
-export function setItem(key, value) {
+export const canStoreURLs = true;
 
+export function getStorage() {
+  // don't support storage by default.
+  return null;
 }
 
-export function getItem(key) {
 
-}
+export class FileStorage {
+  constructor(filePath) {
+    this.path = filePath;
+  }
+
+  setItem(key, value, cb) {
+    lockfile.lock(this.path, this._lockfileOptions(), (err, release) => {
+      if (err) {
+        return cb(err);
+      }
+
+      cb = this._releaseAndCb(release, cb);
+      this._getData((err, data) => {
+        if (err) {
+          return cb(err);
+        }
+
+        data[key] = value;
+        this._writeData(data, (err) => cb(err));
+      });
+    });
+  }
+
+  getItem(key, cb) {
+    this._getData((err, data) => {
+      if (err) {
+        return cb(err);
+      }
+      cb(null, data[key]);
+    });
+  }
+
+  removeItem(key, cb) {
+    lockfile.lock(this.path, this._lockfileOptions(), (err, release) => {
+      if (err) {
+        return cb(err);
+      }
+
+      cb = this._releaseAndCb(release, cb);
+      this._getData((err, data) => {
+        if (err) {
+          return cb(err);
+        }
+
+        delete data[key];
+        this._writeData(data, (err) => cb(err));
+      });
+    });
+  }
+
+  _lockfileOptions() {
+    return {
+      realpath: false,
+      retries: {
+        retries: 5,
+        minTimeout: 20
+      }
+    };
+  }
+
+  _releaseAndCb(release, cb) {
+    return (err) => {
+      if (err) {
+        // @TODO consider combining error from release callback
+        release(() => cb(err));
+        return;
+      }
+
+      release(cb);
+    };
+  }
 
-export function removeItem(key) {
+  _writeData(data, cb) {
+    const opts = {
+      encoding: "utf8",
+      mode: 0o660,
+      flag: "w"
+    };
+    writeFile(this.path, JSON.stringify(data), opts, (err) => cb(err));
+  }
 
+  _getData(cb) {
+    readFile(this.path, "utf8", (err, data) => {
+      if (err) {
+        // return empty data if file does not exist
+        err.code === "ENOENT" ? cb(null, {}) : cb(err);
+        return;
+      } else {
+        try {
+          data = !data.trim().length ? {} : JSON.parse(data);
+        } catch (error) {
+          cb(error);
+          return;
+        }
+        cb(null, data);
+      }
+    });
+  }
 }

lib/upload.js

@@ -6,9 +6,9 @@ import { Base64 } from "js-base64";
 
 // We import the files used inside the Node environment which are rewritten
 // for browsers using the rules defined in the package.json
-import {newRequest, resolveUrl} from "./node/request";
-import {getSource} from "./node/source";
-import * as Storage from "./node/storage";
+import { newRequest, resolveUrl } from "./node/request";
+import { getSource } from "./node/source";
+import { getStorage } from "./node/storage";
 
 const defaultOptions = {
   endpoint: null,
@@ -26,13 +26,18 @@ const defaultOptions = {
   overridePatchMethod: false,
   retryDelays: null,
   removeFingerprintOnSuccess: false,
-  uploadLengthDeferred: false
+  uploadLengthDeferred: false,
+  urlStorage: null,
+  fileReader: null
 };
 
 class Upload {
   constructor(file, options) {
     this.options = extend(true, {}, defaultOptions, options);
 
+    // The storage module used to store URLs
+    this._storage = this.options.urlStorage;
+
     // The underlying File/Blob object
     this.file = file;
 
@@ -82,10 +87,15 @@ class Upload {
       return;
     }
 
+    if (this.options.resume && this._storage == null) {
+      this._storage = getStorage();
+    }
+
     if (this._source) {
       this._start(this._source);
     } else {
-      getSource(file, this.options.chunkSize, (err, source) => {
+      const fileReader = this.options.fileReader || getSource;
+      fileReader(file, this.options.chunkSize, (err, source) => {
         if (err) {
           this._emitError(err);
           return;
@@ -193,34 +203,44 @@ class Upload {
     }
 
     // Try to find the endpoint for the file in the storage
-    if (this.options.resume) {
+    if (this._hasStorage()) {
       this._fingerprint = this.options.fingerprint(file, this.options);
-      let resumedUrl = Storage.getItem(this._fingerprint);
+      this._storage.getItem(this._fingerprint, (err, resumedUrl) => {
+        if (err) {
+          this._emitError(err);
+          return;
+        }
 
-      if (resumedUrl != null) {
-        this.url = resumedUrl;
-        this._resumeUpload();
-        return;
-      }
+        if (resumedUrl != null) {
+          this.url = resumedUrl;
+          this._resumeUpload();
+        } else {
+          this._createUpload();
+        }
+      });
+    } else {
+      // An upload has not started for the file yet, so we start a new one
+      this._createUpload();
     }
-
-    // An upload has not started for the file yet, so we start a new one
-    this._createUpload();
   }
 
   abort() {
     if (this._xhr !== null) {
       this._xhr.abort();
       this._source.close();
-      this._aborted = true;
     }
+    this._aborted = true;
 
     if (this._retryTimeout != null) {
       clearTimeout(this._retryTimeout);
       this._retryTimeout = null;
     }
   }
 
+  _hasStorage() {
+    return this.options.resume && this._storage;
+  }
+
   _emitXhrError(xhr, err, causingErr) {
     this._emitError(new DetailedError(err, causingErr, xhr));
   }
@@ -322,8 +342,12 @@ class Upload {
         return;
       }
 
-      if (this.options.resume) {
-        Storage.setItem(this._fingerprint, this.url);
+      if (this._hasStorage()) {
+        this._storage.setItem(this._fingerprint, this.url, (err) => {
+          if (err) {
+            this._emitError(err);
+          }
+        });
       }
 
       this._offset = 0;
@@ -363,10 +387,14 @@ class Upload {
 
     xhr.onload = () => {
       if (!inStatusCategory(xhr.status, 200)) {
-        if (this.options.resume && inStatusCategory(xhr.status, 400)) {
+        if (this.options.resume && this._storage && inStatusCategory(xhr.status, 400)) {
           // Remove stored fingerprint and corresponding endpoint,
           // on client errors since the file can not be found
-          Storage.removeItem(this._fingerprint);
+          this._storage.removeItem(this._fingerprint, (err) => {
+            if (err) {
+              this._emitError(err);
+            }
+          });
         }
 
         // If the upload is locked (indicated by the 423 Locked status code), we