From 6dbb7e0ea863dc9b9d5dbaeb413b1c294254810e Mon Sep 17 00:00:00 2001 From: Matteo Visconti di Oleggio Castello Date: Thu, 4 Jul 2024 12:49:20 -0700 Subject: [PATCH] FIX avoid using jQuery for getting numpy arrays (#544) * FIX avoid using jQuery for getting numpy arrays * Remove old offset declaration --- cortex/webgl/data.py | 8 +- cortex/webgl/resources/js/datamodel.js | 126 ++++++++++++++++++------- 2 files changed, 94 insertions(+), 40 deletions(-) diff --git a/cortex/webgl/data.py b/cortex/webgl/data.py index 3bf704f54..dc15eccb8 100644 --- a/cortex/webgl/data.py +++ b/cortex/webgl/data.py @@ -21,12 +21,14 @@ class Package(object): """Package the data into a form usable by javascript""" def __init__(self, data): self.dataset = dataset.normalize(data) - self.uniques = data.uniques(collapse=True) + self.uniques = list(data.uniques(collapse=True)) + self.subjects = set() self.brains = dict() self.images = dict() for brain in self.uniques: name = brain.name + self.subjects.add(brain.subject) self.brains[name] = brain.to_json(simple=True) if isinstance(brain, (dataset.Vertex, dataset.VertexRGB)): encdata = brain.vertices @@ -60,10 +62,6 @@ def views(self): metadata.append(meta) return metadata - @property - def subjects(self): - return set(braindata.subject for braindata in self.uniques) - def reorder(self, subjects): indices = dict((k, np.load(os.path.splitext(v)[0]+".npz")) for k, v in subjects.items()) for brain in self.uniques: diff --git a/cortex/webgl/resources/js/datamodel.js b/cortex/webgl/resources/js/datamodel.js index 862bd0c05..38e0c2b52 100644 --- a/cortex/webgl/resources/js/datamodel.js +++ b/cortex/webgl/resources/js/datamodel.js @@ -84,6 +84,7 @@ NParray.fromJSON = function(json) { return NParray.fromData(data, json.dtype, json.shape); } NParray.fromURL = function(url, callback) { + console.log("Loading numpy array from " + url); loadCount++; $("#dataload").show(); var hideload = function() { @@ -91,43 +92,98 @@ NParray.fromURL = function(url, callback) { if (loadCount == 0) $("#dataload").hide(); }; - $.ajax(url, { - headers: {Range: 'bytes=0-1023'}, - dataType: 'text', - success: function(data, status, headerxhr) { - if (data.slice(1, 6) != 'NUMPY') - throw "Invalid npy file" - console.log("npy version "+data.charCodeAt(6)+"."+data.charCodeAt(7)); - var nbytes = data.charCodeAt(8) + (data.charCodeAt(9) << 8); - var info = parse_dict(data.slice(10, 10+nbytes)); - var shape = info.shape.slice(1, info.shape.length-1).split(','); - shape = shape.map(function(num) { return parseInt(num.trim()) }); - var array = new NParray(dtypeNames[info.descr], shape); - - if (headerxhr.status == 206) { - var length = array.dtype.BYTES_PER_ELEMENT * array.size; - var increment = array._slice * array.dtype.BYTES_PER_ELEMENT || length; - var offset = nbytes + 10; - Stream(url, length, increment, offset).progress(array.update.bind(array)).done(hideload); - } else if (headerxhr.status == 200) { - //Server doesn't support partial data, returned the whole thing - var chars = new Uint8Array(data.length - nbytes - 10); - for (var i = 0, il = chars.length; i < il; i++) - chars[i] = data.charCodeAt(i+nbytes+10); - array.update(chars.buffer); - hideload(); - } else throw "Invalid response from server"; - callback(array); - }, - }); + + // With thanks to https://gist.github.com/nvictus/88b3b5bfe587d32ac1ab519fd0009607 + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.setRequestHeader("Range", "bytes=0-1023"); + xhr.responseType = "arraybuffer"; + + xhr.onload = function() { + if (xhr.status !== 200 && xhr.status !== 206) { + console.error("Failed to fetch file: status " + xhr.status); + hideload(); + return; + } + + function asciiDecode(buf) { + return String.fromCharCode.apply(null, new Uint8Array(buf)); + } + + function readUint16LE(buffer) { + var view = new DataView(buffer); + var val = view.getUint8(0); + val |= view.getUint8(1) << 8; + return val; + } + + var buffer = xhr.response; + console.log("Got buffer: " + buffer); + console.log("Buffer length: " + buffer.byteLength); + + // Check we got a valid npy file + var magic = asciiDecode(buffer.slice(0, 6)); + if (magic.slice(1, 6) != 'NUMPY') + throw "Invalid npy file" + console.log("Magic: "+magic); + + // OK, now let's parse the header and get some info + var version = new Uint8Array(buffer.slice(6, 8)); + console.log("npy version " + version); + + var headerLength = readUint16LE(buffer.slice(8, 10)); + console.log("Header length: " + headerLength); + + var headerStr = asciiDecode(buffer.slice(10, 10 + headerLength)); + console.log("Header: " + headerStr); + + // Create the array object + var info = parse_dict(asciiDecode(buffer.slice(10, 10 + headerLength))); + var shape = info.shape.slice(1, info.shape.length-1).split(','); + shape = shape.map(function(num) { return parseInt(num.trim()) }); + var array = new NParray(dtypeNames[info.descr], shape); + + console.log("Array shape: " + array.shape); + console.log("Array size: " + array.size); + console.log("Array dtype: " + array.dtype); + + if (xhr.status == 206) { + // We got partial data, we're streaming it + var length = array.dtype.BYTES_PER_ELEMENT * array.size; + var increment = array._slice * array.dtype.BYTES_PER_ELEMENT || length; + var offset = 10 + headerLength; + Stream(url, length, increment, offset).progress(array.update.bind(array)).done(hideload); + } else if (xhr.status == 200) { + // Server doesn't support partial data, returned the whole thing + array.update(buffer.slice(10 + headerLength)); + hideload(); + } + console.log("Data length: " + array.data.length); + console.log("Data: " + array.data); + + // Finish up + hideload(); + callback(array); + } + + xhr.onerror = function() { + console.error("Failed to fetch file"); + hideload(); + }; + + xhr.send(); } NParray.prototype.update = function(buffer) { - if (this.shape.length > 1) { - this.data = new this.dtype(buffer, 0, (++this.available)*this._slice); - this.loaded.notify(this.available); - } else { - this.data = new this.dtype(buffer); - this.loaded.resolve(); + try { + if (this.shape.length > 1) { + this.data = new this.dtype(buffer, 0, (++this.available)*this._slice); + this.loaded.notify(this.available); + } else { + this.data = new this.dtype(buffer); + this.loaded.resolve(); + } + } catch (e) { + console.error("Error updating array:", e); } } NParray.prototype.view = function() {