scijs / ndarray Goto Github PK

Does this module has methods similar to reshape(A, m, n) in matlab? If so, how can I use it?

Would be nice if ndarray could take any ndarray-ish object and automatically coerce it to an ndarray. For example, if you feed it any object with data,shape,stride,offset properties, it should convert it into an ndarray with the correct interface automatically.

I took a look at the generated code and I added comments for the varying bits. It seems that the varying parts are quite small and the same for each set of dimensions. I don't see any parts that have combinations between dtype and shape. I think the string concatenated code could be greatly reduced, or possibly eliminated if you were willing to manually build out a set of dimensional arrays.

It looks like you sometimes define stride and shape as arrays, and sometimes as a varying number of scalars (_shape0, _shape2...)

If you always dealt with shape and stride as arrays, you could write generic code to deal with these cases.

Am I missing something here?

'use strict'
function View2dfloat32(a,b0,b1,c0,c1,d){this.data=a
                                        this._shape0=b0|0
                                        this._shape1=b1|0
                                        this._stride0=c0|0
                                        this._stride1=c1|0
                                        this.offset=d|0}
var proto=View2dfloat32.prototype;
proto.dtype='float32';
proto.dimension=2;
function VStride2dfloat32(v) {this._v=v;} 
var aproto=VStride2dfloat32.prototype;

//varying
aproto.length=2;
aproto.toJSON=function VStride2dfloat32_toJSON(){
    return [this._v._stride0,this._v._stride1];};
aproto.valueOf = aproto.toString= function VStride2dfloat32_toString(){
    return [this._v._stride0,this._v._stride1].join();};
Object.defineProperty(aproto,0,{
    get:function() { return this._v._stride0;},
    set:function(v){ return this._v._stride0=v|0;},
    enumerable:true});
Object.defineProperty(aproto,1,{
    get:function(){ return this._v._stride1;},
    set:function(v){return this._v._stride1=v|0;},
    enumerable:true});
//end varying
aproto.concat=Array.prototype.concat;
aproto.join=Array.prototype.join;
aproto.slice=Array.prototype.slice;
aproto.toString=Array.prototype.toString;
aproto.indexOf=Array.prototype.indexOf;
aproto.lastIndexOf=Array.prototype.lastIndexOf;
aproto.forEach=Array.prototype.forEach;
aproto.every=Array.prototype.every;
aproto.some=Array.prototype.some;
aproto.filter=Array.prototype.filter;
aproto.map=Array.prototype.map;
aproto.reduce=Array.prototype.reduce;
aproto.reduceRight=Array.prototype.reduceRight;

//varying
Object.defineProperty(proto,'stride',{
    get:function VStride2dfloat32_get(){
        return new VStride2dfloat32(this);},
    set: function VStride2dfloat32_set(v){
        this._stride0=v[0]|0;
        this._stride1=v[1]|0;
        return v;}});

function VShape2dfloat32(v) {this._v=v;} 
var aproto=VShape2dfloat32.prototype;
aproto.length=2;
aproto.toJSON=function VShape2dfloat32_toJSON(){
    return [this._v._shape0,this._v._shape1];};
aproto.valueOf=aproto.toString=function VShape2dfloat32_toString(){
    return [this._v._shape0,this._v._shape1].join()};

Object.defineProperty(aproto,0,{
    get:function(){return this._v._shape0;},
    set:function(v){return this._v._shape0=v|0;},
    enumerable:true});
Object.defineProperty(aproto,1,{
    get:function(){return this._v._shape1;},
    set:function(v){return this._v._shape1=v|0;},
    enumerable:true});

//end varying
aproto.concat=Array.prototype.concat;
aproto.join=Array.prototype.join;
aproto.slice=Array.prototype.slice;
aproto.toString=Array.prototype.toString;
aproto.indexOf=Array.prototype.indexOf;
aproto.lastIndexOf=Array.prototype.lastIndexOf;
aproto.forEach=Array.prototype.forEach;
aproto.every=Array.prototype.every;
aproto.some=Array.prototype.some;
aproto.filter=Array.prototype.filter;
aproto.map=Array.prototype.map;
aproto.reduce=Array.prototype.reduce;
aproto.reduceRight=Array.prototype.reduceRight;
//varying
Object.defineProperty(proto,'shape',{
    get:function VShape2dfloat32_get(){return new VShape2dfloat32(this);},
    set: function VShape2dfloat32_set(v){
        this._shape0=v[0]|0;
        this._shape1=v[1]|0;
        return v};});
Object.defineProperty(proto,'size',{
    get:function View2dfloat32_size(){
        return this._shape0*this._shape1;}});

Object.defineProperty(proto,'order', {
    get:function View2dfloat32_order(){
        return (Math.abs(this._stride0)>Math.abs(this._stride1))?[1,0]:[0,1];}});
proto.set=function View2dfloat32_set(i0,i1,v){
    return this.data[this.offset+this._stride0*i0+this._stride1*i1]=v;};
proto.get=function View2dfloat32_get(i0,i1){
    return this.data[this.offset+this._stride0*i0+this._stride1*i1];};
proto.index=function View2dfloat32_index(i0,i1){
    return this.offset+this._stride0*i0+this._stride1*i1;};
proto.hi=function View2dfloat32_hi(i0,i1){
    var arg2 = (typeof i0!=='number'||i0<0)?this._shape0:i0|0;
    var arg3 = (typeof i1!=='number'||i1<0)?this._shape1:i1|0;
    return new View2dfloat32(this.data,
                             arg1, 
                             arg2,
                             this._stride0,
                             this._stride1,
                             this.offset);};

proto.lo=function View2dfloat32_lo(i0,i1){
    var b = this.offset, d = 0, 
        a0 = this._shape0, a1 = this._shape1, 
        c0 = this._stride0, c1 = this._stride1;
    if(typeof i0==='number'&&i0>=0){
        d=i0|0;
        b+=c0*d;
        a0-=d}
    if(typeof i1==='number'&&i1>=0){
        d=i1|0;
        b+=c1*d;
        a1-=d}
    return new View2dfloat32(this.data,a0,a1,c0,c1,b)};
proto.step=function View2dfloat32_step(i0,i1){
    var a0=this._shape0,a1=this._shape1,
    b0=this._stride0,b1=this._stride1,
    c=this.offset,d=0,ceil=Math.ceil;
    if(typeof i0==='number'){
        d=i0|0;
        if(d<0){
            c+=b0*(a0-1);
            a0=ceil(-a0/d);}
        else{
            a0=ceil(a0/d);}
        b0*=d;}
    if(typeof i1==='number'){
        d=i1|0;
        if(d<0){
            c+=b1*(a1-1);
            a1=ceil(-a1/d);}
        else{
            a1=ceil(a1/d);}
        b1*=d};
    return new View2dfloat32(this.data,a0,a1,b0,b1,c);}
proto.transpose=function View2dfloat32_transpose(i0,i1){
    i0=(i0===undefined?0:i0|0);
    i1=(i1===undefined?1:i1|0);
    var a=this.shape,b=this.stride;
    return new View2dfloat32(this.data,a[i0],a[i1],b[i0],b[i1],this.offset);};
proto.pick=function View2dfloat32_pick(i0,i1){
    var a=[],b=[],c=this.offset;
    if(typeof i0==='number'&&i0>=0){
        c=(c+this._stride0*i0)|0}
    else{
        a.push(this._shape0);b.push(this._stride0);}

    if(typeof i1==='number'&&i1>=0){
        c=(c+this._stride1*i1)|0;}
    else{
        a.push(this._shape1);
        b.push(this._stride1);}
    var ctor=CTOR_LIST[a.length+1];
    return ctor(this.data,a,b,c);};

//end varying
return function construct_View2dfloat32(data,shape,stride,offset){
    return new View2dfloat32(data,shape[0],shape[1],stride[0],stride[1],offset);};

The get() method does not throw an error when using floating point numbers for the array indices. I know that it's kind of obvious to not do this, but I ran into a bug that was quite difficult to trace because the ndarray was returning data that seemed valid, but was just sometimes wrong.

Throwing an error when floating point numbers are used to access data seems quite helpful. I assume that this goes for the set method as well (and maybe others).

With the mass of ndarray extensions it's hard to find the functions you need. I would be happy about some simple document (can be in the wiki or just a Markdown file somewhere) which lists all functions from all extensions so that I immediately know what exists and where to find it.

How to convert an ndarray back to a buffer/array/…? Including all transformations, that is, so not just foo.data.

const fs = require('fs');

const ndarray = require('ndarray');
const show = require('ndarray-show');

const matrix = ndarray([
    0, 0, 0,
    1, 0, 0,
    0, 0, 0
], [3, 3]);
console.log(show(matrix, 2));
/*
 0  0  0
 1  0  0
 0  0  0
*/
console.log('-'.repeat(72));
const modified = matrix.transpose(1, 0);
console.log(show(modified, 2));
/*
 0  1  0
 0  0  0
 0  0  0
*/
console.log('-'.repeat(72));
console.log(modified.data);
/* [
 0, 0, 0,
 1, 0, 0,
 0, 0, 0
] */

// I want to somehow get the transposed matrix as an array or buffer:

/* [
    0, 1, 0,
    0, 0, 0,
    0, 0, 0
]
*/

Is there a better way than .picking and .getting each value individually and manually adding it to an array/buffer? I feel like I’m missing something obvious here…

For matrices, it makes sense that A.transpose() would transpose the only two dimensions. It's a special case which is undesirable, but there's no other possibility and it's clearly what's intended, which is desirable. Not sure whether this would be a minor or major version increment since .transpose() was technically a noop before (I believe?). Just a suggestion to make the API as unsurprising as possible for newcomers, but disagreement welcome.

I love the idea of using the hi and lo functions to get a subsection of the array, so that I can pass that subsection to a function and it can work on it oblivious to where it is in the array.

Unfortunately, that doesn't quite work properly. I have a 3D array of cells. For every cell, I need to check all the cells in the cardinal directions around it. I figured I could just use hi() and lo() to give me a 3x3x3 view of the cells around the cell I'm working on (using arr.hi(x-1,y-1,z-1).lo(2,2,2). I was hoping (because the documentation said nothing on edge cases) that when I'm on the edge of one of the dimensions (say, x = 0), this new view would have a view where x=0 pointed to theoretical x=-1 in the parent array. And I figured calling get(0, 1, 1) on that subview would give me null or undefined, as there is nothing to retrieve from the empty space beyond the edge of the array.

Instead, I find the subarray is badly shifted: the hi() function reverts to 0 instead of allowing a negative view range, and now I get a view of the 3D array one off in the +X direction, causing the whole operation to just be wrong.

Not sure how to do an "oblivious array operation" when there's something like this going on. I'm going to have to actually give the for-each function the whole array now or something.

Basically what I think should happen: Allow a view to be partially outside the range of the theoretical array, and return undefined when you try and query it. Throw an exception when you try and set out there.

Hey Mikola, do you know of a fast way to add a column to a 2d array?

This is just a shot in the dark; I have a feeling that I need to allocate a new typed array to achieve this.

I'm sure that sweet.js can help this codebase. You can make it a compile time only dependency.

String concatentation to build up functions is very awkward.
With sweet.js you can have regular javascript looking code, and then macros which perform the string concatenation for you at runtime if you are worried about code size. You just shouldn't be dealing with all of these strings, it's incredibly error prone. I'm taking a shot at the conversion.

this may be #9 put in other words
or in other words: a way to compose a ndarray by "glueing" slices

Using this library, I have found frequently that I need to set an entire vector at a time when manipulating the ndarray. For example:

const n = ndarray([1, 2, 3, 4, 5, 6, 7, 8], [2, 2, 2]);
const newVector = n.pick(0, 0);
for (let i = 0; i < n.shape[3]; i++) {
    n.set(1, 1, j, newVector.get(j));
}

Is there an intended way to handle this redundancy? Ideally:

const n = ndarray([1, 2, 3, 4, 5, 6, 7, 8], [2, 2, 2]);
const newVector = n.pick(0, 0);
n.set(1, 1, newVector);

Context:
Writing an image processing library. I wrote a median filter for the image. I need iterate over all the pixels and set them to the median of their neighbors.

might be a good thing to have the modules listed here available as a wiki page http://0fps.net/2013/06/01/ndarray-modules/

tensorflow is now available in a javascript flavour, only supporting browsers via webGL rn, but with intents to run out of node.js too (altho only for CUDA? 👃 )

how regal it would be if the ndarray module complex had a GPU backend!

Some solutions have been proposed to reduce the string concatenation (#11, #12). I'd like to pitch another alternative: code generation from a dynamically generated AST using escodegen.

You could have something like this:

const Sum = (exprs) => exprs.reduce((prev, curr) => BinOp(prev, '+', curr));

const strideTimesX = (i) => BinOp(
    Member(parse('this.stride'), Literal(i), true),
    '*',
    'x' + i
);

const getFactory = (d, type) => FunctionDecl(
    className(d, type) + '_get', args,
    Return(
      Member(
        parse('this.data'),
        Sum( range(d).map(strideTimesX).concat(thisOffset) ),
        true))));

esprima or acorn can also be used to generate the AST for the static portions of the code:

// from view.pick
let vars = acorn.parse('var a = [], b = [], c = this.offset;').body[0].expression;

FunctionDecl('pick', range(n).map((i) => 'x' + i),
  [ vars,
  //...
    Return(BinOp('a', '+', 'b')) ] // <-- not accurate, just an example

This would allow for runtime generation of the classes and retention of efficiency, but in a more controlled and maintainable way. It also reads somewhat intuitively (I've been playing around with escodegen, the above are working examples).

Let me know what you think. You can see updates in my fork patgrasso/ndarray (check out the escodegen branch).

I'd like to include ndarray in a website as a simple script, without browserify or anything, and I don't want to distribute it myself, so I was looking for it at some CDNs but couldn't find it. I think it would be nice if it's added to https://www.jsdelivr.com

On the README, I understand the description for hi/lo as:

lo() is just a start offset for each dimension. hi() is the opposite, that is, it truncates by the amount for each dimension. But in fact, hi() is also based on absolute indices and not by an amount. So, I think this should be changed also examples with output added.

The docs for transpose claim it works in-place, but it does not.

Hi!
For now, node’s Buffers are interpreted by ndarrays as having simple UInt8 type, where each value is integer within 0..255 range, and is accessed as buffer[index]. There seems to be a lot of applications of buffers with other types, like float, or int32. For example, node-speaker takes int16LE buffers, which is default for PCM-streams encoding, AFAIK, and other libs work with float-typed buffers.
It would be super-awesome ndarrays take all the routine work of handling buffer types, as far there are lots of them.
Or what is the most appropriate solution for that? Current status quo is a bit frustrating.
As a workaround now, it is possible to create "generic" wrapper for buffers, but is seems like duplicating of ndarray’s work.

I have 245 dicom images (.dcm), each of size 512x512 that I am reading via the daikon.js package. I get the single image's data as a stream[1-D array] using a certain function. Now Im trying to create a ndarray volume which represents the images and I do it like this:

    var r = series.images[0].getRows(); // 512
    var c = series.images[0].getCols();  // 512
    var no = series.images.length;  // 245

    var volume_buffer = new Array(r*c*no); // Create a 1-D array of the volume

    // A fast loop to traverse the image data
    var i = no, j = 0,k = 0; 
    while(i--){
      // This gives the 1-D standard Array which represents the image
      // If the parameter is false, it gives a Float32Array
      var slice_buffer = series.images[j++].getInterpretedData(true); // 512*512 length
      // volume_buffer[j-1] = slice_buffer;
      var l = slice_buffer.length, m = 0;
      while(l--){
        volume_buffer[k++] = slice_buffer[m++];
      }
    }

    var volume = ndarray(volume_buffer,[r,c,no]); // Create a ndarray from this.
    var slice = volume.pick(null,null,115); // Select a slice
    // var slice = ndarray(new Float32Array(volume_buffer[115]),[r,c]);
    // Plot a heatmap of the image.

But when I look at the image, it does not show up properly, If I try another approach like this (simply changing the above code a little) :

    var r = series.images[0].getRows(); // Get the row size of image 1
    var c = series.images[0].getCols();  // Get the column size of image 1
    var no = series.images.length;  // Get the number of images

    var volume_buffer = new Array(no); // Create a 1-D array of the volume

    var i = no, j = 0,k = 0; 
    while(i--){
      // This gives the 1-D standard Array which represents the image
      // If the parameter is false, it gives a Float32Array
      var slice_buffer = series.images[j++].getInterpretedData(true);       
      volume_buffer[j-1] = slice_buffer;
      // var l = slice_buffer.length, m = 0;
      // while(l--){
      //  volume_buffer[k++] = slice_buffer[m++];
      // }
    }

    // var volume = ndarray(volume_buffer,[r,c,no]); // Create a ndarray from this.
    // var slice = volume.pick(null,null,115); // Select a slice
    var slice = ndarray(new Float32Array(volume_buffer[115]),[r,c]);
    // Plot a heatmap of the image.

This works, but its not what I need. So is there a way to properly load the images as a volume?

The array that I am trying to store should visually look something like this :

[
  [
    [... 512 length ...],
    [... 512 length ...],
    ...
    512 length
  ],
  245 length
  ...
]

Sweet.js could really help the code quality of ndarray. http://sweetjs.org/
The adhoc string concatenation that you use looks error prone and hard to edit. sweet.js wouldn't be needed at runtime, you could use it at just buildtime.

You can also use sweet.js to provide a more python like interface for fancy indexing and operator overloading for equality testing. partial assignment too.
nd_arr[2:30] = nd_arr2[0:28]

I am trying to use ndarray.hi() to manipulate the first 100 column of an image. YThe interesting thing i, for some image it is behaving perfectly and for some it is no. For the non-working cases, it seems that the column is divided into some steps and dispersed all over the image. I am using the following piece of code:

let img = ndarray(image, [w, h, 4]); // image is an Uint8ClampedArray
let panel = img.hi(null, 100);
let red = panel.pick(null, null, 0);
console.log(img.shape);

For example, the follwoing image not working perperly :

The image size (got from img.shape) is [700, 492, 4].

Also, here is an example where it woks properly (size is [3096, 4128, 4]):

What is the reason behind such anomaly and how can I fix it ? Please help me out here.

Is there a simple way to reshape an array?

Thanks,

John

I have found myself wanting to increment an index in place, but I have to call get first, increment, then set.

I understand that using arr.pick() allows you to pick out a subarray along an orthogonal axis, however, how can I pick out an angled (non-orthogonal) slice?

Hi,

Is there a way/module to perform dot product of 2 ndarrays?

Best regards

Since lo()/hi() accept nulls this should be added to the Readme as it is quite convenient.

I was just wondering why get-pixels and save-pixels were separate - same with ndarray-pack and ndarray-unpack.

I think it might be neat if there was a "super-pack" that could be included, like ndarray-signals and then you could use ndsignals.convolve(), etc.

I need a function that, given an index in the underlying array, returns the ndarray index: [i,j,...];

I implemented it in my code (without the compilation logic of ndarray), and thought it might be useful within this project, since index is already there.

function unindex(ndarr, n)
{
    const result = new Array( ndarr.shape.length );

    n = n - ndarr.offset;
    ndarr.stride.forEach( (stride, i)=>{
        result[i] = Math.floor( n / stride );
        n %= stride;
    });

    return result;
};

It would be interesting to see if Proxies: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy

Can be used to make performant ndarrays with an indexer syntax.

var mat = ndarray(new Float64Array([1, 0, 0, 1]), [2,2])
console.log(mat[1][2]);

It seems like the overhead might one proxy per dimension. Could be an interesting experiment! :)

It wasn't immediately obvious how to actually loop over this array from the examples. Something like this would be helpful to newcomers:

var mat = ndarray([1, 2, 3, 4, 5, 6, 7, 8, 9], [3,3])
var m = mat.hi(2, 2)

console.log('at: %s', m.get(2, 2));
for(var i=0; i < m.shape[0]; ++i) {
  for(var j=0; j < m.shape[1]; ++j) {
    console.log('(%s, %s): %s', i, j + m.offset, mat.get(i, j + m.offset));
  }
}

I can open a PR to add it to the Readme, if you're interested, or if there's a better way to improve this, so it's more generic.

@timoxley and I, being mere mortals, had trouble understanding the operation of .hi() and .lo().

Brain dump

• the way they're introduced in the documentation makes them a little confusing, in that they're introduced solely as a mechanism to get a "view" of a larger array, and then only later explained fully in their operation
• hi and lo seem to suggest a spatial relationship, which makes hi sound like the upper-left corner of the array, and lo the lower-right
• the names offset and limit might make the operation of the functions clearer

Code I used to understand how it works

var ndarray = require("ndarray");

var whole_array = ndarray(new Buffer(8 * 8), [8, 8]),
    view        = whole_array.hi(7, 6).lo(5, 4);

var x = 0, y = 0;

for (x=0;x<whole_array.shape[0];++x) {
  for (y=0;y<whole_array.shape[1];++y) {
    whole_array.set(x, y, 0);
  }
}

for (x=0;x<view.shape[0];++x) {
  for (y=0;y<view.shape[1];++y) {
    view.set(x, y, 1);
  }
}

for (x=0;x<whole_array.shape[0];++x) {
  for (y=0;y<whole_array.shape[1];++y) {
    process.stdout.write(whole_array.get(x, y) + " ");
  }

  process.stdout.write("\n");
}

0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0
0 0 0 0 1 1 0 0
0 0 0 0 1 1 0 0
0 0 0 0 0 0 0 0

Not sure why, but on Opera 12 and Opera next, there are strange errors relating to the shape/stride accessors. It probably is related to the crazy hacks going on with the implicit object constructors. One way to fix this maybe is to generate a separate set of code for opera/debug environments that is slower but simpler.

Browserify automatically inserts the buffer polyfill if any instance of Buffer is used. We can avoid this by switching out the references to Buffer in this module, but we need a light weight method to detect if an object is a Buffer.

Something that computes the flat index from an index tuple given strides and steps and offset. I believe you could just add

  //view.index(i0, ...):
  code.push([
"proto.index=function ",className,"_index(", args.join(","), "){"].join(""))
  code.push("return index_str}")

Before the view.set, and all would be good. But then again, I cannot claim to have fully understood the way the final code is generated, so this may be wrong.

Hi, numpy is implemented in C++ for performance whereas this is implemented in JavaScript itself. Do you think there will any performance hit for writing it in C++?

Or we could have written it in C++ and then add v8 wrapper?

How? Tried the suggested ndarray-unpack on a 1D ndarray, didn't work.
thanks.

Is there a reason that the many additional packages you have are not added t the prototype for ndarray. It would make access to the loaded extensions easier and more consistent.

Thanks,

John

im trying to create a list with some specific rules.
ex ) x :2, y:5
0, 0
0, 1
0, 2
0, 3
0, 4
1, 0
1, 1
1, 2
1, 3
1, 4

it should have 2 sets, y should be repeated for x times.
so the table shape should be [2, x * y].
1D data should look like [0, 1, 0, 2, 0, 3, 0, 4, 1, 0, 1, 1, 1, 2, 1, 3, 1, 4]

it works fine until here but when i try to get myNdarray.get(0, 5), i expected 1 but gives 2 instead.

    let x = 2;
    let y = 4;

    let datArr = []
    for (let xx = 0; xx < x; xx++){
        for (let yy = 0; yy < y; yy++){
            datArr.push(xx)
            datArr.push(yy)
            // narr.
        }
    }
    let narr = ndarray(new Float32Array(datArr), [2, x*y]);
    console.log(narr)
    console.log(narr.get(0, 5))
    console.log(narr.get(1, 5))

here is my test code, and

this is the log

NdArrayTest.js:35 View2dfloat32 {data: Float32Array(20), shape: Array(2), stride: Array(2), offset: 0}data: Float32Array(20) [0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 1, 0, 1, 1, 1, 2, 1, 3, 1, 4]offset: 0shape: (2) [2, 10]stride: (2) [10, 1]order: (...)size: (...)proto: Object
NdArrayTest.js:36 2
NdArrayTest.js:37 2

what is happening?
am i using the wrong code?

For some cases it would be nice to use masked arrays, similar to numpy (or.. better), so this is for typed arrays. I guess if you have undefined values in standard arrays, then ndarray leaves these alone, right?

I think this is a huge task but I still wanted to open an issue to see if anyway else is interested as well.