vectorho / co-webhdfs Goto Github PK

var _ = require('codash'),
    co = require('co'),
    LineReader = require('co-stream').LineReader,
    Writer = require('co-stream').Writer,
    WebHdfsClient = require('co-webhdfs');

co(function *() {
    var client = new WebHdfsClient({ namenode_hosts: ['10.10.0.141', '10.10.0.140'] });
    // Or, if you don't want to enable failover:
    // var client = new WebHdfsClient({ namenode_host: '10.10.0.140' });

    var home = yield* client.getHomeDirectory();
    var files = yield* client.listStatus(home);
    console.log(_.map(files, function (f) { return f.pathSuffix }));

    yield* client.mkdirs(home + '/tmp');

    var filename = home + '/tmp/foo.txt';

    // Data style.
    yield* client.del(filename);
    yield* client.create(filename, 'foooooo\n', { overwrite: true });
    yield* client.append(filename, 'barr\n');
    console.log(yield* client.open(filename));

    // Stream style.
    var stream = new Writer(yield* client.createWriteStream(filename, { overwrite: true }));
    yield stream.writeline('xxxxxxx');
    yield _.sleep(100);
    yield stream.writeline('gggggg');
    yield stream.end();
    yield _.sleep(1000);

    var reader = new LineReader(yield* client.createReadStream(filename));
    for (var line; line = yield* reader.read();) {
        console.log('line: ', line);
    }
}).then(function () {
    console.log('finished...');
}, function (err) {
    console.log(err);
    console.log(err.stack);
});

// e.g. http://<HOST>:<PORT>/webhdfs/v1/<PATH>?delegation=<TOKEN>&op=...
var client = new WebHdfsClient({ token: 'kerberos_token', ... });

var client = new WebHdfsClient({ ... });
client.addAuth("cookies");
...
// cookies访问是有时效性的，所以client后续操作返回401响应，则让用户重新登录获取cookies

WebHdfsClient(options)
  options:
    user: user name, default 'hadoop'
    namenode_host: ip/hotname of namenode, default 'localhost'
    namenode_port: port of webhdfa, default 50070

create: function *(path, data, hdfsoptions)
createWriteStream: function *(path, hdfsoptions)
  path: path of file
  data: data to write
  hdfsoptions:
    overwrite: <true|false>
    replication: <SHORT>]
    blocksize: <LONG>
    permission: <OCTAL>
    buffersize: <INT>

append: function *(path, data, hdfsoptions)
createAppendStream: function *(path, hdfsoptions)
  path: path of file
  data: data to write
  hdfsoptions:
    buffersize: <INT>

open: function *(path, hdfsoptions)
createReadStream: function *(path, hdfsoptions)
  path: path of file
  hdfsoptions:
    buffersize: <INT>
    offset:<LONG>
    length:<LONG>

mkdirs: function *(path, hdfsoptions)
  path: path of file
  hdfsoptions:
    permission: <OCTAL>

rename: function *(from, to)

del: function *(path)

getFileStatus: function *(path)

listStatus: function *(path)

getContentSummary: function *(path)

getFileChecksum: function *(path)

getHomeDirectory: function *()