teambition / merge2 Goto Github PK

View Code? Open in Web Editor NEW

171.0 17.0 15.0 149 KB

Merge multiple streams into one stream in sequence or parallel (~119M/month downloads).

License: MIT License

JavaScript 100.00%

merge2's Introduction

merge2

Merge multiple streams into one stream in sequence or parallel.

Install

Install with npm

npm install merge2

Usage

const gulp = require('gulp')
const merge2 = require('merge2')
const concat = require('gulp-concat')
const minifyHtml = require('gulp-minify-html')
const ngtemplate = require('gulp-ngtemplate')

gulp.task('app-js', function () {
  return merge2(
      gulp.src('static/src/tpl/*.html')
        .pipe(minifyHtml({empty: true}))
        .pipe(ngtemplate({
          module: 'genTemplates',
          standalone: true
        })),
      gulp.src([
        'static/src/js/app.js',
        'static/src/js/locale_zh-cn.js',
        'static/src/js/router.js',
        'static/src/js/tools.js',
        'static/src/js/services.js',
        'static/src/js/filters.js',
        'static/src/js/directives.js',
        'static/src/js/controllers.js'
      ])
    )
    .pipe(concat('app.js'))
    .pipe(gulp.dest('static/dist/js/'))
})

const stream = merge2([stream1, stream2], stream3, {end: false})
//...
stream.add(stream4, stream5)
//..
stream.end()

// equal to merge2([stream1, stream2], stream3)
const stream = merge2()
stream.add([stream1, stream2])
stream.add(stream3)

// merge order:
//   1. merge `stream1`;
//   2. merge `stream2` and `stream3` in parallel after `stream1` merged;
//   3. merge 'stream4' after `stream2` and `stream3` merged;
const stream = merge2(stream1, [stream2, stream3], stream4)

// merge order:
//   1. merge `stream5` and `stream6` in parallel after `stream4` merged;
//   2. merge 'stream7' after `stream5` and `stream6` merged;
stream.add([stream5, stream6], stream7)

// nest merge
// equal to merge2(stream1, stream2, stream6, stream3, [stream4, stream5]);
const streamA = merge2(stream1, stream2)
const streamB = merge2(stream3, [stream4, stream5])
const stream = merge2(streamA, streamB)
streamA.add(stream6)

API

const merge2 = require('merge2')

merge2()

merge2(options)

merge2(stream1, stream2, ..., streamN)

merge2(stream1, stream2, ..., streamN, options)

merge2(stream1, [stream2, stream3, ...], streamN, options)

return a duplex stream (mergedStream). streams in array will be merged in parallel.

mergedStream.add(stream)

mergedStream.add(stream1, [stream2, stream3, ...], ...)

return the mergedStream.

mergedStream.on('queueDrain', function() {})

It will emit 'queueDrain' when all streams merged. If you set end === false in options, this event give you a notice that should add more streams to merge or end the mergedStream.

stream

option Type: Readable or Duplex or Transform stream.

options

option Type: Object.

end - Boolean - if end === false then mergedStream will not be auto ended, you should end by yourself. Default: undefined
pipeError - Boolean - if pipeError === true then mergedStream will emit error event from source streams. Default: undefined
objectMode - Boolean . Default: true

objectMode and other options(highWaterMark, defaultEncoding ...) is same as Node.js Stream.

License

MIT © Teambition

merge2's People

Contributors

Stargazers

Watchers

Forkers

plugn avvs ksheedlo adventure-yunfei adamquadmon linusu krmax44 yiidot f1nzer ronycohen joshuaramsamooj jeniex mythofsisyphus guieco boenrobot

merge2's Issues

As of gulp 4, merged streams have 0 files if they weren't piped

For example, this doesn't work.

const debug = require('gulp-debug');
const merge = require('merge2');
const gulp = require('gulp');

return merge(
  gulp.src('file/a.js'),
  gulp.src('file/b.js')
).pipe(debug({ title: 'Files:' })) // outputs "Files: 0"

But this does.

const debug = require('gulp-debug');
const merge = require('merge2');
const gulp = require('gulp');

return merge(
  gulp.src('file/a.js').pipe(debug({ title: 'Files in a:' })), // outputs "Files in a: 1"
  gulp.src('file/b.js').pipe(debug({ title: 'Files in b:' })) // outputs "Files in b: 1"
).pipe(debug({ title: 'Files:' })) // outputs "Files: 0" // outputs "Files: 2"

I noticed this when I switched from the alpha version of gulp 4 to the one installed via gulp@next (which is 4.0.0).

Running streams in sequential

I'm trying to execute a couple of streams sequential, run first stream, wait for it to complete, then run the next stream. The results I'm getting are not what I do expect.

Following is my gulp task:

gulp.task('build', ['clean', 'compile'], function () {
    var extension = gulp.src(['README.md', 'LICENSE.txt', 'images/**/*', '!images/**/*.pdn', 'vss-extension.json'], { base: '.' })
        .pipe(debug({ title: 'Copying extension files:' }))
        .pipe(gulp.dest(_buildRoot));

    const getDirectories = fs.readdirSync(_tasksRoot).filter(function (file) {
        return fs.statSync(_tasksRoot + '/' + file).isDirectory();
    });

    const stream = merge2(extension, { end: false })

    getDirectories.forEach(task => {
        var taskPath = path.join(_tasksRoot, task);
        var taskBuild = path.join(_buildRoot, task)

        gutil.log(taskPath);

        var taskFiles = path.join(task, '**/*');

        gutil.log(taskFiles);

        stream.add(gulp.src([taskFiles, '!**/*.ts', '!**/package.json'])
            .pipe(debug({ title: 'Task files copy for ' + task }))
            .pipe(gulp.dest(taskBuild)));

        //getExternalModules();
    });

    return stream.end();
});

The output I do get in the console is the following:

C:\Repos\tfs-ansible-tower-extension>gulp build
[15:22:56] Using gulpfile C:\Repos\tfs-ansible-tower-extension\gulpfile.js
[15:22:56] Starting 'clean'...
[15:22:56] Finished 'clean' after 28 ms
[15:22:56] Starting 'compile'...
[15:22:56] Compiling playbook-task
[15:22:56] Compiling run-playbook
[15:22:56] Finished 'compile' after 244 ms
[15:22:56] Starting 'build'...
[15:22:56] C:\Repos\tfs-ansible-tower-extension\Tasks\playbook-task
[15:22:56] playbook-task\**\*
[15:22:56] C:\Repos\tfs-ansible-tower-extension\Tasks\run-playbook
[15:22:56] run-playbook\**\*
[15:22:56] Finished 'build' after 7.64 ms
[15:22:56] Task files copy for playbook-task playbook-task\AnsibleTower.js
[15:22:56] Copying extension files: README.md
[15:22:56] Task files copy for run-playbook run-playbook\icon.png
[15:23:00] Task files copy for playbook-task playbook-task\icon.png
[15:23:00] Copying extension files: LICENSE.txt
[15:23:00] Task files copy for run-playbook run-playbook\task.json
[15:23:00] Task files copy for run-playbook 2 items
[15:23:00] Task files copy for playbook-task playbook-task\task.js
[15:23:00] Copying extension files: images\logo.png
[15:23:00] Task files copy for playbook-task playbook-task\task.json
[15:23:00] Task files copy for playbook-task playbook-task\test
[15:23:00] Task files copy for playbook-task 5 items
[15:23:00] Copying extension files: vss-extension.json
[15:23:00] Copying extension files: 4 items

Now this is not looking to be synced.
Am I misunderstanding the usage of this library or not using it correctly?
How do I get each of my copy operations executed sequentially one after another?

Thanks

Node 0.10 crashes "Use of const in strict mode"

I have an application that runs on Node 0.10. When we try to import this package it crashes with the following error:

/Users/ken/unpm-code/chameleon-ui/node_modules/merge2/index.js:9
const Stream = require('stream')
^^^^^
SyntaxError: Use of const in strict mode.
    at Module._compile (module.js:439:25)
    at Object.Module._extensions..js (module.js:474:10)
    at Module.load (module.js:356:32)
    at Function.Module._load (module.js:312:12)
    at Module.require (module.js:364:17)
    at require (module.js:380:17)
    at Object.<anonymous> (/Users/ken/unpm-code/chameleon-ui/node_modules/unitest/index.js:7:13)
    at Module._compile (module.js:456:26)
    at Object.Module._extensions..js (module.js:474:10)
    at Module.load (module.js:356:32)

This crash is occurring because merge2 uses const, which behaves differently in node 0.10. Please consider adding a build step that compiles ES6 to ES5 for older versions of node at publish time.

Strange work with concated streams

Windows, node.js v 0.12.2

for gulp-merge (npm link direct to this project).

Example

File structure:

test1
|- script1.js
|- script2.js
|- ...
test2
...
test5

gulpfile.js:

gulp.task('test', function() {
    var streamsArr = [];

    for (var stream, i = 1; i < 5; i++) {
        stream = gulp.src(path.join(__dirname, 'test' + i, '**/*'))
            .pipe(through.obj(function(file, enc, callback) {
                setTimeout(function() {
                    callback(null, file);

                }, Math.ceil(Math.random()*100));
            }))
            .pipe(require('gulp-concat')('script.js'))
            .on('end', (function(i) {
                console.log('end of the stream ' + i);
            }).bind(null, i));

        streamsArr.push(stream);
    }

    require('gulp-merge')(streamsArr)
        .pipe(through.obj(function(file, enc, callback){
            console.log('merged stream transform:', path.relative(__dirname, file.path));

            callback(null, file);

        }, function(callback) {
            console.log('merged stream flush');

            callback();
        }))
        .on('end', function() {
            console.log('end of the merged stream');
        });
});

console:

end of the stream 4
end of the stream 3
merged stream transform: test1\script.js
end of the stream 1
merged stream transform: test2\script.js
end of the stream 2

Somehow, part of a stream is lost. Also did not work flush and end event of merged stream.

for merge2 i have
TypeError: Cannot read property 'endEmitted' of undefined at pipe (D:\Projects\test\node_modules\merge2\index.js:57:32) <...>

Please help with sourcemaps

How and where init and write sourcempas using merge2?

...
    return merge(
        require('gulp-bem-preset-browserjs')(bundle, {includePaths: ['bem']}),
        require('gulp-bem-preset-bemtree')(bundle, {ym: true}),
        require('gulp-bem-preset-bemhtml')(bundle, {ym: true, engine: {elemJsInstances: true}})
      )
      .pipe(gulpif(env === 'production', babel({ presets: ['es2015'], compact: false })))
      .pipe(gulpif(env === 'production', uglify()))
      .pipe(concat(bundle.name + '.min.js')),
...

Streams participating in multiple merges get lost

I have to build multiple CSS files and create bundles of all that files with multiple themes. Some file is included in multiple bundles. But after script is run that file is missing in concatenation results. When I replace merge2 with merge-stream it works, but concatenated files have random order.

const gulp = require('gulp');
const merge = require('merge2');
const less = require('gulp-less');
const concat = require('gulp-concat');
const rename = require('gulp-rename');

gulp.task('build', () => {
    const files = ['file1', 'file2', 'file3'];
    const themes = ['light', 'dark'];

    const getThemedStream = (theme, file) => {
        return gulp.src(`./${file}.less`)
            .pipe(less())
            .pipe(rename(`${file}.${theme}.css`))
            .pipe(gulp.dest('./result'));
    };

    const getConcatStream = (theme, streams) => {
        return merge(...streams)
            .pipe(concat(`merged.${theme}.css`))
            .pipe(gulp.dest('./result'));
    };

    const commonStream = gulp.src(`./common.less`)
        .pipe(less())
        .pipe(rename(`common.css`))
        .pipe(gulp.dest('./result'));

    return merge(commonStream, ...themes.map((theme) => {
        const streams = files.map((file) => getThemedStream(theme, file));
        streams.push(commonStream);
        return getConcatStream(theme, streams);
    }));
});

Task randomly fails to complete

Sometimes my gulp task will prematurely exit when I have gulp-modernizr included:

var gulp = require('gulp');
var addsrc = require('gulp-add-src');
var concat = require("gulp-concat");
var gulpMerge = require('gulp-merge');
var gulpif = require('gulp-if');
var modernizr = require('gulp-modernizr');
var ngAnnotate = require('gulp-ng-annotate');
var rewriteCSS = require('gulp-rewrite-css');
var sass = require('gulp-sass');
var sourcemaps = require('gulp-sourcemaps');
var templateCache = require('gulp-angular-templatecache');
var uglify = require('gulp-uglify');
var watch = require('gulp-watch');


var doUglify = true;
var dest = './assets/dist';


function jsSrcs(){
    return gulpMerge(
        gulp.src([
            './assets/bower-libs/jquery/dist/jquery.js',
            './assets/bower-libs/angular/angular.js',
            './assets/bower-libs/angular-recaptcha/release/angular-recaptcha.js',
            './assets/bower-libs/angular-route/angular-route.js',
            './assets/bower-libs/isotope/dist/isotope.pkgd.js',
            './assets/bower-libs/isotope-packery/packery-mode.pkgd.js',
            './assets/bower-libs/querystring/querystring.js'
        ]),
        gulp.src('./assets/js/**/*.tpl.html').pipe(templateCache({root: '/js/', standalone: true})).pipe(concat('templates.js')),
        gulp.src([
            './assets/js/**/*.js'
        ])
    );
}


gulp.task('js', function() {
  return gulpMerge(jsSrcs().pipe(modernizr({tests: ['videoautoplay']}))).add(jsSrcs())
    .pipe(sourcemaps.init())
    .pipe(concat('bundle.min.js'))
    .pipe(gulpif(doUglify, ngAnnotate()))
    .pipe(gulpif(doUglify, uglify()))
    .pipe(sourcemaps.write('./'))
    .pipe(gulp.dest(dest));
});

Command output:

$ gulp js
(node:89371) fs: re-evaluating native module sources is not supported. If you are using the graceful-fs module, please update it to a more recent version.
[10:44:03] Using gulpfile ~/dev/project/gulpfile.js
[10:44:03] Starting 'js'...

>> Explicitly including these tests:
>> videoautoplay

Looking for Modernizr references

>> Ready to build using these settings:
>> addTest, html5printshiv, testProp, fnBind

Building your customized Modernizr...OK

If successful, after that last line there should be something like:

[10:42:59] Finished 'js' after 2.0 s

Getting a TypeError "Cannot read property '_readableState' of undefined at PauseStream"

I am running merge2 for multiple streaming of task. My tasks involve gulp-image-resize, gulp-imagemin and gulp-webp. Actually I am running an Array forEach Loop inside the tasks.

My coding on gulpfile is the following:

gulp.task('mergeimg',function(){
 var bannerImages = [2000,1600,1200,800,533];

 var imgmin =  bannerImages.forEach(function(size){
    return gulp.src( '../html/assets/unoptimized/banners/**/*.{png,jpg,jpeg}')
    .pipe(imageResize({
       width: size,
       crop:true,
 }))
 .pipe(imagemin({
    		progressive: true,
    		use: [pngquant()],
        use:[imageminMozjpeg({quality:85})],

   }))
 .pipe(rename(function(path){
    path.basename += '_'+size
}))
 .pipe(debug())
   .pipe(gulp.dest('../html/assets/temp/' ))
 });
 var imgretina =   bannerretina.forEach(function(size){
  return gulp.src( '../html/assets/unoptimized/banners/**/*.{png,jpg,jpeg}')
   .pipe(imageResize({
      width: size * 2,
    crop:true,
 }))
 .pipe(imagemin({
      	progressive: true,
        use: [pngquant()],
       use:[imageminMozjpeg({quality:85})],
}))
  .pipe(rename(function(path){
      	path.basename += '_'+size+'@2x'
	}))
  .pipe(debug())
  .pipe(gulp.dest('../html/assets/temp/' ))
  });
var webPimg = bannerImages.forEach(function(size){
   return gulp.src( '../html/assets/unoptimized/banners/**/*.{png,jpg,jpeg}')
   .pipe(webp())
    .pipe(rename(function(path){
      	path.basename += '_'+size
}))
  .pipe(debug())
  .pipe(gulp.dest('../html/assets/temp/' ))
 });
  var mergedStream = merge( imgmin, imgretina, webPimg );
 return mergedStream;
});

The tasks are executing, all the images are being resized and compressed according to my need.
But I am geeting a error saying:

TypeError: Cannot read property '_readableState' of undefined at pauseStreams

I don't know why these all I am getting. Maybe my mistake. Can anyone help me to have a solution. Thanks in advance.

The entire error code:

[13:26:00] Starting 'mergeimg'...
[13:26:00] 'mergeimg' errored after 28 ms
[13:26:00] TypeError: Cannot read property '_readableState' of undefined
    at pauseStreams (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\node
_modules\merge2\index.js:98:18)
    at addStream (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\node_mo
dules\merge2\index.js:31:25)
    at merge2 (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\node_modul
es\merge2\index.js:90:30)
    at Gulp.<anonymous> (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\
gulpfile.js:183:24)
    at module.exports (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\no
de_modules\orchestrator\lib\runTask.js:34:7)
    at Gulp.Orchestrator._runTask (E:\xamp\htdocs\damithemes\capture-kids\gulp_a
utomation\node_modules\orchestrator\index.js:273:3)
    at Gulp.Orchestrator._runStep (E:\xamp\htdocs\damithemes\capture-kids\gulp_a
utomation\node_modules\orchestrator\index.js:214:10)
    at Gulp.Orchestrator.start (E:\xamp\htdocs\damithemes\capture-kids\gulp_auto
mation\node_modules\orchestrator\index.js:134:8)
    at C:\Users\Da mirecle\AppData\Roaming\npm\node_modules\gulp\bin\gulp.js:129
:20
    at _combinedTickCallback (internal/process/next_tick.js:131:7)
[13:26:01] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_533.we
bp
[13:26:01] gulp-debug: 1 item
[13:26:01] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_2000.w
ebp
[13:26:01] gulp-debug: 1 item
[13:26:01] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_1200.w
ebp
[13:26:01] gulp-debug: 1 item
[13:26:01] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_800.we
bp
[13:26:01] gulp-debug: 1 item
[13:26:01] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_1600.w
ebp
[13:26:01] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_800.jp
g
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 2%)
[13:26:02] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_1200.j
pg
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 1.4%)
[13:26:02] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_533@2x
.jpg
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 1.6%)
[13:26:02] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_533.jp
g
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 3.1%)
[13:26:02] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_1600.j
pg
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[13:26:02] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_2000@2
x.jpg
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[13:26:02] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_800@2x
.jpg
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[13:26:02] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_1200@2
x.jpg
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[13:26:02] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_2000.j
pg
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[13:26:02] gulp-debug: 1 item
[13:26:02] gulp-debug: ..\html\assets\unoptimized\banners\child-with-bags_1600@2
x.jpg
[13:26:02] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[13:26:02] gulp-debug: 1 item

Accept fn for lazy sequential streams

I haven't looked into the implementation of merge2 yet but wondered whether it would be possible to accept a function (() => Stream), allowing streams to be lazy, e.g.

const pipeline1 = () => readable1.pipe(transform1); // Lazy
const pipeline2 = () => readable2.pipe(transform2); // Lazy
merge(pipeline1, pipeline2).pipe(writable); // `readable2` is not read from until `pipeline1` is finished

Error events are not merged / propagated

Hi! I discovered merge2 off of the recommendation of another project... and I generally like the simplicity of merge2's stream composition API, but I got bit quite severely when I found that errors are not propagated.

Right now, I have to wrap all my usages of merge2, to listen to errors on source streams and then re-emit them on the merged stream as a workaround.

IMHO - there are probably a few classic error propagation strategies to offer when using merged streams:

Eager errors - emit error event as soon as source stream emits one. This should probably be the default, and matches the asynchronous behaviour for Promise.all(...) and Promise.race(...) (the aggregate operation errors as soon as the first source operation errors).
Deferred errors - bottle up errors until all streams have either ended or error'd. Option to aggregate errors / expose the full list of errors.

License

Hi,

Can you please update the package.json to include the license?

Using a domain property in MakeCallback is deprecated

(node:23160) [DEP0097] DeprecationWarning: Using a domain property in MakeCallback is deprecated. Use the async_context variant of MakeCallback or the AsyncResource class instead.
    at emitMakeCallbackDeprecation (domain.js:123:13)
    at FSReqCallback.topLevelDomainCallback (domain.js:134:5)
    at FSReqCallback.callbackTrampoline (internal/async_hooks.js:121:14)
[20:49:24] Finished 'min' after 15 s

Data from streams are not merged

Consider this example:

/// <reference path="../typings/tape/tape.d.ts" />
/// <reference path="../typings/node/node.d.ts" />
var test = require("tape");
var merge2 = require('merge2');
var inherits = require("util").inherits;

var Readable = require("stream").Readable;
inherits(CharsStream, Readable);

function CharsStream(options) {
    Readable.call(this);
    this.timeout = options.timeout || 100;
    this.count = options.count || 1;
}

CharsStream.prototype._read = (function () {
    var c = "a".charCodeAt(0);
    return function () {
        var letter = String.fromCharCode(c++);
        var isEnd = (c > "d".charCodeAt(0));
        if (isEnd) {
            this.push(null);
            return;
        }
        var self = this;
        setTimeout(function () {
            var o = new Array(~~self.count + 1).join(letter);
            self.push(o);
        }, ~~(Math.random() * self.timeout));
    };
})();

test("merge streams", function (t) {
    var s1 = new CharsStream({count: 1});
    var s2 = new CharsStream({count: 2});
    var s = merge2([s1, s2]);
    s.on("data", function (chunk) {
        console.log(chunk.toString());
    });
    s.on("end", t.end);
})

Run:
Results:

bb
cc
a

Where are other values: aa, b, c?

Next run:

a
bb
c

Where are other values: aa, b, cc?

Is it expected behavior?

merge2 sometimes misses streams on('end') events

in some scenarios, especially with gulp.src() resolving to an empty stream, merge2 misses the on('end') event.

The problem is solved by adding the on('end') immediately on addStream:

  function addStream() {
    for (var i=0; i<arguments.length; i++) {
        (function(stream) {
            function onend() {
                stream.removeListener('merge2UnpipeEnd', onend);
                stream.removeListener('end', onend);
                console.log('stream ended', stream._id);
                if (stream._merge2_onend) {
                    stream._merge2_onend();
                }
                stream._merge2_finished = true;
            }
            stream.on('merge2UnpipeEnd', onend);
            stream.on('end', onend);
        })(arguments[i]);
    }
...
    function pipe(stream) {
      if (stream._merge2_finished) {
        console.log('pipe', stream._id, 'already finished');
        next();
      } else {
        console.log('pipe', stream._id, 'go');
        stream._merge2_onend = function() {
          console.log('pipe', stream._id, 'end');
          next();
        }
        stream.pipe(mergedStream, {end: false});
      }
/*
      function onend() {
        stream.removeListener('merge2UnpipeEnd', onend);
        stream.removeListener('end', onend);
        next();
      }
      stream.on('merge2UnpipeEnd', onend);
      stream.on('end', onend);
      stream.pipe(mergedStream, {end: false});
*/
    }

gulp task starts but does not end

I have the next files structure:

src/
    themes/
        default/
            1.png
            2.png
        oneMoreTheme/
            3.png
            4.png
dist
    themes/
        default/
        oneMoreTheme/

and the next gulpfile:

  'use strict';

  var gulp = require('gulp'),
    _ = require('lodash'),
    plugins = require('gulp-load-plugins')();

  gulp.task('images', function () {
    var streams = {},
      themes = [
        'default',
        'oneMoreTheme'
      ];

    streams.themesImages = [];

    _.forEach(themes, function(theme) {
      streams.themesImages.push(gulp.src('src/themes/' + theme + '/*.*')
      .pipe(gulp.dest('dist/themes/' + theme + '/')));
    });

    streams.allThemeImages = plugins.merge(streams.themesImages);

    return streams.allThemeImages;
  });

When I use gulp-merge plugin I have two problems I can not understand:

Why do I see in console the next text when task "images" starts:

"D:\home\projects\storyBook>gulp images
[15:05:23] Using gulpfile D:\home\projects\storyBook\gulpfile.js
[15:05:23] Starting 'images'..."

But when task is finished I do NOT see the next text:

"[12:13:56] Finished 'images' after 72 ms"

Why do not I get this text? I do streams merge and then return merged stream to have synchronous task. Maybe I do something wrong? Only sometimes I get this message.

If I add one code line to my privous code:

streams.allThemeImages = plugins.merge(streams.themesImages);
return streams.allThemeImages.pipe(gulp.dest('temp/')); // HERE IS NEW CODE LINE

I expect that in temp folder I'll have the next images:

temp/ 
    1.png 
    2.png 
    3.png 
    4.png

Because I merged streams, first one contains 1.png and 2.png images and second one contains 3.png and 4.png. But in reality I get the next files:

temp/ 
    1.png 
    2.png

Sometimes I get only 1.png inside temp folder and sometimes all work well and I get all images inside temp folder.
What is wrong?

When I use merge2 plugin insted of gulp-merge, all works well and I have not these two problems! Why?

Is gulp-merge deprecated in favor of merge2?

It appears to be, but the gulp-merge package on NPM doesn't say anything about that.

Can no longer pipe to merge src

Hi,

Can you please provide some insight to why the gulp-merge plugin was deprecated?
This has been a project dependency for some considerable time.

This change cause issues with any historical code base in any git branch that references gulp-merge.
It also means any other developer that works on the code will experience the same issue.

The issue i am seeing is if i add a pipe to the first source, the second source is ignored

var gulp = require('gulp'),
    gulpConcat = require('gulp-concat'),
    gulpMainBowerFiles = require('gulp-main-bower-files'),
    gulpMerge = require('gulp-merge');

gulp.task('css', function () {

  return gulpMerge (
    gulp.src('./bower.json')
    .pipe(gulpMainBowerFiles('**/*.*css', {
      includeDev: 'exclusive',
      group: 'css'
    })),
    gulp.src('./source/css/styles.css')
  )
  .pipe(gulpConcat('stylesheet.css'))
  .pipe(gulp.dest('./build/css'))

});

However if i ever if i reverse the sources to be merged the pipes are obeyed and both sources are merged.

How can I make the first source pass through the pipe?

Thanks

install instructions

The install instructions have the wrong package
npm install --save-dev gulp-sequence

needs to be changed to: npm install --save-dev gulp-merge

Getting an error "Cannot read property '_readableState' of undefined" with multiple tasks

I am using gulp for image optimization, OS is Windows 7.
My script is

var merge = require('merge2');
gulp.task('mergeimg',function(){

  var bannerImages = [2000,1600,1200,800,533];
  var bannerretina = [2000,1600,1200,800,533];


  var imgmin =  bannerImages.forEach(function(size){
    return gulp.src( '../html/assets/unoptimized/banners/**/*.{png,jpg,jpeg}')
    .pipe(imageResize({
       width: size,
       crop:true,

     }))

     .pipe(imagemin({
    		progressive: true,
    		use: [pngquant()],
        use:[imageminMozjpeg({quality:85})],

    	}))

    .pipe(rename(function(path){
    		path.basename += '_'+size

    	}))
      .pipe(gulp.dest('../html/assets/temp/' ))

  });

  var imgretina =   bannerretina.forEach(function(size){
      return gulp.src( '../html/assets/unoptimized/banners/**/*.{png,jpg,jpeg}')
      .pipe(imageResize({
         width: size * 2,
         crop:true,

       }))

       .pipe(imagemin({
      		progressive: true,
      		use: [pngquant()],
          use:[imageminMozjpeg({quality:85})],

      	}))
       .pipe(rename(function(path){
      		path.basename += '_'+size+'@2x'

      	}))
        .pipe(gulp.dest('../html/assets/temp/' ))

    });

    var webPimg = bannerImages.forEach(function(size){
      return gulp.src( '../html/assets/unoptimized/banners/**/*.{png,jpg,jpeg}')
      .pipe(webp())
       .pipe(rename(function(path){
      		path.basename += '_'+size

      	}))
        .pipe(gulp.dest('../html/assets/temp/' ))

    });

    var mergedStream = merge( imgmin, imgretina, webPimg );

    return mergedStream;

});

Tasks are performing as per my need, but getting an error message...

[18:40:16] 'mergeimg' errored after 34 ms
[18:40:16] TypeError: Cannot read property '_readableState' of undefined
    at pauseStreams (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\node
_modules\merge2\index.js:98:17)
    at addStream (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\node_mo
dules\merge2\index.js:31:25)
    at merge2 (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\node_modul
es\merge2\index.js:90:30)
    at Gulp.<anonymous> (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\
gulpfile.js:240:24)
    at module.exports (E:\xamp\htdocs\damithemes\capture-kids\gulp_automation\no
de_modules\orchestrator\lib\runTask.js:34:7)
    at Gulp.Orchestrator._runTask (E:\xamp\htdocs\damithemes\capture-kids\gulp_a
utomation\node_modules\orchestrator\index.js:273:3)
    at Gulp.Orchestrator._runStep (E:\xamp\htdocs\damithemes\capture-kids\gulp_a
utomation\node_modules\orchestrator\index.js:214:10)
    at Gulp.Orchestrator.start (E:\xamp\htdocs\damithemes\capture-kids\gulp_auto
mation\node_modules\orchestrator\index.js:134:8)
    at C:\Users\Da mirecle\AppData\Roaming\npm\node_modules\gulp\bin\gulp.js:129
:20
    at _combinedTickCallback (internal/process/next_tick.js:73:7)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 1.6%)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 1.4%)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 2%)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 3.1%)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)
[18:40:18] gulp-imagemin: Minified 1 image (saved 858 B - 1.1%)

Can't find my error. Highly appreciate any help. Thanks in advance.

Cannot switch to old mode

Hi!

I've a weird issue when using merge2. It seems it comes from the fact I'm splitting my stream pipes. But I'm not sure how to fix this, and how to do this differently...

var streams = require('merge2')(),
requireTree = function(name) {
    var stream = gulp.src( components[name] + '/component.scss' );
    if(fs.existsSync(components[name] + '/require.json')) {
        var deps = require(components[name] + '/require.json'),
            i, j;
        for(i=deps.length-1, j=0; i>=j; --i) {
            requireTree(deps[i]);
            stream.pipe( insert.append('@import "' + components[deps[i]] + '/component.scss";') );
        }
    }
    stream.pipe( gulp.dest(rootPath + '/cache/scss/' + name) );
    streams.add(stream);
};

Stream errors in merge2 1.2.0

I've been struggling with this for a few hours now. I've started a new project and using Gulp 3.9.1 and merge2 1.2.0 my old build scripts are failing because gulp.src doesn't seem to return correct streams to merge.

merge(
 gulp.src(['*.txt'], {base: '.'})
 .pipe(debug({ title: 'txtfiles:' })),
 gulp.src(['*.css'], {base: '.'})
 .pipe(debug({ title: 'cssfiles:' }))
)
.pipe(debug({ title: 'merge:' }))
.pipe(zip('myzip.zip'))

I'm getting all css files double (i.e. I'm getting css files from the txt files stream as well as the css stream) and in the zip file you'll find double entries. Then if one of the streams returns no files (let's say change to '*.foobar') then all streams return 0 files.

0.3.3 with gulp-sass produces Error: Only readable stream can be merged.

Trying to merge two streams, one of which has been piped through gulp-sass, produces Error: Only readable stream can be merged.

I had tried a few other gulp plugins and they all worked, so I first posted an issue to gulp-sass about this: dlmanning/gulp-sass#237 (includes sample code and repo)

But after looking into it further, the same code actually works fine if using merge2 <= 0.3.1

With 0.3.2 I get:

[03:09:26] Using gulpfile /var/www/gulp-sass-merge2/gulpfile.js
[03:09:26] Starting 'default'...

/var/www/gulp-sass-merge2/node_modules/merge2/index.js:57
      if (stream._readableState.endEmitted) return next();
                               ^
TypeError: Cannot read property 'endEmitted' of undefined
    at pipe (/var/www/gulp-sass-merge2/node_modules/merge2/index.js:57:32)
    at mergeStream (/var/www/gulp-sass-merge2/node_modules/merge2/index.js:65:46)
    at next (/var/www/gulp-sass-merge2/node_modules/merge2/index.js:47:7)
    at DestroyableTransform.onend (/var/www/gulp-sass-merge2/node_modules/merge2/index.js:54:9)
    at DestroyableTransform.emit (events.js:117:20)
    at /var/www/gulp-sass-merge2/node_modules/gulp/node_modules/vinyl-fs/node_modules/through2/node_modules/readable-stream/lib/_stream_readable.js:965:16
    at process._tickCallback (node.js:448:13)

And with 0.3.3, as per the linked issue:

[02:41:42] Using gulpfile /var/www/gulp-sass-merge2/gulpfile.js
[02:41:42] Starting 'default'...
[02:41:42] 'default' errored after 4.51 ms
[02:41:42] Error: Only readable stream can be merged.
    at pauseStreams (/var/www/gulp-sass-merge2/node_modules/merge2/index.js:92:13)
    at addStream (/var/www/gulp-sass-merge2/node_modules/merge2/index.js:29:25)
    at merge2 (/var/www/gulp-sass-merge2/node_modules/merge2/index.js:84:30)
    at Gulp.<anonymous> (/var/www/gulp-sass-merge2/gulpfile.js:9:9)
    at module.exports (/var/www/gulp-sass-merge2/node_modules/gulp/node_modules/orchestrator/lib/runTask.js:34:7)
    at Gulp.Orchestrator._runTask (/var/www/gulp-sass-merge2/node_modules/gulp/node_modules/orchestrator/index.js:273:3)
    at Gulp.Orchestrator._runStep (/var/www/gulp-sass-merge2/node_modules/gulp/node_modules/orchestrator/index.js:214:10)
    at Gulp.Orchestrator.start (/var/www/gulp-sass-merge2/node_modules/gulp/node_modules/orchestrator/index.js:134:8)
    at /usr/lib/node_modules/gulp/bin/gulp.js:129:20
    at process._tickCallback (node.js:448:13)

So now I'm not sure if there's a problem with this plugin, the gulp-sass plugin, or if for some reason they just don't play well together?

Gulp streams out of order

I've tried a handful of different possible solutions to the problem I'm having, and just can't seem to get the streams to run sequentially in order. I added some logging to the stream so that I can see the order in which things are being read. I used the docs to write the below code a few different ways, but same results each time. Am I missing something? What might I be doing wrong? Thank you in advance!

Package:

"paths": {
    "src": {
        "scss": "./src/assets/scss/"
    },
    "dev": {
        "css": "./dev/assets/css/"
    },
    "dist": {
        "css": "./dist/assets/css/"
    }
},
"devDependencies": {
    "browser-sync": "^2.26.7",
    "fancy-log": "^1.3.3",
    "gulp": "^4.0.2",
    "gulp-autoprefixer": "^6.1.0",
    "gulp-clean-css": "^4.2.0",
    "gulp-environments": "^0.1.2",
    "gulp-flatten": "^0.4.0",
    "gulp-group-css-media-queries": "^1.2.0",
    "gulp-if-env": "^0.1.0",
    "gulp-load-plugins": "^2.0.0",
    "gulp-notify": "^3.0.0",
    "gulp-plumber": "^1.1.0",
    "gulp-rename": "^1.2.3",
    "gulp-sass": "^4.0.2",
    "gulp-sourcemaps": "^2.6.4",
    "gulp-strip-css-comments": "^2.0.0",
    "merge2": "^1.2.4"
}

Gulp:

const { src, dest, lastRun, watch, series } = require('gulp');
const $ = require('gulp-load-plugins') ({
    pattern: ['*'],
    scope: ['devDependencies']
});
const pkg = require('./package.json');
const browserSync = require('browser-sync').create();

function css() {
    return $.merge2(
        src([
            pkg.paths.src.scss + '**/*.scss',
            '!' + pkg.paths.src.scss + 'projects/example-project/example.scss'
        ])
            .on('end', function() {
                $.fancyLog('cssDev');
            })
            .pipe($.plumber({
                errorHandler: $.notify.onError('Error: <%= error.message %>')
            }))
            .pipe($.sourcemaps.init())
            .pipe($.sass())
            .pipe($.stripCssComments({
                preserve: false
            }))
            .pipe($.autoprefixer({
                cascade: false
            }))
            .pipe($.groupCssMediaQueries())
            .pipe($.sourcemaps.write('./'))
            .pipe($.flatten({
                includeParents: 1
            }))
            .pipe(dest(pkg.paths.dev.css))
            .pipe(browserSync.stream()),
        src([
            pkg.paths.dev.css + '**/*.css'
        ])
            .on('end', function() {
                $.fancyLog('cssDist');
            })
            .pipe($.ifEnv.includes('sandbox', 'production', $.cleanCss({
                level: {
                    2: {
                        restructureRules: true
                    }
                }
            })))
            .pipe($.ifEnv.includes('sandbox', 'production', $.rename({
                suffix: '.min'
            })))
            .pipe($.ifEnv.includes('sandbox', 'production', $.flatten({
                includeParents: 1
            })))
            .pipe($.ifEnv.includes('sandbox', 'production', dest(pkg.paths.dist.css)))
    );
};

exports.css = css;

Command:

gulp css --env production

Output:

Using gulpfile ~/Sites/red-hat/webdms/gulpfile.js
Starting 'css'...
cssDist
cssDev
Finished 'css' after 2.22 s

teambition / merge2 Goto Github PK

merge2's Introduction

merge2

Install

Usage

API

merge2()

merge2(options)

merge2(stream1, stream2, ..., streamN)

merge2(stream1, stream2, ..., streamN, options)

merge2(stream1, [stream2, stream3, ...], streamN, options)

mergedStream.add(stream)

mergedStream.add(stream1, [stream2, stream3, ...], ...)

mergedStream.on('queueDrain', function() {})

stream

options

License

merge2's People

Contributors

Stargazers

Watchers

Forkers

merge2's Issues

Recommend Projects

Recommend Topics

Recommend Org