Gulp: wait for pipeline to finish

[tcurdt]

I am struggling a little to create a reference to a collection of files as a parameter to a plugin in another pipeline. I am not sure which way to go:

Gulp.task('collection', function() {

  return Gulp.src('some/**/*.md')
    .pipe($.frontmatter());

});

Gulp.task('templates', [ 'collection' ], function() {

  var other = Gulp.src('other/**/*.md')
    .pipe($.template({
      collection: -> reference to the finished collection
    }))

  ...
});

or better

Gulp.task('templates', [ 'collection' ], function() {

  var collection = Gulp.src('some/**/*.md')
    .pipe($.frontmatter());

  var other = Gulp.src('other/**/*.md')
    .pipe($.template({
      collection: -> reference to the finished collection
    }))

  ...
});

I would love to do this in a single task but given the pipeline architecture that doesn't seem so easy.
Any pointers much appreciated.

[TrySound]

@tcurdt

gulp.task('task', function () {
  return Promise.all([
    new Promise(function(resolve, reject) {
      gulp.src(src + '/*.md')
        .pipe(plugin())
        .on('error', reject)
        .pipe(gulp.dest(dist))
        .on('end', resolve)
    }),
    new Promise(function(resolve, reject) {
      gulp.src(src + '/*.md')
        .pipe(plugin())
        .on('error', reject)
        .pipe(gulp.dest(dist))
        .on('end', resolve)
    })
  ]).then(function () {
    // Other actions
  });
});

[tcurdt]

@TrySound oh - that easy?

I thought that only returning a pipeline would run it. What runs the pipeline like this?
Doesn't gulp.src().pipe().pipe().on() just create the pipeline?

Doesn't it need to be

gulp.task('task', function () {

  var collection;

  return Promise.all([
    new Promise(function(resolve, reject) {
      collection = gulp.src(src + '/*.md')
        .pipe(plugin())
        .on('error', reject)
        .pipe(gulp.dest(dist))
        .on('end', resolve)
    })
  ]).then(function () {

    var other = Gulp.src('other/**/*.md')
      .pipe($.template({
        collection: -> reference to the finished collection
      }))

    return merge(collection, other);

  });
});

[TrySound]

Mm.. No. You just need to say your task when it ends. You can return stream or promise or use callback of task.

[tcurdt]

@TrySound but this doesn't even associate the the pipeline with the promise:

new Promise(function(resolve, reject) {
  gulp.src(src + '/*.md')
    .pipe(plugin())
    .on('error', reject)
    .pipe(gulp.dest(dist))
    .on('end', resolve)
})

In the promise it builds the pipeline but it's not connected to the task via promise.
Is it maybe just missing a return?

[TrySound]

@tcurdt If you want to make a few tasks in parallel use Promise.all, in series use promise reducing. Returning just one promise is the same as returning stream.

[TrySound]

Or what about you?

[tcurdt]

@TrySound hm - interesting. just tried this

gulp.task('test', function() {

  gulp.src("design/**/*")
    .pipe($.tap(function(file){
      console.log(file.path);
    }));

  return null;
});

and it does run the pipeline. I don't understand how/why yet - but seems like you are right then.

[TrySound]

@tcurdt Task finished before pipeline.

[tcurdt]

Right - but I am not getting what starts the pipeline yet.
Maybe need to dig into the source to understand this :-/

[contra]

Creating the pipeline starts the pipeline, it runs immediately. This is how streams work, check the node docs for more info.

[tcurdt]

So something like this should actually be quite easily possible:

// execute pipeline and return files when finished
var collection = collect(function(){
  return gulp.src(src + '/*.md')
    .pipe($.frontmatter());
});

Gulp.src('other/**/*.md')
  .pipe($.template({
    collection: collection
  }))
  ...

[tcurdt]

I have tried it like this - but the pipe does not fire the error/end handlers.

function pipeToCollection(pipe) {
  var collection = [];
  new Promise(function(resolve, reject) {

    console.log("create collection");

    pipe()
    .on('error', reject)
    // add to collection
    .on('end', resolve)

  }).then(function(){

    console.log("create collection - done");

  });
  return collection;
}

var posts = pipeToCollection(function(){
  return gulp.src('content/posts/**/*.md', { base: 'content/posts' })
    .pipe($.frontmatter());
});

[tcurdt]

While I the end/error handlers are now firing it became obvious (doh!) that the function (of course) returns before the then and the end of the pipeline. So that feels a little like back to square one.

[tcurdt]

I guess I could try with https://github.com/luciotato/waitfor but is there really no better way?

[tcurdt]

Actually - I guess this should work:

function pipeToCollection(pipe, callback) {

  var collection = [];

  pipe()
  .on('error', function(){
    // error handling
  })
  .pipe($.tap(function(file){
    collection.push(file)
  }))
  .on('end', function(){
    callback(collection)
  })
}

pipeToCollection(function(){
  return gulp.src('...'})
    .pipe($.frontmatter())
}, function(posts){

  var pages = gulp.src('...')
    .pipe($.template({
    posts: posts
  }))

});

[tcurdt]

Only gets ugly (because of the nesting) when there are multiple collections used.

[contra]

Yes, streams are asynchronous. Treat them like any other async thing, use a module (promises, async, whatever you want) to collect data from multiple streams.

[rahbari]

function Async(p) {
   return new Promise((res, rej) => p.on('error', err => rej(err)).on('end', () => res()));
}

gulp.task('task', async () => {

  await Async(gulp.src(src + '/*.md')
      .pipe(plugin())
      .pipe(gulp.dest(dist)));

  await Async(gulp.src(src + '/*.md')
      .pipe(plugin())
      .pipe(gulp.dest(dist)));
});