Meteor-files: Can I have a demo app with S3 upload code?

Created on 21 Jul 2016 · 11Comments · Source: veliovgroup/Meteor-Files

Hello, I have read this guide: https://github.com/VeliovGroup/Meteor-Files/wiki/AWS-S3-Integration

I have created S3 bucket, key, secret, cloudfront.
I created a lib folder in meteor and inserted 3 files:
meteor-files.js:

this.Images = new Meteor.Files({
  debug: true,
  collectionName: 'Images',
  allowClientCode: false, // Disallow remove files from Client
  onBeforeUpload: function (file) {
    // Allow upload files under 10MB, and only in png/jpg/jpeg formats
    if (file.size <= 10485760 && /png|jpg|jpeg/i.test(file.extension)) {
      return true;
    } else {
      return 'Please upload image, with size equal or less than 10MB';
    }
  }
});

if (Meteor.isServer) {
  Images.denyClient();
  Meteor.publish('files.images.all', function () {
    return Images.find().cursor;
  });

} else {

  Meteor.subscribe('files.images.all');
}

stream-files.js

this.Videos = new FilesCollection({collectionName: 'Videos'});

// Upload sample files on server's startup:
if (Meteor.isServer) {
  Meteor.startup(function () {
    Images.load('https://raw.githubusercontent.com/VeliovGroup/Meteor-Files/master/logo.png', {
      fileName: 'logo.png'
    });
    Videos.load('http://www.sample-videos.com/video/mp4/240/big_buck_bunny_240p_5mb.mp4', {
      fileName: 'Big-Buck-Bunny.mp4'
    });
  });

  Meteor.publish('files.images.all', function () {
    return Images.find().cursor;
  });
  Meteor.publish('files.videos.all', function () {
    return Videos.find().cursor;
  });

} else {
  Meteor.subscribe('files.images.all');
  Meteor.subscribe('files.videos.all');
}

AWS-S3-Integration.js

var knox, bound, client, Request, cfdomain, Collections = {};

if (Meteor.isServer) {
  // Fix CloudFront certificate issue
  // Read: https://github.com/chilts/awssum/issues/164
  process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;

  knox    = Npm.require('knox');
  Request = Npm.require('request');
  bound = Meteor.bindEnvironment(function(callback) {
    return callback();
  });
  cfdomain = 'https://xxx.cloudfront.net'; // <-- Change to your Cloud Front Domain
  client = knox.createClient({
    key: 'xxx',
    secret: 'yyy',
    bucket: 'zzz',
    region: 'jjj'
  });
}

Collections.files = new FilesCollection({
  debug: false, // Change to `true` for debugging
  throttle: false,
  storagePath: 'assets/app/uploads/uploadedFiles',
  collectionName: 'uploadedFiles',
  allowClientCode: false,
  onAfterUpload: function(fileRef) {
    // In onAfterUpload callback we will move file to AWS:S3
    var self = this;
    _.each(fileRef.versions, function(vRef, version) {
      // We use Random.id() instead of real file's _id 
      // to secure files from reverse engineering
      // As after viewing this code it will be easy
      // to get access to unlisted and protected files
      var filePath = "files/" + (Random.id()) + "-" + version + "." + fileRef.extension;
      client.putFile(vRef.path, filePath, function(error, res) {
        bound(function() {
          var upd;
          if (error) {
            console.error(error);
          } else {
            upd = {
              $set: {}
            };
            upd['$set']["versions." + version + ".meta.pipeFrom"] = cfdomain + '/' + filePath;
            upd['$set']["versions." + version + ".meta.pipePath"] = filePath;
            self.collection.update({
              _id: fileRef._id
            }, upd, function(error) {
              if (error) {
                console.error(error);
              } else {
                // Unlink original files from FS
                // after successful upload to AWS:S3
                self.unlink(self.collection.findOne(fileRef._id), version);
              }
            });
          }
        });
      });
    });
  },
  interceptDownload: function(http, fileRef, version) {
    var path, ref, ref1, ref2;
    path = (ref = fileRef.versions) != null ? (ref1 = ref[version]) != null ? (ref2 = ref1.meta) != null ? ref2.pipeFrom : void 0 : void 0 : void 0;
    if (path) {
      // If file is moved to S3
      // We will pipe request to S3
      // So, original link will stay always secure
      Request({
        url: path,
        headers: _.pick(http.request.headers, 'range', 'accept-language', 'accept', 'cache-control', 'pragma', 'connection', 'upgrade-insecure-requests', 'user-agent')
      }).pipe(http.response);
      return true;
    } else {
      // While file is not yet uploaded to S3
      // We will serve file from FS
      return false;
    }
  }
});

if (Meteor.isServer) {
  // Intercept File's collection remove method
  // to remove file from S3
  var _origRemove = Collections.files.remove;

  Collections.files.remove = function(search) {
    var cursor = this.collection.find(search);
    cursor.forEach(function(fileRef) {
      _.each(fileRef.versions, function(vRef) {
        var ref;
        if (vRef != null ? (ref = vRef.meta) != null ? ref.pipePath : void 0 : void 0) {
          client.deleteFile(vRef.meta.pipePath, function(error) {
            bound(function() {
              if (error) {
                console.error(error);
              }
            });
          });
        }
      });
    });
    // Call original method
    _origRemove.call(this, search);
  };
}

Of course I have inserted my S3 cfdomain, key, secret, bucket, region into the code. When I run the app and submit images it just upload to my server folder only: app\ .meteor\localbuild\programs\serverassetsapp\uploads\Images.
How can I upload my images to the S3? Can anyone show me a demo app with this S3 upload code please... Thank you very much.

Source

MinhTri66

All 11 comments

Here is production code: https://github.com/VeliovGroup/Meteor-Files/blob/master/demo/lib/files.collection.coffee

dr-dimitru on 21 Jul 2016

@dr-dimitru Thanks, but when I insert that code on the server it said: Exception in setInterval callback: ReferenceError: _app is not defined.

I am new to meteor so can you give me a simplest code to upload to the S3 bucket, please...

Can I upload directly to the S3 bucket without cloudfront like this package: https://github.com/CulturalMe/meteor-slingshot/

I still don't know how this package upload a file. Correct me if I am wrong:

User select a file.
It check the file-size, file-extension and upload to server folder 'serverassetsapp\uploads\Images'. This process use the RAM and CPU of the client, right?
The server auto upload to the S3 cloudfront bucket? This process use the RAM and CPU of the server, right?

What is the code to auto upload file from server to the S3 cloudfront? I use react to render the client. I want the file is not uploaded until user click the upload button. So I use this code

formSubmit(e) {
  var upload = Images.insert({
    file: e.target.files[0],
    streams: 'dynamic',
    chunkSize: 'dynamic'
  }, false);

  upload.on('start', function () {
  });

  upload.on('end', function (error, fileObj) {
    if (error) {
      console.error('Error uploading', error.message);
      alert('Error during upload: ' + error);
    } else {
      alert('File "' + fileObj.name + '" successfully uploaded');
    }
  });

  upload.start();
}

This code uploaded the image to the server folder. But what is the code for auto uploading the image to the S3 cloudfront after that? Can you give me that code in javascript rather than coffescript please...

Thank you and sorry for my bad English.

MinhTri66 on 22 Jul 2016

You've asked for example of demo app, I gave it to you (_sorry it's CoffeeScript_)
S3 integration example is well tested, do you have any errors?
"What is the code to auto upload file from server to the S3 cloudfront" - See onAfterUpload in S3 integration example. This hook is responsible for upload to S3

I'll repeat it here:

  onAfterUpload: function(fileRef) {
    // In onAfterUpload callback we will move file to AWS:S3
    var self = this;
    _.each(fileRef.versions, function(vRef, version) {
      // We use Random.id() instead of real file's _id 
      // to secure files from reverse engineering
      // As after viewing this code it will be easy
      // to get access to unlisted and protected files
      var filePath = "files/" + (Random.id()) + "-" + version + "." + fileRef.extension;
      client.putFile(vRef.path, filePath, function(error, res) {
        bound(function() {
          var upd;
          if (error) {
            console.error(error);
          } else {
            upd = {
              $set: {}
            };
            upd['$set']["versions." + version + ".meta.pipeFrom"] = cfdomain + '/' + filePath;
            upd['$set']["versions." + version + ".meta.pipePath"] = filePath;
            self.collection.update({
              _id: fileRef._id
            }, upd, function(error) {
              if (error) {
                console.error(error);
              } else {
                // Unlink original files from FS
                // after successful upload to AWS:S3
                self.unlink(self.collection.findOne(fileRef._id), version);
              }
            });
          }
        });
      });
    });
  }

dr-dimitru on 22 Jul 2016

👍1

@dr-dimitru Thank you, I have uploaded successfully :)

One more thing, after I uploaded image to server, how can I check the type, size, width and height of the image before upload it to the S3. I want to check the file on the server, not on the client because user can edit the code on the client.

And can you show me how to resize and create thumbnail after image was uploaded to the server? Can I upload that thumbnail to the S3 with the default image?.

Thank you very much.

MinhTri66 on 22 Jul 2016

The onBeforeUpload is isomorphic and runs on both Server and Client. Moreover it checks passed data for every received chunk, so you stay safe from upload interceptions.

dr-dimitru on 23 Jul 2016

👍1

For image resize example check out here and here, _sorry for CoffeeScript_

dr-dimitru on 23 Jul 2016

👍1

Hi @MinhTri66 any news on your end?

dr-dimitru on 24 Jul 2016

Hi @dr-dimitru Sorry for late. I am trying to find a best way to work with images on the server of Meteor.
I have read the resize example CoffeeScript that you showed to me above. It uses the
CollectionFS/Meteor-cfs-graphicsmagick and read the image by the path image = gm(fileRef.path) , but when I run it return error. . .

Exception in callback of async function: Error: cfs:graphicsmagick could not find "graphicsMagick" or "imageMagick".

I am trying to find another way to work with images in Meteor. Do you know any other packages for this? Tell me know please. . . Thank you.

MinhTri66 on 24 Jul 2016

Is there any reason why can not install imageMagick or graphicsMagick on your server?

dr-dimitru on 24 Jul 2016

👍1

Oh that is my mistake! I didn't think imageMagick or graphicsMagick is a sofware that I can setup it on the server. I thought it was a package of meteor or javascript that I must install with the cmd, haha

Thank you very much for your support and guidance for me. You are a great man, this is a awesome package. Thank you again.

We can close the issue now.

MinhTri66 on 24 Jul 2016

@MinhTri66 glad you've solved it. Thank you for using this package 👍 . And do not hesitate to ask any other question.

Please, support this project by: