Meteor-files: File not uploded google storage bucket but I can see it in .meteor.../assets/app/upload folder

[nikhilgoswami]

I'm having an issue:

I get no errors on client and server I get file is uploaded successfully but then I cannot see it on google cloud storage bucket, but I can see it in assets/app/upload folder of .meteor
This started happening after I cloned project from github added settings.json. Ironically in 1st project the upload works and I can see it on google bucket but in new cloned project I get this defect. Any Idea what could be missing.

Documentation is missing something or incorrect (have typos, etc.):

I think the documentation for google is a bit old and needs to be updated.

[dr-dimitru]

@nikhilgoswami we need more info, please follow our issue template:

• Give an expressive description of what is went wrong
• Version of Meteor-Files you're experiencing this issue
• Version of Meteor you're experiencing this issue
• Where this issue appears? OS (Mac/Win/Linux)? Browser name and its version?
• Is it Client or Server issue?
• Post Client and/or Server logs with enabled debug option, you can enable "debug" mode in Constructor

Thank you
//cc @jankapunkt

[nikhilgoswami]

metoer files version ostrio:[email protected]
Meteor version 1.8.1
linux ubuntu
no error on client or server (defect is that image is not added to google bucket)

[dr-dimitru]

@nikhilgoswami thank you for reporting about this, could you post your implementation of integration with google cloud. This tutorial is made by contributors: @salmanhasni, @amos-whitewolf, and @jeremywrnr — hope they can tell you more about Google Cloud integration

[salmanhasni]

@nikhilgoswami can you please enable the debug flag true
debug: true in FilesCollection({..}); and let me know what you get it

[salmanhasni]

@nikhilgoswami I have made the change to documentation, lets wait for reviewers to have a look over it meanwhile you can try it.
https://github.com/VeliovGroup/Meteor-Files/pull/698

[nikhilgoswami]

Console logs when debug is set to true
`I20190627-11:11:36.553(5.5)? [FilesCollection] [File Start Method] sakal.png - FBExzCK8JbuwfedG5
I20190627-11:11:36.561(5.5)? [FilesCollection] [Upload] [DDP Start Method] Got #-1/1 chunks, dst: sakal.png
I20190627-11:11:36.750(5.5)? [FilesCollection] [Upload] [DDP] Got #1/1 chunks, dst: sakal.png
I20190627-11:11:36.758(5.5)? [FilesCollection] [Upload] [DDP] Got #-1/1 chunks, dst: sakal.png
I20190627-11:11:36.759(5.5)? [FilesCollection] [Upload] [finish(ing)Upload] -> assets/app/uploads/productImageAdmin/FBExzCK8JbuwfedG5.png
I20190627-11:11:36.767(5.5)? [FilesCollection] [Upload] [finish(ed)Upload] -> assets/app/uploads/productImageAdmin/FBExzCK8JbuwfedG5.png
I20190627-11:11:36.961(5.5)? [FilesCollection] [_preCollectionCursor.observe] [changed]: FBExzCK8JbuwfedG5
I20190627-11:11:37.110(5.5)? [FilesCollection] [_preCollectionCursor.observe] [removed]: FBExzCK8JbuwfedG5

My Implementation

var gcloud, gcs, bucket, bucketMetadata, Request, bound, Collections = {};
var google = require('../../../settings.json');
import {
Random
} from 'meteor/random';

if (Meteor.isServer) {
var Future = Npm.require('fibers/future');
gcloud = Npm.require('@google-cloud/storage')({
projectId: google.projectId, // <-- Replace this with your project ID
keyFilename: Meteor.rootPath + google.keyfilePath // <-- Replace this with the path to your key.json
});
gcs = gcloud;
bucket = gcs.bucket(google.bucketName); // <-- Replace this with your bucket name
bucket.getMetadata(function (error, metadata, apiResponse) {
if (error) {
console.error(error);
}
});
Request = Npm.require('request');
bound = Meteor.bindEnvironment(function (callback) {
return callback();
});
Meteor.publish('productImageAdmin.all', function () {
return productImageAdmin.find().cursor;
});
Meteor.methods({
'removeProductAdminImage': function (fileObj,_id) {
var future = new Future()
try {
productImageAdmin.remove({_id})
future.return('success')
} catch (e) {
future.throw(e)
}
// console.log(link)
return future.wait();
}

});
}

export const productImageAdmin = new FilesCollection({
debug: true, // Set to true to enable debugging messages
storagePath: 'assets/app/uploads/productImageAdmin',
collectionName: 'productImageAdmin',
downloadRoute: 'https://storage.googleapis.com/nikhilsbucket',
allowClientCode: false,
onbeforeunloadMessage() {
return 'Upload is still in progress! Upload will be aborted if you leave this page!';
},
onBeforeUpload(file) {
// Allow upload files under 10MB, and only in png/jpg/jpeg formats
// Note: You should never trust to extension and mime-type here
// as this data comes from client and can be easily substitute
// to check file's "magic-numbers" use mmmagic or file-type package
// real extension and mime-type can be checked on client (untrusted side)
// and on server at onAfterUpload hook (trusted side)
if (file.size <= 10485760 && /png|jpe?g/i.test(file.ext)) {
return true;
}
Bert.alert({
title: 'Error occurred',
message: 'Please upload image, with size equal or less than 10MB',
type: 'danger',
style: 'growl-top-right',
icon: 'fa-remove'
});
return 'Please upload image, with size equal or less than 10MB';
},
onAfterUpload: function (fileRef) {
// In the onAfterUpload callback, we will move the file to Google Cloud Storage
var self = this;
_.each(fileRef.versions, function (vRef, version) {
// We use Random.id() instead of real file's _id
// to secure files from reverse engineering
// As after viewing this code it will be easy
// to get access to unlisted and protected files
var filePath = "productImageAdmin/" + fileRef._id +"." + fileRef.extension;
// console.log(filePath);
// Here we set the neccesary options to upload the file, for more options, see
// https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.36.0/storage/bucket?method=upload
var options = {
destination: filePath,
resumable: true,
public:true
};

        bucket.upload(fileRef.path, options, function (error, file) {
            bound(function () {
                var upd;
                if (error) {
                    console.error(error);
                } else {
                    upd = {
                        $set: {}
                    };
                    upd['$set']["versions." + version + ".meta.pipePath"] = filePath;
                    self.collection.update({
                        _id: fileRef._id
                    }, upd, function (error) {
                        if (error) {
                            console.error(error);
                        } else {
                            // Unlink original files from FS
                            // after successful upload to Google Cloud Storage
                            self.unlink(self.collection.findOne(fileRef._id), version);
                        }
                    });
                }
            });
        });
    });
},

interceptDownload: function (http, fileRef, version) {
var path, ref, ref1, ref2;
path = (ref = fileRef.versions) != null ? (ref1 = ref[version]) != null ? (ref2 = ref1.meta) != null ? ref2.pipePath : void 0 : void 0 : void 0;
var vRef = ref1;
if (path) {
// If file is moved to Google Cloud Storage
// We will pipe request to Google Cloud Storage
// So, original link will stay always secure
var remoteReadStream = getReadableStream(http, path, vRef);
this.serve(http, fileRef, vRef, version, remoteReadStream);
return true;
} else {
// While the file has not been uploaded to Google Cloud Storage, we will serve it from the filesystem
return false;
}
}
});

if (Meteor.isServer) {
// Intercept file's collection remove method to remove file from Google Cloud Storage
var _origRemove = productImageAdmin.remove;

productImageAdmin.remove = function (search) {
    var cursor = this.collection.find(search);
    cursor.forEach(function (fileRef) {
        _.each(fileRef.versions, function (vRef) {
            var ref;
            if (vRef != null ? (ref = vRef.meta) != null ? ref.pipePath : void 0 : void 0) {
                bucket.file(vRef.meta.pipePath).delete(function (error) {
                    bound(function () {
                        if (error) {
                            console.error(error);
                        }
                    });
                });
            }
        });
    });
    // Call the original removal method
    _origRemove.call(this, search);
};

}

function getReadableStream(http, path, vRef) {
var array, end, partial, remoteReadStream, reqRange, responseType, start, take;

if (http.request.headers.range) {
    partial = true;
    array = http.request.headers.range.split(/bytes=([0-9]*)-([0-9]*)/);
    start = parseInt(array[1]);
    end = parseInt(array[2]);
    if (isNaN(end)) {
        end = vRef.size - 1;
    }
    take = end - start;
} else {
    start = 0;
    end = vRef.size - 1;
    take = vRef.size;
}

if (partial || (http.params.query.play && http.params.query.play === 'true')) {
reqRange = {
start: start,
end: end
};
if (isNaN(start) && !isNaN(end)) {
reqRange.start = end - take;
reqRange.end = end;
}
if (!isNaN(start) && isNaN(end)) {
reqRange.start = start;
reqRange.end = start + take;
}
if ((start + take) >= vRef.size) {
reqRange.end = vRef.size - 1;
}
if ((reqRange.start >= (vRef.size - 1) || reqRange.end > (vRef.size - 1))) {
responseType = '416';
} else {
responseType = '206';
}
} else {
responseType = '200';
}

if (responseType === "206") {
    remoteReadStream = bucket.file(path).createReadStream({
        start: reqRange.start,
        end: reqRange.end
    });
} else if (responseType === "200") {
    remoteReadStream = bucket.file(path).createReadStream();
}

return remoteReadStream;

}

[nikhilgoswami]

This error was caused after I install [email protected] and [email protected] and @material-ui/core using meteor yarn add. Funny thing is even after reverting back the changes doesn't resolve the problem it persists.

[nikhilgoswami]

Thanks for your help. It turns out meteor yarn was the culprit deleting node_modules and using good old meteor npm install is working smooth for now. If the problem persist I will reopen the issue.

[dr-dimitru]

Seems like we all should follow best practices.
@nikhilgoswami I'm glad it's solved
@salmanhasni thank you for contribution we all were waiting for ;)