Google-api-nodejs-client: Drive download returning token JSON object

[tomaspinho]

Hello, I'm having a problem downloading from the Drive using this API (v3). The data returned from the API is of the form:

{
  "access_token" : "HUGE_TOKEN_STRING",
  "token_type" : "Bearer",
  "expires_in" : 3600
}

My code is roughly as follows (parts ommited due to confidentiality reasons):

  var buffer = new Buffer(0);
  drive.files.get({
      auth: auth,
      alt: 'media',
      fileId: id,
  })
  .on('error', function (err) {
    logger.error("Error " + err);
    callback(err, null);
  })
  .on('data', function (data) {
    logger.info("Got data!");
    buffer = Buffer.concat([buffer, data], buffer.length + data.length);
  })
  .on('end', function () {
    logger.info("End event!");
    callback(null, buffer);
  });

auth is an object created using a derivation of this example: https://developers.google.com/drive/v3/web/quickstart/nodejs#step_3_set_up_the_sample

Am I doing something wrong, or has the API changed somehow?

[tomaspinho]

To anyone facing the same problem, in order to actually get the data, you should attach a callback(err, reponse) similarly to the other API endpoints:

drive.files.get({
  auth: auth,
  alt: 'media',
  fileId: id,
}, function (err, response) {
  // response has the file's contents
});

The fix above does not work for binary files, as the callback response is passed as a string.

To the docs mantainer, I would suggest editing https://developers.google.com/drive/v3/web/manage-downloads (Node.js tab) to reflect this.

[tomaspinho]

In order to get this working as expected for the time being, I suggest a downgrade to v2 of the API and using a second request to download the file:

  drive.files.get({
    auth: auth,
    fields: 'downloadUrl',
    fileId: id,
  }, function (err, file) {
    var url;
    var requestSettings = {
      method: 'GET',
      headers : {
        Authorization: 'Bearer ' + auth.credentials.access_token,
      },
      encoding: null, // Does not try to parse file contents as UTF-8
    };

    if (err) {
      throw err;
    }

    url = file.downloadUrl;
    if (!url) {
      throw 'No download url was returned by the Google Drive API';
    }

    requestSettings.url = url;
    request(requestSettings, function (err, response, body) {
      if (err) {
        throw err;
      }
      // body is a buffer with the file's contents
    });
  });
}

[jmdobry]

It appears the recommended way to download a file using the v3 API in Node.js is to pipe to a stream, as seen here: https://developers.google.com/drive/v3/web/manage-downloads The example writes to an fs writable stream, but I imagine you could stream the download to anything that implements stream.Writable

Could you give that a try?

[tcosentino]

I have run into this as well. When trying to pipe to a stream I am getting the error Cannot read property 'pipe' of undefined. I can get the data via callback like mentioned by @tomaspinho but have the same issue of the binary contents being parsed as a string.

I add my auth when creating my drive object and then tried the following code:

  var output = fs.createWriteStream("./output/accounts/2016/" + file.name);
  drive.files.get({
      alt: 'media',
      fileId: id,
  }).pipe(output);

Any recommendations?

[Adrion]

I had this problem too.
It seems to happen when we try to get a token several times. In my case I call authorize before listing files and another time when I export/get the file.

To follow the google example :
You can store the oauth2Client directly into google object and test it before calling another auth request.

/**
 * Create an OAuth2 client with the given credentials, and then execute the
 * given callback function.
 *
 * @param {function} callback The callback to call with the authorized client.
 */
function authorize(callback) {
  if(google._options.auth) {
    return callback();
  }

  var clientSecret = credentials.installed.client_secret;
  var clientId = credentials.installed.client_id;
  var redirectUrl = credentials.installed.redirect_uris[0];
  var auth = new googleAuth();
  var oauth2Client = new auth.OAuth2(clientId, clientSecret, redirectUrl);

  // Check if we have previously stored a token.
  fs.readFile(TOKEN_PATH, function(err, token) {
    if (err) {
      getNewToken(oauth2Client, callback);
    } else {
      oauth2Client.credentials = JSON.parse(token);
      google.options({ auth: oauth2Client });
      callback();
    }
  });
}

Callback doesn't need anymore auth parameter.

function listFiles() {
  var service = google.drive('v3');
  service.files.list({
    pageSize: 10,
    fields: "nextPageToken, files(id, name)"
  }, function(err, response) {
    if (err) {
      console.log('The API returned an error: ' + err);
      return;
    }
    var files = response.files;
    if (files.length == 0) {
      console.log('No files found.');
    } else {
      console.log('Files:');
      for (var i = 0; i < files.length; i++) {
        var file = files[i];
        console.log('%s (%s)', file.name, file.id);
      }
    }
  });
}

[Aspie96]

I had the same problem with this code:

drive.files.get({
    fileId: myId,
    alt: "media"
}).on("end", function() {
    // Stuff
}).on("error", function(err) {
    // Stuff
}).pipe(stream);

The very fist time calling it, I got the access token (which is a big problem, since the response is streamed to the user).

I sorted the problem out requiring the file's data before its content:

drive.files.get({ fileId: myId }, function(err, file) {
    if(err) {
        // Stuff
    } else {
        drive.files.get({
            fileId: file.id,
            alt: "media"
        }).on("end", function() {
            // Stuff
        }).on("error", function(err) {
            // Stuff
        }).pipe(stream);
    }
});

This, though, requires two requests to the server.

Is there any better solution?

Also, I tried to make a request (with the old code that downloads the file directly), wait an hour and make another request. It had the same problem.
My guess is that this happens because after an hour the token expires and get transparently renewed with the next request (I haven't checked it to be true, it's just my guess. If someone knows better then me, please do tell me), but, for some reason, this doesn't happen correctly when the request is a download one.

The problem with the solution I used is that if what I wrote is right, then there is a possibility that the token still get sent to the user: imagine a situation where the token is just about to expire and the first request gets sent. By the time the second request gets sent (even if it's very short), the token might have expire already and it has not been renewed.
I know it is a pretty unlucky event, but I really don't want to expose the token, so "impossible" would be better than "unlucky".

I find it weird that Google hasn't solved the problem yet.
Or maybe I am doing something wrong, am I?

[Aspie96]

It appears the recommended way to download a file using the v3 API in Node.js is to pipe to a stream, as seen here: https://developers.google.com/drive/v3/web/manage-downloads The example writes to an fs writable stream, but I imagine you could stream the download to anything that implements stream.Writable.

@jmdobry, I tried to do exactly that, and it doesn't work. Does that mean there is some mistake in my code? Or is it a bug in the APIs?

[Aspie96]

Ok, I found out something (hope it is useful).

I tried "piping" a file metadata request, like that:

drive.files.get({ fileId: myId }).on("end", function() {
    // Stuff
}).on("error", function(err) {
    // Stuff
}).pipe(stream);

And I even tried listing files:

drive.files.get({
    pageSize: 10,
    fields: "nextPageToken, files(id, name)"
}).on("end", function() {
    // Stuff
}).on("error", function(err) {
    // Stuff
}).pipe(stream);

And guess what: I got the exact same problem.

But it works exactly fine the standard way:

drive.files.list({
    pageSize: 10,
    fields: "nextPageToken, files(id, name)"
}, function(err, response) {
    console.log(response);
});

That means the problem is not in download requests themselves, but in streming the results, in general.

I even tried:

drive.files.list({
    fileId: myId,
    alt: "media"
}, function(err, response) {
    console.log(response);  // Yes, I am displaying binary data. It's for testing…
});

And it works correctly, but that means I gotta have the full content of the binary file in a variable, and then manually send it to the client, not exactly the best.

[AKP]

I ended up at the same position of displaying a png image binary in the console and not being able to save it into the location. Please share the optimal way of doing it if you have found it. It seems so complex to download and save a file when it should be very simple.

[Aspie96]

@AKP I don't have a way to perform the download, but the problem, in this case, is not just that it's harder to download a file. It's a security issue as well.

[Aspie96]

I know the product is free and I am really thankful for it.
But I find it kinda amazing this issue has not been solved yet.

[jmdobry]

I know the product is free and I am really thankful for it.
But I find it kinda amazing this issue has not been solved yet.

It's not just free, but Open Source, meaning anyone can contribute features and fixes.

I've never used the Drive API before, but here's a sample that I think shows how to download a file via the Drive v3 API: https://github.com/google/google-api-nodejs-client/pull/617

I tested it and was able to download a file from my Google Drive.

[Aspie96]

I tested it and was able to download a file from my Google Drive.

Of course.

The Google Drive API do work.

The problem occurs when one uses objects returned by createAPIRequest (using pipe and on) and the token has expired.
Whileas the callback function, if specified, is always called a the right moment, the returned object refers not to the very first request made (which does not return the needed data when the token has to be refreshed).
I hope I managed to explain the problem.

The solution would be, instead of returning a request, to return an object that acts like a request (having methods pipe and on), but isn't: such an object would accept the functions as parameters and only bind them to the latest request that is made.

Now, personally, I am not able to do such thing because the code is quite complex and I can't find where the token is refreshed automatically.
It would be really helpful if someone who knows the code more tried to solve the issue.
I am willing to help, I just don't know if I can.

[jmdobry]

Let me rephrase to make sure I understand:

If you make the request and provide a callback, then the request will successfully refresh auth tokens if necessary, but if you make the request and instead use the returned stream object, it doesn't refresh auth tokens like it should. Does that sum it up?

[Aspie96]

Does that sum it up?

Yes, I know, I do understand it.

That is exactly what the problem is.

it doesn't refresh auth tokens like it should

Well, actually the token is indeed refreshed (if it wasn't, further requests would have the same problem).

Just, the returned stream object does not refer to the correct response (but to the first one, containing the new token).

[tcosentino]

@Aspie96 Here is the working code I ended up with. It has been a while so I don't remember exactly why I did everything but I know this is working.

Refreshing to token manually before hand was necessary as was using a rate limiter to prevent making too many calls quickly. Does this help you out?

var downloadFiles = function (drive, year, callback) {
    GDriveFile.find({ type: "account." + year }, function (err, files) {
        if (err) {
        callback.call(this, err);
        return;
      }

        var finished = _.after(files.length, function () {
            setTimeout(callback.bind(null, null, files), 100);
        });

        var limiter = new RateLimiter(5, 'second');

      jwtClient.refreshAccessToken(function(err, tokens) {
        _.each(files, function (file) {
            var output = fs.createWriteStream(accountDirectory + file.name);

            limiter.removeTokens(1, function(err, remainingRequests) {
                var test = drive.files.get({
                  fileId: file.gID,
                  alt: 'media'
                }).on('end', function() {
                   finished.call();
                }).on('error', function(err) {
                    process.exit(err);
                }).pipe(output);

          test.on('finish', function () {
            output.close();
          });
            });
        });
      });
    });
};

GDriveFile is a mongoose model that I am using to store file information (including the google id). Anything else confusing let me know

[jmdobry]

does not refer to the correct response

That I don't understand. Let me take another look at the code.

[jmdobry]

In my example in #617 I force the second drive.files.get request to use the same tokens as the first drive.files.get request, and it works fine. I'm the guessing the problem is that one shouldn't have to do that, right?

[Aspie96]

That I don't understand. Let me take another look at the code.

I just meant what you wrote yourself: it streams the new token instead of the actual file content (or whatever has been required).

I'm the guessing the problem is that one shouldn't have to do that, right?

No, that's ot the biggest problem.

Aside from having to perform two requests (therefore slowing the process down), the first request may be done just right before the token expires.

If that happened (although unlikely), the access token would be returned.

Anyways, I solved with #619.

[jmdobry]

Problem is I can't merge #619 in its current state. The next time I regenerate this library the changes in #619 would be deleted.

[SamuelePilleri]

What's wrong with #619? How should I fix it?

[jmdobry]

I may need to re-open this issue, but I'd like to see if 12.4.0 works for anybody as far as the encoding goes. You should now be able to do:

drive.files.get({
  fileId: 'asxKJod9s79',
  alt: 'media'
}, {
  encoding: null // Make sure we get the binary data
}, function (err, buffer) {
  // Nice, the buffer is actually a Buffer!
});

[SamuelePilleri]

Users should be informed that using pipe() is not safe as it might leak sensitive data.

Moreover, I must point out that fixes introduced in #623 could lead, under certain conditions where large files are involved, to the complete _server crash_ since the buffer might eventually drain the server memory completely.

A new approach has to be discussed and it must use pipe() in some way.

See:
http://stackoverflow.com/questions/8974375/whats-the-maximum-size-of-a-node-js-buffer
https://nodejs.org/api/buffer.html#buffer_buffer_kmaxlength

[jmdobry]

Agreed, pipe should not leak credentials. Opened #625

[yetithefoot]

Until callbacks hell from #625 not resolved documntation frustrates users about piping.

[robotpapier]

Hi,
I have similar issue.
Turned out my problem was the access token getting expired.
So I added a manual refresh before the oauth2Client callback in the node.js quickstart google api example link

replaced

function authorize(credentials, callback) {
  var clientSecret = credentials.installed.client_secret;
  var clientId = credentials.installed.client_id;
  var redirectUrl = credentials.installed.redirect_uris[0];
  var auth = new googleAuth();
  var oauth2Client = new auth.OAuth2(clientId, clientSecret, redirectUrl);

  // Check if we have previously stored a token.
  fs.readFile(TOKEN_PATH, function(err, token) {
    if (err) {
      getNewToken(oauth2Client, callback);
    } else {
      oauth2Client.credentials = JSON.parse(token);
      callback(oauth2Client);
    }
  });
}

by

function authorize(credentials, callback) {
  var clientSecret = credentials.installed.client_secret;
  var clientId = credentials.installed.client_id;
  var redirectUrl = credentials.installed.redirect_uris[0];
  var auth = new googleAuth();
  var oauth2Client = new auth.OAuth2(clientId, clientSecret, redirectUrl);

  // Check if we have previously stored a token.
  fs.readFile(TOKEN_PATH, function(err, token) {
    if (err) {
      getNewToken(oauth2Client, callback);
    } else {

      oauth2Client.credentials = JSON.parse(token);

      // === ALWAYS refresh token and do the callback only when it is refreshed
      oauth2Client.refreshAccessToken(function(err, token) {
        // your access_token is now refreshed and stored in oauth2Client
        // store these new tokens in a safe place (e.g. database)
        callback(oauth2Client);
      });
    }
  });
}

Don't know if it's the best way but hope it helps.

[AliakseiBychyk]

There is an important nuance. In documentation in quickstart.js sample, the defined value in SCOPES is not enough for downloading files. It should be 'https://www.googleapis.com/auth/drive' added to the SCOPES array.