Google-api-nodejs-client: High memory usage when uploading large file.

[datnguyencse]

Env: Nodejs v8.9.1, googleapis v28.1.0, Windows 10.
I tried to upload a large files around 2GB, I notice that the process is up to 3GB RAM. My code upload that file to Drive successful on 8GB ram system, failed on 1GB ram system.

      var fileMetadata = {
         "name": fileName
      }

      var media = {
          mimeType: mineType,
          body: fs.createReadStream(path)
      }

      self.drive.files.insert({
          resource: fileMetadata,
          media: media,
          fields: "id"
      }, function (err, file) {
        if (err) {
          logger.error("Error: " + err.message)
        }
        logger.info("Finished, id: " + file.data.id)
      })

8GB RAM

1GB RAM

[fengmao]

what auth method do you use ? can you post a gist url of your code?

[datnguyencse]

Hi Fengmao, I'm using Oauth2:

let googleAuth = new OAuth2(
            token.client_id,
            token.client_secret
          )

          googleAuth.setCredentials({
            access_token: token.access_token,
            refresh_token: token.refresh_token,
            expiry_date: token.expiry_date
          })

          // auto refresh token itself
          this.drive = google.drive({ version: 'v3', auth: googleAuth })

[ziro-official]

I wrote my own code to reduce ram usage, for who need. Less than 500MB ram for any large file.

  import parseRange from 'http-range-parse';
  import request from 'request';
  import fs from 'fs';
  /**
   * Divide the file to multi path for upload
   * @returns {array} array of chunk info
   */
  _getChunks(filePath, start) {
    var allsize = fs.statSync(filePath).size;
    var sep = allsize < (150 * 1024 * 1024) ? allsize : (150 * 1024 * 1024) - 1;
    var ar = [];
    for (var i = start; i < allsize; i += sep) {
      var bstart = i;
      var bend = i + sep - 1 < allsize ? i + sep - 1 : allsize - 1;
      var cr = 'bytes ' + bstart + '-' + bend + '/' + allsize;
      var clen = bend != allsize - 1 ? sep : allsize - i;
      var stime = allsize < (150 * 1024 * 1024) ? 5000 : 10000;
      ar.push({
        bstart : bstart,
        bend : bend,
        cr : cr,
        clen : clen,
        stime: stime,
      });
    }
    return ar;
  }

  /**
   * Upload one chunk to server: this api is using for onedrive and google
   * @returns {string} file id if any
   */
  _uploadChunk(filePath, chunk, mineType, uploadUrl) {
    logger.info(`Uploading new chunk: ${chunk.clen} bytes`)
    return new Promise((resolve, reject) => {
      request.put({
        url: uploadUrl,
        headers: {
            'Content-Length': chunk.clen,
            'Content-Range': chunk.cr,
            'Content-Type': mineType,
        },
        body: fs.createReadStream(filePath, {
          encoding: null,
          start   : chunk.bstart,
          end     : chunk.bend + 1
        })
      }, function(error, response, body) {
        if (error) {
          logger.info(`Upload chunk failed, Error from request module: ${error.message}`)
          return reject(error)
        }

        logger.info(`Upload chunk finish: ${chunk.clen} bytes`)
        let headers = response.headers
        if (headers && headers.range) {
          logger.warn(`Look like google drive recieve not enough data, got headers range: ${JSON.stringify(headers.range)}`)
          let range = parseRange(headers.range)
          if (range && range.last != chunk.bend) {
            // range is diff, need to return to recreate chunks
            return resolve(range)
          }
        }

        if (!body) {
          logger.warn(`Upload chunk return empty body.`)
          return resolve(null)
        }

        body = JSON.parse(body)
        if (body && body.id) {
          logger.info(`Got file id ${body.id}`)
          return resolve(body.id)
        } else {
          logger.info(`Got file id null`)
          return resolve(null)
        }
      })
    })
  }

  /**
   * Upload a file to google drive: Passed
   * @returns {path} File's path
   */
  _uploadGoogleDriveFile(filePath, file) {
    let self = this
    logger.info(`Uploading ${file.fileName} to googlde drive`)
    return new Promise((resolve, reject) => {
      isGoogleExpiried(self.token).then(token => {
        let options = {
          method: 'POST',
          url: 'https://www.googleapis.com/upload/drive/v3/files',
          qs: { uploadType: 'resumable' },
          headers: 
           { 'Postman-Token'            : '1d58fdd0-0698-45fa-a45d-fc703bff724a',
             'Cache-Control'            : 'no-cache',
             'X-Upload-Content-Length'  : file.size,
             'X-Upload-Content-Type'    : file.mineType,
             'Content-Type'             : 'application/json',
             'Authorization'            : `Bearer ${token.access_token}` },
          body: { name: file.fileName},
          json: true
        }

        request(options, async function (error, response, body) {
          if (error) {
            return reject(error)
          }

          if (!response) {
            return reject(`Get drive resumable url return undefined headers`)
          }

          if (!response.headers || !response.headers.location || response.headers.location.length <= 0) {
            return reject(`Get drive resumable url return invalid headers: ${response.headers}`)
          }

          logger.info(`Uploading file ${file.fileName} to Google drive.`)
          let chunks = self._getChunks(filePath, 0)
          let fileId = null
          try {
            logger.info(`Upload total ${chunks.length} chunks`)
            logger.info(JSON.stringify(chunks))
            let i = 0
            while (i < chunks.length) {
              logger.info('Uploading chunk ' + i + ' start: ' + chunks[i].bstart)
              // last chunk will return the file id
              fileId = await self._uploadChunk(filePath, chunks[i], file.mineType, response.headers.location)
              if((typeof fileId === "object") && (fileId !== null)) {
                logger.info('Got object from chunk upload, recreate chunks')
                chunks = self._getChunks(filePath, fileId.last)
                logger.info(JSON.stringify(chunks))
                i = 0
              } else {
                i++
              }

              logger.info('')         
            }

            if (fileId && fileId.length > 0) {
              return resolve(fileId)
            } else {
              return reject(new Error("Uploaded and got invalid id for file " + file.fileName))
            }
          } catch (er) {
            logger.error(`Uploading chunks for file ${file.fileName} failed: ${er.message}`)
            return reject(er)
          }
        })
      }).catch(err => {
        console.log("Sending request to get resumable url: " + err.message)
        return reject(err)
      })
    })
  }

[ghjbnm]

I think it's an issue with axios/follow-redirects.
There is a buffer array in the globalAgent that seems to grow without ever being deallocated.
As a workaround using the following code in the onProgress function of the readStream appears to work:

https.globalAgent.sockets["www.googleapis.com:443::::::::::"][0]._httpMessage._redirectable._requestBodyBuffers=[];
https.globalAgent.sockets["www.googleapis.com:443::::::::::"][0]._httpMessage._redirectable._requestBodyLength=0;

#

@JustinBeckwith
Somewhat related to this issue, google.options({ maxContentLength: (128 * 1024 * 1024 * 1024) }); doesn't work for me. I have to pass it to the insert method directly.

[ghjbnm]

Related axios issue:
https://github.com/axios/axios/issues/1045

Just tested using maxRedirects: 0 as option, seems to work. Probably should be set this way as default, if there isn't any reason not to.

[dipu-bd]

I am using NodeJS.ReadableStream to upload file from my server. I do not have access to locally downloaded file. Is there any way to solve this issue for me?

[ghjbnm]

Did you try using maxRedirects: 0, works fine for me.
Should look somewhat like this:

google.youtube('v3')["videos"]["insert"](
    {
        auth: oauth2Client,
        part: 'id,snippet,status',
        requestBody:requestBody,
        media:
        {
            body: readStream
        }
    },
    {
        maxContentLength: 128 * 1024 * 1024 * 1024,
        maxRedirects: 0
    });

[dipu-bd]

it worked! thank you @ghjbnm

[JustinBeckwith]

Pretty sure we have this fixed on master, and it's going to be in v37 :)