Node-oracledb: Retriving CLOB Object Data is Causing RSS Memory Leak.

Created on 5 Apr 2017 · 11Comments · Source: oracle/node-oracledb

Using oracledb/1.13.0

Fetching CLOB Object Increasing RSS Memory And It's Not Releasing The Memory Causing Memory Leak In The Application.

I am accessing empDetails From Employee Table which is a CLOB Object. Sample Application Code Has Been Provided Below Which Is Causing The Memory Leak.

Employee Table
empid - Number
empDetails - CLOB

//My Config File config.js

    poolAlias: "app1", //Name Of The Oracle Pool
    user: "test", // Oracle user name
    password: "test", // Oracle user password
    connectString: "test11", // Oralce connection string (SID)
    poolMax: 10,
    poolMin: 1,
    poolIncrement: 1,
    poolTimeout: 1
};

//Common Wrapper written to use by my application oracle_db.js

var oracledb = require('oracledb'); // orawrap for oracle
var logger = config.logger;
var oracle_db = {};
var pool;
var async = require('async');
// return all CLOBs as Strings
oracledb.fetchAsString = [ oracledb.CLOB ];
///////////////////////////
// CREATE CONNECTION POOL//
///////////////////////////
oracle_db.init = function(callback) {
    logger.info("initializing oracle db pool...");
    oracledb.createPool(config.ORACLE, function(err, pool) {
        if (err) {
            console.log("Failed to create oracle db pool..." + pool.poolAlias);
            callback(err);
        } else {
            callback();
        }
    });
};
//////////////////////////
// TERMINATING POOL     //
//////////////////////////
oracle_db.terminateConnection = function(callback) {
    logger.info("Terminating oracle db pool...");
    pool = oracledb.getPool(config.ORACLE.poolAlias);
    pool.terminate(function(err) {
        if (err) {
            console.log("Failed to Terminating oracle db pool..." + pool.poolAlias);
            callback(err);
        } else {
            callback();
        }
    });
};
//////////////////////
// GET A CONNECTION //
//////////////////////
oracle_db.doConnect = function(getConnectionCb) {
    var maxRetries = 3;
    pool = oracledb.getPool(config.ORACLE.poolAlias);
    async.retry(maxRetries, function(cb) {
        pool.getConnection(function(err, conn) {
            if (err) {
                console.log("Failed to create connection...");
                cb(err);
                return;
            }
            conn.execute('select 1 from dual', function(err, result) {
                if (err) {
                    console.log("Invalid Connection...");
                    conn.release(function() {
                        cb(err);
                    });
                } else {
                    cb(null, conn);
                }
            });
        });
    }, function(err, conn) {
        if (err) {
            getConnectionCb(err);
        } else {
            getConnectionCb(null, conn);
        }
    });
};
/////////////
// EXECUTE //
/////////////
oracle_db.executeSql = function(connection, options, callback) {
    if (!options.bindParams) {
        options.bindParams = {};
    }
    if (!options.opts) {
        options.opts = {};
    }
    connection.execute(options.sql, options.bindParams, options.opts, function(err, result) {
        logger.info("Executing sql: " + options.sql + " , bind params: " + options.bindParams + " , options: " + options.opts);
        // Something went wrong - handle the data and release the connection
        if (err) {
            console.log("ERROR: Unable to execute the SQL: ", err);
            return callback(err);
        }
        // Return the result to the request initiator
        // console.log("INFO: Result from Database: ", result)
        return callback(err, result);
    });
};
////////////
// COMMIT //
////////////
oracle_db.doCommit = function(connection, callback) {
    connection.commit(function(err) {
        if (err) {
            console.log("ERROR: Unable to COMMIT transaction: ", err);
        }
        return callback(err, connection);
    });
};
//////////////
// ROLLBACK //
//////////////
oracle_db.doRollback = function(connection, callback) {
    connection.rollback(function(err) {
        if (err) {
            console.log("ERROR: Unable to ROLLBACK transaction: ", err);
        }
        return callback(err, connection);
    });
};
//////////////////////////
// RELEASE A CONNECTION //
//////////////////////////
oracle_db.doRelease = function(connection) {
    connection.release(function(err) {
        if (err) {
            console.log("ERROR: Unable to RELEASE the connection: ", err);
        }
        return;
    });
};
module.exports = oracle_db;

// Below Application Code Using The Above Library

var config = require('config'),
    oracle_db = require('oracle_db');

adminDB.getEmployees = function(apiname, userName, empRequest, modelCallback) {
    var employees=[];
    var failureResponse={};
oracle_db.doConnect(function(err, connection) {
        if (err) {           
            failureResponse.errorMsg = "DB_CONNECTION_ERROR";
             oracle_db.doRelease(connection, function(err) {
                     modelCallback(failureResponse,null);
                  });

        } else {
            oracle_db.executeSql(connection, {
                sql: "select empid,empDetails from Emplyee",
                bindParams: {},
                opts: {
                    resultSet: true,
                    prefetchRows: 1000,
                    autoRelease: false
                }
            }, function(err, results) {
                if (err) {                  
                   oracle_db.doRelease(connection, function(err) {
                     modelCallback(failureResponse,null);
                  });
                } else {
                    results.resultSet.getRows(1000, function(err, rows) {
                        if (rows && rows.length > 0) {
                            for (var i = 0; i < rows.length; i++) {
                                var response_1 = {
                                    "empid": rows[i][0].trim(),
                                    "empDetails":rows[i][1]
                                };
                                employees.push(response_1);
                            }
                           results.resultSet.close(function(err) {
  oracle_db.doRelease(connection, function(err) {
    modelCallback(null, employees);
  });
});
                        } else {
                                                    results.resultSet.close(function(err) {
  oracle_db.doRelease(connection, function(err) {
    modelCallback(null, employees);
  });
});

                        }
                    });
                }
            });
        }
    });

question

Source

talasila66

Most helpful comment

@talasila66 I don't see a leak. I see high initial memory utilization that decreases over time.

With async.times, you're doing a synchronous loop adding 1 million callback functions to libuv's work queue. That's what's taking up all the memory and why the memory decreases over time.

Also, you're passing the result in the next call. This basically acts as an accumulator, storing the values of the results in an array so that they can be passed along to the final callback at the end. This will consume more memory for each execution of the iteratee.

You can redefine the table using a VARCHAR2(64) and you'll see the same memory issues. The problem is not related to CLOBs. Here's what I got running your program as is with a CLOB:

10-04-2017 12:05:06.966:Execution Completed
10-04-2017 12:05:06.966:INFO: RSS MB 503.33203125
10-04-2017 12:05:06.966:INFO: RSS Peak MB 1704.609375
10-04-2017 12:05:06.967:INFO: GC Run Count 304

And using a VARCHAR2(64):

10-04-2017 12:13:05.624:Execution Completed
10-04-2017 12:13:05.624:INFO: RSS MB 1199.34765625
10-04-2017 12:13:05.624:INFO: RSS Peak MB 1892.62109375
10-04-2017 12:13:05.624:INFO: GC Run Count 311

Obviously, both of the results are bad. I just wanted to point out that the CLOB test used less memory.

What's a better way to run this program? Throwing all the work over to libuv isn't good. Instead, use timesSeries, to keep the queue in the JS thread. Remember, a connection can only do one thing at a time anyway.

            async.timesSeries(1000000, function(n, next) {
                conn.execute("select c,c,c,c,c,c,c,c,c,c,c,c,c from t", {}, {}, function(err_2, results) {
                    if (err_2) {
                        ...
                    } else {
                        next(null, results.rows[0][0]);
                    }
                });
            }, function(err_3, values) {
                ...
            });

With just this one change here's the difference (using a CLOB):

10-04-2017 12:35:30.482:Execution Completed
10-04-2017 12:35:30.482:INFO: RSS MB 183.83984375
10-04-2017 12:35:30.482:INFO: RSS Peak MB 183.92578125
10-04-2017 12:35:30.483:INFO: GC Run Count 313

... a few seconds later after gc types 2 & 4

10-04-2017 12:36:02.167:INFO: RSS MB 48.1953125

Even better? Don't accumulate the results if you don't need them:

            async.timesSeries(1000000, function(n, next) {
                conn.execute("select c,c,c,c,c,c,c,c,c,c,c,c,c from t", {}, {}, function(err_2, results) {
                    if (err_2) {
                        ...
                    } else {
                        next(null); // Not passing results here
                    }
                });
            }, function(err_3, values) {
                ...
            });

This basically accumulates undefined, which is minimal. Here are the results:

10-04-2017 13:26:28.407:Execution Completed
10-04-2017 13:26:28.407:INFO: RSS MB 61.5234375
10-04-2017 13:26:28.407:INFO: RSS Peak MB 68.1953125
10-04-2017 13:26:28.407:INFO: GC Run Count 1609

... a few seconds later after gc types 2 & 4

10-04-2017 13:27:36.210:INFO: RSS MB 40.3125

The biggest take away is to avoid throwing a bunch of work over to libuv's work queue. I've only seen bad things happen when developers do this.

dmcghan on 10 Apr 2017

👍2

All 11 comments

@talasila66 Is emplyee.empDetails the CLOB? If so, I don't see where you stream it to the end. See this section of the doc on closing LOBs to prevent this type of leak.

Given the following table:

drop table t;

create table t (c clob);

-- 64 bytes;
insert into t (c) values ('0123456789012345678901234567890123456789012345678901234567890123');

commit;

The following test can be used to observe the behavior of closing LOBs vs not closing LOBS. Just toggle the boolean at the letLobDrain top to see the difference. Note you'll need to run npm install gc-stats to run the test (or just comment out portions that use the module).

const letLobDrain = false;

const oracledb = require('oracledb');
const config = require('./dbConfig.js');
const gc = (require('gc-stats'))();
let rssPeak = 0;
let gcRunCount = 0;
let lobsOpened = 0;
let lobsClosed = 0;
let conn;

gc.on('stats', function(stats) {
  gcRunCount += 1;
  console.log('GC Ran (type ' + stats.gctype + ')');
});

function logMemory() {
  const mem = process.memoryUsage();
  const rss = mem.rss/1024/1024;

  if (rss > rssPeak) {
    rssPeak = rss;
  }

  console.log('RSS MB', rss);
  console.log('LOBs opened/closed', lobsOpened + '/' + lobsClosed);
}

logMemory();

const interval = setInterval(logMemory, 1000);

oracledb.createPool(config)
  .then(() => {
    let promiseChain = Promise.resolve({rows: false});

    for (let idx = 0; idx < 64000; idx += 1) {
      promiseChain = promiseChain
        .then(result => {
          return runTest();
        });
    }

    return promiseChain;
  })
  .catch(err => {
    console.log(err);
  })
  .then(() => {
    clearInterval(interval);

    logMemory();

    console.log('RSS Peak MB', rssPeak);
    console.log('GC Run Count', gcRunCount);

    return oracledb.getPool().close();
  })
  .catch(err => {
    console.log(err);
  })
  .then(() => {
    console.log('Pool closed');
  });

function runTest() {
  return new Promise((resolve, reject) => {
    oracledb.getConnection()
      .then(c => {
        conn = c;

        return conn.execute(
         `select c
          from t`
        );
      })
      .then(result => {
        return new Promise((resolve, reject) => {
          lobsOpened += 1;

          if (!letLobDrain) {
            resolve();
            return;
          }

          if (result.rows) {
            let lob = result.rows[0][0];

            lob.on('data', () => {});
            lob.on('end', resolve);
            lob.on('close', () => {lobsClosed += 1;});
            lob.on('error', reject);
          } 
        });
      })
      .catch(err => {
        reject();
        console.log(err);
      })
      .then(() => {
        return conn.close();
      })
      .catch(err => {
        console.log(err);
      })
      .then(() => {
        resolve();
      });
  });
}

You should see that when letLobDrain is set to true, there is no memory leak.

dmcghan on 5 Apr 2017

@dmcghan Thanks For The Reply. Yes emplyee.empDetails is the CLOB data. Fetching The CLOB data as string using oracledb.fetchAsString = [ oracledb.CLOB ];

Am I missed something to close the CLOB data after fetching?

talasila66 on 5 Apr 2017

@talasila66

Am I missed something to close the CLOB data after fetching?

Sorry, I missed the oracledb.fetchAsString = [ oracledb.CLOB ];. That changes everything. :)

Could you please show your code properly formatted? See this for details:
https://guides.github.com/features/mastering-markdown/#GitHub-flavored-markdown

dmcghan on 5 Apr 2017

@talasila66 Why do you think it is leaking vs not yet cleaned up by the garbage collector - what tooling are you using to measure with? How are you running the app and for how long?

You seemed to have skipped a few questions in the issue template, e.g Oracle version info that is always useful when looking at issues . . . .

(Also why is your poolTimeout so small - why bother using a pool? You might have reasons but it would be not so common. I added some tuning tips recently in the doc: https://github.com/oracle/node-oracledb/blob/master/doc/api.md#connpooling that you may want to check out)

cjbj on 6 Apr 2017

@talasila66 I saw you logged https://github.com/nodejs/node/issues/12208 make sure you reproduce the issue with a current version of Node.js.

cjbj on 6 Apr 2017

@cjbj Thanks For The Reply Unable Install oracledb/1.13.0 on Linux6 platform it's giving the error as gc c++ 11 compiler is required, Went through your pre-requisites(https://github.com/oracle/node-oracledb/blob/master/INSTALL.md#Prerequisites) mentioned is there anyway I can do it?

Below are my System Details

system details

node-v6.10.2-linux-x64
Linux Version-Linux 2.6.32-504.el6.x86_64 x86_64
Kernel Version 2.6.32-504.el6.x86_64
gcc Version gcc (GCC) 4.4.7 20120313 (Red Hat 4.4.7-11)
Oracle Client 12.1.0.2.0

But Linux6 supports only gcc 4.4.x

Below are the error details

make: Entering directory `/projects/development/Zain/Manju/FIX/smartdealer/node_modules/oracledb/build'
CXX(target) Release/obj.target/oracledb/src/njs/src/njsOracle.o
In file included from ../src/njs/src/njsOracle.h:56,
from ../src/njs/src/njsOracle.cpp:53:
../../nan/nan.h:43:3: error: #error This version of node/NAN/v8 requires a C++11 compiler
In file included from /u/mmyadav/.node-gyp/6.10.2/include/node/node.h:42,
from ../src/njs/src/njsOracle.cpp:51:
/u/mmyadav/.node-gyp/6.10.2/include/node/v8.h:345: error: expected unqualified-id before ‘using’
/u/mmyadav/.node-gyp/6.10.2/include/node/v8.h: In constructor ‘v8::MaybeLocal::MaybeLocal()’:
/u/mmyadav/.node-gyp/6.10.2/include/node/v8.h:362: error: ‘nullptr’ was not declared in this scope

talasila66 on 6 Apr 2017

this is how I solved it on centos 6: https://github.com/oracle/node-oracledb/issues/488#issuecomment-238148826

sagiegurari on 6 Apr 2017

@talasila66, as pointed out by @sagiegurari you will need a newer compiler because Node / V8 needs it for Node 4+. This is independent of node-oracledb. INSTALL's pre-reqs links to my steps for getting a new compiler, or check @sagiegurari's link

cjbj on 8 Apr 2017

After Upgrading as suggested, I can see a significant Improvement which is very Good, But Still I Can Find RSS Memory Leak And CPU Utilization is very high.

@sagiegurari Thank You I can upgrade successfully with the suggestions provided.

@cjbj Thanks For The Inputs Provided.

@dmcghan Thanks For Providing Sample Test Program, Just Ran the same and provided the results.

My System Details

node-v6.10.2
Red Hat Enterprise Linux Server release 6.7 (Santiago)
Kernel Version 2.6.32-573.el6.x86_64
gcc Version gcc-4.8.2-15.el6.x86_64
Oracle Client 12.1.0

//Logs Intially When Program Started

10-04-2017 14:06:52.448:INFO: RSS MB 26.8515625
10-04-2017 14:06:52.456:INFO: GC Ran (type 1)
10-04-2017 14:06:52.457:INFO: GC Ran (type 1)
10-04-2017 14:07:04.653:INFO: RSS MB 1896.15625
10-04-2017 14:07:04.708:INFO: GC Ran (type 1)
10-04-2017 14:07:04.780:INFO: GC Ran (type 1)
10-04-2017 14:07:04.863:INFO: GC Ran (type 1)
10-04-2017 14:07:05.018:INFO: GC Ran (type 1)
10-04-2017 14:07:05.141:INFO: GC Ran (type 1)
10-04-2017 14:07:05.655:INFO: RSS MB 1884.359375
10-04-2017 14:07:05.924:INFO: GC Ran (type 1)
10-04-2017 14:07:06.656:INFO: RSS MB 1884.359375
10-04-2017 14:07:07.657:INFO: RSS MB 1447.26171875
10-04-2017 14:07:08.657:INFO: RSS MB 1269.3046875
10-04-2017 14:07:09.658:INFO: RSS MB 1269.3046875
10-04-2017 14:07:10.659:INFO: RSS MB 1268.8359375
10-04-2017 14:07:11.661:INFO: RSS MB 1268.84375

//Logs After Execution Completed

10-04-2017 14:24:45.532:Execution Completed
10-04-2017 14:24:45.532:INFO: RSS MB 792.01171875
10-04-2017 14:24:45.533:INFO: RSS Peak MB 1896.15625
10-04-2017 14:24:45.533:INFO: GC Run Count 786
10-04-2017 14:24:45.533:INFO: GC Ran (type 1)
10-04-2017 14:24:45.582:INFO: RSS MB 792.01171875
10-04-2017 14:24:46.584:INFO: RSS MB 792.01171875
10-04-2017 14:24:47.586:INFO: RSS MB 792.01171875
10-04-2017 14:24:48.588:INFO: RSS MB 792.01171875
10-04-2017 14:24:49.589:INFO: RSS MB 792.01171875
10-04-2017 14:24:50.590:INFO: RSS MB 792.01171875
10-04-2017 14:24:51.594:INFO: RSS MB 792.01171875
10-04-2017 14:24:52.596:INFO: RSS MB 792.01171875
10-04-2017 14:24:53.598:INFO: RSS MB 792.01171875
10-04-2017 14:24:54.599:INFO: RSS MB 792.01171875
10-04-2017 14:24:55.601:INFO: RSS MB 792.01171875
10-04-2017 14:24:56.603:INFO: RSS MB 792.01171875
10-04-2017 14:24:57.605:INFO: RSS MB 792.01171875
10-04-2017 14:24:58.607:INFO: RSS MB 792.01171875
10-04-2017 14:24:59.609:INFO: RSS MB 792.01171875
10-04-2017 14:25:00.726:INFO: RSS MB 792.01171875
10-04-2017 14:25:01.766:INFO: RSS MB 792.01171875
10-04-2017 14:25:02.767:INFO: RSS MB 792.01171875
10-04-2017 14:25:03.768:INFO: RSS MB 792.01171875
10-04-2017 14:25:04.769:INFO: RSS MB 792.01171875
10-04-2017 14:25:05.770:INFO: RSS MB 792.01171875
10-04-2017 14:25:06.771:INFO: RSS MB 792.01171875
10-04-2017 14:25:07.772:INFO: RSS MB 792.01171875
10-04-2017 14:25:08.773:INFO: RSS MB 792.01171875
10-04-2017 14:25:09.774:INFO: RSS MB 792.01171875
10-04-2017 14:25:10.775:INFO: RSS MB 792.01171875
10-04-2017 14:25:11.776:INFO: RSS MB 792.01171875
10-04-2017 14:25:11.782:INFO: GC Ran (type 4)
10-04-2017 14:25:12.777:INFO: RSS MB 792.01171875
10-04-2017 14:25:13.778:INFO: RSS MB 792.01171875
10-04-2017 14:25:14.779:INFO: RSS MB 792.01171875
10-04-2017 14:25:15.780:INFO: RSS MB 792.01171875
10-04-2017 14:25:16.781:INFO: RSS MB 792.01171875
10-04-2017 14:25:17.781:INFO: RSS MB 792.01171875
10-04-2017 14:25:17.792:INFO: GC Ran (type 2)
10-04-2017 14:25:18.783:INFO: RSS MB 672.55859375
10-04-2017 14:25:19.784:INFO: RSS MB 672.55859375
10-04-2017 14:25:20.785:INFO: RSS MB 672.55859375
10-04-2017 14:25:21.786:INFO: RSS MB 672.55859375
10-04-2017 14:25:22.787:INFO: RSS MB 672.55859375
10-04-2017 14:25:23.788:INFO: RSS MB 672.55859375
10-04-2017 14:25:23.801:INFO: GC Ran (type 4)
10-04-2017 14:25:24.789:INFO: RSS MB 672.55859375
10-04-2017 14:25:25.790:INFO: RSS MB 672.55859375
10-04-2017 14:25:26.792:INFO: RSS MB 672.55859375
10-04-2017 14:25:27.793:INFO: RSS MB 672.55859375
10-04-2017 14:25:28.795:INFO: RSS MB 672.55859375
10-04-2017 14:25:29.796:INFO: RSS MB 672.55859375

//Waiting Some More Time

10-04-2017 14:48:00.627:INFO: GC Ran (type 1)
10-04-2017 14:48:01.629:INFO: RSS MB 625.5859375
10-04-2017 14:48:02.630:INFO: RSS MB 625.58984375
10-04-2017 14:48:03.631:INFO: RSS MB 625.59375
10-04-2017 14:48:04.632:INFO: RSS MB 625.59765625
10-04-2017 14:48:05.633:INFO: RSS MB 625.6015625

CPU Utilization 64.3%

Given the following table:

drop table t;

create table t (c clob);

-- 64 bytes;
insert into t (c) values ('0123456789012345678901234567890123456789012345678901234567890123');

commit;

Program Executed Is

var oracledb = require('oracledb');
var config = require('./dbConfig.js');
var gc = (require('gc-stats'))();
var rssPeak = 0;
var gcRunCount = 0;
var async = require('async');
var moment = require('moment');
//var logger = config.logger;
// return all CLOBs as Strings
oracledb.fetchAsString = [oracledb.CLOB];

gc.on('stats', function(stats) {
    gcRunCount += 1;
    console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+':INFO: GC Ran (type ' + stats.gctype + ')');
});

function logMemory() {
    var mem = process.memoryUsage();
    var rss = mem.rss / 1024 / 1024;
    if (rss > rssPeak) {
        rssPeak = rss;
    }
    console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+':INFO: RSS MB', rss);
}

logMemory();

var interval = setInterval(logMemory, 1000);

oracledb.createPool(config.ORACLE, function(err, pool) {
    if (err) {
        console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+":ERROR: Failed to create oracle db pool...");
    } else {
        try {
            runTest();
        } catch (err) {
            clearInterval(interval);
            logMemory();
            console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+':INFO: RSS Peak MB', rssPeak);
            console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+':INFO: GC Run Count', gcRunCount);
            return oracledb.getPool().close();
        }
    }
});

function runTest() {
    oracledb.getConnection(function(err_0, conn) {
        if (err_0) {
            console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+":ERROR: Unable to connect: ", err_0);
        } else {
            async.times(1000000, function(n, next) {
                conn.execute("select c,c,c,c,c,c,c,c,c,c,c,c,c from t", {}, {}, function(err_2, results) {
                    if (err_2) {
                        console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+":ERROR: Unable to execute the SQL: ", err_2);
                        next(err_2);
                    } else {
                        next(null, results.rows[0][0]);
                    }
                });
            }, function(err_3, values) {
                console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+":Execution Completed");
                logMemory();
                console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+':INFO: RSS Peak MB', rssPeak);
                console.log(moment().format('DD-MM-YYYY HH:mm:ss.SSS')+':INFO: GC Run Count', gcRunCount);
            });
        }
    });
}

talasila66 on 10 Apr 2017

@talasila66 I don't see a leak. I see high initial memory utilization that decreases over time.

With async.times, you're doing a synchronous loop adding 1 million callback functions to libuv's work queue. That's what's taking up all the memory and why the memory decreases over time.

You can redefine the table using a VARCHAR2(64) and you'll see the same memory issues. The problem is not related to CLOBs. Here's what I got running your program as is with a CLOB:

10-04-2017 12:05:06.966:Execution Completed
10-04-2017 12:05:06.966:INFO: RSS MB 503.33203125
10-04-2017 12:05:06.966:INFO: RSS Peak MB 1704.609375
10-04-2017 12:05:06.967:INFO: GC Run Count 304

And using a VARCHAR2(64):

10-04-2017 12:13:05.624:Execution Completed
10-04-2017 12:13:05.624:INFO: RSS MB 1199.34765625
10-04-2017 12:13:05.624:INFO: RSS Peak MB 1892.62109375
10-04-2017 12:13:05.624:INFO: GC Run Count 311

Obviously, both of the results are bad. I just wanted to point out that the CLOB test used less memory.

            async.timesSeries(1000000, function(n, next) {
                conn.execute("select c,c,c,c,c,c,c,c,c,c,c,c,c from t", {}, {}, function(err_2, results) {
                    if (err_2) {
                        ...
                    } else {
                        next(null, results.rows[0][0]);
                    }
                });
            }, function(err_3, values) {
                ...
            });

With just this one change here's the difference (using a CLOB):

10-04-2017 12:35:30.482:Execution Completed
10-04-2017 12:35:30.482:INFO: RSS MB 183.83984375
10-04-2017 12:35:30.482:INFO: RSS Peak MB 183.92578125
10-04-2017 12:35:30.483:INFO: GC Run Count 313

... a few seconds later after gc types 2 & 4

10-04-2017 12:36:02.167:INFO: RSS MB 48.1953125

Even better? Don't accumulate the results if you don't need them:

            async.timesSeries(1000000, function(n, next) {
                conn.execute("select c,c,c,c,c,c,c,c,c,c,c,c,c from t", {}, {}, function(err_2, results) {
                    if (err_2) {
                        ...
                    } else {
                        next(null); // Not passing results here
                    }
                });
            }, function(err_3, values) {
                ...
            });

This basically accumulates undefined, which is minimal. Here are the results:

10-04-2017 13:26:28.407:Execution Completed
10-04-2017 13:26:28.407:INFO: RSS MB 61.5234375
10-04-2017 13:26:28.407:INFO: RSS Peak MB 68.1953125
10-04-2017 13:26:28.407:INFO: GC Run Count 1609

... a few seconds later after gc types 2 & 4

10-04-2017 13:27:36.210:INFO: RSS MB 40.3125

The biggest take away is to avoid throwing a bunch of work over to libuv's work queue. I've only seen bad things happen when developers do this.

dmcghan on 10 Apr 2017

👍2

@dmcghan Thank You

talasila66 on 12 Apr 2017

Was this page helpful?

0 / 5 - 0 ratings

Related issues

NJS-069 node-oracledb 4.0.1 requires Node.js 8.16 or later in vuetify app

rheinripper · 4Comments

Problem with large numbers

xpro666 · 3Comments

Question why getting Error: NJS-046: poolAlias "xxxPool" already exists in the connection pool cache

rstor1 · 4Comments

windows 10, and Visual Studio 2015 (14.0) install issues

ainthek · 3Comments

Installing node-oracledb 1.6 on Mojave

sanfords · 3Comments