Rocket.chat: Unhandled Promise Rejections

Created on 24 Apr 2019 · 13Comments · Source: RocketChat/Rocket.Chat

With: #14220 and newer version of node.js exiting by default on unhandled promise rejections... we now need to capture and solve every case of a unhandled promise rejection.

[x] subscription.receiver is not iterable

TypeError: subscription.receiver is not iterable
    at Promise.asyncApply (app/lib/server/lib/sendNotificationsOnMessage.js:39:34)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
Exiting due to an unhandled promise rejection

[ ] MongoError: ns not found

{ MongoError: ns not found
    at /app/bundle/programs/server/npm/node_modules/meteor/npm-mongo/node_modules/mongodb-core/lib/connection/pool.js:581:63
    at authenticateStragglers (/app/bundle/programs/server/npm/node_modules/meteor/npm-mongo/node_modules/mongodb-core/lib/connection/pool.js:504:16)
    at Connection.messageHandler (/app/bundle/programs/server/npm/node_modules/meteor/npm-mongo/node_modules/mongodb-core/lib/connection/pool.js:540:5)
    at emitMessageHandler (/app/bundle/programs/server/npm/node_modules/meteor/npm-mongo/node_modules/mongodb-core/lib/connection/connection.js:310:10)
    at Socket.<anonymous> (/app/bundle/programs/server/npm/node_modules/meteor/npm-mongo/node_modules/mongodb-core/lib/connection/connection.js:453:17)
    at emitOne (events.js:116:13)
    at Socket.emit (events.js:211:7)
    at addChunk (_stream_readable.js:263:12)
    at readableAddChunk (_stream_readable.js:250:11)
    at Socket.Readable.push (_stream_readable.js:208:10)
    at TCP.onread (net.js:597:20)
  operationTime: Timestamp { _bsontype: 'Timestamp', low_: 20, high_: 1556245520 },
  ok: 0,
  errmsg: 'ns not found',
  code: 26,
  codeName: 'NamespaceNotFound',
  '$clusterTime':
   { clusterTime: Timestamp { _bsontype: 'Timestamp', low_: 20, high_: 1556245520 },
     signature: { hash: [Object], keyId: 0 } },
  name: 'MongoError',
  [Symbol(mongoErrorContextSymbol)]: {} }
Exiting due to an unhandled promise rejection

[ ] mongo network related

Exception in setInterval callback: { Error: read ECONNRESET
    at _errnoException (util.js:992:11)
    at TCP.onread (net.js:618:25)
  name: 'MongoNetworkError',
  errorLabels: [ 'TransientTransactionError' ],
  [Symbol(mongoErrorContextSymbol)]: {} }
Exception while invoking method 'UserPresence:online' { Error: read ECONNRESET
    at _errnoException (util.js:992:11)
    at TCP.onread (net.js:618:25)
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
  name: 'MongoNetworkError',
  errorLabels: [ 'TransientTransactionError' ],
  [Symbol(mongoErrorContextSymbol)]: {} }
Exception while invoking method 'public-settings/get' { Error: read ECONNRESET
    at _errnoException (util.js:992:11)
    at TCP.onread (net.js:618:25)
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at MethodInvocation.public-settings/get (app/lib/server/publications/settings.js:8:50)
    at MethodInvocation.methodsMap.(anonymous function) (app/lib/server/lib/debug.js:73:34)
    at MethodInvocation.methodMap.(anonymous function) (packages/rocketchat_monitoring.js:2731:30)
    at maybeAuditArgumentChecks (packages/ddp-server/livedata_server.js:1767:12)
    at DDP._CurrentMethodInvocation.withValue (packages/ddp-server/livedata_server.js:719:19)
    at Meteor.EnvironmentVariable.EVp.withValue (packages/meteor.js:1304:12)
    at DDPServer._CurrentWriteFence.withValue (packages/ddp-server/livedata_server.js:717:46)
    at Meteor.EnvironmentVariable.EVp.withValue (packages/meteor.js:1304:12)
    at Promise (packages/ddp-server/livedata_server.js:715:46)
    at new Promise (<anonymous>)
    at Session.method (packages/ddp-server/livedata_server.js:689:23)
    at packages/ddp-server/livedata_server.js:559:43
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
  name: 'MongoNetworkError',
  errorLabels: [ 'TransientTransactionError' ],
  [Symbol(mongoErrorContextSymbol)]: {} }
{ Error: read ECONNRESET
    at _errnoException (util.js:992:11)
    at TCP.onread (net.js:618:25)
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Function.Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
    at Promise.asyncApply (packages/mongo/mongo_driver.js:1042:14)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at MethodInvocation.public-settings/get (app/lib/server/publications/settings.js:8:50)
    at MethodInvocation.methodsMap.(anonymous function) (app/lib/server/lib/debug.js:73:34)
    at MethodInvocation.methodMap.(anonymous function) (packages/rocketchat_monitoring.js:2731:30)
    at maybeAuditArgumentChecks (packages/ddp-server/livedata_server.js:1767:12)
    at DDP._CurrentMethodInvocation.withValue (packages/ddp-server/livedata_server.js:719:19)
    at Meteor.EnvironmentVariable.EVp.withValue (packages/meteor.js:1304:12)
    at DDPServer._CurrentWriteFence.withValue (packages/ddp-server/livedata_server.js:717:46)
    at Meteor.EnvironmentVariable.EVp.withValue (packages/meteor.js:1304:12)
    at Promise (packages/ddp-server/livedata_server.js:715:46)
    at new Promise (<anonymous>)
    at Session.method (packages/ddp-server/livedata_server.js:689:23)
    at packages/ddp-server/livedata_server.js:559:43
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
 => awaited here:
    at Promise.await (/app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:60:12)
    at SynchronousCursor._nextObject (packages/mongo/mongo_driver.js:1091:12)
    at SynchronousCursor.forEach (packages/mongo/mongo_driver.js:1105:22)
    at SynchronousCursor.map (packages/mongo/mongo_driver.js:1115:10)
    at SynchronousCursor.fetch (packages/mongo/mongo_driver.js:1139:17)
    at Cursor.(anonymous function) [as fetch] (packages/mongo/mongo_driver.js:887:44)
    at Statistics.findLast (app/models/server/models/Statistics.js:23:42)
    at Promise.asyncApply (app/metrics/server/lib/metrics.js:98:32)
    at /app/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
  name: 'MongoNetworkError',
  errorLabels: [ 'TransientTransactionError' ],
  [Symbol(mongoErrorContextSymbol)]: {} }
Exiting due to an unhandled promise rejection

[ ] error loading cert.pem https://github.com/RocketChat/Rocket.Chat/issues/14280

bug

Source

geekgonecrazy

👍1

Most helpful comment

@theorenck with #14220 these exceptions cause Rocket.Chat to crash instead of carrying on.

So need solved, or we need to make the exiting on unhandled optional at first or 1.0 will lead to a bad experience. Not everyone is using autorestart on crash

cc @rodrigok

geekgonecrazy on 26 Apr 2019

👍2

All 13 comments

@rodrigok here is first since the change. Keeping an eye out for more

geekgonecrazy on 24 Apr 2019

@rodrigok found two more. Both popped up right around the time that an unexpected primary election happened. Added both above. MongoError: ns not found happened on 3 out of the 4 instances. The other happened on just one.

geekgonecrazy on 26 Apr 2019

I keep getting the following UnhandledPromiseRejectionWarning :

UnhandledPromiseRejectionWarning: TypeError: Cannot convert undefined or null to object
at Function.keys (<anonymous>)
at getStaticFileInfo (packages/webapp/webapp_server.js:493:33)
at Promise.asyncApply (packages/webapp/webapp_server.js:416:16)

It comes from app/assets/server/assets.js in this line :
WebAppInternals.staticFilesMiddleware(WebAppInternals.staticFiles, req, res, next);

I don't know if it's a problem in my setup, but after checking the webapp package, there is no WebAppInternals.staticFiles, I think it should be WebAppInternals.staticFilesByArch

ramrami on 26 Apr 2019

@theorenck with #14220 these exceptions cause Rocket.Chat to crash instead of carrying on.

So need solved, or we need to make the exiting on unhandled optional at first or 1.0 will lead to a bad experience. Not everyone is using autorestart on crash

cc @rodrigok

geekgonecrazy on 26 Apr 2019

👍2

Have another one in SAML - https://github.com/RocketChat/Rocket.Chat/issues/14280

geekgonecrazy on 28 Apr 2019

@geekgonecrazy Hi, I found one bug too related to this. When adding a WebDAV server if the password is incorrect, the latest develop server crashes. I earlier thought it's a bug with the webdav-client library so I opened an issue there https://github.com/perry-mitchell/webdav-client/issues/156 (Maybe it still is a problem with the library only... but in any case, our server should not crash right?)
The issue also has the logs and relevant code. The weird thing is that the problematic code is inside of try... catch block and while resolving the issue I checked that the catch block is run but still the server crashes :(

Maybe #13649 is related to this.

deepin-screen-recorder_google-chrome_20190505183113

ubarsaiyan on 5 May 2019

@ramrami i'm running into the exact same problem as you. did you maybe find a solution for this?

errorroutine on 23 May 2019

@errorroutine the solution is to replace WebAppInternals.staticFiles with WebAppInternals.staticFilesByArch in app/assets/server/assets.js.

It is fixed here https://github.com/RocketChat/Rocket.Chat/pull/14462/commits/af279a70a48759b3463b3c0881affaa79754253b but not merged yet, it's planned for 1.1.0.

ramrami on 23 May 2019

❤1

@geekgonecrazy I met the same bug
,please give me a hand
https://github.com/RocketChat/Rocket.Chat/issues/17020

564064202 on 29 Mar 2020

I get UnHandledPromiseRejection in the following setup:

rocket.chat-3.0.10 installed from tar.gz via ansible
2 instances
Prometheus is enabled and every instance has it's own prometheus port via "Environment=PROMETHEUS_PORT=xxx" in the systemd unit file
Startup of the first instance is fine, but when I start the second instance I get the following in my logs:

Apr 02 10:46:59 eval-rocket RocketChat[64498]: Starting Email Intercepter...
Apr 02 10:47:01 eval-rocket RocketChat[64498]: === UnHandledPromiseRejection ===
Apr 02 10:47:01 eval-rocket RocketChat[64498]: Error [ERR_SERVER_ALREADY_LISTEN] [ERR_SERVER_ALREADY_LISTEN]: Listen method has been called more than once without closing.
Apr 02 10:47:01 eval-rocket RocketChat[64498]:     at Server.listen (net.js:1387:11)
Apr 02 10:47:01 eval-rocket RocketChat[64498]:     at app/metrics/server/lib/metrics.js:185:9
Apr 02 10:47:01 eval-rocket RocketChat[64498]:     at /var/lib/rocket.chat/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40 {
Apr 02 10:47:01 eval-rocket RocketChat[64498]:   code: 'ERR_SERVER_ALREADY_LISTEN'
Apr 02 10:47:01 eval-rocket RocketChat[64498]: }
Apr 02 10:47:01 eval-rocket RocketChat[64498]: ---------------------------------
Apr 02 10:47:01 eval-rocket RocketChat[64498]: Errors like this can cause oplog processing errors.
Apr 02 10:47:01 eval-rocket RocketChat[64498]: Setting EXIT_UNHANDLEDPROMISEREJECTION will cause the process to exit allowing your service to automatically restart the process
Apr 02 10:47:01 eval-rocket RocketChat[64498]: Future node.js versions will automatically exit the process
Apr 02 10:47:01 eval-rocket RocketChat[64498]: =================================
Apr 02 10:47:01 eval-rocket RocketChat[64498]: === UnHandledPromiseRejection ===
Apr 02 10:47:01 eval-rocket RocketChat[64498]: Error [ERR_SERVER_ALREADY_LISTEN] [ERR_SERVER_ALREADY_LISTEN]: Listen method has been called more than once without closing.
Apr 02 10:47:01 eval-rocket RocketChat[64498]:     at Server.listen (net.js:1387:11)
Apr 02 10:47:01 eval-rocket RocketChat[64498]:     at app/metrics/server/lib/metrics.js:185:9
Apr 02 10:47:01 eval-rocket RocketChat[64498]:     at /var/lib/rocket.chat/bundle/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40 {
Apr 02 10:47:01 eval-rocket RocketChat[64498]:   code: 'ERR_SERVER_ALREADY_LISTEN'
Apr 02 10:47:01 eval-rocket RocketChat[64498]: }
Apr 02 10:47:01 eval-rocket RocketChat[64498]: ---------------------------------
Apr 02 10:47:01 eval-rocket RocketChat[64498]: Errors like this can cause oplog processing errors.
Apr 02 10:47:01 eval-rocket RocketChat[64498]: Setting EXIT_UNHANDLEDPROMISEREJECTION will cause the process to exit allowing your service to automatically restart the process
Apr 02 10:47:01 eval-rocket RocketChat[64498]: Future node.js versions will automatically exit the process
Apr 02 10:47:01 eval-rocket RocketChat[64498]: =================================

But "Environment=PROMETHEUS_PORT=xxx" is working, every node process is listening on it's own prometheus port.

photoninger on 2 Apr 2020

So i just hit this same issue at the time our servers had an election. It appears when an election occurs the servers never recover.

Is anyone else looking into this? If not i will...

ndroo on 22 Jul 2020

@ndroo My prod RC instance finally just survived 2 Mongo primary elections last night with my latest config changes. See my last 2 comments on this forum post for details: https://forums.rocket.chat/t/messages-not-displaying-and-greyed-out/2035

dkoo761 on 18 Dec 2020

@geekgonecrazy Here are a couple UnHandledPromiseRejections I'm currently experiencing on RC 3.7.1:

1) When users message Rocket.Cat:

Dec 18 03:29:01 *** rocketchat[25979]: === UnHandledPromiseRejection ===
Dec 18 03:29:02 *** rocketchat[25979]: TypeError: receiver.emails.some is not a function
Dec 18 03:29:02 *** rocketchat[25979]: at app/lib/server/lib/sendNotificationsOnMessage.js:159:19
Dec 18 03:29:02 *** rocketchat[25979]: at /opt/Rocket.Chat/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40

2) After new Mongo primary election when USE_NATIVE_OPLOG=true (which is needed when using MongoDB Atlas):

Dec 17 16:05:20 **** rocketchat[25979]: === UnHandledPromiseRejection ===
Dec 17 16:05:20 **** rocketchat[25979]: MongoError: not master
Dec 17 16:05:20 **** rocketchat[25979]: at Connection. (/opt/Rocket.Chat/programs/server/npm/node_modules/meteor/npm-mongo/node_modules/mongodb/lib/core/connection/pool.js:451:61)
Dec 17 16:05:20 **** rocketchat[25979]: at Connection.emit (events.js:210:5)
Dec 17 16:05:20 **** rocketchat[25979]: at Connection.EventEmitter.emit (domain.js:475:20)
Dec 17 16:05:20 **** rocketchat[25979]: at processMessage (/opt/Rocket.Chat/programs/server/npm/node_modules/meteor/npm-mongo/node_modules/mongodb/lib/core/connection/connection.js:452:10)
Dec 17 16:05:20 **** rocketchat[25979]: at TLSSocket. (/opt/Rocket.Chat/programs/server/npm/node_modules/meteor/npm-mongo/node_modules/mongodb/lib/core/connection/connection.js:621:15)
Dec 17 16:05:20 **** rocketchat[25979]: at TLSSocket.emit (events.js:210:5)
Dec 17 16:05:20 **** rocketchat[25979]: at TLSSocket.EventEmitter.emit (domain.js:475:20)
Dec 17 16:05:20 **** rocketchat[25979]: at addChunk (_stream_readable.js:309:12)
Dec 17 16:05:20 **** rocketchat[25979]: at readableAddChunk (stream_readable.js:290:11)
Dec 17 16:05:20 **** rocketchat[25979]: at TLSSocket.Readable.push (stream_readable.js:224:10)
Dec 17 16:05:20 **** rocketchat[25979]: at TLSWrap.onStreamRead (internal/stream_base_commons.js:182:23)
Dec 17 16:05:20 **** rocketchat[25979]: => awaited here:
Dec 17 16:05:20 **** rocketchat[25979]: at Function.Promise.await (/opt/Rocket.Chat/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
Dec 17 16:05:20 **** rocketchat[25979]: at app/models/server/raw/NotificationQueue.ts:60:15
Dec 17 16:05:20 **** rocketchat[25979]: at /opt/Rocket.Chat/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40
Dec 17 16:05:20 **** rocketchat[25979]: => awaited here:
Dec 17 16:05:20 **** rocketchat[25979]: at Function.Promise.await (/opt/Rocket.Chat/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/promise_server.js:56:12)
Dec 17 16:05:20 **** rocketchat[25979]: at app/notification-queue/server/NotificationQueue.ts:48:21
Dec 17 16:05:20 **** rocketchat[25979]: at /opt/Rocket.Chat/programs/server/npm/node_modules/meteor/promise/node_modules/meteor-promise/fiber_pool.js:43:40 {
Dec 17 16:05:20 **** rocketchat[25979]: operationTime: Timestamp { bsontype: ‘Timestamp’, low: 6, high: 1608221119 },
Dec 17 16:05:20 **** rocketchat[25979]: ok: 0,
Dec 17 16:05:20 **** rocketchat[25979]: code: 10107,
Dec 17 16:05:20 **** rocketchat[25979]: codeName: ‘NotWritablePrimary’,
Dec 17 16:05:20 **** rocketchat[25979]: ‘$clusterTime’: {
Dec 17 16:05:20 **** rocketchat[25979]: clusterTime: Timestamp { bsontype: ‘Timestamp’, low: 8, high: 1608221119 },
Dec 17 16:05:20 **** rocketchat[25979]: signature: { hash: [Binary], keyId: [Long] }
Dec 17 16:05:20 **** rocketchat[25979]: },
Dec 17 16:05:20 **** rocketchat[25979]: name: ‘MongoError’,
Dec 17 16:05:20 **** rocketchat[25979]: [Symbol(errorLabels)]: Set { ‘RetryableWriteError’ }
Dec 17 16:05:20 **** rocketchat[25979]: }