Newman: collection execution and script execution timeouts occur when api options timeout or timeoutScript are omitted

[cchretien]

1. Newman Version (can be found via newman -v): 3.10.0
2. OS details (type, version, and architecture): Windows 7 Enterprise 64Bit, node 8.11.3 64Bit
3. Are you using Newman as a library, or via the CLI? Library
4. Did you encounter this recently, or has this bug always been there: Bug was not present in 3.8.3
5. Expected behaviour: No timeouts during long collection or test script execution when options are omitted. This breaks backwards compatibility.

1. Command / script used to run Newman: See attached script
   timeout-bug.js.txt

2. Sample collection, and auxiliary files (minus the sensitive details): See attached collection
   timeout.postman_collection.json.txt

3. Screenshots (if applicable): See attached logs
   timeout-bug-loop.log
   timeout-bug-client-delay.log

[cchretien]

One thing I noticed is if I explicitly set the timeoutScript to 0 it still triggers a timeout as if it were omitted.

[kunagpal]

@cchretien This is because the global timeout still takes precedence in this case, you'll have to set timeout: 0 explicitly. A similar case has been discussed and resolved here: https://community.getpostman.com/t/error-callback-timed-out-why/1508/19

[cchretien]

Even if I set the timeoutand timeoutScriptto 0as you suggested with the script/collection/environment provided I get the timeout.

var Newman = require('newman');

var opts = {
    collection: './timeout.postman_collection.json',
    folder: 'timeout loop',
    environment: {
        "id": "d2dce2c9-3ada-4c8f-9af9-0281eb6d3336",
        "name": "timeout",
        "values": [
            {
                "key": "LOOPS",
                "value": "19",
                "description": "",
                "type": "text",
                "enabled": true
            },
            {
                "key": "TIMEOUT",
                "value": "180000",
                "description": "",
                "type": "text",
                "enabled": true
            }
        ],
        "_postman_variable_scope": "environment",
        "_postman_exported_at": "2018-07-13T18:49:04.858Z",
        "_postman_exported_using": "Postman/6.0.10"
    },
    delayRequest: 100,
    timeoutRequest: 180000,
    insecure: true,
    bail: false,
    suppressExitCode: true,
    reporters: ['cli'],
    reporter: {
        cli: {
            noBanner: true,
            noSummary: false
        }
    }
};


function runNewman(opts) {
    var numErrs = 0;
    try {
        Newman.run(opts, function (err, summary) {
            console.log('collection run complete!');

            if (err) {
                numErrs += 1;
                console.log(`encountered error`, numErrs);
                console.log(err.stack);
            }
        });

    } catch (e) {
        numErrs =+ 1;
        console.log(`encountered error`, numErrs);
        console.log(e.stack);
    }

}

// script timeout with defaults
console.log('run script timeout\n********');
opts.folder = 'client delay';
opts.timeout = 0;
opts.timeoutScript = 0;
runNewman(opts);

See attached log set-timeout-and-timeoutScript-to-zero.log

[kunagpal]

@cchretien The error in your updated log is a script timeout error, which will have to be separately overridden by providing timeoutScript: 0. In a nutshell, the timeout behaviour is as follows:

1. timeout: This is the global timeout value, the time in milliseconds for the entire collection run to complete.
2. timeoutRequest: Similar to the above, but this value is applied individually across all requests in the collection run.
3. timeoutScript: Similar to the above, but this value is applied individually across all scripts in the collection run.

For any of the 3 config values above, specifying a value of 0 will result in the timeout being set to Infinity.

[cchretien]

In my script above, I did set timeoutScript: 0. Unless, I'm missing something, my last example indicates the following settings as i relates to timeouts...

timeout = 0
timeoutRequest = 180000
timeoutScript = 0

I'll wait for the next 3.x release with this fix but I'm not entirely convinced that the issues I've been reporting will be fixed.

[codenirvana]

@cchretien This has been fixed in Newman v4.0.0.

[cchretien]

Still broken. Using the same collection and toying with the same script that i've attached originally.

var Newman = require('newman');

var opts = {
    collection: './timeout.postman_collection.json',
    folder: 'client delay',
    environment: {
        "id": "d2dce2c9-3ada-4c8f-9af9-0281eb6d3336",
        "name": "timeout",
        "values": [
            {
                "key": "LOOPS",
                "value": "19",
                "description": "",
                "type": "text",
                "enabled": true
            },
            {
                "key": "TIMEOUT",
                "value": "180000",
                "description": "",
                "type": "text",
                "enabled": true
            }
        ],
        "_postman_variable_scope": "environment",
        "_postman_exported_at": "2018-07-13T18:49:04.858Z",
        "_postman_exported_using": "Postman/6.0.10"
    },
    delayRequest: 100,
    timeout: 0,
    timeoutRequest: 180000,
    timeoutScript: 0,
    insecure: true,
    bail: false,
    suppressExitCode: true,
    reporters: ['cli'],
    reporter: {
        cli: {
            noBanner: true,
            noSummary: false
        }
    }
};


function runNewman(opts) {
    var numErrs = 0;
    try {
        Newman.run(opts, function (err, summary) {
            console.log('collection run complete!');

            if (err) {
                numErrs += 1;
                console.log(`encountered error`, numErrs);
                //console.log(err);
                console.log(err.stack);
            }
        });

    } catch (e) {
        numErrs =+ 1;
        console.log(`encountered error`, numErrs);
        //console.log(e);
        console.log(e.stack);
    }

}

// script timeout with defaults
console.log('run script timeout\n********');
runNewman(opts);

Output:

c:\Work\newman>node --version
v8.11.3

c:\Work\newman>node_modules\.bin\newman -v
4.0.0

c:\Work\newman>
c:\Work\newman>
c:\Work\newman>
c:\Work\newman>node timeout-bug.js
run script timeout
********
timeout

□ client delay
└ Delay Response with client delay
  GET https://postman-echo.com/delay/10 [200 OK, 362B, 11.1s]

┌─────────────────────────┬──────────┬──────────┐
│                         │ executed │   failed │
├─────────────────────────┼──────────┼──────────┤
│              iterations │        0 │        0 │
├─────────────────────────┼──────────┼──────────┤
│                requests │        1 │        0 │
├─────────────────────────┼──────────┼──────────┤
│            test-scripts │        0 │        0 │
├─────────────────────────┼──────────┼──────────┤
│      prerequest-scripts │        0 │        0 │
├─────────────────────────┼──────────┼──────────┤
│              assertions │        0 │        0 │
├─────────────────────────┴──────────┴──────────┤
│ total run duration: 0ms                       │
├───────────────────────────────────────────────┤
│ total data received: 14B (approx)             │
├───────────────────────────────────────────────┤
│ average response time: 11.1s                  │
└───────────────────────────────────────────────┘
collection run complete!
encountered error 1
Error: Script execution timed out.
    at ContextifyScript.Script.runInContext (vm.js:59:29)
    at Object.runInContext (vm.js:120:6)
    at PostmanSandbox.emitter._dispatch (c:\Work\newman\node_modules\uvm\lib\uvm\bridge.js:113:16)
    at PostmanSandbox.dispatch (c:\Work\newman\node_modules\uvm\lib\uvm\index.js:104:30)
    at PostmanSandbox.execute (c:\Work\newman\node_modules\postman-sandbox\lib\index.js:106:14)
    at PostmanCollectionRun.<anonymous> (c:\Work\newman\node_modules\postman-runtime\lib\runner\extensions\event.command.js:289:27)
    at c:\Work\newman\node_modules\async\dist\async.js:1135:9
    at replenish (c:\Work\newman\node_modules\async\dist\async.js:1011:17)
    at c:\Work\newman\node_modules\async\dist\async.js:1016:9
    at _asyncMap (c:\Work\newman\node_modules\async\dist\async.js:1133:5)

c:\Work\newman>

[cchretien]

@kunagpal,

Can this be reopened and fixed?