At first glance this issue seems to be related with #1764 but I don't see how a network issue can cause the hanging here (directConnect is set to true). So here is the setup:
protractor-config.js
'use strict';
/* globals jasmine, browser */
exports.config = {
allScriptsTimeout: 300000, // 5 min
// Spec patterns are relative to this directory.
specs: [
'protractor/*.js'
],
capabilities: {
'browserName': 'chrome',
version: '',
platform: 'ANY'
},
directConnect: true,
baseUrl: 'http://localhost:3000/', // gets overwritten
troubleshoot: true,
framework: 'jasmine',
onPrepare: function beforeProtractorRuns() {
browser.driver.manage().window().setSize(1280, 1024);
// Disable animations so e2e tests run more quickly
var disableNgAnimate = function () {
angular.module('disableNgAnimate', []).run([
'$animate',
function ($animate) {
$animate.enabled(false);
}
]);
};
browser.addMockModule('disableNgAnimate', disableNgAnimate);
// see: https://github.com/angular/protractor/issues/60
require('jasmine-reporters');
var capsPromise = browser.getCapabilities();
capsPromise.then(function setOutputDir(caps) {
var browserName = caps.caps_.browserName.toUpperCase();
var browserVersion = caps.caps_.version;
var filePrefix = browserName + '-' + browserVersion + '-';
jasmine.getEnv().addReporter(
new jasmine.JUnitXmlReporter(
'build/reports/testresults',
true, //consolidate: save nested describes within the same file as their parent
true, //useDotNotation: separate suite names with dots rather than spaces (ie "Class.init" not "Class init")
filePrefix
)
);
});
},
jasmineNodeOpts: {
isVerbose: true,
showColors: true,
includeStackTrace: true,
defaultTimeoutInterval: 300000, // 5 min
}
};
Here is an extract of the essential parts of the Jenkins Job console log:
⋮
13:28:22 DEBUG - Running with --troubleshoot
13:28:22 DEBUG - Protractor version: 2.1.0
13:28:22 DEBUG - Your base url for tests is http://localhost:57505/
13:28:22 Using ChromeDriver directly...
13:28:22 [launcher] Running 1 instances of WebDriver
13:28:22 DEBUG - WebDriver session successfully started with capabilities { caps_:
13:28:22 { acceptSslCerts: true,
13:28:22 applicationCacheEnabled: false,
13:28:22 browserConnectionEnabled: false,
13:28:22 browserName: 'chrome',
13:28:22 chrome: { userDataDir: '/tmp/.com.google.Chrome.AuA2qr' },
13:28:22 cssSelectorsEnabled: true,
13:28:22 databaseEnabled: false,
13:28:22 handlesAlerts: true,
13:28:22 javascriptEnabled: true,
13:28:22 locationContextEnabled: true,
13:28:22 mobileEmulationEnabled: false,
13:28:22 nativeEvents: true,
13:28:22 platform: 'Linux',
13:28:22 rotatable: false,
13:28:22 takesHeapSnapshot: true,
13:28:22 takesScreenshot: true,
13:28:22 version: '44.0.2403.125',
13:28:22 webStorageEnabled: true } }
13:28:30 Testing foobar
13:28:30 Setup
13:28:30 should display foobar - pass
⋮ <--- more describe-s and it-s
13:28:35 Finished in 12.616 seconds
13:28:35 2 tests, 9 assertions, 0 failures
13:28:35
13:28:35 [launcher] 0 instance(s) of WebDriver still running
13:28:35 [launcher] chrome #1 passed
⋮ <--- other tests scenarios also run successfully
13:28:47 DEBUG - Running with --troubleshoot
13:28:47 DEBUG - Protractor version: 2.1.0
13:28:47 DEBUG - Your base url for tests is http://localhost:34976/
13:28:47 Using ChromeDriver directly...
13:28:47 [launcher] Running 1 instances of WebDriver <--- hanging from here,
can't figure out why,
wish I had logs : [
15:05:23 Xvfb stopping <--- canceled build manually,
see time difference, no timeouts triggered,
neither from Protractor nor from WebDriver : [
15:05:23 Build has been canceled
Note that it is not always the same tests scenario which leads to the hanging.
Same problems with node 0.10.33 as with 0.12.7.
Using Ubuntu 12.04.5 LTS (GNU/Linux 3.13.0-37-generic x86_64).
$ file /opt/google/chrome/chrome
/opt/google/chrome/chrome: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), for GNU/Linux 2.6.26, BuildID[sha1]=0xe4fc0257b42110fc269782274f1e774718135349, stripped
I opened that other issue. In my case, I was running protractor inside of Vagrant and had port 4444 forwarded in my Vagrantfile. protractor silently failed/hung in this scenario
I had the same issue but it got fixed when I ran webdriver-manager start in the foreground ie
webdriver-manager start &
protractor conf.js #hangs
but
webdriver-manager start
protractor conf.js #works
Oddly both styles seem to work now. I'm not sure but I think the nature of the tests may also determine whether protractor hangs or not. I had a test that was infinitely trying to reach a nonexistent location because I hadn't set the baseUrl. Protractor showed the same symptoms.
I'm getting hanging behaviour, running tests with sharding on Codeship CI.
I run the same tests with Firefox and Chrome - I've split these into 2 separate config files but other than the browser setting they're the same.
The process sometimes hangs at the end of the Chrome file, and sometimes at the end of the Firefox one (and, if I'm very lucky, sometimes doesn't hang :))
Consistently seems to be "1 instances" left running (rather than 2 or more).
It's run as a foreground process so no help from @caninemwenja 's comment...
I've got a similar issue, where it hangs at
Using ChromeDirver directly...
[launcher] Running 1 instances of WebDriver
about 50% of the time. It will hang there indefinitely. However, even if it hangs you can start other instances which may or may not succeed in the same manner.
I've experienced the issue both with and without directConnect so that's not part of the issue
Yeah, I've just confirmed that the same things happens with directConnect and standalone selenium.
Also seeing the problem, but only on our Ubuntu build servers. Haven't seen the issue yet on the Windows servers. Using Google Chrome 44.0.2403.130 and ChromeDriver 2.15.322448.
I don't know how significant it is, but its also our Ubuntu build servers which are affected. We have fedora, ubuntu, and OSX systems, but its only the Ubuntu servers which hang at the ChromeDriver start up.
Ubuntu for me too. Using Chromium 43.0.2357.130 and ChromeDriver 2.16. It works now though (with no apparent change).
We're on Ubuntu too (- using Codeship and their servers are based on Ubuntu Trusty)
We're on Ubuntu as well (updated issue description with system information "Ubuntu 12.04.5 LTS (GNU/Linux 3.13.0-37-generic x86_64)").
The ubuntu servers also use Xvfb (X Virtual Frame Buffer) to use the browser without a window manager.
Interesting; i've experienced some weird hang ups on Debian (wheezy) when using Xvfb. They _seemed_ to be due to the browser not being able to attach to the headless display. Is everyone here attempting to run things via Xvfb?
If you're having problems racing with Xvfb startup try using xdpyinfo to wait for X to be ready:
MAX=120 # About 60 seconds
CT=0
while ! xdpyinfo >/dev/null 2>&1; do
sleep 0.50s
CT=$(( CT + 1 ))
if [ "$CT" -ge "$MAX" ]; then
LOG "FATAL: $0: Gave up waiting for X server $DISPLAY"
exit 11
fi
done
LOG "X is available"
Also, you can get more logs from chromedriver itself. Those often have more details about what went wrong (but often don't have enough useful info, either). Often we have to look into the chrome debug logs, too. See http://stackoverflow.com/questions/31662828/how-to-access-chromedriver-logs-for-protractor-test for suggestions on how to get additional chromedriver logs.
Yup, using xvfb-run on ubuntu and (obviously) not using it on the windows build servers so that sounds like a definite possibility. @tullmann do you use your script in conjunction with xvfb-run or Xvfb?
We have put xvfb in our init scripts, so as far as I know it is running soon after startup. Unless there is something else that needs to be done to make sure that it's "Ready" to accept connections, then I think xvfb is "ready" in my case.
@tullmann We also have xvfb running from the beginning and some of the tests run fine before the hanging. But thanks for the chromedriver logs hint. Will try that as soon as possible.
I'm using windows server 2012. If I run webdriver-manager start in the foreground there is no problem. But if I run it in the background, similar "hanging behavior" will happen- about 1.5 minute for each browser.
I also try standalone selenium server so that I can skip the step webdriver-manager start. The test runs without problem if I start it locally. But if I remotely use psexec on other machine to run the test, it will run in background and the hanging behavior will happen again.
I managed to enable chrome driver logs (using this method).
Here is the result:
[0,014][INFO]: COMMAND InitSession {
"desiredCapabilities": {
"browserName": "chrome",
"count": 1,
"platform": "ANY",
"version": ""
}
}
[0,014][INFO]: Populating Preferences file: {
"alternate_error_pages": {
"enabled": false
},
"autofill": {
"enabled": false
},
"browser": {
"check_default_browser": false
},
"distribution": {
"import_bookmarks": false,
"import_history": false,
"import_search_engine": false,
"make_chrome_default_for_user": false,
"show_welcome_page": false,
"skip_first_run_ui": true
},
"dns_prefetching": {
"enabled": false
},
"profile": {
"content_settings": {
"pattern_pairs": {
"https://*,*": {
"media-stream": {
"audio": "Default",
"video": "Default"
}
}
}
},
"default_content_settings": {
"geolocation": 1,
"mouselock": 1,
"notifications": 1,
"popups": 1,
"ppapi-broker": 1
},
"password_manager_enabled": false
},
"safebrowsing": {
"enabled": false
},
"search": {
"suggest_enabled": false
},
"translate": {
"enabled": false
}
}
[0,014][INFO]: Populating Local State file: {
"background_mode": {
"enabled": false
},
"ssl": {
"rev_checking": {
"enabled": false
}
}
}
[0,015][INFO]: Launching chrome: /opt/google/chrome/google-chrome --disable-background-networking --disable-client-side-phishing-detection --disable-component-update --disable-default-apps --disable-hang-monitor --disable-prompt-on-repost --disable-sync --disable-web-resources --enable-logging --ignore-certificate-errors --load-extension=/tmp/.com.google.Chrome.NqAJ9w/internal --log-level=0 --metrics-recording-only --no-first-run --password-store=basic --remote-debugging-port=12199 --safebrowsing-disable-auto-update --safebrowsing-disable-download-protection --test-type=webdriver --use-mock-keychain --user-data-dir=/tmp/.com.google.Chrome.LqJPTu data:,
[0,016][DEBUG]: DevTools request: http://127.0.0.1:12199/json/version
[0,077][DEBUG]: DevTools request failed
[1:1:0909/154445:ERROR:image_metadata_extractor.cc(111)] Couldn't load libexif.
Xlib: extension "RANDR" missing on display ":2".
[0,127][DEBUG]: DevTools request: http://127.0.0.1:12199/json/version
[0,128][DEBUG]: DevTools request failed
[0,178][DEBUG]: DevTools request: http://127.0.0.1:12199/json/version
[0,179][DEBUG]: DevTools request failed
Xlib: extension "RANDR" missing on display ":2".
[0,229][DEBUG]: DevTools request: http://127.0.0.1:12199/json/version
[0,230][DEBUG]: DevTools request failed
[58787:58787:0909/154445:ERROR:sandbox_linux.cc(345)] InitializeSandbox() called with multiple threads in process gpu-process
[0,280][DEBUG]: DevTools request: http://127.0.0.1:12199/json/version
In my logs folder I count the occurences in all log files:
$ grep -nr "ERROR:sandbox_linux.cc(345)] InitializeSandbox() called with multiple threads in process gpu-process" . | wc -l
161
$ grep -nr "ERROR:image_metadata_extractor.cc(111)] Couldn't load libexif." . | wc -l
161
You can hide/avoid the "multiple threads in process gpu-process" by adding --disable-gpu to the chrome command line. Generally when running under XVFB, you're not leveraging a GPU anyway, so disabling it shouldn't be bad. Anyway, I believe this (and the RANDR and the libexif messages) are harmless messages and just distracting.
Its odd that chromedriver isn't timing out (it looks like you're waiting an hour or more?). You might get more useful information from the chrome logs about what's going on. (The "DevTools request" stuff is from chromedriver trying to establish a basic connection to chrome -- it just polls repeatedly until it gets a connection). It looks like chromedriver receives no reply, but doesn't timeout either ... might be worth comparing this log to a "normal" case to see what the differences are in your setup.)
To get more chrome debug logging add the following arguments to chrome's arguments (in your protractor config): enable-logging, v=1 and userDataDir=<somedir> where <somedir> is a new directory private to this run. (You can leave off userDataDir and chrome will pick a random directory in /tmp, but it can be annoying to figure out which one...)
Seems like enabling chrome logs fixes a race condition. The build job with chrome logging enabled refuses to hang while the build job without logging still hangs on a regular basis.
Also enabled logging and still had it hang, but the log has perhaps something useful at the end:
(google-chrome:13094): GConf-WARNING **: Client failed to connect to the D-BUS daemon:
//bin/dbus-launch terminated abnormally without any error message
[13132:13132:0929/003546:ERROR:sandbox_linux.cc(345)] InitializeSandbox() called with multiple threads in process gpu-process
The gpu-process issue has been discussed earlier, but could the d-bus message be related?
I'm hoping that this is fixed with the new version of chromedriver in Protractor 2.3.0 and higher. Can anyone confirm?
@juliemr we've had it running for a few days now and a few hundred builds on 2.4 and haven't seen it hang, so yes, that seems to have fixed it.
Whee! Closing - please open up a new issue if this crops up again.
I'm sorry for the late reply and also sorry for letting you know that the hanging persists with protractor 2.4. Shall I really open up a new issue even if we still do not know what is really the root cause of the problem?
I've also had hanging since 2.4. Oddly, I had had an extended period without any hanging before it happened a few times last week.
This issue still occurs. on protractor 2.5.1
I can confirm that this issues still occurs on most recent versions of Protractor (used versions corresponding to updates for chromedriver & webdriver).
I've been having this problem running on an Ubuntu CI environment with parallel machines ("containers") for each build. Recently ran with --troubleshoot and observed the following.
Output for a successful container:
DEBUG - Running with --troubleshoot
DEBUG - Protractor version: 1.8.0
DEBUG - Your base url for tests is undefined
Using ChromeDriver directly...
[launcher] Running 1 instances of WebDriver
DEBUG - WebDriver session successfully started with capabilities { caps_:
...
Output for a hanging container:
DEBUG - Running with --troubleshoot
DEBUG - Protractor version: 1.8.0
DEBUG - Your base url for tests is undefined
Using ChromeDriver directly...
[launcher] Running 1 instances of WebDriver command protractor protractor.conf.js --troubleshoot --suite=container_suite took more than 10 minutes since last output
It appears that a webdriver session never gets started. @juliemr, can this issue be reopened?
So we were able to ssh in to both a successful container and a hanging container simultaneously and view running processes. This is the output of running ps auxwf.
Successful container:
ubuntu 17473 2.0 0.0 85024 5988 ? S 15:51 0:26 | \_ sshd: ubuntu@pts/0
ubuntu 33618 0.0 0.0 14820 1508 pts/0 Ss+ 16:03 0:00 | \_ /bin/bash ./circle_scripts/test_override.sh
ubuntu 33934 6.7 0.0 728856 85596 pts/0 Rl+ 16:03 0:39 | \_ node /home/ubuntu/nvm/v0.10.33/bin/protractor protractor.conf.js --suite=container_suite
ubuntu 33938 1.8 0.0 378332 10212 pts/0 Sl+ 16:03 0:10 | \_ /home/ubuntu/nvm/v0.10.33/lib/node_modules/protractor/selenium/chromedriver --port=48603
ubuntu 33941 8.1 0.0 710400 90864 pts/0 Sl+ 16:03 0:47 | \_ /opt/google/chrome/chrome --disable-setuid-sandbox --disable-background-networking --disable-client-side-phishing-detection --disable-component-update --disable-default-apps --disable-hang-mo
ubuntu 33949 0.0 0.0 9664 620 pts/0 S+ 16:03 0:00 | \_ cat
ubuntu 33950 0.0 0.0 9664 616 pts/0 S+ 16:03 0:00 | \_ cat
ubuntu 33953 0.0 0.0 341532 28224 pts/0 S+ 16:03 0:00 | \_ /opt/google/chrome/chrome --type=zygote --enable-logging --log-level=0 --user-data-dir=/tmp/.com.google.Chrome.1GMsD3
ubuntu 33954 0.0 0.0 28012 1972 pts/0 S+ 16:03 0:00 | | \_ /opt/google/chrome/nacl_helper
ubuntu 33958 0.0 0.0 341532 8064 pts/0 S+ 16:03 0:00 | | \_ /opt/google/chrome/chrome --type=zygote --enable-logging --log-level=0 --user-data-dir=/tmp/.com.google.Chrome.1GMsD3
ubuntu 34001 36.0 0.1 938400 204744 pts/0 Rl+ 16:03 3:29 | | \_ /opt/google/chrome/chrome --type=renderer --enable-logging --log-level=0 --test-type=webdriver --lang=en-US --user-data-dir=/tmp/.com.google.Chrome.1GMsD3 --disable-client-side-ph
ubuntu 34025 0.0 0.0 761540 35692 pts/0 Sl+ 16:03 0:00 | | \_ /opt/google/chrome/chrome --type=renderer --enable-logging --log-level=0 --test-type=webdriver --lang=en-US --user-data-dir=/tmp/.com.google.Chrome.1GMsD3 --extension-process --en
ubuntu 33994 0.0 0.0 434308 36752 pts/0 Sl+ 16:03 0:00 | \_ /opt/google/chrome/chrome --type=gpu-process --channel=33941.0.931696333 --enable-logging --log-level=0 --user-data-dir=/tmp/.com.google.Chrome.1GMsD3 --supports-dual-gpus=false --gpu-dri
Hanging container:
ubuntu 4408 2.2 0.0 85472 7140 ? S 15:48 0:27 | \_ sshd: ubuntu@pts/0
ubuntu 20741 0.0 0.0 14820 1508 pts/0 Ss+ 15:59 0:00 | \_ /bin/bash ./circle_scripts/test_override.sh
ubuntu 21057 1.1 0.0 670188 36728 pts/0 Sl+ 15:59 0:06 | \_ node /home/ubuntu/nvm/v0.10.33/bin/protractor protractor.conf.js --suite=container_suite
ubuntu 21061 0.0 0.0 378208 6560 pts/0 Sl+ 15:59 0:00 | \_ /home/ubuntu/nvm/v0.10.33/lib/node_modules/protractor/selenium/chromedriver --port=56618
ubuntu 21064 0.0 0.0 556412 47968 pts/0 Sl+ 15:59 0:00 | \_ /opt/google/chrome/chrome --disable-setuid-sandbox --disable-background-networking --disable-client-side-phishing-detection --disable-component-update --disable-default-apps --disable-hang-mo
ubuntu 21072 0.0 0.0 9664 616 pts/0 S+ 15:59 0:00 | \_ cat
ubuntu 21073 0.0 0.0 9664 620 pts/0 S+ 15:59 0:00 | \_ cat
ubuntu 21076 0.0 0.0 341532 28224 pts/0 S+ 15:59 0:00 | \_ /opt/google/chrome/chrome --type=zygote --enable-logging --log-level=0 --user-data-dir=/tmp/.com.google.Chrome.1bhwCh
ubuntu 21077 0.0 0.0 28012 1964 pts/0 S+ 15:59 0:00 | | \_ /opt/google/chrome/nacl_helper
ubuntu 21080 0.0 0.0 341532 7764 pts/0 S+ 15:59 0:00 | | \_ /opt/google/chrome/chrome --type=zygote --enable-logging --log-level=0 --user-data-dir=/tmp/.com.google.Chrome.1bhwCh
ubuntu 21117 0.0 0.0 434308 36912 pts/0 Sl+ 15:59 0:00 | \_ /opt/google/chrome/chrome --type=gpu-process --channel=21064.0.776113428 --enable-logging --log-level=0 --user-data-dir=/tmp/.com.google.Chrome.1bhwCh --supports-dual-gpus=false --gpu-dri
ubuntu 21118 0.2 0.0 556412 13244 pts/0 S+ 15:59 0:01 | \_ /opt/google/chrome/chrome --disable-setuid-sandbox --disable-background-networking --disable-client-side-phishing-detection --disable-component-update --disable-default-apps --disable-han
The only discernible difference is the preference of chrome instances with type=renderer in the successful container.
If you're running into problems with chrome occasionally hanging at startup, its probably a Chrome bug. See https://code.google.com/p/chromium/issues/detail?id=309093 where Google ran into this problem in their chrome testing setup, and worked around it in their test infrastructure.
Basically, chrome is mulit-threaded, and relies on some standard gconf libraries, one of those libraries does a fork+exec to start up "dbus" if it is not already running. Doing a fork+exec in a multi-threaded application is bad because you will occasionally fork while a different thread has the malloc lock (or other critical lock) and the child will then deadlock when it tries to acquire that lock, and everything grinds to a halt. Generally on a desktop, dbus is already running, but in many stripped-down test environments dbus does not get started.
Our work-around is to make sure dbus is running by having our scripts launch that XVFB run its child processes via dbus-launch --exit-with-session. (We also have a script that polls for X to be ready before proceeding, that seems to have helped, but we're less confident its strictly necessary).
Here's a lightly modified version of the xvfb wrapper script we use that starts xvfb, dbus, and waits for X to be ready: https://gist.github.com/tullmann/2d8d38444c5e81a41b6d
And here's the waitForX script that depends on:
https://gist.github.com/tullmann/476cc71169295d5c3fe6
@tullmann How do you use those scripts? Do you start up xvfb before every run? Or do you have those as part of startup scripts? Are they just wrappers around Xvfb?
@Callmenorm We use the bb-xfvb script to wrap each call to the protractor script in a private XVFB instance. So if you normally run protractor foo.js, you can do bb-xvfb protractor foo.js to run it under an XVFB instance. (The script is just a wrapper around xvfb-run.)
If you're starting XVFB (or a real X server) in some other way (well before you get around to starting protractor), you will want to use the "dbus-launch --exit-with-session" and/or waitForX scripts as necessary in your environment.
Thanks @tullmann! We will give it a try in the next sprint. Update: Yep, the scripts seem to do the trick.
Anyway, protractor should timeout in case of a deadlock during dbus startup.
Thank you @tullmann !
I have just integrated your scripts as part of our testing on Codeship. It's now run once, and for the first time in ages all of the tests finished :)
I'll monitor it over the coming week and will shout if any issues, but in the meantime: THANK YOU :)
Likewise @tullmann's scripts have solved our CI builds timing out randomly. Thanks!
@tullmann, you're a hero.
we also have problems with hanging chrome browsers, running in docker containers, which are used as ci-agents (teamcity). When the container is started, it also starts xvfb as a service and runs for several days.
Now Protractor starts Chrome himself with the "--directConnect=true" option and also starts several browsers in a single test. Some builds run smoothly, some hang indefinitly.
@tullmann any idea how i can integrate your scripts?
@Sabartius make sure when you start xvfb that you also start dbus. I don't think my scripts are specifically useful in your scenario, so you'll need to figure out a different way to make sure dbus is running. Its probably as simple as just having your container run dbus-launch in the right place (see the man page for more details).
Aww - after 2 weeks without an issue, my last 3 builds have hung.
Haven't yet had time to investigate why. Anyone else seen this @tullmann @jas13 @Callmenorm @borisdiakur ?
Seems to have fixed itself at some point this morning... very strange
@tullmann How do you get (install) dbus-launch? I have installed dbus but there isn't a dbus-launch command there. I'm using debian:jessie based docker container.
Looks like its part of the dbus-x11 package:
# apt-cache search dbus-launch
dbus-x11 - simple interprocess messaging system (X11 deps)
Not sure this is chrome specific. I have the same issue when running against firefox.
It probably isn't because sometimes it hangs even when using remote selenium grid.
I'm currently running protractor version 3.0.0, and have been seeing hanging builds specifically when running chrome (with directConnect) on Xvfb. I enabled chromedriver logs and chrome logs using the methods mentioned in this thread earlier.
I see this at the end of chrome's logs:
[9504:9504:0419/185420:ERROR:sandbox_linux.cc(338)] InitializeSandbox() called with multiple threads in process gpu-process
[9504:9504:0419/213227:ERROR:x11_util.cc(82)] X IO error received (X server probably went away)
[9407:9461:0419/213227:WARNING:channel.cc(358)] RawChannel write error
And this at the end of chromedriver's logs:
[9504:9504:0419/185420:ERROR:sandbox_linux.cc(338)] InitializeSandbox() called with multiple threads in process gpu-process
[9504:9504:0419/213227:ERROR:x11_util.cc(82)] X IO error received (X server probably went away)
We start Xvfb at the beginning of our test suite before running four end-to-end test suites - two with Firefox and two with Chrome. Often a build will have already run a few of these suites (maybe even a chrome suite) before hanging. The hanging is not as often as others report though - in my last test, one out of ten repeated builds got hung.
I'll report back on this thread after trying to run the test suites with --disable-gpu. I'll also try to locate Xvfb logs to see if something went wrong with it.
So.. reporting back:
Running with --disable-gpu does not fix the hanging. Also, we're running Xvfb using a simple wrapper around it with pyvirtualdisplay, and getting hold of Xvfb's output would have meant not using that wrapper, so I abandoned that approach.
I went on to run the tests inside a dbus-launch --exit-with-session wrapper, and it worked out all great.
I also came across this bug report: SeleniumHQ/docker-selenium#87. It seems to indicate that simply setting DBUS_SESSION_BUS_ADDRESS to /dev/null should prevent chrome from hanging - I'll test that approach too and report back.
It looks like setting DBUS_SESSION_BUS_ADDRESS to /dev/null alone is sufficient to prevent chrome from deadlocking. I ran a whole bunch of iterations of my test suite, and not a single hang.
The solution of @jrharshath worked for me. Thank you very much!
Wow! Thanks!
I set the environment variable with export DBUS_SESSION_BUS_ADDRESS=/dev/null and no longer have any hanging chromedriver processes!
I ran one file that had an empty it statement with 12 concurrent browser sessions, and looped it 6000 times. After almost 2 hours of continuously running, there was not a single hanging process.
[12:20:33] Protractor ran 6000 files (6000 specs) -- 6000 passed
[12:20:33]
[12:20:33] Finished 'protractor' after 1.92 h
Thanks a ton @jrharshath!
@cnishina the problem for Protractor here is that we should time out, instead of hanging, with a reasonable error, if chrome can't start up. Can you take this one?
Hi everyone! I plan to look into the hanging error and improve error reporting.
On a side note, I just created a Jenkins server this weekend to try out a few Protractor tests. It appears there are Xvfb issues. If you are using Jenkins, there is an Xvfb plugin. This is my setup:
/usr/bin/webdriver-manager updatenpm installnpm testConsole output snippets:
Xvfb starting$
+ npm install
npm WARN deprecated [email protected]: Please update to minimatch 3.0.2 or higher to avoid a RegExp DoS issue
npm WARN [email protected] No repository field.
+ npm test
> [email protected] test /var/lib/jenkins/workspace/protractor-cookbook-jasmine-junit-reports/tests
> jasmine
Started
> [email protected] pretest /var/lib/jenkins/workspace/protractor-cookbook-jasmine-junit-reports/jasmine-junit-reports
> webdriver-manager update
[11:06:07] I/downloader - selenium standalone: downloading version 2.53.1
[11:06:07] I/downloader - curl -o /var/lib/jenkins/workspace/protractor-cookbook-jasmine-junit-reports/jasmine-junit-reports/node_modules/protractor/node_modules/webdriver-manager/selenium/selenium-server-standalone-2.53.1.jar https://selenium-release.storage.googleapis.com/2.53/selenium-server-standalone-2.53.1.jar
[11:06:07] I/downloader - chromedriver: downloading version 2.22
[11:06:07] I/downloader - curl -o /var/lib/jenkins/workspace/protractor-cookbook-jasmine-junit-reports/jasmine-junit-reports/node_modules/protractor/node_modules/webdriver-manager/selenium/chromedriver_2.22linux64.zip https://chromedriver.storage.googleapis.com/2.22/chromedriver_linux64.zip
[11:06:07] I/update - chromedriver: unzipping /var/lib/jenkins/workspace/protractor-cookbook-jasmine-junit-reports/jasmine-junit-reports/node_modules/protractor/node_modules/webdriver-manager/selenium/chromedriver_2.22linux64.zip
[11:06:07] I/update - chromedriver: setting permissions to 0755 for /var/lib/jenkins/workspace/protractor-cookbook-jasmine-junit-reports/jasmine-junit-reports/node_modules/protractor/node_modules/webdriver-manager/selenium/chromedriver_2.22
> [email protected] test /var/lib/jenkins/workspace/protractor-cookbook-jasmine-junit-reports/jasmine-junit-reports
> protractor conf.js
[11:06:09] I/direct - Using ChromeDriver directly...
[11:06:09] I/launcher - Running 1 instances of WebDriver
Started
.F.F
4 specs, 2 failures
Finished in 5.971 seconds
[11:06:15] I/launcher - 0 instance(s) of WebDriver still running
[11:06:15] I/launcher - chrome #01 failed 2 test(s)
[11:06:15] I/launcher - overall: 2 failed spec(s)
[11:06:15] E/launcher - Process exited with error code 1
Xvfb stopping
I'm facing the same issue on Windows. When I try to debug my tests from VSCode Protractor hangs forever at
I/launcher - Running 4 instances of WebDriver
My VSCode launch config runs a powershell script starting the SUT and selenium server in separate processes (using the Start-Process cmdlet). Only after both respond with HTTP 200 the launch config invokes protractor. VSCode debug setup is similar to that of Webstorm as described in the protractor docs. VSCode eventually invokes protractor like follows:
node --debug-brk=<randomPort> node_mdoules\protractor\bin\protractor src\test\protractorConf.js
I also tried using node_modules\protractor\built\cli.js but got the same result. Experimenting with protractor's --debug, debug and --debuggingServer command line options also got me nowhere.
Simply executing the tests from gulp (using gulp-angular-protractor) works fine though.
The only difference I can see between the two (apart from debugger config) is that when running via gulp selenium, browsersync and protractor share standardInput/Output, while in debug mode they are separate. But that wouldn't make a difference, would it?
I'm facing a similar problem on Windows 7 without xvfb. Tests will run but the process never terminate:
tsc && concurrently \"http-server -s\" \"protractor protractor.config.js\"
[1] [14:24:54] I/local - Starting selenium standalone server...
[1] [14:24:54] I/launcher - Running 1 instances of WebDriver
[1] [14:24:55] I/local - Selenium standalone server started at http://10.153.242.35:57125/wd/hub
[1] AppDir: ./
[1] Suite: QuickStart E2E Tests
[1] passed - should display: My First Angular 2 App
[1] Suite passed: QuickStart E2E Tests
[1] [14:25:01] I/local - Shutting down selenium standalone server.
[1] [14:25:01] I/launcher - 0 instance(s) of WebDriver still running
[1] [14:25:01] I/launcher - phantomjs #01 passed
[1] protractor protractor.config.js exited with code 0
If I use the following command (copied from the angular quickstart):
tsc && concurrently \"http-server -s\" \"protractor protractor.config.js\" --kill-others --success first
[1] AppDir: ./
[1] Suite: QuickStart E2E Tests
[1] passed - should display: My First Angular 2 App
[1] Suite passed: QuickStart E2E Tests
[1] [14:30:48] I/local - Shutting down selenium standalone server.
[1] [14:30:48] I/launcher - 0 instance(s) of WebDriver still running
[1] [14:30:48] I/launcher - phantomjs #01 passed
[1] protractor protractor.config.js exited with code 0
--> Sending SIGTERM to other processes..
It will hang at the last command.
I got the same issue on running protractor tests in gitlab CI via docker executor. At the beginning tests run fast and everything is good but after some period of time (30-40 min) tests become extremely slow and protractor hangs out.
First it fails with Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL and then WebDriverError: no such sessionbecause of page crush.
I use official docker container openjdk:7 and install node, chrome, xvfb and set DBUS_SESSION_BUS_ADDRESS =/dev/null. Our application ( on play framework) is also run in this container. Docker runs with privileged mode and volumes /dev/shm:/dev/shm
Xvfb :21 -screen 0 1920x1280x24 &
export DISPLAY=:21.0
grunt test
Protractor 4.0.9, chrome 53, chromederiver 2.24
logs: chromedriver.txt
We also had this issue and I can confirm that doing export DBUS_SESSION_BUS_ADDRESS=/dev/null helps 👍
Protractor 5.0.0
Chromedriver 2.26
Ubuntu 16.04
Docker 1.13.0
We have the protractor hanging after
14:50:50 [14:50:48] I/launcher - Running 1 instances of WebDriver
14:50:50 [14:50:48] I/hosted - Using the selenium server at http://localhost:7840/wd/hub
even when using the DBUS_SESSION_BUS_ADDRESS workaround. It happens about 20-30% of the cases in our Jenkins builds (even on the very same Jenkins slave).
node.js 6.9.5
protractor 5.1.1
chromedriver 2.28
selenium 3.1.1
webdriver-manager 12.0.2
chrome 56.0.2924.87
(we have fixed these versions from constantly changing and potentially breaking things)
Jenkins slaves are running on CentOS 7
The build does not use docker (yet).
protractor is started by a npm script (in package.json):
"ci-e2e": "bash -c \"set -o pipefail ; export DBUS_SESSION_BUS_ADDRESS=/dev/null ; protractor protractor-conf.js --seleniumAddress http://localhost:${npm_package_config_seleniumPort}/wd/hub 2>&1 | tee .ci/protractor.log\"",
(of course there are pre and post scripts as well to start/stop things)
chromedriver logs are enabled by the following shell script line (run by the pre npm script):
nohup webdriver-manager start --versions.standalone $SELENIUM_STANDALONE_EXPECTED_VERSION --versions.chrome $SELENIUM_CHROMEDRIVER_EXPECTED_VERSION --chrome_logs ${THIS_SCRIPT_DIR}/../.ci/chromedriver.log --seleniumPort $SELENIUM_PORT >.ci/webdriver-manager_start.log 2>&1 &
(there is a 5 seconds sleep after that)
Xvfb is started before webdriver in a similar style (nohup + background)
I just checked the chromedriver log and it shows:
[0.096][INFO]: COMMAND InitSession {
"alwaysMatch": {
"count": 1,
"platform": "ANY",
"version": "ANY"
},
"capabilities": {
"desiredCapabilities": {
"browserName": "chrome",
"chromeOptions": {
"args": [ "--no-sandbox", "--test-type=browser" ],
"prefs": {
"download": {
"default_directory": "/tmp/",
"prompt_for_download": false
}
}
},
"count": 1,
"platform": "ANY",
"version": "ANY"
},
"requiredCapabilities": {
}
},
"desiredCapabilities": {
"browserName": "chrome",
"chromeOptions": {
"args": [ "--no-sandbox", "--test-type=browser" ],
"prefs": {
"download": {
"default_directory": "/tmp/",
"prompt_for_download": false
}
}
},
"count": 1,
"platform": "ANY",
"version": "ANY"
},
"firstMatch": [ {
"browserName": "chrome",
"chromeOptions": {
"args": [ "--no-sandbox", "--test-type=browser" ],
"prefs": {
"download": {
"default_directory": "/tmp/",
"prompt_for_download": false
}
}
}
} ],
"requiredCapabilities": {
}
}
[0.099][INFO]: Populating Preferences file: {
"alternate_error_pages": {
"enabled": false
},
"autofill": {
"enabled": false
},
"browser": {
"check_default_browser": false
},
"distribution": {
"import_bookmarks": false,
"import_history": false,
"import_search_engine": false,
"make_chrome_default_for_user": false,
"show_welcome_page": false,
"skip_first_run_ui": true
},
"dns_prefetching": {
"enabled": false
},
"download": {
"default_directory": "/tmp/",
"prompt_for_download": false
},
"profile": {
"content_settings": {
"pattern_pairs": {
"https://*,*": {
"media-stream": {
"audio": "Default",
"video": "Default"
}
}
}
},
"default_content_setting_values": {
"geolocation": 1
},
"default_content_settings": {
"geolocation": 1,
"mouselock": 1,
"notifications": 1,
"popups": 1,
"ppapi-broker": 1
},
"password_manager_enabled": false
},
"safebrowsing": {
"enabled": false
},
"search": {
"suggest_enabled": false
},
"translate": {
"enabled": false
}
}
[0.100][INFO]: Populating Local State file: {
"background_mode": {
"enabled": false
},
"ssl": {
"rev_checking": {
"enabled": false
}
}
}
[0.107][INFO]: Launching chrome: /opt/google/chrome/google-chrome --disable-background-networking --disable-client-side-phishing-detection --disable-default-apps --disable-hang-monitor --disable-popup-blocking --disable-prompt-on-repost --disable-sync --disable-web-resources --enable-automation --enable-logging --ignore-certificate-errors --load-extension=/tmp/.org.chromium.Chromium.ekVmsH/internal --log-level=0 --metrics-recording-only --no-first-run --no-sandbox --password-store=basic --remote-debugging-port=12521 --safebrowsing-disable-auto-update --test-type=browser --use-mock-keychain --user-data-dir=/tmp/.org.chromium.Chromium.6KPDEf data:,
Does anyone know how to track this down or how can I provide more debugging information (when it happens)?
Is there some more specific bug report for this somewhere?
Could updating any of the above SW components (e.g. chrome, chrome driver) help?
I updated chromedriver to 2.29 and chrome to 57. It still happened.
I realized that --enable-logging is passed to chrome (https://www.chromium.org/for-testers/enable-logging) therefore I checked chrome_debug.log in its user data dir (also passed to chrome).
[7072:7072:0406/121917.369242:WARNING:audio_manager.cc(321)] Multiple instances of AudioManager detected
[7072:7072:0406/121917.369280:WARNING:audio_manager.cc(278)] Multiple instances of AudioManager detected
[7146:7146:0406/121917.570450:ERROR:gl_surface_glx.cc(406)] glxQueryVersion failed
[7146:7146:0406/121917.570506:ERROR:gl_initializer_x11.cc(136)] GLSurfaceGLX::InitializeOneOff failed.
[7146:7146:0406/121917.572327:ERROR:child_thread_impl.cc(762)] Request for unknown Channel-associated interface: ui::mojom::GpuMain
[7146:7146:0406/121917.572698:ERROR:gpu_child_thread.cc(330)] Exiting GPU process due to errors during initialization
However on successful runs this also happens and the log continues:
[32566:32598:0406/144120.367781:ERROR:browser_gpu_channel_host_factory.cc(113)] Failed to launch GPU process.
[32566:32598:0406/144120.367832:ERROR:browser_gpu_channel_host_factory.cc(113)] Failed to launch GPU process.
[32566:32598:0406/144120.367863:ERROR:browser_gpu_channel_host_factory.cc(113)] Failed to launch GPU process.
[32566:32598:0406/144120.367888:ERROR:browser_gpu_channel_host_factory.cc(113)] Failed to launch GPU process.
[32566:32598:0406/144120.367912:ERROR:browser_gpu_channel_host_factory.cc(113)] Failed to launch GPU process.
(then the logs from the tests follow)
It seems that https://bugs.chromium.org/p/chromedriver/issues/detail?id=1298 is the related bug report, which was closed without any real solution.
I have this random issue too.
npm logs:
> NODE_ENV=e2e node_modules/.bin/protractor protractor.others.conf.js
[10:28:22] I/hosted - Using the selenium server at http://selenium-52640-others:4444/wd/hub
[10:28:22] I/launcher - Running 1 instances of WebDriver
ERROR: Job failed: execution took longer than 20m0s seconds
Selenimu logs:
31/05/2017 10:28:0810:28:08.341 INFO - Selenium Server is up and running
31/05/2017 10:28:2210:28:22.780 INFO - SessionCleaner initialized with insideBrowserTimeout 0 and clientGoneTimeout 1800000 polling every 180000
31/05/2017 10:28:2210:28:22.815 INFO - Executing: [new session: Capabilities [{count=1, browserName=chrome, chromeOptions={prefs={download={directory_upgrade=true, default_directory=/tmp/, prompt_for_download=false}}}, version=ANY, platform=ANY}]])
31/05/2017 10:28:2210:28:22.836 INFO - Creating a new session for Capabilities [{count=1, browserName=chrome, chromeOptions={prefs={download={directory_upgrade=true, default_directory=/tmp/, prompt_for_download=false}}}, version=ANY, platform=ANY}]
31/05/2017 10:28:23Starting ChromeDriver 2.27.440175 (9bc1d90b8bfa4dd181fbbf769a5eb5e575574320) on port 4676
31/05/2017 10:28:23Only local connections are allowed.
Then nothing until timeout.
Great news for everyone in the thread: from version 59 Chrome will be supporting headless mode, so xvfb will not be needed anymore. Details are here: https://developers.google.com/web/updates/2017/04/headless-chrome
Hm, the issue with headless chrome (v. 59) is, that sendkey interactions are not supported (you can not enter data into an input field 👎 )
See https://bugs.chromium.org/p/chromedriver/issues/detail?id=1772
I am getting similar issue where 1 instance of webdriver remains forever until I abort the test run in jenkins. Protractor hangs at
I/launcher - 1 instance(s) of WebDriver still running
Setup (protractor | headless chrome | docker | jenkins)
protractor 5.3.2
chromedriverVersion: '2.40.565383'
chrome 'version' => '68.0.3440.75'
platform=Linux 3.10.0-862.3.2.el7.x86_64 x86_64)
Docker image :
FROM node:9-stretch
RUN apt-get update && \
DEBIAN_FRONTEND="noninteractive" \
apt-get install -y --no-install-recommends \
chromium\
dbus-x11\
dumb-init\
libgconf-2-4 \
openjdk-8-jre-headless \
&& rm -rf /var/lib/apt/lists/*
USER node
ENV CHROME_BIN /usr/bin/chromium
ENV DBUS_SESSION_BUS_ADDRESS /dev/null
protractor.conf.js
directConnect: true,
capabilities: {
shardTestFiles: true,
maxInstances: 6,
'browserName': 'chrome',
'chromeOptions': {
args: ['no-sandbox','headless','disable-gpu','window-size=1366,768','enable-logging' ,'v=1','disable-dev-shm-usage'],
}
},
Tried using : ENV DBUS_SESSION_BUS_ADDRESS /dev/null as suggested in docker file but error still exist.
Any progress with this issue? I am using protractor 5.4.0 with "protractor-cucumber-framework": "6.0.0", and if some test fails in parallel mode, instance of webdriver is not shut down and hangs indefinitely.
seeing this as well
Version 5.4.0
Google Chrome 69.0.3497.100
'--disable-infobars',
'--disable-notifications',
'--window-size=1440,900',
'--headless',
'--no-sandbox',
'--disable-gpu',
'--enable-logging',
'v=1'
Also seeing this. At first it would only happen intermittently and could be fixed with a restart, now it seems to happen every time.
Protractor 5.4.1
Chrome 69.0.3497.100
/** Protractor configuration */
exports.config = {
...
capabilities: {
browserName: 'chrome',
loggingPrefs: {
browser: 'SEVERE',
},
chromeOptions: {
args: [
'--allow-running-insecure-content',
'--use-fake-device-for-media-stream',
'--use-fake-ui-for-media-stream',
'--window-size=1920,1080',
'--enable-gpu-rasterization',
'--enable-impl-side-painting',
'--no-first-run',
'--no-sandbox',
'--incognito',
'--headless',
'--disable-infobars',
'--disable-session-crashed-bubble',
'--noerrdialogs',
'--disable-touch-adjustment',
'--disable-touch-events',
'--disable-pinch',
'--overscroll-history-navigation=0',
'--disable-setuid-sandbox',
'--js-flags="--expose-gc"',
]
}
},
directConnect: true,
framework: 'jasmine',
getPageTimeout: 5000,
allScriptsTimeout: 5000,
...
};
(https://github.com/angular/protractor/issues/4966)
Same issue when running parallel tests. Commenting onComplete solved it for us.
I am still facing same error.
I am not using onComplete
below is my config:
`
exports.config = {
framework: 'jasmine2',
getPageTimeout: pageLoadTimeout,
allScriptsTimeout: allScriptsTimeout,
customerCode:customerCode,
useAllAngular2AppRoots: true,
testData:webpackHelper.getTestData(),
//directConnect: true, // this runs selenium server on the fly and it has faster execution + parallel execution efficiently
//and tests are more stable with local server started instead of directConnection.
baseUrl: 'http://localhost:3000',
capabilities: {
browserName: 'chrome',
shardTestFiles: true,
maxInstances: 5,
chromeOptions: {
args: [
'disable-extensions',
'disable-web-security',
'--start-fullscreen', // enable for Mac OS
'--headless', // start on background
'--disable-gpu',
'--window-size=2880,1800'
]
},
'moz:firefoxOptions': {
args: ['--headless']
}
},
jasmineNodeOpts: {
defaultTimeoutInterval: extendedDefaultTimeoutInterval,
isVerbose: true,
showTiming: true,
includeStackTrace: true,
realtimeFailure: true,
showColors: true
},
suites: {
regression: [
// login logout tests
testBaseDir + 'login.test.js',
],
/**
* This suite is for development environment and always all dev tests will be executed.
*/
development: [
testBaseDir + 'dev1.js',
testBaseDir + 'dev2.js'
]
},
onCleanUp: function (results) {
retry.onCleanUp(results);
},
onPrepare() {
retry.onPrepare();
//console.log('Running instance at '+ new Date());
jasmine.getEnv().addReporter(new SpecReporter({
displayStacktrace: true,
displaySpecDuration: true,
displaySuiteNumber: true
}));
browser.manage().timeouts().pageLoadTimeout(pageLoadTimeout);
browser.manage().timeouts().implicitlyWait(2000);
let jasmineReporters = require('jasmine-reporters');
let junitReporter = new jasmineReporters.JUnitXmlReporter({
savePath: protractorPath,
// conslidate all true:
// output/junitresults.xml
//
// conslidate all set to false:
// output/junitresults-example1.xml
// output/junitresults-example2.xml
consolidateAll: true
});
jasmine.getEnv().addReporter(new JSONReporter({
file: 'target/jasmine-results.json', // by default it writes to jasmine.json
beautify: true,
indentationLevel: 4 // used if beautify === true
}));
jasmine.getEnv().addReporter(junitReporter);
var AllureReporter = require('jasmine-allure-reporter');
jasmine.getEnv().addReporter(new AllureReporter({
resultsDir: 'target/allure-results'
}));
jasmine.getEnv().afterEach(function(done){
browser.takeScreenshot().then(function (png) {
allure.createAttachment('Screenshot', function () {
return new Buffer(png, 'base64')
}, 'image/png')();
done();
})
});
//browser.driver.manage().window().maximize(); // disable for Mac OS
browser.get(browser.baseUrl);
return browser.wait(() => {
return browser.getCurrentUrl().then(url => {
return /login/.test(url);
});
}, pageResolveTimeout);
},
beforeLaunch: function () {
},
afterLaunch: function() {
return retry.afterLaunch(maxRetryForFailedTests);
}
};
`
Please help me , I am stuck
Guys & girls, seems like I have just found a workaroung, not very nice but at least working:
webdriver-manager clean
webdriver-manager update
and you are ready to go!
Most helpful comment
It looks like setting
DBUS_SESSION_BUS_ADDRESSto/dev/nullalone is sufficient to prevent chrome from deadlocking. I ran a whole bunch of iterations of my test suite, and not a single hang.