Chromedp: There's currently no way to wait for a condition except with WaitFunc and a selector

Created on 7 Aug 2020 · 4Comments · Source: chromedp/chromedp

What I want is to evaluate some JS repeatedly until it returns the expected result. There seems to be no other way to do that but with a clumsy code like this, using WaitFunc and a random selector:

chromedp.Query("random-selector",
    func(s *chromedp.Selector) {
        chromedp.WaitFunc(func(ctx context.Context, cur *cdp.Frame, ids ...cdp.NodeID) ([]*cdp.Node, error) {
            if len(ids) == 0 {
                return nil, nil
            }
            var result string
            if err := chromedp.Evaluate(someJavascript, &result).Do(ctx); err != nil {
                return nil, err
            } else if result != whatWeAreWaitingFor {
                return nil, nil
            }
            return []*cdp.Node{}, nil
        })(s)
    },
    chromedp.AtLeast(0),
)

It would be great if one could wait for a JS condition using a simpler action.

Source

opennota

😕1

Most helpful comment

FWIW, puppeteer has waitForFunction:

https://devdocs.io/puppeteer/index#pagewaitforfunctionpagefunction-options-args

opennota on 7 Aug 2020

👍2

All 4 comments

FWIW, puppeteer has waitForFunction:

https://devdocs.io/puppeteer/index#pagewaitforfunctionpagefunction-options-args

opennota on 7 Aug 2020

👍2

There should be something like the puppeteer API

yoyo00xx on 6 Mar 2021

@opennota It seems that you just want the wait part of chromedp.Query. It's implemented like this:

https://github.com/chromedp/chromedp/blob/e2970556e3d05f3259c464faeed1ec0e862f0560/query.go#L161-L166

Borrow this implementation, your code can be changed to (I have extended it to be runnable, and it's based on chromedp v0.6.5):

package main

import (
    "context"
    "fmt"
    "log"
    "net/http"
    "net/http/httptest"
    "time"

    "github.com/chromedp/chromedp"
)

func main() {
    ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        fmt.Fprint(w, `
<html>
<script>
let v = 0;
setTimeout(()=>{v = 100}, 1000);
</script>
<body>
hello world
</body>
</html>
`)
    }))
    defer ts.Close()

    ctx, cancel := chromedp.NewContext(context.Background(), chromedp.WithDebugf(log.Printf))
    defer cancel()
    ctx, cancel = context.WithTimeout(ctx, 5*time.Second)
    defer cancel()

    if err := chromedp.Run(ctx,
        chromedp.Navigate(ts.URL),
        chromedp.ActionFunc(func(ctx context.Context) error {
            for {
                select {
                case <-ctx.Done():
                    return ctx.Err()
                // you can customize the interval below
                case <-time.After(5 * time.Millisecond):
                }
                var result bool
                err := chromedp.Evaluate(`v === 100`, &result).Do(ctx)
                if err != nil {
                    return err
                }
                if result {
                    return nil
                }
            }
        }),
    ); err != nil {
        log.Fatal(err)
    }
}

The problem of this implementation is that it involves many roundtrips on the WebSocket connection. I have looked into the implementation of https://devdocs.io/puppeteer/index#pagewaitforfunctionpagefunction-options-args, and found that the wait part is totally happens in the browser, and there is just one roundtrip on the WebSocket connection.

The following implementation is borrowed from https://devdocs.io/puppeteer/index#pagewaitforfunctionpagefunction-options-args (It uses Runtime.callFunctionOn and requires an executionContextId, which is not exported by chromedp. We will use chrome.Query to get the executionContextId):

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "net/http"
    "net/http/httptest"

    "github.com/chromedp/cdproto/cdp"
    "github.com/chromedp/cdproto/runtime"
    "github.com/chromedp/chromedp"
)

func main() {
    ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        fmt.Fprint(w, `
<html>
<script>
let v = 0;
setTimeout(()=>{v = 100}, 3000);
</script>
<body>
hello world
</body>
</html>
`)
    }))
    defer ts.Close()

    ctx, cancel := chromedp.NewContext(context.Background(), chromedp.WithDebugf(log.Printf))
    defer cancel()

    // copied from puppeteer
    waitForPredicatePageFunction := `async function waitForPredicatePageFunction(predicateBody, polling, timeout, ...args) {
    const predicate = new Function('...args', predicateBody);
    let timedOut = false;
    if (timeout)
        setTimeout(() => (timedOut = true), timeout);
    if (polling === 'raf')
        return await pollRaf();
    if (polling === 'mutation')
        return await pollMutation();
    if (typeof polling === 'number')
        return await pollInterval(polling);
    /**
     * @returns {!Promise<*>}
     */
    async function pollMutation() {
        const success = await predicate(...args);
        if (success)
            return Promise.resolve(success);
        let fulfill;
        const result = new Promise((x) => (fulfill = x));
        const observer = new MutationObserver(async () => {
            if (timedOut) {
                observer.disconnect();
                fulfill();
            }
            const success = await predicate(...args);
            if (success) {
                observer.disconnect();
                fulfill(success);
            }
        });
        observer.observe(document, {
            childList: true,
            subtree: true,
            attributes: true,
        });
        return result;
    }
    async function pollRaf() {
        let fulfill;
        const result = new Promise((x) => (fulfill = x));
        await onRaf();
        return result;
        async function onRaf() {
            if (timedOut) {
                fulfill();
                return;
            }
            const success = await predicate(...args);
            if (success)
                fulfill(success);
            else
                requestAnimationFrame(onRaf);
        }
    }
    async function pollInterval(pollInterval) {
        let fulfill;
        const result = new Promise((x) => (fulfill = x));
        await onTimeout();
        return result;
        async function onTimeout() {
            if (timedOut) {
                fulfill();
                return;
            }
            const success = await predicate(...args);
            if (success)
                fulfill(success);
            else
                setTimeout(onTimeout, pollInterval);
        }
    }
}
`

    var exeCtxID runtime.ExecutionContextID
    // run
    if err := chromedp.Run(ctx,
        chromedp.Navigate(ts.URL),
        // to get the runtime.ExecutionContextID
        chromedp.Query("random-selector",
            func(s *chromedp.Selector) {
                chromedp.WaitFunc(func(ctx context.Context, cur *cdp.Frame, id runtime.ExecutionContextID, ids ...cdp.NodeID) ([]*cdp.Node, error) {
                    exeCtxID = id
                    return []*cdp.Node{}, nil
                })(s)
            },
            chromedp.AtLeast(0),
        ),
        chromedp.ActionFunc(func(ctx context.Context) error {
            predicateBody, err := json.Marshal(`return (()=>v === 100)(...args);`)
            if err != nil {
                return err
            }
            timeout, err := json.Marshal(5000)
            if err != nil {
                return err
            }
            polling, err := json.Marshal("raf")
            if err != nil {
                return err
            }
            // I just ignore the result below. It should parsed to check whether it's timeout
            _, _, err = runtime.CallFunctionOn(waitForPredicatePageFunction).
                WithExecutionContextID(exeCtxID).
                WithReturnByValue(false).
                WithAwaitPromise(true).
                WithUserGesture(true).
                WithArguments([]*runtime.CallArgument{
                    {Value: predicateBody},
                    {Value: polling},
                    {Value: timeout},
                }).Do(ctx)
            return err
        }),
    ); err != nil {
        log.Fatal(err)
    }
}