Colly: csrfAttackDetected on DZone

[luckylittle]

Hi,

I am trying to create scraper for http://dzone.com and i am having issues logging in. For simplicity, these are the steps that i am doing:

1. Go to https://dzone.com/users/login.html and extract form[role=form] input[type=hidden][name=TH_CSRF], for example 8195347781222493544.
2. On the same page, i post {"TH_CSRF": <TH_CSRF>, "_spring_security_remember_me": "true", "j_username": "<USERNAME>", "j_password": "<PASSWORD>"}.
3. I get redirected to https://dzone.com/static/csrfAttackDetected.html and everything after that fails.

Note: I tried with one collector as well as cloned collector.

Please advise!

[luckylittle]

Code:

package main

import (
    "log"

    "github.com/gocolly/colly"
    "github.com/gocolly/colly/debug"
)

/*
Simple workflow for testing the procedure of logging on to Dzone.com and downloading the random refcardz PDF file #279342.
*/

// variable for TH_CSRF token
var thCsrf string

func main() {

    // initiate a new collector
    c := colly.NewCollector(colly.Debugger(&debug.LogDebugger{}))

    // extract TH_CSRF token for the session
    c.OnHTML("form[role=form] input[type=hidden][name=TH_CSRF]", func(e *colly.HTMLElement) {
        thCsrf := e.Attr("value")
        log.Println(thCsrf)
        return
    })

    // attach callbacks for /users/login.html
    c.OnResponse(func(r *colly.Response) {
        log.Println("TH_CSRF response received", r.StatusCode, r.Request)
    })

    // visit the login page to extract the TH_CSRF token
    c.Visit("https://dzone.com/users/login.html")

    // initiate clone of "c" collector
    d := c.Clone()

    // authenticate and use the CSRF token from previous step
    err := d.Post("https://dzone.com/users/login.html", map[string]string{"TH_CSRF": thCsrf, "_spring_security_remember_me": "true", "j_username": "[email protected]", "j_password": "password123456"})
    if err != nil {
        log.Fatal(err)
    }

    // save the PDF
    d.OnResponse(func(r *colly.Response) {
        r.Headers.Set("Content-Type", "application/x-www-form-urlencoded")
        log.Println("/asset/download/279342 response:", r.StatusCode, r.Request)
        r.Save("279342.pdf")
    })

    // visit refcard download link, that redirects to the actual PDF file
    d.Visit("https://dzone.com/asset/download/279342")

}

Debug:

[000001] 1 [     1 - request] map["url":"https://dzone.com/users/login.html"] (20.092µs)
[000002] 1 [     1 - response] map["url":"https://dzone.com/users/login.html" "status":"OK"] (1.545247196s)
2019/02/23 12:00:19 TH_CSRF response received 200 &{https://dzone.com/users/login.html 0xc420184238 0xc42001fe80 1 GET <nil>  1 0xc4200a1380 false <nil> }
[000003] 1 [     1 - html] map["selector":"form[role=form] input[type=hidden][name=TH_CSRF]" "url":"https://dzone.com/users/login.html"] (1.547319144s)
2019/02/23 12:00:19 7291120165991301399
[000004] 1 [     1 - scraped] map["url":"https://dzone.com/users/login.html"] (1.547411513s)
[000005] 2 [     1 - request] map["url":"https://dzone.com/users/login.html"] (1.54751484s)
[000006] 2 [     1 - response] map["url":"https://dzone.com/static/csrfAttackDetected.html" "status":"OK"] (2.466044655s)
[000007] 2 [     1 - scraped] map["url":"https://dzone.com/static/csrfAttackDetected.html"] (2.466186341s)
[000008] 2 [     2 - request] map["url":"https://dzone.com/asset/download/279342"] (2.467179936s)
[000009] 2 [     2 - response] map["url":"https://dzone.com/users/login.html" "status":"OK"] (3.615490698s)
2019/02/23 12:00:21 /asset/download/279342 response: 200 &{https://dzone.com/users/login.html 0xc4205a2038 0xc420412090 1 GET <nil>  2 0xc4200a01a0 false <nil> }
[000010] 2 [     2 - scraped] map["url":"https://dzone.com/users/login.html"] (3.616186555s)

[luckylittle]

I am closing it. I figured it out myself. Here is the working code:

package main

import (
    "fmt"
    "log"
    "time"

    "github.com/gocolly/colly"
)

var thCsrf string

func main() {
    c := colly.NewCollector()
    c.OnHTML("form[role=form] input[type=hidden][name=TH_CSRF]", func(e *colly.HTMLElement) {
        thCsrf := e.Attr("value")
        err := c.Post("https://dzone.com/j_spring_security_check", map[string]string{"TH_CSRF": thCsrf, "_spring_security_remember_me": "true", "j_username": "[email protected]", "j_password": "password123456"})
        if err != nil {
            log.Fatal(err)
        }
        return
    })
    c.Visit("https://dzone.com/users/login.html")
    // ---------------------------------------------------------------------------------------------
    d := c.Clone()
    d.SetRequestTimeout(180 * time.Second)

    d.OnResponse(func(r *colly.Response) {
        fmt.Println(r.Request.URL.String())
        r.Save("279342.pdf")
    })

    d.Visit("https://dzone.com/asset/download/279342")
}