【Golang】使用Golang为老婆编写一个爬虫工具

Cookie: SESSIONID=3c818c6d-97a1-407a-9512-9de604dde83d; proxyUser=null; acw_tc=2760827b16125811058633717e8fce8be9948b90272f2fb598d2313cae0ba2; Hm_lvt_8aa7b2541dc5c9cfc124ff1d86098aaa=1612579177,1612579299,1612579497,1612581149; Hm_lpvt_8aa7b2541dc5c9cfc124ff1d86098aaa=1612581149  

go get -u github.com/gocolly/colly/v2

package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/gocolly/colly/v2"
)
func PostLogin() (err error) {	
    c := colly.NewCollector(
        colly.UserAgent(sameUserAgent),
    )
    var requestData = make(map[string]string, 3)
    requestData["username"] = username
    requestData["password"] = password
    c.OnResponse(func(r *colly.Response) {
        fmt.Println("response received", r.StatusCode)
        if r.StatusCode == 200 {
            fmt.Println("Firstly,Login Action And Redirect Success...")
        }
    })
    c.OnRequest(func(r *colly.Request) {
        r.Headers.Add("Cookie", cookie)
    })

    c.OnError(func(r *colly.Response, err error) {
        // fmt.Println(r)
        fmt.Println(err)
        fmt.Println("Login Action And Redirect occurs error... Please Contact 287572291@qq.com")
    })

    //1.Login Action
    err = c.Post(loginURL, requestData)
    c.Wait()
}

// Response is the representation of a HTTP response made by a Collector
type Response struct {
    // StatusCode is the status code of the Response
    StatusCode int
    // Body is the content of the Response
    Body []byte
    // Ctx is a context between a Request and a Response
    Ctx *Context
    // Request is the Request object of the response
    Request *Request
    // Headers contains the Response's HTTP headers
    Headers *http.Header
    // Trace contains the HTTPTrace for the request. Will only be set by the
    // collector if Collector.TraceHTTP is set to true.
    Trace *HTTPTrace
}

  <script>window.addEventListener('load', function(){document.forms[0].submit();});</script>

package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/gocolly/colly/v2"
)
func PostLogin() (err error) {
    //1.Login Action Start
	c := colly.NewCollector(
		colly.UserAgent(sameUserAgent),
	)
	var requestData = make(map[string]string, 3)
	requestData["username"] = username
	requestData["password"] = password
	c.OnResponse(func(r *colly.Response) {
		fmt.Println("response received", r.StatusCode)
		if r.StatusCode == 200 {
			fmt.Println("Firstly,Login Action And Redirect Success...")
		}
	})
	c.OnRequest(func(r *colly.Request) {
		r.Headers.Add("Cookie", cookie)
	})

	c.OnError(func(r *colly.Response, err error) {
		// fmt.Println(r)
		fmt.Println(err)
		fmt.Println("Login Action And Redirect occurs error... Please Contact 287572291@qq.com")
	})

	//1.Login Action End
	err = c.Post(loginURL, requestData)
	c.Wait()

    //2.After Login Action, the Redirection Of "window.location.href" Start
	c.OnResponse(func(r *colly.Response) {
		fmt.Println("response received", r.StatusCode)
		if r.StatusCode == 200 {
			fmt.Println("After Login Action,by the \"window.location.href\",Get Cookie success...")
		}
		AfterLoginActionCookie = r.Headers.Get("Set-Cookie")
	})

	c.OnError(func(r *colly.Response, err error) {
		// log.Println(r)
		fmt.Println(err)
		fmt.Printf("%s Redirect occurs error... Please Contact 287572291@qq.com", AfterLoginSuccessURL)
	})

	//2.After Login Action, the Redirection Of "window.location.href" End
	err = c.Request("GET", AfterLoginSuccessURL, nil, nil, nil)
	c.Wait()

    //3.Obtain AfterLoginSuccess-Cookie Start
	c.OnResponse(func(r *colly.Response) {
		fmt.Println("response received", r.StatusCode)
		fmt.Println("Get Redirect Cookie success...")
		LoginSuccessCookie = r.Request.Headers.Get("cookie")
		fmt.Printf("After Login Success,Obtain Full Cookie: %s", LoginSuccessCookie)
	})

	client = http.DefaultClient
	client.CheckRedirect = func(req *http.Request, via []*http.Request) (err error) {
		for _,v:=range via{
			fmt.Println(v.URL)
		}
		return
	}

	c.OnRequest(func(r *colly.Request) {
		r.Headers.Add("Cookie", AfterLoginActionCookie)
	})

	c.OnError(func(r *colly.Response, err error) {
		log.Println(r)
		log.Println(err)
		log.Printf("%s After Login Success,Obtain Full Cookie occurs error... Please Contact 287572291@qq.com",
			LoginSessionIDURL)
	})

	//3.Obtain AfterLoginSuccess-Cookie End
	err = c.Request("GET", LoginSessionIDURL, nil, nil, nil)
	c.Wait()
	return
}

//3.Obtain AfterLoginSuccess-Cookie Start
c.OnResponse(func(r *colly.Response) {
	fmt.Println("response received", r.StatusCode)
	fmt.Println("Get Redirect Cookie success...")
	LoginSuccessCookie = r.Request.Headers.Get("cookie")
	fmt.Printf("After Login Success,Obtain Full Cookie: %s", LoginSuccessCookie)
})

var projectData ProjectResponse	
c.OnResponse(func(r *colly.Response) {
    log.Println("response received", r.StatusCode)
    // fmt.Println(string(r.Body))
    err = json.Unmarshal(r.Body, &projectData)
    if err != nil {
        fmt.Println(err)
    }
    // fmt.Println("反序列化结果：", projectData)
})
type ProjectResponse struct{
    Code  string  
    Action  string  
    Msg  string  
    Data  ProjectData 
    Status bool 
}

go get -u github.com/tealeg/xlsx

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"strings"

	"github.com/gocolly/colly/v2"
	"github.com/tealeg/xlsx"
)
var result []DeviceStatusResult

// LoopObtain 循环项目
func Export() (err error) {
	// jsonResult,err:=json.Marshal(result)
	// fmt.Println(string(jsonResult))

	file := xlsx.NewFile()
	sheet, err := file.AddSheet("Sheet1")
	for _, v := range result {
		row := sheet.AddRow()
		row.SetHeightCM(1) //设置每行的高度
		for i, d := range v.DeviceInfo {
			if i == 0 {
				cell := row.AddCell()
				cell.Value = v.ProjectName
				cell = row.AddCell()
				cell.Value = d.DeviceNo
				cell = row.AddCell()
				cell.Value = d.Status
			} else {
				row := sheet.AddRow()
				row.SetHeightCM(1) //设置每行的高度
				cell := row.AddCell()
				cell.Value = ""
				cell = row.AddCell()
				cell.Value = d.DeviceNo
				cell = row.AddCell()
				cell.Value = d.Status
			}
		}
	}

	err = file.Save("设备状态表.xlsx")
	fmt.Println("Excel Export Success...")
	return
}