其实本章算不上逆向教程 只是介绍golang的colly框架而已
列表页分析
根据关键字搜索
通过抓包分析可知
下一页所请求的参数如下
上图标红的代表所需参数
所以其实我们真正需要的也就是SearchSql
但是你多观察即可知 这个SearchSql在第一页访问的时候就传给了
所以也就是
爬取第一页获取第二页的请求参数——爬取第二页获取第三页的请求参数
详情页分析
详情页链接在列表页给你的链接是错误的
真是的链接如下图所示
由此图可知我们需要很多参数才能完成爬取
但是列表页给我们的链接也不是毫无用处
我们用正则提取中间所需要的参数即可完成爬取
代码
由于代码是用Go写的 用colly 还不是特别熟练
package main
import (
"fmt"
"github/gocolly/colly"
"regexp"
"time"
)
func main() {
PostUrl := "https://knski/kns8/Brief/GetGridTableHtml"
c, contentCollector := initCollector()
c.OnRequest(func(r *colly.Request) {
fmt.Println(r)
fmt.Println(r.Method)
r.Headers.Set("Accept", "text/html, */*; q=0.01")
r.Headers.Set("Accept-Language", "zh-CN,zh;q=0.9")
r.Headers.Set("Connection", "keep-alive")
r.Headers.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
r.Headers.Set("Origin", "https://knski")
r.Headers.Set("Referer", "https://knski/kns8/defaultresult/index")
r.Headers.Set("Sec-Fetch-Dest", "empty")
r.Headers.Set("Sec-Fetch-Mode", "cors")
r.Headers.Set("Sec-Fetch-Site", "same-origin")
r.Headers.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36")
r.Headers.Set("X-Requested-With", "XMLHttpRequest")
})
// 解析列表页
c.OnHTML(".result-table-list tbody", func(element *colly.HTMLElement) {
element.ForEach("tr", func(i int, element *colly.HTMLElement) {
// 获取链接地址
titleUrl, _ := element.DOM.Find("a").Attr("href")
// 获取时间
titleDate := element.DOM.Find("td.date").Text()
// 拿到标题名
s := element.DOM.Find("a").Text()
// 切片取值
if len(s) < 0 {
_ = fmt.Errorf("匹配失败,长度为:%v\n", len(s))
}
titleName := s[:len(s)-4]
// 正则取 decode的值
//deCodeRe := `DbCode=(.*?)&`
// 正则 fileName 的值
fileNameRe := `FileName=(.*?)&`
// 正则 取DbName 的值
dbNameRe := `DbName=(.*?)&`
isMatched, _ := regexp.MatchString(fileNameRe, titleUrl)
if !isMatched {
_ = fmt.Errorf("匹配失败,%v\n", isMatched)
}
fileName := regexp.MustCompile(fileNameRe).FindStringSubmatch(titleUrl)[1]
dbName := regexp.MustCompile(dbNameRe).FindStringSubmatch(titleUrl)[1]
trueUrl := fmt.Sprintf(`https://knski/kcms/detail/detail.aspx?dbcode=CCND&dbname=%s&filename=%s&uniplatform=NZKPT`, dbName, fileName)
fmt.Println(titleName)
fmt.Println(titleDate)
fmt.Println(trueUrl)
_ = contentCollector.Visit(trueUrl)
})
sqlVal, _ := element.DOM.Find("#sqlVal").Attr("value")
_ = c.Post(PostUrl, map[string]string{
"IsSearch": "false",
"QueryJson": `{"Platform":"","DBCode":"CFLS","KuaKuCode":"CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN","QNode":{"QGroup":[{"Key":"Subject","Title":"","Logic":1,"Items":[{"Title":"主题","Name":"SU","Value":"矿","Operate":"%=","BlurType":""}],"ChildItems":[]}]}}`,
"SearchSql": sqlVal,
"PageName": "defaultresult",
"HandlerId": "16",
"DBCode": "CFLS",
"KuaKuCodes": "CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN",
"CurPage": "2",
"RecordsCntPerPage": "20",
"CurDisplayMode": "listmode",
"CurrSortField": "",
"CurrSortFieldType": "desc",
"IsSortSearch": "false",
"IsSentenceSearch": "false",
"Subject": "",
})
})
// 解析详情页
contentCollector.OnHTML(".brief", func(element *colly.HTMLElement) {
contentHtml := element.Text
fmt.Printf(contentHtml)
})
_ = c.Post(PostUrl, map[string]string{
"IsSearch": "true",
"QueryJson": `{"Platform":"","DBCode":"CFLS","KuaKuCode":"CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN","QNode":{"QGroup":[{"Key":"Subject","Title":"","Logic":1,"Items":[{"Title":"主题","Name":"SU","Value":"矿","Operate":"%=","BlurType":""}],"ChildItems":[]}]}}`,
"PageName": "defaultresult",
"DBCode": "CFLS",
"KuaKuCodes": "CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN",
"CurPage": "1",
"RecordsCntPerPage": "20",
"CurDisplayMode": "listmode",
"CurrSortField": "",
"CurrSortFieldType": "desc",
"IsSentenceSearch": "false",
"Subject": "",
})
}
func initCollector() (*colly.Collector, *colly.Collector) {
// 1. 创建收集器
c := colly.NewCollector(
colly.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36"),
colly.AllowURLRevisit(),
)
// 设置抓取频率限制
_ = c.Limit(&colly.LimitRule{
DomainGlob: "*",
RandomDelay: 5 * time.Second, // 随机延迟
})
// 设置clone内容 解析
contentCollector := c.Clone()
return c, contentCollector
}
更多推荐
JS逆向实战8——某网实战(基于golang-colly)
发布评论