golang源码分析：爬虫 colly（part I）

package main
import (  "fmt"
  "github.com/gocolly/colly/v2")
func main() {  // Instantiate default collector  c := colly.NewCollector(    // Visit only domains: hackerspaces.org, wiki.hackerspaces.org    colly.AllowedDomains("hackerspaces.org", "wiki.hackerspaces.org"),  )  // On every a element which has href attribute call callback  c.OnHTML("a[href]", func(e *colly.HTMLElement) {    link := e.Attr("href")    // Print link    fmt.Printf("Link found: %q -> %s\n", e.Text, link)    // Visit link found on page    // Only those links are visited which are in AllowedDomains    c.Visit(e.Request.AbsoluteURL(link))  })
  // Before making a request print "Visiting ..."  c.OnRequest(func(r *colly.Request) {    fmt.Println("Visiting", r.URL.String())  })
  // Start scraping on https://hackerspaces.org  c.Visit("https://hackerspaces.org/")}

colly_test.go           http_backend.go         storagecontext.go              http_trace.go           unmarshal.goLICENSE.txt             context_test.go         http_trace_test.go      unmarshal_test.goREADME.md               debug                   proxy                   xmlelement.goVERSION                 extensions              queue                   xmlelement_test.go_examples               go.mod                  request.gocmd                     go.sum                  response.gocolly.go                htmlelement.go  

  func main()     app := cli.App("colly", "Scraping Framework for Gophers")    scraper.WriteString(htmlCallbackTemplate)    app.Run(os.Args)

type Debugger interface {  // Init initializes the backend  Init() error  // Event receives a new collector event.  Event(e *Event)}

 func (l *LogDebugger) Event(e *Event)      l.logger.Printf("[%06d] %d [%6d - %s] %q (%s)\n", i, e.CollectorID, e.RequestID, e.Type, e.Values, time.Since(l.start))

    func (w *WebDebugger) Init() error       w.Address = "127.0.0.1:7676"      http.HandleFunc("/status", w.statusHandler)

    func (w *WebDebugger) Event(e *Event)         w.CurrentRequests[e.RequestID] = requestInfo{      URL:         e.Values["url"],      Started:     time.Now(),      ID:          e.RequestID,      CollectorID: e.CollectorID,    }

 func (w *WebDebugger) indexHandler(wr http.ResponseWriter, r *http.Request)      function fetchStatus() {        $.getJSON("/status", function(data) {

func (w *WebDebugger) statusHandler(wr http.ResponseWriter, r *http.Request)       jsonData, err := json.MarshalIndent(w, "", "  ")

    genFirefoxUA,    genChromeUA,    genEdgeUA,    genOperaUA,    genMobileUcwebUA,    genMobileNexus10UA,    ffVersions    chromeVersions    osStrings

    func Referer(c *colly.Collector)

  func URLLengthFilter(c *colly.Collector, URLLengthLimit int)

type roundRobinSwitcher struct {  proxyURLs []*url.URL  index     uint32}func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error)func RoundRobinProxySwitcher(ProxyURLs ...string) (colly.ProxyFunc, error) 

type Storage interface {  // Init initializes the storage  Init() error  // AddRequest adds a serialized request to the queue  AddRequest([]byte) error  // GetRequest pops the next request from the queue  // or returns error if the queue is empty  GetRequest() ([]byte, error)  // QueueSize returns with the size of the queue  QueueSize() (int, error)}

type Queue struct {  // Threads defines the number of consumer threads  Threads int  storage Storage  wake    chan struct{}  mut     sync.Mutex // guards wake and running  running bool}

func (q *Queue) IsEmpty() boolfunc (q *Queue) AddURL(URL string) error func (q *Queue) AddRequest(r *colly.Request) errorfunc (q *Queue) storeRequest(r *colly.Request) error func (q *Queue) Size() (int, error) func (q *Queue) Run(c *colly.Collector) errorfunc (q *Queue) Stop()func (q *Queue) loop(c *colly.Collector, requestc chan<- *colly.Request, complete <-chan struct{}, errc chan<- error){   req, err = q.loadRequest(c) }func (q *Queue) loadRequest(c *colly.Collector) (*colly.Request, error){  copy(copied, buf)}

func (q *Queue) Run(c *colly.Collector) error{    for i := 0; i < q.Threads; i++ {      go independentRunner(requestc, complete)    }    go q.loop(c, requestc, complete, errc)}

type InMemoryQueueStorage struct {  // MaxSize defines the capacity of the queue.  // New requests are discarded if the queue size reaches MaxSize  MaxSize int  lock    *sync.RWMutex  size    int  first   *inMemoryQueueItem  last    *inMemoryQueueItem}  

func (q *InMemoryQueueStorage) Init() errorfunc (q *InMemoryQueueStorage) AddRequest(r []byte) error func (q *InMemoryQueueStorage) GetRequest() ([]byte, error)func (q *InMemoryQueueStorage) QueueSize() (int, error)

type inMemoryQueueItem struct {  Request []byte  Next    *inMemoryQueueItem}

func independentRunner(requestc <-chan *colly.Request, complete chan<- struct{})       req.Do()

type Storage interface {  // Init initializes the storage  Init() error  // Visited receives and stores a request ID that is visited by the Collector  Visited(requestID uint64) error  // IsVisited returns true if the request was visited before IsVisited  // is called  IsVisited(requestID uint64) (bool, error)  // Cookies retrieves stored cookies for a given host  Cookies(u *url.URL) string  // SetCookies stores cookies for a given host  SetCookies(u *url.URL, cookies string)}

type InMemoryStorage struct {  visitedURLs map[uint64]bool  lock        *sync.RWMutex  jar         *cookiejar.Jar}  

func (s *InMemoryStorage) Init() error