Generating a HAR file about chromedp HOT 21 OPEN

import (
	"container/list"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/url"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/chromedp/cdproto/har"
	"github.com/chromedp/cdproto/network"
	"github.com/chromedp/chromedp"
)

type nvPair har.NameValuePair
type hRequest har.Request
type hResponse har.Response

var start = time.Now()

// String function to print the values stored in the
// har request structure.
func (r hRequest) String() string {
	sb := strings.Builder{}
	fmt.Fprintf(&sb, "Method : %s\n", r.Method)
	fmt.Fprintf(&sb, "Url : %s\n", r.URL)
	fmt.Fprintf(&sb, "Headers : \n")
	for _, header := range r.Headers {
		fmt.Fprintf(&sb, "%s: %s\n", header.Name, header.Value)
	}
	fmt.Fprintf(&sb, "Query string : \n")
	for _, qs := range r.QueryString {
		fmt.Fprintf(&sb, "%s: %s\n", qs.Name, qs.Value)
	}
	fmt.Fprintf(&sb, "\n")
	return sb.String()
}

func (r hResponse) String() string {
	sb := strings.Builder{}
	fmt.Fprintf(&sb, "Status : %d\n", r.Status)
	fmt.Fprintf(&sb, "Status Text : %s\n", r.StatusText)
	fmt.Fprintf(&sb, "HTTP Version : %s\n", r.HTTPVersion)
	for _, header := range r.Headers {
		fmt.Fprintf(&sb, "%s: %s\n", header.Name, header.Value)
	}
	return sb.String()
}

// process requests and return a structured data
func processRequest(r *network.EventRequestWillBeSent) *hRequest {
	req := hRequest{}
	// http method
	req.Method = r.Request.Method
	// http request url
	req.URL = r.Request.URL
	// http version.
	req.HTTPVersion = ""
	// Associated headers for the request.
	req.Headers = []*har.NameValuePair{}
	// headers from the *network.EventRequestWillBeSent are in the form,
	// map[key:value]. this needs to be converted to the form of a
	// har.NameValuePair
	for header := range r.Request.Headers {
		h := har.NameValuePair{}
		h.Name = header
		h.Value = r.Request.Headers[header].(string)
		req.Headers = append(req.Headers, &h)
	}
	// Store cookie details.
	req.Cookies = []*har.Cookie{}
	// Url Query stirngs details.
	req.QueryString = []*har.NameValuePair{}
	u, err := url.Parse(req.URL)
	if err != nil {
		log.Printf("[E] Invalid URL data recived : %v", err)
	}
	// Query strings are of the format name = []values when
	// received from the network.EventRequestWillBeSent. This
	// needs to be converted to the form of multiple name, value
	// pairs.
	for name := range u.Query() {
		if len(name) != 0 {
			values := u.Query()[name]
			for _, val := range values {
				req.QueryString = append(req.QueryString, &har.NameValuePair{
					Name:  name,
					Value: val,
				})
			}
		}
	}
	// req.Postdata points to the post data.
	req.PostData = nil
	//if req.Method == "POST" {
	//// Process the post data of the form *har.PostData
	//}
	log.Printf("Post Data : %s", r.Request.PostData)
	// TODO : to implement headersize and bodySize for the request
	req.HeadersSize = 0
	req.BodySize = 0
	return &req
}

func processResponse(r *network.EventResponseReceived) *hResponse {
	res := hResponse{}
	res.Status = r.Response.Status
	res.StatusText = r.Response.StatusText
	res.HTTPVersion = r.Response.Protocol
	// TODO : implement cookie information.
	res.Cookies = nil
	res.Headers = []*har.NameValuePair{}
	// headers from the *network.EventRequestWillBeSent are in the form,
	// map[key:value]. this needs to be converted to the form of a
	// har.NameValuePair
	for header := range r.Response.Headers {
		h := har.NameValuePair{}
		h.Name = header
		h.Value = r.Response.Headers[header].(string)
		res.Headers = append(res.Headers, &h)
	}
	// response content
	res.Content = &har.Content{}
	res.Content.MimeType = r.Response.MimeType
	res.Content.Size = 0

	// Redirect URL
	res.RedirectURL = ""
	res.HeadersSize = 0
	res.BodySize = 0
	return &res
}

//RunHeadlessBrowser executes URL in headless browser
func RunHeadlessBrowser(ctx context.Context, url string) error {

	var ws sync.WaitGroup
	nRequests := list.New()
	nResponses := list.New()

	chromedp.ListenTarget(ctx, func(v interface{}) {
		switch v.(type) {
		case *network.EventRequestWillBeSent:
			ws.Add(1)
			go func(r *network.EventRequestWillBeSent) {
				req := processRequest(r)
				rm := map[network.RequestID]*hRequest{}
				rm[r.RequestID] = req
				nRequests.PushBack(rm)
				ws.Done()
			}(v.(*network.EventRequestWillBeSent))
			break

		case *network.EventResponseReceived:
			ws.Add(1)
			go func(r *network.EventResponseReceived) {
				res := processResponse(r)
				rm := map[network.RequestID]*hResponse{}
				rm[r.RequestID] = res
				nResponses.PushBack(rm)
				ws.Done()
			}(v.(*network.EventResponseReceived))
			break
		case *network.EventDataReceived:
			// Fired when data chunk was received over the network.
			go func() {
				edr := v.(*network.EventDataReceived)
				log.Printf("Data Received : %d\n", edr.DataLength)
			}()
			// case *network.EventLoadingFinished:
			// 	go func() {
			// 		lf := v.(*network.EventLoadingFinished)
			// 		log.Printf("Loading finished : %f\n", lf.EncodedDataLength)
			// 	}()
			// case *network.EventLoadingFailed:
			// 	// Fired when HTTP request has failed to load.
			// 	go func() {
			// 		lf := v.(*network.EventLoadingFailed)
			// 		log.Printf("Loading finished : %s\n", lf.ErrorText)
			// 	}()
		}
	})

	chromedp.Run(ctx, network.Enable())
	err := chromedp.Run(ctx,
		chromedp.Navigate(url),
	)

	if err != nil {
		log.Printf("Failed run. %v\n", err)
		return err
	}

	// Wait for all the go routines to complete
	ws.Wait()

	for e := nRequests.Front(); e != nil; e = e.Next() {
		r := e.Value.(map[network.RequestID]*hRequest)
		for k := range r {
			e, err := json.Marshal(r[k])
			if err != nil {
				log.Printf("Error : %v\n", err)
			}
			log.Printf("\n%s\n", string(e))
		}
	}

	// for e := nResponses.Front(); e != nil; e = e.Next() {
	// 	r := e.Value.(*hResponse)
	// 	log.Printf("%s", *r)
	// }

	return nil
}