package data import ( "strings" "time" "github.com/google/btree" "git.scraperwall.com/scw/grs" ) type RequestData struct { ID string `bson:"_id,omitempty" json:"_id,omitempty" form:"_id,omitempty" url:"_id,omitempty"` Connection string `bson:"connection" json:"connection" form:"connection,omitempty" url:"connection,omitempty"` IpDst string `bson:"ip_dst" json:"ip_dst" form:"ip_dst,omitempty" url:"ip_dst,omitempty"` TcpSeq uint32 `bson:"tcp_seq" json:"tcp_seq" form:"tcp_seq,omitempty" url:"tcp_seq,omitempty"` XForwardedFor string `bson:"x_forwarded_for" json:"x_forwarded_for,omitempty" form:"x_forwarded_for,omitempty" url:"x_forwarded_for,omitempty"` XRealIP string `bson:"x_real_ip" json:"x_real_ip,omitempty" form:"x_real_ip,omitempty" url:"x_real_ip,omitempty"` Origin string `bson:"origin" json:"origin,omitempty" form:"origin,omitempty" url:"origin,omitempty"` PortDst uint32 `bson:"port_dst" json:"port_dst,omitempty" form:"port_dst,omitempty" url:"port_dst,omitempty"` Referer string `bson:"referer" json:"referer,omitempty" form:"referer,omitempty" url:"referer,omitempty"` XRequestedWith string `bson:"x_requested_with" json:"x_requested_with,omitempty" form:"x_requested_with.omitempty" url:"x_requested_with,omitempty"` PortSrc uint32 `bson:"port_src" json:"port_src,omitempty" form:"port_src,omitempty" url:"port_src,omitempty"` Url string `bson:"url" json:"url,omitempty" form:"url,omitempty" url:"url,omitempty"` Reverse string `bson:"reverse" json:"reverse,omitempty" form:"reverse,omitempty" url:"reverse,omitempty"` AcceptEncoding string `bson:"accept_encoding" json:"accept_encoding,omitempty" form:"accept_encoding,omitempty" url:"accept_encoding,omitempty"` AcceptLanguage string `bson:"accept_language" json:"accept_language,omitempty" form:"accept_language,omitempty" url:"accept_language,omitempty"` CreatedAt time.Time `bson:"created_at" json:"created_at,omitempty" form:"created_at,omitempty" url:"created_at,omitempty"` SortTime time.Time `bson:"-" json:"-" form:"-" url:"-"` IpSrc string `bson:"ip_src" json:"ip_src,omitempty" form:"ip_src,omitempty" url:"ip_src,omitempty"` Method string `bson:"method" json:"method,omitempty" form:"method,omitempty" url:"method,omitempty"` UserAgent string `bson:"user_agent" json:"user_agent,omitempty" form:"user_agent,omitempty" url:"user_agent,omitempty"` Accept string `bson:"accept" json:"accept,omitempty" form:"accept,omitempty" url:"accept,omitempty"` Cookie string `bson:"cookie" json:"cookie,omitempty" form:"cookie,omitempty" url:"cookie,omitempty"` Host string `bson:"host" json:"host,omitempty" form:"host,omitempty" url:"host,omitempty"` HostName string `bson:"hostname" json:"hostname,omitempty" form:"hostname,omitempty" url:"hostname,omitempty"` Protocol string `bson:"protocol" json:"protocol,omitempty" form:"protocol,omitempty" url:"protocol,omitempty"` Source string `bson:"source" json:"source,omitempty" form:"source,omitempty" url:"source,omitempty"` Via string `bson:"via" json:"via,omitempty" form:"via,omitempty" url:"via,omitempty"` Customer string `bson:"customer" json:"customer,omitempty" form:"customer,omitempty" url:"customer,omitempty"` DataRaw map[string]interface{} `bson:"data_raw" json:"data_raw,omitempty" form:"data_raw,omitempty" url:"data_raw,omitempty"` Weight uint64 `bson:"weight" json:"weight,omitempty" form:"weight,omitempty" url:"weight,omitempty"` BrowserAge time.Duration `bson:"bage" json:"browser_age,omitempty" form:"browser_age,omitempty" url:"browser_age,omitempty"` Grs grs.GRS `bson:"grs" json:"grs,omitempty" form:"grs,omitempty" url:"grs,omitempty"` } func (a *RequestData) Less(b btree.Item) bool { return a.CreatedAt.Before(b.(*RequestData).CreatedAt) } func (a *RequestData) ToRequest() *Request { return &Request{ Url: a.Url, Reverse: a.Reverse, IpSrc: a.IpSrc, IpDst: a.IpDst, PortSrc: a.PortSrc, PortDst: a.PortDst, TcpSeq: a.TcpSeq, CreatedAt: a.CreatedAt.UnixNano(), XForwardedFor: a.XForwardedFor, XRealIP: a.XRealIP, Method: a.Method, Origin: a.Origin, Referer: a.Referer, UserAgent: a.UserAgent, Source: a.Source, Host: a.Host, Protocol: a.Protocol, Connection: a.Connection, XRequestedWith: a.XRequestedWith, AcceptEncoding: a.AcceptEncoding, AcceptLanguage: a.AcceptLanguage, Accept: a.Accept, Cookie: a.Cookie, Via: a.Via, Customer: a.Customer, } } // Path returns the Url without any query string func (a *RequestData) Path() string { idx := strings.Index(a.Url, "?") if idx >= 0 { return a.Url[0:idx] } return a.Url }