added AJP13 protocol

defaults/munchclient

@@ -5,4 +5,4 @@ NATS_QUEUE=requests
 NATS_URL=nats://127.0.0.1:4222
 PROMISCUOUS=false
 USE_X_FORWARDED=true
-
+PROTOCOL=http

main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/nats-io/nats"
 	"github.com/nats-io/nats/encoders/protobuf"
 
+	"git.scraperwall.com/scw/ajp13"
 	"git.scraperwall.com/scw/data"
 	"git.scraperwall.com/scw/ip"
 )
@@ -34,10 +35,12 @@ var (
 	natsQueue             = flag.String("nats-queue", "requests", "The NATS queue name")
 	sleepFor              = flag.Duration("sleep", 0, "Sleep this long between sending data (only when replaying a file)")
 	requestsFile          = flag.String("requests", "", "CSV file containing requests (IP and URL)")
+	protocol              = flag.String("protocol", "http", "which protocol to parse: http or ajp13")
 	useXForwardedAsSource = flag.Bool("use-x-forwarded", false, "Use the IP address in X-Forwarded-For as source")
 	configFile            = flag.String("config", "", "The location of the TOML config file")
-	beQuiet               = flag.Bool("quiet", true, "Be quiet")
-	doVersion             = flag.Bool("version", false, "Show version information")
+
+	beQuiet   = flag.Bool("quiet", true, "Be quiet")
+	doVersion = flag.Bool("version", false, "Show version information")
 
 	natsEC  *nats.EncodedConn
 	count   uint64
@@ -65,6 +68,7 @@ type Config struct {
 	RequestsFile          string
 	UseXForwardedAsSource bool
 	Quiet                 bool
+	Protocol              string
 }
 
 type duration struct {
@@ -88,6 +92,7 @@ func (c Config) print() {
 	fmt.Printf("SleepFor:              %s\n", c.SleepFor.String())
 	fmt.Printf("RequestsFile:          %s\n", c.RequestsFile)
 	fmt.Printf("UseXForwardedAsSource: %t\n", c.UseXForwardedAsSource)
+	fmt.Printf("Protocol:              %s\n", c.Protocol)
 	fmt.Printf("Quiet:                 %t\n", c.Quiet)
 }
 
@@ -200,81 +205,155 @@ func processPacket(packet gopacket.Packet) {
 		return
 	}
 
-	data := data.Request{}
+	request := data.Request{
+		IpSrc:     ipSrc,
+		IpDst:     ipDst,
+		PortSrc:   uint32(portSrc),
+		PortDst:   uint32(portDst),
+		TcpSeq:    uint32(sequence),
+		CreatedAt: packet.Metadata().CaptureInfo.Timestamp.UnixNano(),
+	}
+
+	switch *protocol {
+	case "http":
+		processHTTP(&request, applicationLayer.Payload())
+	case "ajp13":
+		processAJP13(&request, applicationLayer.Payload())
+	}
+
+	if config.UseXForwardedAsSource && request.XForwardedFor != "" {
+		if strings.Contains(request.XForwardedFor, ",") {
+			ips := strings.Split(request.XForwardedFor, ",")
+			for i := len(ips) - 1; i >= 0; i-- {
+				ipRaw := strings.TrimSpace(ips[i])
+				ipAddr := net.ParseIP(ipRaw)
+				if ipAddr != nil && !ipPriv.IsPrivate(ipAddr) {
+					request.Source = ipRaw
+					break
+				}
+			}
+		} else {
+			ipAddr := net.ParseIP(strings.TrimSpace(request.XForwardedFor))
+
+			if !ipPriv.IsPrivate(ipAddr) {
+				request.Source = request.XForwardedFor
+			}
+		}
+	}
+
+	if request.Source == request.IpSrc && request.XRealIP != "" {
+		request.Source = request.XRealIP
+	}
 
-	reader := bufio.NewReader(strings.NewReader(string(applicationLayer.Payload())))
+	natsEC.Publish(config.NatsQueue, &request)
+}
+
+func processAJP13(request *data.Request, appData []byte) error {
+	a, err := ajp13.Parse(appData)
+	if err != nil {
+		return fmt.Errorf("Failed to parse AJP13 request: %s", err)
+	}
+
+	request.Url = a.URI
+	request.Method = a.Method()
+	request.Host = a.Server
+	request.Protocol = a.Version
+	request.Origin = a.RemoteAddr.String()
+
+	if v, ok := a.Header("Referer"); ok {
+		request.Referer = v
+	}
+
+	if v, ok := a.Header("Connection"); ok {
+		request.Connection = v
+	}
+
+	if v, ok := a.Header("X-Forwarded-For"); ok {
+		request.XForwardedFor = v
+	}
+
+	if v, ok := a.Header("X-Real-IP"); ok {
+		request.XRealIP = v
+	}
+
+	if v, ok := a.Header("X-Requested-With"); ok {
+		request.XRequestedWith = v
+	}
+
+	if v, ok := a.Header("Accept-Encoding"); ok {
+		request.AcceptEncoding = v
+	}
+
+	if v, ok := a.Header("Accept-Language"); ok {
+		request.AcceptLanguage = v
+	}
+
+	if v, ok := a.Header("User-Agent"); ok {
+		request.UserAgent = v
+	}
+
+	if v, ok := a.Header("Accept"); ok {
+		request.Accept = v
+	}
+
+	if v, ok := a.Header("Cookie"); ok {
+		request.Cookie = v
+	}
+
+	if v, ok := a.Header("X-Forwarded-Host"); ok {
+		if v != request.Host {
+			request.Host = v
+		}
+	}
+
+	return nil
+}
+
+func processHTTP(request *data.Request, appData []byte) error {
+	reader := bufio.NewReader(strings.NewReader(string(appData)))
 
 	req, err := http.ReadRequest(reader)
 	if err != nil {
-		return
+		return fmt.Errorf("Failed to parse HTTP header: %s", err)
 	}
 
-	data.IpSrc = ipSrc
-	data.IpDst = ipDst
-	data.PortSrc = uint32(portSrc)
-	data.PortDst = uint32(portDst)
-	data.TcpSeq = uint32(sequence)
-	data.CreatedAt = time.Now().UnixNano()
-	data.Url = req.URL.String()
-	data.Method = req.Method
-	data.Referer = req.Referer()
-	data.Host = req.Host
-	data.Protocol = req.Proto
-	data.Origin = data.Host
+	request.Url = req.URL.String()
+	request.Method = req.Method
+	request.Referer = req.Referer()
+	request.Host = req.Host
+	request.Protocol = req.Proto
+	request.Origin = request.Host
 	if _, ok := req.Header["Connection"]; ok {
-		data.Connection = req.Header["Connection"][0]
+		request.Connection = req.Header["Connection"][0]
 	}
 	if _, ok := req.Header["X-Forwarded-For"]; ok {
-		data.XForwardedFor = req.Header["X-Forwarded-For"][0]
+		request.XForwardedFor = req.Header["X-Forwarded-For"][0]
 	}
 	if _, ok := req.Header["X-Real-IP"]; ok {
-		data.XRealIP = req.Header["X-Real-IP"][0]
+		request.XRealIP = req.Header["X-Real-IP"][0]
 	}
 	if _, ok := req.Header["X-Requested-With"]; ok {
-		data.XRequestedWith = req.Header["X-Requested-With"][0]
+		request.XRequestedWith = req.Header["X-Requested-With"][0]
 	}
 	if _, ok := req.Header["Accept-Encoding"]; ok {
-		data.AcceptEncoding = req.Header["Accept-Encoding"][0]
+		request.AcceptEncoding = req.Header["Accept-Encoding"][0]
 	}
 	if _, ok := req.Header["Accept-Language"]; ok {
-		data.AcceptLanguage = req.Header["Accept-Language"][0]
+		request.AcceptLanguage = req.Header["Accept-Language"][0]
 	}
 	if _, ok := req.Header["User-Agent"]; ok {
-		data.UserAgent = req.Header["User-Agent"][0]
+		request.UserAgent = req.Header["User-Agent"][0]
 	}
 	if _, ok := req.Header["Accept"]; ok {
-		data.Accept = req.Header["Accept"][0]
+		request.Accept = req.Header["Accept"][0]
 	}
 	if _, ok := req.Header["Cookie"]; ok {
-		data.Cookie = req.Header["Cookie"][0]
-	}
-	data.Source = data.IpSrc
-
-	if config.UseXForwardedAsSource && data.XForwardedFor != "" {
-		if strings.Contains(data.XForwardedFor, ",") {
-			ips := strings.Split(data.XForwardedFor, ",")
-			for i := len(ips) - 1; i >= 0; i-- {
-				ipRaw := strings.TrimSpace(ips[i])
-				ipAddr := net.ParseIP(ipRaw)
-				if ipAddr != nil && !ipPriv.IsPrivate(ipAddr) {
-					data.Source = ipRaw
-					break
-				}
-			}
-		} else {
-			ipAddr := net.ParseIP(strings.TrimSpace(data.XForwardedFor))
-
-			if !ipPriv.IsPrivate(ipAddr) {
-				data.Source = data.XForwardedFor
-			}
-		}
+		request.Cookie = req.Header["Cookie"][0]
 	}
 
-	if data.Source == data.IpSrc && data.XRealIP != "" {
-		data.Source = data.XRealIP
-	}
+	request.Source = request.IpSrc
 
-	natsEC.Publish(config.NatsQueue, &data)
+	return nil
 }
 
 // replayFile takes a file containing a list of requests (SourceIP Url) and queues the requests
@@ -344,6 +423,7 @@ func loadConfig() {
 	config.SleepFor.Duration = *sleepFor
 	config.RequestsFile = *requestsFile
 	config.UseXForwardedAsSource = *useXForwardedAsSource
+	config.Protocol = *protocol
 	config.Quiet = *beQuiet
 
 	if *configFile == "" {

munchclient.toml

@@ -9,3 +9,5 @@
 # Quiet = true
 # SleepFor = 100µs
 # RequestsFile = requests.csv
+# Protocol = http
+# Protocol = ajp13