- munchclient can now use Apache log files as source to feed the queue

main.go

@@ -14,9 +14,11 @@ import (
 	"time"
 
 	"github.com/BurntSushi/toml"
+	"github.com/Songmu/axslogparser"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
+	"github.com/hpcloud/tail"
 	"github.com/kr/pretty"
 	"github.com/nats-io/nats"
 	"github.com/nats-io/nats/encoders/protobuf"
@@ -39,6 +41,7 @@ var (
 	protocol              = flag.String("protocol", "http", "which protocol to parse: http or ajp13")
 	useXForwardedAsSource = flag.Bool("use-x-forwarded", false, "Use the IP address in X-Forwarded-For as source")
 	trace                 = flag.Bool("trace", false, "Trace the packet capturing")
+	apacheLog             = flag.String("apache-log", "", "Parse an Apache Log file")
 	configFile            = flag.String("config", "", "The location of the TOML config file")
 
 	beQuiet   = flag.Bool("quiet", true, "Be quiet")
@@ -72,6 +75,7 @@ type Config struct {
 	Quiet                 bool
 	Protocol              string
 	Trace                 bool
+	ApacheLog             string
 }
 
 type duration struct {
@@ -94,6 +98,7 @@ func (c Config) print() {
 	fmt.Printf("NatsQueue:             %s\n", c.NatsQueue)
 	fmt.Printf("SleepFor:              %s\n", c.SleepFor.String())
 	fmt.Printf("RequestsFile:          %s\n", c.RequestsFile)
+	fmt.Printf("Apache Log:            %s\n", c.ApacheLog)
 	fmt.Printf("UseXForwardedAsSource: %t\n", c.UseXForwardedAsSource)
 	fmt.Printf("Protocol:              %s\n", c.Protocol)
 	fmt.Printf("Quiet:                 %t\n", c.Quiet)
@@ -144,12 +149,94 @@ func main() {
 	// What should I do?
 	if config.RequestsFile != "" {
 		replayFile()
+	} else if config.ApacheLog != "" {
+		apacheLogCapture(config.ApacheLog)
 	} else if config.Live {
 		fmt.Printf("live capture (%s, %s) to %s\n", config.Interface, config.Filter, config.NatsURL)
 		liveCapture()
 	}
 }
 
+func apacheLogCapture(logfile string) {
+	if _, err := os.Stat(logfile); err != nil {
+		log.Fatalf("%s: %s", logfile, err)
+	}
+
+	t, err := tail.TailFile(logfile, tail.Config{
+		Follow:   true,                                 // follow the file
+		ReOpen:   true,                                 // reopen log file when it gets closed/rotated
+		Logger:   tail.DiscardingLogger,                // don't log anything
+		Location: &tail.SeekInfo{Offset: 0, Whence: 2}, // start at the end of the file
+	})
+	if err != nil {
+		log.Fatalf("%s: %s", logfile, err)
+	}
+
+	var p axslogparser.Parser
+	parserSet := false
+
+	for line := range t.Lines {
+		l := line.Text
+
+		if !parserSet {
+			p, _, err = axslogparser.GuessParser(l)
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+			parserSet = true
+		}
+
+		logEntry, err := p.Parse(l)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
+		remote := logEntry.Host
+		if *useXForwardedAsSource && logEntry.ForwardedFor != "" {
+			remote = logEntry.ForwardedFor
+		}
+
+		// only use the first host in case there are multiple hosts in the log
+		if cidx := strings.Index(remote, ","); cidx >= 0 {
+			remote = remote[0:cidx]
+		}
+
+		// extract the virtual host
+		var virtualHost string
+		vhost := logEntry.VirtualHost
+		if vhost != "" {
+			vhostAndPort := strings.Split(logEntry.VirtualHost, ":")
+			virtualHost = vhostAndPort[0]
+		}
+
+		request := data.Request{
+			IpSrc:         remote,
+			IpDst:         "127.0.0.1",
+			PortSrc:       0,
+			PortDst:       0,
+			TcpSeq:        0,
+			CreatedAt:     logEntry.Time.UnixNano(),
+			Url:           logEntry.RequestURI,
+			Method:        logEntry.Method,
+			Host:          virtualHost,
+			Protocol:      logEntry.Protocol,
+			Origin:        remote,
+			Source:        remote,
+			Referer:       logEntry.Referer,
+			XForwardedFor: logEntry.ForwardedFor,
+			UserAgent:     logEntry.UserAgent,
+		}
+
+		if config.Trace {
+			log.Printf("[%s] %s\n", request.Source, request.Url)
+		}
+
+		natsEC.Publish(config.NatsQueue, &request)
+	}
+}
+
 func liveCapture() {
 	ipPriv = ip.NewIP()
 
@@ -458,6 +545,7 @@ func loadConfig() {
 	config.RequestsFile = *requestsFile
 	config.UseXForwardedAsSource = *useXForwardedAsSource
 	config.Protocol = *protocol
+	config.ApacheLog = *apacheLog
 	config.Quiet = *beQuiet
 	config.Trace = *trace