replay apache file

build-rpm-centos7.sh

@@ -23,7 +23,7 @@ install -v -m 644 defaults/$DEFAULTS_FILE $DESTDIR/usr/share/$BINARY/$DEFAULTS_F
 install -v -m 644 $SERVICE_FILE $DESTDIR/etc/systemd/system
 install -v -m 644 $BINARY.toml $DESTDIR/etc/$BINARY/
 install -v -m 644 $BINARY.toml $DESTDIR/usr/share/$BINARY/
-install -v -m 644 ca-chain.cert.pem $DESTDIR/etc/$BINARY/
+# install -v -m 644 ca-chain.cert.pem $DESTDIR/etc/$BINARY/
 
 
 fpm -s dir -t $PKG_TYPE -C $DESTDIR --name $BINARY \

main.go

@@ -50,6 +50,7 @@ var (
 	useXForwardedAsSource = flag.Bool("use-x-forwarded", false, "Use the IP address in X-Forwarded-For as source")
 	trace                 = flag.Bool("trace", false, "Trace the packet capturing")
 	apacheLog             = flag.String("apache-log", "", "Parse an Apache Log file")
+	apacheReplay          = flag.String("apache-replay", "", "Apache log file to replay into the system")
 	nginxLog              = flag.String("nginx-log", "", "Nginx log file to tail")
 	nginxFormat           = flag.String("nginx-format", "", "The nginx log file format")
 	hostName              = flag.String("hostname", "", "Override the captured hostname with this one")
@@ -94,6 +95,7 @@ type Config struct {
 	Protocol              string
 	Trace                 bool
 	ApacheLog             string
+	ApacheReplay          string
 	NginxLog              string
 	NginxLogFormat        string
 	HostName              string
@@ -124,6 +126,7 @@ func (c Config) print() {
 	fmt.Printf("SleepFor:              %s\n", c.SleepFor.String())
 	fmt.Printf("RequestsFile:          %s\n", c.RequestsFile)
 	fmt.Printf("Apache Log:            %s\n", c.ApacheLog)
+	fmt.Printf("Apache Replay:         %s\n", c.ApacheReplay)
 	fmt.Printf("Nginx Log:             %s\n", c.NginxLog)
 	fmt.Printf("Nginx Log Format:      %s\n", c.NginxLogFormat)
 	fmt.Printf("HostName:              %s\n", c.HostName)
@@ -178,6 +181,8 @@ func main() {
 	// What should I do?
 	if config.RequestsFile != "" {
 		replayFile()
+	} else if config.ApacheReplay != "" {
+		apacheLogReplay(config.ApacheReplay)
 	} else if config.ApacheLog != "" {
 		apacheLogCapture(config.ApacheLog)
 	} else if config.Live {
@@ -338,6 +343,201 @@ func nginxLogCapture(logfile, format string) {
 	}
 }
 
+func apacheLogReplay(logfile string) {
+	file, err := os.Open(logfile)
+	if err != nil {
+		log.Fatalf("%s: %s", logfile, err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+
+	var p axslogparser.Parser
+	parserSet := false
+
+	var tOffset time.Duration
+
+	for scanner.Scan() {
+		l := scanner.Text()
+
+		if err := scanner.Err(); err != nil {
+			log.Fatal(err)
+		}
+
+		if !parserSet {
+			p, _, err = axslogparser.GuessParser(l)
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+			parserSet = true
+		}
+
+		logEntry, err := p.Parse(l)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
+		if tOffset == 0 {
+			tOffset = time.Now().Sub(logEntry.Time)
+		}
+
+		ts := logEntry.Time.Add(tOffset)
+		if ts.After(time.Now()) {
+			time.Sleep(ts.Sub(time.Now()))
+		}
+
+		// fmt.Println(l)
+		remote := logEntry.Host
+		if *useXForwardedAsSource && logEntry.ForwardedFor != "" {
+			remote = logEntry.ForwardedFor
+		}
+
+		// only use the first host in case there are multiple hosts in the log
+		if cidx := strings.Index(remote, ","); cidx >= 0 {
+			remote = remote[0:cidx]
+		}
+
+		// extract the virtual host
+		var virtualHost string
+		vhost := logEntry.VirtualHost
+		if vhost != "" {
+			vhostAndPort := strings.Split(logEntry.VirtualHost, ":")
+			virtualHost = vhostAndPort[0]
+		} else {
+			if config.HostName != "" {
+				vhost = config.HostName
+			} else {
+				vhost = "[not available]"
+			}
+		}
+
+		request := data.Request{
+			IpSrc:         remote,
+			IpDst:         "127.0.0.1",
+			PortSrc:       0,
+			PortDst:       0,
+			TcpSeq:        0,
+			CreatedAt:     (logEntry.Time.Add(tOffset)).UnixNano(),
+			Url:           logEntry.RequestURI,
+			Method:        logEntry.Method,
+			Host:          virtualHost,
+			Protocol:      logEntry.Protocol,
+			Origin:        remote,
+			Source:        remote,
+			Referer:       logEntry.Referer,
+			XForwardedFor: logEntry.ForwardedFor,
+			UserAgent:     logEntry.UserAgent,
+		}
+
+		if config.Trace {
+			log.Printf("[%s] %s\n", request.Source, request.Url)
+		}
+
+		count++
+		publishRequest(config.NatsQueue, &request)
+
+	}
+}
+
+/*
+func apacheLogReplay(logfile string) {
+	if _, err := os.Stat(logfile); err != nil {
+		log.Fatalf("%s: %s", logfile, err)
+	}
+
+	file, err := os.Open(logfile)
+	if err != nil {
+		log.Fatalf("%s: %s", logfile, err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+
+	var p axslogparser.Parser
+	parserSet := false
+
+	var tOffset time.Duration
+
+	for scanner.Scan() {
+		l := scanner.Text()
+
+		if err := scanner.Err(); err != nil {
+			log.Fatal(err)
+		}
+
+		if !parserSet {
+			p, _, err = axslogparser.GuessParser(l)
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+			parserSet = true
+		}
+
+		logEntry, err := p.Parse(l)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
+		remote := logEntry.Host
+		if *useXForwardedAsSource && logEntry.ForwardedFor != "" {
+			remote = logEntry.ForwardedFor
+		}
+
+		// only use the first host in case there are multiple hosts in the log
+		if cidx := strings.Index(remote, ","); cidx >= 0 {
+			remote = remote[0:cidx]
+		}
+
+		// extract the virtual host
+		var virtualHost string
+		vhost := logEntry.VirtualHost
+		if vhost != "" {
+			vhostAndPort := strings.Split(logEntry.VirtualHost, ":")
+			virtualHost = vhostAndPort[0]
+		} else {
+			if config.HostName != "" {
+				vhost = config.HostName
+			} else {
+				vhost = "[not available]"
+			}
+		}
+
+		if tOffset == 0 {
+			tOffset = time.Now().Sub(logEntry.Time)
+		}
+
+		request := data.Request{
+			IpSrc:         remote,
+			IpDst:         "127.0.0.1",
+			PortSrc:       0,
+			PortDst:       0,
+			TcpSeq:        0,
+			CreatedAt:     (logEntry.Time.Add(tOffset)).UnixNano(),
+			Url:           logEntry.RequestURI,
+			Method:        logEntry.Method,
+			Host:          virtualHost,
+			Protocol:      logEntry.Protocol,
+			Origin:        remote,
+			Source:        remote,
+			Referer:       logEntry.Referer,
+			XForwardedFor: logEntry.ForwardedFor,
+			UserAgent:     logEntry.UserAgent,
+		}
+
+		if config.Trace {
+			log.Printf("[%s] %s\n", request.Source, request.Url)
+		}
+
+		count++
+		publishRequest(config.NatsQueue, &request)
+	}
+}
+*/
+
 func apacheLogCapture(logfile string) {
 	if _, err := os.Stat(logfile); err != nil {
 		log.Fatalf("%s: %s", logfile, err)
@@ -487,32 +687,33 @@ func writeLogToWatch(r *data.Request) {
 		h["X-Forwarded-For"] = r.XForwardedFor
 	}
 
-	if r.XRequestedWith != "" {
-		h["X-Requested-With"] = r.XRequestedWith
-	}
 	if r.XRequestedWith != "" {
 		h["X-Requested-With"] = r.XRequestedWith
 	}
 
 	data := map[string]interface{}{
 		"request": map[string]interface{}{
-			"time":    time.Unix(0, r.CreatedAt),
-			"address": r.Source,
-			// "scheme":  r.Protocol,
-			"method":  r.Method,
-			"url":     r.Url,
-			"headers": h,
+			"time":     time.Unix(0, r.CreatedAt),
+			"address":  r.Source,
+			"protocol": r.Protocol,
+			"scheme":   "https",
+			"method":   r.Method,
+			"url":      r.Url,
+			"headers":  h,
 		},
-		"response": map[string]interface{}{"status": 200},
+		"response": map[string]interface{}{"status": "200"},
 	}
 
 	jdata, err := json.Marshal(data)
 
 	client := &http.Client{}
 
+	fmt.Println(string(jdata))
 	buf := bytes.NewBuffer(jdata)
 	req, err := http.NewRequest("POST", "https://log.access.watch/1.1/log", buf)
-	req.Header.Add("Api-Key", *accessWatchKey)
+	req.Header.Add("Api-Key", config.AccessWatchKey)
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Content-Type", "application/json")
 	resp, err := client.Do(req)
 	if err != nil {
 		log.Println(err)
@@ -850,6 +1051,7 @@ func loadConfig() {
 	config.UseXForwardedAsSource = *useXForwardedAsSource
 	config.Protocol = *protocol
 	config.ApacheLog = *apacheLog
+	config.ApacheReplay = *apacheReplay
 	config.NginxLog = *nginxLog
 	config.NginxLogFormat = *nginxFormat
 	config.HostName = *hostName