apache parser update. use VirtualHost if available

Makefile

@@ -5,7 +5,7 @@ all:
 	docker run --rm -v $(PWD):/root/go/src/git.scraperwall.com/scw/munchclient -w /root/go/src/git.scraperwall.com/scw/munchclient registry.scw.systems/centos7-build:latest make compile
 
 all-debian:
-	docker run --rm -v $(PWD):/go/src/git.scraperwall.com/scw/munchclient -w /go/src/git.scraperwall.com/scw/munchclient registry.scw.systems/debian-build:latest make compile
+	docker run --rm -v $(PWD):/go/src/git.scraperwall.com/scw/munchclient -w /go/src/git.scraperwall.com/scw/munchclient scraperwall/debian-build:latest make compile
 
 compile:
 	env GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags "-s -X main.Version=$(VERSION) -X main.BuildDate=$(BUILD)"

apache.go

@@ -59,6 +59,9 @@ func apacheLogReplay(logfile string) {
 
 		// fmt.Println(l)
 		remote := logEntry.Host
+		if logEntry.VirtualHost != "" {
+			remote = logEntry.VirtualHost
+		}
 		if *useXForwardedAsSource && logEntry.ForwardedFor != "" {
 			remote = logEntry.ForwardedFor
 		}

build-rpm-centos7.sh

@@ -45,13 +45,13 @@ docker run --rm -i -t -v $PWD:/scw -v $PWD/rpmmacros:/root/.rpmmacros -w /scw $D
 rpm_file="$OUTDIR/$BINARY-$VERSION-$ITERATION.x86_64.rpm"
 
 docker run --rm -i -t -v $PWD:/scw -v $PWD/rpmmacros:/root/.rpmmacros -w /scw $DOCKER_IMAGE \
-  /usr/bin/rpm --define "_gpg_name <tobias@scraperwall.com>" --addsign $rpm_file
+  /usr/bin/rpm --define "_gpg_name <rpm@scraperwall.com>" --addsign $rpm_file
 
 ok=$?
 
 rm -rf $DESTDIR
 
-if [ $ok -eq 0 -a -f "$rpm_file" ]; then
+if [ $ok -eq 0 -a -f "$rpm_file" -a -d "$RPM_DIR" ]; then
 	# rm -f "$RPM_DIR/$BINARY-*.rpm"
   cp "$rpm_file" "$RPM_DIR"
 

vendor/github.com/Songmu/axslogparser/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+## [v1.1.0](https://github.com/Songmu/axslogparser/compare/v1.0.0...v1.1.0) (2018-08-29)
+
+* [incompatible] [bugfix] s/RemoteUser/RemoteLogname/g [#14](https://github.com/Songmu/axslogparser/pull/14) ([Songmu](https://github.com/Songmu))
+* care remote_user which contains whitespace in apache log [#13](https://github.com/Songmu/axslogparser/pull/13) ([Songmu](https://github.com/Songmu))
+* Support to multi ips into a remote host field [#12](https://github.com/Songmu/axslogparser/pull/12) ([softctrl](https://github.com/softctrl))
+
+## [v1.0.0](https://github.com/Songmu/axslogparser/compare/5cfe5b4ad944...v1.0.0) (2018-04-04)
+
+* Accept other than '-' as RemoteUser [#11](https://github.com/Songmu/axslogparser/pull/11) ([ulyssessouza](https://github.com/ulyssessouza))
+* Fix typo in error message [#9](https://github.com/Songmu/axslogparser/pull/9) ([yano3](https://github.com/yano3))
+* Consider the case where a host field is at the beginning of the LTSV log in `GuessParser` [#8](https://github.com/Songmu/axslogparser/pull/8) ([Songmu](https://github.com/Songmu))
+* Readme [#7](https://github.com/Songmu/axslogparser/pull/7) ([Songmu](https://github.com/Songmu))
+* add tests for LTSV [#6](https://github.com/Songmu/axslogparser/pull/6) ([Songmu](https://github.com/Songmu))
+* enhance testing [#5](https://github.com/Songmu/axslogparser/pull/5) ([Songmu](https://github.com/Songmu))
+* better error handling [#4](https://github.com/Songmu/axslogparser/pull/4) ([Songmu](https://github.com/Songmu))
+* Adjust field names [#3](https://github.com/Songmu/axslogparser/pull/3) ([Songmu](https://github.com/Songmu))
+* Adjust LTSV parser [#2](https://github.com/Songmu/axslogparser/pull/2) ([Songmu](https://github.com/Songmu))
+* Better parsing for apache log [#1](https://github.com/Songmu/axslogparser/pull/1) ([Songmu](https://github.com/Songmu))

vendor/github.com/Songmu/axslogparser/Makefile

@@ -3,7 +3,7 @@ test: deps
 
 deps:
 	go get -d -v -t ./...
-	go get github.com/golang/lint/golint
+	go get golang.org/x/lint/golint
 	go get golang.org/x/tools/cmd/cover
 	go get github.com/mattn/goveralls
 

vendor/github.com/Songmu/axslogparser/apache.go

@@ -16,10 +16,10 @@ type Apache struct {
 }
 
 var logRe = regexp.MustCompile(
-	`^(?:(\S+)\s)?` + // %v(The canonical ServerName/virtual host)
+	`^(?:(\S+(?:,\s\S+)*)\s)?` + // %v(The canonical ServerName/virtual host) - 192.168.0.1 or 192.168.0.1,192.168.0.2, 192.168.0.3
 		`(\S+)\s` + // %h(Remote Hostname) $remote_addr
 		`(\S+)\s` + // %l(Remote Logname)
-		`(\S+)\s` + // $remote_user
+		`([\S\s]+)\s` + // $remote_user
 		`\[(\d{2}/\w{3}/\d{2}(?:\d{2}:){3}\d{2} [-+]\d{4})\]\s` + // $time_local
 		`(.*)`)
 
@@ -30,11 +30,18 @@ func (ap *Apache) Parse(line string) (*Log, error) {
 		return nil, fmt.Errorf("failed to parse apachelog (not matched): %s", line)
 	}
 	l := &Log{
-		VirtualHost: matches[1],
-		Host:        matches[2],
-		RemoteUser:  matches[3],
-		User:        matches[4],
+		VirtualHost:   matches[1],
+		Host:          matches[2],
+		RemoteLogname: matches[3],
+		User:          matches[4],
 	}
+	if l.Host == "-" && l.VirtualHost != "" {
+		l.Host = l.VirtualHost
+		l.VirtualHost = ""
+		l.User = fmt.Sprintf("%s %s", l.RemoteLogname, l.User)
+		l.RemoteLogname = "-"
+	}
+
 	l.Time, _ = time.Parse(clfTimeLayout, matches[5])
 	var rest string
 

vendor/github.com/Songmu/axslogparser/axslogparser.go

@@ -13,24 +13,25 @@ type Parser interface {
 
 // Log is the struct stored parsed result of single line of accesslog
 type Log struct {
-	VirtualHost  string `ltsv:"vhost"`
-	Host         string
-	RemoteUser   string
-	User         string
-	Time         time.Time `ltsv:"-"`
-	TimeStr      string    `ltsv:"time"`
-	Request      string    `ltsv:"req"`
-	Status       int
-	Size         uint64
-	Referer      string
-	UserAgent    string `ltsv:"ua"`
-	ReqTime      *float64
-	AppTime      *float64
-	TakenSec     *float64 `ltsv:"taken_sec"` // Hatena specific
-	ForwardedFor string
-	RequestURI   string `ltsv:"uri"`
-	Protocol     string
-	Method       string
+	VirtualHost     string `ltsv:"vhost"`
+	Host            string
+	RemoteLogname   string `ltsv:"ident"`
+	User            string
+	Time            time.Time `ltsv:"-"`
+	TimeStr         string    `ltsv:"time"`
+	Request         string    `ltsv:"req"`
+	Status          int
+	Size            uint64
+	Referer         string
+	UserAgent       string `ltsv:"ua"`
+	ReqTime         *float64
+	ReqTimeMicroSec *float64 `ltsv:"reqtime_microsec"`
+	AppTime         *float64
+	TakenSec        *float64 `ltsv:"taken_sec"` // Hatena specific
+	ForwardedFor    string
+	RequestURI      string `ltsv:"uri"`
+	Protocol        string
+	Method          string
 }
 
 func (l *Log) breakdownRequest() error {

vendor/github.com/Songmu/axslogparser/ltsv.go

@@ -21,7 +21,7 @@ func (lv *LTSV) Parse(line string) (*Log, error) {
 	}
 	l.Time, _ = time.Parse(clfTimeLayout, strings.Trim(l.TimeStr, "[]"))
 	if err := l.breakdownRequest(); err != nil {
-		return nil, errors.Wrap(err, "failed to parse ltsvlog (invalid request): %s")
+		return nil, errors.Wrap(err, "failed to parse ltsvlog (invalid request)")
 	}
 	return l, nil
 }

vendor/vendor.json

@@ -44,10 +44,10 @@
 			"revisionTime": "2017-06-26T11:06:00Z"
 		},
 		{
-			"checksumSHA1": "VPx3ayTuqB+mhEw5hQ3PnQodWO0=",
+			"checksumSHA1": "d7dugXZ0lIbH3nH92qw83i8X6J0=",
 			"path": "github.com/Songmu/axslogparser",
-			"revision": "bf78871227132ce86c3b82f4f154539c657fcd55",
-			"revisionTime": "2018-04-04T03:19:36Z"
+			"revision": "31874ad7dc8d453b39e3494f8979cedfaa386199",
+			"revisionTime": "2019-03-12T01:56:35Z"
 		},
 		{
 			"checksumSHA1": "XHHhMG76dY41wot8n9QwMm0XZoQ=",