added v2

go.mod

@@ -5,9 +5,12 @@ go 1.12
 require (
 	git.scraperwall.com/scw/geoip v1.0.2
 	git.scraperwall.com/scw/grs v0.0.0-20170601133330-3236d4163146
-	github.com/golang/protobuf v1.3.2
-	github.com/google/btree v1.0.0
+	github.com/golang/protobuf v1.5.2
+	github.com/google/btree v1.0.1
+	github.com/oschwald/geoip2-golang v1.5.0 // indirect
 	github.com/scraperwall/asndb v1.2.0
 	github.com/scraperwall/geoip v1.1.0
+	github.com/scraperwall/ipmeta v1.2.5 // indirect
+	golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect
 	gopkg.in/jmcvetta/napping.v3 v3.2.0 // indirect
 )

go.sum

@@ -4,31 +4,198 @@ git.scraperwall.com/scw/grs v0.0.0-20170601133330-3236d4163146 h1:pmtzyKPIrtqUmc
 git.scraperwall.com/scw/grs v0.0.0-20170601133330-3236d4163146/go.mod h1:tBzymrSfvUbefC/s0jnUtcHjpcZbH7+7f755VdGGhwM=
 git.scraperwall.com/scw/ip v0.0.0-20181127150108-80dbbe18947e h1:TJIzDykAwl9XbHGRGIp2gSKmUgCzhAcQDeEjXGSCblE=
 git.scraperwall.com/scw/ip v0.0.0-20181127150108-80dbbe18947e/go.mod h1:VWpgAIAUfNlojwRfuTrv2xh4EHYHSO0jV5CKEJyktrg=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.7.4/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/ip2location/ip2location-go v8.3.0+incompatible h1:QwUE+FlSbo6bjOWZpv2Grb57vJhWYFNPyBj2KCvfWaM=
+github.com/ip2location/ip2location-go v8.3.0+incompatible/go.mod h1:3JUY1TBjTx1GdA7oRT7Zeqfc0bg3lMMuU5lXmzdpuME=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
+github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
+github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
+github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
+github.com/minio/minio-go/v7 v7.0.14 h1:T7cw8P586gVwEEd0y21kTYtloD576XZgP62N8pE130s=
+github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo=
 github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8=
 github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE=
 github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug=
 github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng=
+github.com/oschwald/geoip2-golang v1.5.0 h1:igg2yQIrrcRccB1ytFXqBfOHCjXWIoMv85lVJ1ONZzw=
+github.com/oschwald/geoip2-golang v1.5.0/go.mod h1:xdvYt5xQzB8ORWFqPnqMwZpCpgNagttWdoZLlJQzg7s=
 github.com/oschwald/maxminddb-golang v1.5.0 h1:rmyoIV6z2/s9TCJedUuDiKht2RN12LWJ1L7iRGtWY64=
 github.com/oschwald/maxminddb-golang v1.5.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
 github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls=
 github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w=
+github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk=
+github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko=
 github.com/scraperwall/asndb v1.2.0 h1:THsXpgeGb5Pdw1WMPBjwTnKRpnf5BVNxa8YKZ7V9bHQ=
 github.com/scraperwall/asndb v1.2.0/go.mod h1:FLEIYBOlrrLIu1gGSUIAGYdwV+DgEW9DQXuDvjvDMmw=
+github.com/scraperwall/asndb/v2 v2.0.0 h1:7JmksWi2socgGj0NiFSk7+Rm+qJ/aNQ8F5RmLCuYWU4=
+github.com/scraperwall/asndb/v2 v2.0.0/go.mod h1:FOrjHkg4x4uMhVZKORrtd+Ud+kjMOs27sMItJyOWVbY=
 github.com/scraperwall/geoip v1.1.0 h1:jvHjpDvW1+PogOkMChLA7BzFjqUJpjI58mAPs9QzKVU=
 github.com/scraperwall/geoip v1.1.0/go.mod h1:/Yg2uchWAnElEme1Aqf5x840A9Qw+3y+xxcybOM1KTs=
+github.com/scraperwall/ipmeta v1.2.5 h1:ObBtj4bI5ZEAFBQPOlKv69jYEl1WBPwSmrHkv1xmAUg=
+github.com/scraperwall/ipmeta v1.2.5/go.mod h1:hmasqfbxURWWkJcFw6P/6AkwhevsEMm2x7C8E5gDtpU=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 h1:Dho5nD6R3PcW2SH1or8vS0dszDaXRxIw55lBX7XiE5g=
 golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef h1:fPxZ3Umkct3LZ8gK9nbk+DWDJ9fstZa2grBn+lWVKPs=
+golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/jmcvetta/napping.v3 v3.2.0 h1:NpSZLAL6VgiyhdqaOkxwVtHXOLrQJZ6fFOMQgp7G8PQ=
 gopkg.in/jmcvetta/napping.v3 v3.2.0/go.mod h1:0dPR4/IGM4+xGT+e48O2yJlg6qofrONCtEAWkurVlZQ=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

request_data.go

@@ -17,13 +17,13 @@ type RequestData struct {
 	XForwardedFor  string                 `bson:"x_forwarded_for" json:"x_forwarded_for,omitempty" form:"x_forwarded_for,omitempty" url:"x_forwarded_for,omitempty"`
 	XRealIP        string                 `bson:"x_real_ip" json:"x_real_ip,omitempty" form:"x_real_ip,omitempty" url:"x_real_ip,omitempty"`
 	Origin         string                 `bson:"origin" json:"origin,omitempty" form:"origin,omitempty" url:"origin,omitempty"`
-	PortDst        uint32                 `bson:"port_dst" json:"port_dst,omitempty" form: "port_dst,omitempty" url:"port_dst,omitempty"`
+	PortDst        uint32                 `bson:"port_dst" json:"port_dst,omitempty" form:"port_dst,omitempty" url:"port_dst,omitempty"`
 	Referer        string                 `bson:"referer" json:"referer,omitempty" form:"referer,omitempty" url:"referer,omitempty"`
 	XRequestedWith string                 `bson:"x_requested_with" json:"x_requested_with,omitempty" form:"x_requested_with.omitempty" url:"x_requested_with,omitempty"`
 	PortSrc        uint32                 `bson:"port_src" json:"port_src,omitempty" form:"port_src,omitempty" url:"port_src,omitempty"`
 	Url            string                 `bson:"url" json:"url,omitempty" form:"url,omitempty" url:"url,omitempty"`
 	Reverse        string                 `bson:"reverse" json:"reverse,omitempty" form:"reverse,omitempty" url:"reverse,omitempty"`
-	AcceptEncoding string                 `bson:"accept_encoding" json:"accept_encoding,omitempty" form:"accept_encoding,omitempty" url:"accept_encoding,omitempty""`
+	AcceptEncoding string                 `bson:"accept_encoding" json:"accept_encoding,omitempty" form:"accept_encoding,omitempty" url:"accept_encoding,omitempty"`
 	AcceptLanguage string                 `bson:"accept_language" json:"accept_language,omitempty" form:"accept_language,omitempty" url:"accept_language,omitempty"`
 	CreatedAt      time.Time              `bson:"created_at" json:"created_at,omitempty" form:"created_at,omitempty" url:"created_at,omitempty"`
 	SortTime       time.Time              `bson:"-" json:"-" form:"-" url:"-"`
@@ -37,7 +37,7 @@ type RequestData struct {
 	Protocol       string                 `bson:"protocol" json:"protocol,omitempty" form:"protocol,omitempty" url:"protocol,omitempty"`
 	Source         string                 `bson:"source" json:"source,omitempty" form:"source,omitempty" url:"source,omitempty"`
 	Via            string                 `bson:"via" json:"via,omitempty" form:"via,omitempty" url:"via,omitempty"`
-	Customer       string                 `bson:"customer" json:"customer,omitempty" form:"customer,omitempty" url:"customer:"omitempty"`
+	Customer       string                 `bson:"customer" json:"customer,omitempty" form:"customer,omitempty" url:"customer,omitempty"`
 	DataRaw        map[string]interface{} `bson:"data_raw" json:"data_raw,omitempty" form:"data_raw,omitempty" url:"data_raw,omitempty"`
 	Weight         uint64                 `bson:"weight" json:"weight,omitempty" form:"weight,omitempty" url:"weight,omitempty"`
 	Grs            grs.GRS                `bson:"grs" json:"grs,omitempty" form:"grs,omitempty" url:"grs,omitempty"`

v2/fingerprint.go

@@ -0,0 +1,51 @@
+package data
+
+import (
+	"time"
+
+	"git.scraperwall.com/scw/geoip"
+	"git.scraperwall.com/scw/grs"
+	"github.com/scraperwall/asndb"
+)
+
+type Fingerprint struct {
+	// ID                  bson.ObjectId `bson:"_id,omitempty" json:"_id"`
+	ClientID      string                 `bson:"cid" json:"cid" form:"cid"`
+	Remote        string                 `bson:"remote" json:"remote" form:"remote"`
+	Url           string                 `bson:"url" json:"url" form:"url"`
+	Fingerprint   string                 `bson:"fp" json:"fp" form:"fp"`
+	CreatedAt     time.Time              `bson:"created_at" json:"created_at" form:"created_at"`
+	Source        string                 `bson:"source" json:"source" form:"source"`
+	XForwardedFor string                 `bson:"x_forwarded_for" json:"x-forwarded-for" form:"x_forwarded_for"`
+	XRealIP       string                 `bson:"x_real_ip" json:"x-real-ip" form:"x_real_ip"`
+	XScwIP        string                 `bson:"x_scw_ip" json:"x-scw-ip" form:"x_scw_ip"`
+	Grs           *grs.GRS               `bson:"grs" json:"grs" form:"grs"`
+	GeoIP         *geoip.GeoIP           `bson:"geoip" json:"geoip" form:"geoip"`
+	Data          map[string]interface{} `bson:"data" json:"data" form:"data"`
+	Bot           bool                   `bson:"bot" json:"b" form:"b"`
+	ASN           *asndb.ASN             `bson:"asn" json:"asn" form:"asn"`
+	Country       string                 `bson:"country" json:"country" form:"country"`
+	Extra         map[string]string      `bson:"extra" json:"extra" form:"extra"`
+	//UserAgent           string        `bson:"user_agent" json:"user_agent" form:"user_agent"`
+	//Adblock             bool          `bson:"adblock" json:"adblock" form:"adblock"`
+	//AvailableResolution []int         `bson:"available_resolution" json:"available_resolution" form:"available_resolution"`
+	//Canvas              string        `bson:"canvas" json:"canvas" form:"canvas"`
+	//ColorDepth          int           `bson:"color_depth" json:"color_depth" form:"color_depth"`
+	//CpuClass            string        `bson:"cpu_class" json:"cpu_class" form:"cpu_class"`
+	//DoNotTrack          string        `bson:"do_not_track" json:"do_not_track" form:"don_not_track"`
+	//HasLiedBrowser      bool          `bson:"has_lied_browser" json:"has_lied_browser" form:"has_lied_browser"`
+	//HasLiedLanguages    bool          `bson:"has_lied_languages" json:"has_lied_languages" form:"has_lied_languages"`
+	//HasLiedOs           bool          `bson:"has_lied_os" json:"has_lied_os" form:"has_lied_os"`
+	//HasLiedResolution   bool          `bson:"has_lied_resolution" json:"has_lied_resolution" form:"has_lied_resolution"`
+	//IndexedDb           int           `bson:"indexed_db" json:"indexed_db" form:"indexed_db"`
+	//JsFonts             []string      `bson:"js_fonts" json:"js_fonts" form:"js_fonts"`
+	//Language            string        `bson:"language" json:"language" form:"language"`
+	//LocalStorage        int           `bson:"local_storage" json:"local_storage" form:"local_storage"`
+	//NavigatorPlatform   string        `bson:"navigator_platform" json:"navigator_platform" form:"navigator_platform"`
+	//Plugins             []string      `bson:"regular_plugins" json:"regular_plugins" form:"regular_plugins"`
+	//Resolution          []int         `bson:"resolution" json:"resolution" form:"resolution"`
+	//SessionStorage      int           `bson:"session_storage" json:"session_storage" form:"session_storage"`
+	//TouchSupport        []interface{} `bson:"touch_support" json:"touch_support" form:"touch_support"`
+	//WebGl               string        `bson:"webgl" json:"webgl" form:"webgl"`
+	//TimeZoneOffset      int           `bson:"timezone_offset" json:"timezone_offset" form:"timezone_offset"`
+}

v2/go.mod

@@ -0,0 +1,16 @@
+module github.com/scraperwall/data/v2
+
+go 1.12
+
+require (
+	git.scraperwall.com/scw/geoip v1.0.2
+	git.scraperwall.com/scw/grs v0.0.0-20170601133330-3236d4163146
+	github.com/golang/protobuf v1.5.2
+	github.com/google/btree v1.0.1
+	github.com/oschwald/geoip2-golang v1.5.0 // indirect
+	github.com/scraperwall/asndb v1.2.0
+	github.com/scraperwall/geoip v1.1.0
+	github.com/scraperwall/ipmeta v1.2.5 // indirect
+	golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect
+	gopkg.in/jmcvetta/napping.v3 v3.2.0 // indirect
+)

v2/ping.go

@@ -0,0 +1,21 @@
+package data
+
+import (
+	"github.com/scraperwall/asndb"
+	"github.com/scraperwall/geoip"
+)
+
+// Ping is used as a primitive way to validate that a browser has JS
+type Ping struct {
+	Timestamp     int64        `bson:"timestamp" json:"timestamp" form:"timestamp"`
+	Checksum      string       `bson:"checksum" json:"checksum" form:"checksum"`
+	Remote        string       `bson:"remote" json:"remote" form:"remote"`
+	URL           string       `bson:"url" json:"url" form:"url"`
+	Source        string       `bson:"source" json:"source" form:"source"`
+	XForwardedFor string       `bson:"x_forwarded_for" json:"x-forwarded-for" form:"x_forwarded_for"`
+	XRealIP       string       `bson:"x_real_ip" json:"x-real-ip" form:"x_real_ip"`
+	XScwIP        string       `bson:"x_scw_ip" json:"x-scw-ip" form:"x_scw_ip"`
+	ASN           *asndb.ASN   `bson:"asn" json:"asn" form:"asn"`
+	GeoIP         *geoip.GeoIP `bson:"geoip" json:"geoip" form:"geoip"`
+	ClientID      string       `bson:"clientid" json:"clientid" form:"clientid"`
+}

v2/request.go

@@ -0,0 +1,35 @@
+package data
+
+import (
+	"time"
+)
+
+func (r *Request) ToRequestData() *RequestData {
+	return &RequestData{
+		Url:            r.Url,
+		Reverse:        r.Reverse,
+		IpSrc:          r.IpSrc,
+		IpDst:          r.IpDst,
+		PortSrc:        r.PortSrc,
+		PortDst:        r.PortDst,
+		TcpSeq:         r.TcpSeq,
+		CreatedAt:      time.Unix(0, r.CreatedAt),
+		XForwardedFor:  r.XForwardedFor,
+		XRealIP:        r.XRealIP,
+		Method:         r.Method,
+		Origin:         r.Origin,
+		Referer:        r.Referer,
+		UserAgent:      r.UserAgent,
+		Source:         r.Source,
+		Host:           r.Host,
+		Protocol:       r.Protocol,
+		Connection:     r.Connection,
+		XRequestedWith: r.XRequestedWith,
+		AcceptEncoding: r.AcceptEncoding,
+		AcceptLanguage: r.AcceptLanguage,
+		Accept:         r.Accept,
+		Cookie:         r.Cookie,
+		Via:            r.Via,
+		Customer:       r.Customer,
+	}
+}

v2/request.pb.go

@@ -0,0 +1,292 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: request.proto
+
+package data
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type Request struct {
+	Url                  string   `protobuf:"bytes,100,opt,name=Url,proto3" json:"Url,omitempty"`
+	Reverse              string   `protobuf:"bytes,200,opt,name=Reverse,proto3" json:"Reverse,omitempty"`
+	IpSrc                string   `protobuf:"bytes,300,opt,name=IpSrc,proto3" json:"IpSrc,omitempty"`
+	IpDst                string   `protobuf:"bytes,400,opt,name=IpDst,proto3" json:"IpDst,omitempty"`
+	PortSrc              uint32   `protobuf:"varint,500,opt,name=PortSrc,proto3" json:"PortSrc,omitempty"`
+	PortDst              uint32   `protobuf:"varint,600,opt,name=PortDst,proto3" json:"PortDst,omitempty"`
+	TcpSeq               uint32   `protobuf:"varint,700,opt,name=TcpSeq,proto3" json:"TcpSeq,omitempty"`
+	CreatedAt            int64    `protobuf:"varint,800,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
+	XForwardedFor        string   `protobuf:"bytes,900,opt,name=XForwardedFor,proto3" json:"XForwardedFor,omitempty"`
+	XRealIP              string   `protobuf:"bytes,1000,opt,name=XRealIP,proto3" json:"XRealIP,omitempty"`
+	Method               string   `protobuf:"bytes,1100,opt,name=Method,proto3" json:"Method,omitempty"`
+	Origin               string   `protobuf:"bytes,1200,opt,name=Origin,proto3" json:"Origin,omitempty"`
+	Referer              string   `protobuf:"bytes,1300,opt,name=Referer,proto3" json:"Referer,omitempty"`
+	UserAgent            string   `protobuf:"bytes,1400,opt,name=UserAgent,proto3" json:"UserAgent,omitempty"`
+	Source               string   `protobuf:"bytes,1500,opt,name=Source,proto3" json:"Source,omitempty"`
+	Host                 string   `protobuf:"bytes,1600,opt,name=Host,proto3" json:"Host,omitempty"`
+	Protocol             string   `protobuf:"bytes,1700,opt,name=Protocol,proto3" json:"Protocol,omitempty"`
+	Connection           string   `protobuf:"bytes,1800,opt,name=Connection,proto3" json:"Connection,omitempty"`
+	XRequestedWith       string   `protobuf:"bytes,1900,opt,name=XRequestedWith,proto3" json:"XRequestedWith,omitempty"`
+	AcceptEncoding       string   `protobuf:"bytes,2000,opt,name=AcceptEncoding,proto3" json:"AcceptEncoding,omitempty"`
+	AcceptLanguage       string   `protobuf:"bytes,2100,opt,name=AcceptLanguage,proto3" json:"AcceptLanguage,omitempty"`
+	Accept               string   `protobuf:"bytes,2200,opt,name=Accept,proto3" json:"Accept,omitempty"`
+	Cookie               string   `protobuf:"bytes,2300,opt,name=Cookie,proto3" json:"Cookie,omitempty"`
+	Via                  string   `protobuf:"bytes,2400,opt,name=Via,proto3" json:"Via,omitempty"`
+	Customer             string   `protobuf:"bytes,2500,opt,name=Customer,proto3" json:"Customer,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Request) Reset()         { *m = Request{} }
+func (m *Request) String() string { return proto.CompactTextString(m) }
+func (*Request) ProtoMessage()    {}
+func (*Request) Descriptor() ([]byte, []int) {
+	return fileDescriptor_7f73548e33e655fe, []int{0}
+}
+
+func (m *Request) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Request.Unmarshal(m, b)
+}
+func (m *Request) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Request.Marshal(b, m, deterministic)
+}
+func (m *Request) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Request.Merge(m, src)
+}
+func (m *Request) XXX_Size() int {
+	return xxx_messageInfo_Request.Size(m)
+}
+func (m *Request) XXX_DiscardUnknown() {
+	xxx_messageInfo_Request.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Request proto.InternalMessageInfo
+
+func (m *Request) GetUrl() string {
+	if m != nil {
+		return m.Url
+	}
+	return ""
+}
+
+func (m *Request) GetReverse() string {
+	if m != nil {
+		return m.Reverse
+	}
+	return ""
+}
+
+func (m *Request) GetIpSrc() string {
+	if m != nil {
+		return m.IpSrc
+	}
+	return ""
+}
+
+func (m *Request) GetIpDst() string {
+	if m != nil {
+		return m.IpDst
+	}
+	return ""
+}
+
+func (m *Request) GetPortSrc() uint32 {
+	if m != nil {
+		return m.PortSrc
+	}
+	return 0
+}
+
+func (m *Request) GetPortDst() uint32 {
+	if m != nil {
+		return m.PortDst
+	}
+	return 0
+}
+
+func (m *Request) GetTcpSeq() uint32 {
+	if m != nil {
+		return m.TcpSeq
+	}
+	return 0
+}
+
+func (m *Request) GetCreatedAt() int64 {
+	if m != nil {
+		return m.CreatedAt
+	}
+	return 0
+}
+
+func (m *Request) GetXForwardedFor() string {
+	if m != nil {
+		return m.XForwardedFor
+	}
+	return ""
+}
+
+func (m *Request) GetXRealIP() string {
+	if m != nil {
+		return m.XRealIP
+	}
+	return ""
+}
+
+func (m *Request) GetMethod() string {
+	if m != nil {
+		return m.Method
+	}
+	return ""
+}
+
+func (m *Request) GetOrigin() string {
+	if m != nil {
+		return m.Origin
+	}
+	return ""
+}
+
+func (m *Request) GetReferer() string {
+	if m != nil {
+		return m.Referer
+	}
+	return ""
+}
+
+func (m *Request) GetUserAgent() string {
+	if m != nil {
+		return m.UserAgent
+	}
+	return ""
+}
+
+func (m *Request) GetSource() string {
+	if m != nil {
+		return m.Source
+	}
+	return ""
+}
+
+func (m *Request) GetHost() string {
+	if m != nil {
+		return m.Host
+	}
+	return ""
+}
+
+func (m *Request) GetProtocol() string {
+	if m != nil {
+		return m.Protocol
+	}
+	return ""
+}
+
+func (m *Request) GetConnection() string {
+	if m != nil {
+		return m.Connection
+	}
+	return ""
+}
+
+func (m *Request) GetXRequestedWith() string {
+	if m != nil {
+		return m.XRequestedWith
+	}
+	return ""
+}
+
+func (m *Request) GetAcceptEncoding() string {
+	if m != nil {
+		return m.AcceptEncoding
+	}
+	return ""
+}
+
+func (m *Request) GetAcceptLanguage() string {
+	if m != nil {
+		return m.AcceptLanguage
+	}
+	return ""
+}
+
+func (m *Request) GetAccept() string {
+	if m != nil {
+		return m.Accept
+	}
+	return ""
+}
+
+func (m *Request) GetCookie() string {
+	if m != nil {
+		return m.Cookie
+	}
+	return ""
+}
+
+func (m *Request) GetVia() string {
+	if m != nil {
+		return m.Via
+	}
+	return ""
+}
+
+func (m *Request) GetCustomer() string {
+	if m != nil {
+		return m.Customer
+	}
+	return ""
+}
+
+func init() {
+	proto.RegisterType((*Request)(nil), "data.Request")
+}
+
+func init() { proto.RegisterFile("request.proto", fileDescriptor_7f73548e33e655fe) }
+
+var fileDescriptor_7f73548e33e655fe = []byte{
+	// 471 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0x3d, 0x6f, 0xd4, 0x40,
+	0x10, 0x86, 0x75, 0xdc, 0x47, 0x72, 0x0b, 0x0e, 0xc9, 0x06, 0xc4, 0x20, 0x84, 0x88, 0x90, 0x22,
+	0x52, 0x5d, 0xc3, 0x2f, 0xb8, 0x5c, 0x88, 0x88, 0x04, 0xe2, 0xe4, 0x23, 0xe1, 0xda, 0x65, 0x3d,
+	0x38, 0x16, 0x8e, 0xd7, 0x19, 0xef, 0x91, 0x3f, 0x40, 0x41, 0x49, 0x41, 0x41, 0x49, 0x41, 0x49,
+	0x41, 0x41, 0x49, 0x41, 0x41, 0x01, 0x12, 0x05, 0x25, 0x05, 0x05, 0x25, 0x05, 0x25, 0x05, 0x05,
+	0x05, 0x9a, 0x1d, 0x9b, 0x4b, 0xd2, 0xf9, 0x7d, 0x9e, 0xd7, 0x6b, 0xcf, 0xee, 0xaa, 0x88, 0xf0,
+	0x70, 0x86, 0x95, 0x1f, 0x94, 0xe4, 0xbc, 0xd3, 0x9d, 0xc4, 0x78, 0x73, 0xfd, 0x73, 0x57, 0x2d,
+	0xc4, 0xc2, 0xf5, 0xb2, 0x6a, 0xef, 0x52, 0x0e, 0xc9, 0x5a, 0x6b, 0xa3, 0x1f, 0xf3, 0xa3, 0xbe,
+	0xcc, 0xf2, 0x09, 0x52, 0x85, 0xf0, 0xa9, 0x15, 0x70, 0x93, 0xf5, 0x45, 0xd5, 0xdd, 0x29, 0x27,
+	0x64, 0xe1, 0xcd, 0x99, 0x20, 0x24, 0x09, 0xde, 0xaa, 0x3c, 0x3c, 0x6f, 0x37, 0x78, 0xab, 0xf2,
+	0xbc, 0xd0, 0xd8, 0x91, 0xe7, 0xfe, 0x6f, 0x16, 0x51, 0xdc, 0xe4, 0x46, 0xf1, 0x3b, 0xdf, 0x3a,
+	0x73, 0xc5, 0x6f, 0x5d, 0x52, 0xbd, 0xfb, 0xb6, 0x9c, 0xe0, 0x21, 0xbc, 0xef, 0x06, 0x53, 0x47,
+	0x7d, 0x55, 0xf5, 0x47, 0x84, 0xc6, 0x63, 0x32, 0xf4, 0xf0, 0xaa, 0xb7, 0xd6, 0xda, 0x68, 0xc7,
+	0x73, 0xa2, 0xd7, 0x55, 0x34, 0xdd, 0x76, 0x74, 0x64, 0x28, 0xc1, 0x64, 0xdb, 0x11, 0x3c, 0x5d,
+	0x08, 0x3f, 0x73, 0x92, 0xf2, 0x97, 0xa7, 0x31, 0x9a, 0x7c, 0x67, 0x0c, 0x3f, 0xa5, 0xd0, 0x64,
+	0xfe, 0xf2, 0x5d, 0xf4, 0xfb, 0x2e, 0x81, 0x2f, 0x8b, 0xc1, 0xd4, 0x91, 0xc5, 0x3d, 0xca, 0xd2,
+	0xac, 0x80, 0xb7, 0x7d, 0x11, 0x12, 0x65, 0xab, 0x1e, 0x21, 0x21, 0xc1, 0x0b, 0xd5, 0x6c, 0x55,
+	0xc8, 0xfc, 0xb7, 0xbb, 0x15, 0xd2, 0x30, 0xc5, 0xc2, 0xc3, 0x1f, 0x91, 0x73, 0xc2, 0x4b, 0x4e,
+	0xdc, 0x8c, 0x2c, 0xc2, 0xf7, 0xb3, 0xb2, 0xa4, 0x44, 0xbd, 0xaa, 0x3a, 0xb7, 0x5d, 0xe5, 0xe1,
+	0xc3, 0xb9, 0x80, 0x43, 0xd0, 0x57, 0xd4, 0xe2, 0x98, 0xcf, 0xcf, 0xba, 0x1c, 0x5e, 0x47, 0x41,
+	0xfc, 0x07, 0xfa, 0x9a, 0x52, 0x23, 0x57, 0x14, 0x68, 0x7d, 0xe6, 0x0a, 0x78, 0xb6, 0x14, 0xf4,
+	0x31, 0xa4, 0x6f, 0xa8, 0xa5, 0x69, 0x7d, 0xdc, 0x98, 0x3c, 0xc8, 0xfc, 0x3e, 0xfc, 0x92, 0xd2,
+	0x29, 0xcc, 0xc5, 0xa1, 0xb5, 0x58, 0xfa, 0x5b, 0x85, 0x75, 0x49, 0x56, 0xa4, 0xf0, 0xf5, 0xbc,
+	0x14, 0x4f, 0xe2, 0x79, 0xf1, 0x8e, 0x29, 0xd2, 0x99, 0x49, 0x11, 0xde, 0x2d, 0x1f, 0x2f, 0x36,
+	0x98, 0xc7, 0x14, 0x02, 0x2f, 0x57, 0x64, 0x4c, 0x89, 0x2c, 0x46, 0xce, 0x3d, 0xce, 0x10, 0xfe,
+	0xd6, 0x42, 0xa2, 0x5e, 0x51, 0xed, 0xbd, 0xcc, 0xc0, 0x0f, 0x2d, 0x17, 0x72, 0x2f, 0x33, 0x3c,
+	0xfd, 0x68, 0x56, 0x79, 0x77, 0x80, 0x04, 0x1f, 0x57, 0x65, 0xfa, 0x06, 0x6c, 0xae, 0xab, 0x0b,
+	0xd6, 0x1d, 0x0c, 0x2a, 0x4b, 0xa6, 0x44, 0x3a, 0x32, 0x79, 0x3e, 0xe0, 0x3b, 0xbe, 0x19, 0xd5,
+	0xa3, 0x85, 0x6d, 0xaa, 0x1e, 0xf6, 0xc2, 0xfd, 0xbf, 0xf9, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xdd,
+	0xf4, 0xb4, 0x9a, 0x10, 0x03, 0x00, 0x00,
+}

v2/request_data.go

@@ -0,0 +1,88 @@
+package data
+
+import (
+	"strings"
+	"time"
+
+	"github.com/google/btree"
+	"github.com/scraperwall/ipmeta"
+)
+
+type RequestData struct {
+	ID             string                 `bson:"_id,omitempty" json:"_id,omitempty" form:"_id,omitempty" url:"_id,omitempty"`
+	Connection     string                 `bson:"connection" json:"connection" form:"connection,omitempty" url:"connection,omitempty"`
+	IpDst          string                 `bson:"ip_dst" json:"ip_dst" form:"ip_dst,omitempty" url:"ip_dst,omitempty"`
+	TcpSeq         uint32                 `bson:"tcp_seq" json:"tcp_seq" form:"tcp_seq,omitempty" url:"tcp_seq,omitempty"`
+	XForwardedFor  string                 `bson:"x_forwarded_for" json:"x_forwarded_for,omitempty" form:"x_forwarded_for,omitempty" url:"x_forwarded_for,omitempty"`
+	XRealIP        string                 `bson:"x_real_ip" json:"x_real_ip,omitempty" form:"x_real_ip,omitempty" url:"x_real_ip,omitempty"`
+	Origin         string                 `bson:"origin" json:"origin,omitempty" form:"origin,omitempty" url:"origin,omitempty"`
+	PortDst        uint32                 `bson:"port_dst" json:"port_dst,omitempty" form:"port_dst,omitempty" url:"port_dst,omitempty"`
+	Referer        string                 `bson:"referer" json:"referer,omitempty" form:"referer,omitempty" url:"referer,omitempty"`
+	XRequestedWith string                 `bson:"x_requested_with" json:"x_requested_with,omitempty" form:"x_requested_with.omitempty" url:"x_requested_with,omitempty"`
+	PortSrc        uint32                 `bson:"port_src" json:"port_src,omitempty" form:"port_src,omitempty" url:"port_src,omitempty"`
+	Url            string                 `bson:"url" json:"url,omitempty" form:"url,omitempty" url:"url,omitempty"`
+	Reverse        string                 `bson:"reverse" json:"reverse,omitempty" form:"reverse,omitempty" url:"reverse,omitempty"`
+	AcceptEncoding string                 `bson:"accept_encoding" json:"accept_encoding,omitempty" form:"accept_encoding,omitempty" url:"accept_encoding,omitempty"`
+	AcceptLanguage string                 `bson:"accept_language" json:"accept_language,omitempty" form:"accept_language,omitempty" url:"accept_language,omitempty"`
+	CreatedAt      time.Time              `bson:"created_at" json:"created_at,omitempty" form:"created_at,omitempty" url:"created_at,omitempty"`
+	SortTime       time.Time              `bson:"-" json:"-" form:"-" url:"-"`
+	IpSrc          string                 `bson:"ip_src" json:"ip_src,omitempty" form:"ip_src,omitempty" url:"ip_src,omitempty"`
+	Method         string                 `bson:"method" json:"method,omitempty" form:"method,omitempty" url:"method,omitempty"`
+	UserAgent      string                 `bson:"user_agent" json:"user_agent,omitempty" form:"user_agent,omitempty" url:"user_agent,omitempty"`
+	Accept         string                 `bson:"accept" json:"accept,omitempty" form:"accept,omitempty" url:"accept,omitempty"`
+	Cookie         string                 `bson:"cookie" json:"cookie,omitempty" form:"cookie,omitempty" url:"cookie,omitempty"`
+	Host           string                 `bson:"host" json:"host,omitempty" form:"host,omitempty" url:"host,omitempty"`
+	HostName       string                 `bson:"hostname" json:"hostname,omitempty" form:"hostname,omitempty" url:"hostname,omitempty"`
+	Protocol       string                 `bson:"protocol" json:"protocol,omitempty" form:"protocol,omitempty" url:"protocol,omitempty"`
+	Source         string                 `bson:"source" json:"source,omitempty" form:"source,omitempty" url:"source,omitempty"`
+	Via            string                 `bson:"via" json:"via,omitempty" form:"via,omitempty" url:"via,omitempty"`
+	Customer       string                 `bson:"customer" json:"customer,omitempty" form:"customer,omitempty" url:"customer,omitempty"`
+	DataRaw        map[string]interface{} `bson:"data_raw" json:"data_raw,omitempty" form:"data_raw,omitempty" url:"data_raw,omitempty"`
+	Weight         uint64                 `bson:"weight" json:"weight,omitempty" form:"weight,omitempty" url:"weight,omitempty"`
+	Meta           *ipmeta.Meta           `bson:"meta" json:"meta,omitempty" url:"meta,omitempty" `
+}
+
+func (a *RequestData) Less(b btree.Item) bool {
+	return a.CreatedAt.Before(b.(*RequestData).CreatedAt)
+}
+
+func (a *RequestData) ToRequest() *Request {
+	return &Request{
+		Url:            a.Url,
+		Reverse:        a.Reverse,
+		IpSrc:          a.IpSrc,
+		IpDst:          a.IpDst,
+		PortSrc:        a.PortSrc,
+		PortDst:        a.PortDst,
+		TcpSeq:         a.TcpSeq,
+		CreatedAt:      a.CreatedAt.UnixNano(),
+		XForwardedFor:  a.XForwardedFor,
+		XRealIP:        a.XRealIP,
+		Method:         a.Method,
+		Origin:         a.Origin,
+		Referer:        a.Referer,
+		UserAgent:      a.UserAgent,
+		Source:         a.Source,
+		Host:           a.Host,
+		Protocol:       a.Protocol,
+		Connection:     a.Connection,
+		XRequestedWith: a.XRequestedWith,
+		AcceptEncoding: a.AcceptEncoding,
+		AcceptLanguage: a.AcceptLanguage,
+		Accept:         a.Accept,
+		Cookie:         a.Cookie,
+		Via:            a.Via,
+		Customer:       a.Customer,
+	}
+}
+
+// Path returns the Url without any query string
+func (a *RequestData) Path() string {
+	idx := strings.Index(a.Url, "?")
+
+	if idx >= 0 {
+		return a.Url[0:idx]
+	}
+
+	return a.Url
+}