From 3a80bf6db6e887180c4963d8e10b1dea175584e3 Mon Sep 17 00:00:00 2001 From: guervild <11190755+guervild@users.noreply.github.com> Date: Tue, 13 Jul 2021 13:27:15 +0200 Subject: [PATCH] Add the applications technologies (#104) --- chrome/chrome.go | 11 +++++--- chrome/helpers.go | 26 +++++++++++++++++++ cmd/report_serve.go | 5 ++-- go.mod | 3 ++- go.sum | 9 +++++++ lib/pagination.go | 2 +- lib/processor.go | 9 ++++--- storage/db.go | 2 +- storage/models.go | 20 +++++++++++++-- web/assets/templates/detail.html | 5 ++++ web/assets/templates/gallery.html | 5 ++++ web/assets_vfsdata.go | 42 +++++++++++++++---------------- 12 files changed, 104 insertions(+), 35 deletions(-) diff --git a/chrome/chrome.go b/chrome/chrome.go index dc22295..a08a9e1 100644 --- a/chrome/chrome.go +++ b/chrome/chrome.go @@ -35,7 +35,7 @@ func NewChrome() *Chrome { } // Preflight will preflight a url -func (chrome *Chrome) Preflight(url *url.URL) (resp *http.Response, title string, err error) { +func (chrome *Chrome) Preflight(url *url.URL) (resp *http.Response, title string, technologies []string, err error) { // purposefully ignore bad certs transport := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -45,7 +45,7 @@ func (chrome *Chrome) Preflight(url *url.URL) (resp *http.Response, title string var erri error proxyURL, erri := url.Parse(chrome.Proxy) if erri != nil { - return nil, "", erri + return nil, "", nil, erri } transport.Proxy = http.ProxyURL(proxyURL) } @@ -73,12 +73,13 @@ func (chrome *Chrome) Preflight(url *url.URL) (resp *http.Response, title string defer resp.Body.Close() title, _ = GetHTMLTitle(resp.Body) + technologies, _ = GetTechnologies(resp) return } // StorePreflight will store preflight info to a DB -func (chrome *Chrome) StorePreflight(url *url.URL, db *gorm.DB, resp *http.Response, title string, filename string) (uint, error) { +func (chrome *Chrome) StorePreflight(url *url.URL, db *gorm.DB, resp *http.Response, title string, technologies []string, filename string) (uint, error) { record := &storage.URL{ URL: url.String(), @@ -97,6 +98,10 @@ func (chrome *Chrome) StorePreflight(url *url.URL, db *gorm.DB, resp *http.Respo record.AddHeader(k, hv) } + for _, v := range technologies { + record.AddTechnologie(v) + } + // get TLS info, if any if resp.TLS != nil { record.TLS = storage.TLS{ diff --git a/chrome/helpers.go b/chrome/helpers.go index 0475bf9..fc1036e 100644 --- a/chrome/helpers.go +++ b/chrome/helpers.go @@ -2,8 +2,11 @@ package chrome import ( "io" + "io/ioutil" + "net/http" "strings" + wappalyzer "github.com/projectdiscovery/wappalyzergo" "golang.org/x/net/html" ) @@ -44,3 +47,26 @@ func GetHTMLTitle(r io.Reader) (string, bool) { return traverse(doc) } + +func GetTechnologies(resp *http.Response) ([]string, error) { + + //defer resp.Body.Close() + + var technologies []string + + data, err := ioutil.ReadAll(resp.Body) + + if err != nil { + + return technologies, err + } + + wappalyzerClient, err := wappalyzer.New() + fingerprints := wappalyzerClient.Fingerprint(resp.Header, data) + + for match := range fingerprints { + technologies = append(technologies, match) + } + + return technologies, nil +} diff --git a/cmd/report_serve.go b/cmd/report_serve.go index ddd72f4..5593b5c 100644 --- a/cmd/report_serve.go +++ b/cmd/report_serve.go @@ -113,7 +113,7 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { fn := lib.SafeFileName(url.String()) fp := lib.ScreenshotPath(fn, url, options.ScreenshotPath) - resp, title, err := chrm.Preflight(url) + resp, title, technologies, err := chrm.Preflight(url) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -121,7 +121,7 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { var rid uint if rsDB != nil { - if rid, err = chrm.StorePreflight(url, rsDB, resp, title, fn); err != nil { + if rid, err = chrm.StorePreflight(url, rsDB, resp, title, technologies, fn); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -167,6 +167,7 @@ func detailHandler(w http.ResponseWriter, r *http.Request) { Preload("TLS"). Preload("TLS.TLSCertificates"). Preload("TLS.TLSCertificates.DNSNames"). + Preload("Technologies"). First(&url, id) // fmt.Printf("%+v\n", url) diff --git a/go.mod b/go.mod index 44944af..ef45046 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/mattn/go-runewidth v0.0.10 // indirect github.com/mattn/go-sqlite3 v1.14.6 // indirect github.com/olekukonko/tablewriter v0.0.5 + github.com/projectdiscovery/wappalyzergo v0.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rivo/uniseg v0.2.0 // indirect github.com/rs/zerolog v1.20.0 @@ -22,7 +23,7 @@ require ( github.com/tomsteele/go-nmap v0.0.0-20191202052157-3507e0b03523 github.com/ugorji/go v1.1.4 // indirect github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 // indirect - golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 + golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect gorm.io/driver/sqlite v1.1.4 gorm.io/gorm v1.20.12 diff --git a/go.sum b/go.sum index 2a7e461..8e7015b 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -144,6 +145,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -188,7 +190,9 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= @@ -208,6 +212,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/projectdiscovery/wappalyzergo v0.0.4 h1:JEgo9JzpOc9zdF0RMt3esz4yap4+SQ5WFr81CfPHA84= +github.com/projectdiscovery/wappalyzergo v0.0.4/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -321,6 +327,8 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -352,6 +360,7 @@ golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210507161434-a76c4d0a0096/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/lib/pagination.go b/lib/pagination.go index 8810563..fd9ce54 100644 --- a/lib/pagination.go +++ b/lib/pagination.go @@ -75,7 +75,7 @@ func (p *Pagination) Page(data interface{}) (*PaginationPage, error) { offset = (p.CurrPage - 1) * p.Limit } - if err := db.Limit(p.Limit).Offset(offset).Find(data).Error; err != nil { + if err := db.Limit(p.Limit).Offset(offset).Preload("Technologies").Find(data).Error; err != nil { return nil, err } diff --git a/lib/processor.go b/lib/processor.go index e4785bc..ae18fc2 100644 --- a/lib/processor.go +++ b/lib/processor.go @@ -29,8 +29,9 @@ type Processor struct { fn string fp string // preflight response - response *http.Response - title string + response *http.Response + title string + technologies []string // persistence id urlid uint // screenshot @@ -88,7 +89,7 @@ func (p *Processor) init() { // preflight invokes the Chrome preflight helper func (p *Processor) preflight() (err error) { p.Logger.Debug().Str("url", p.URL.String()).Msg("preflighting") - p.response, p.title, err = p.Chrome.Preflight(p.URL) + p.response, p.title, p.technologies, err = p.Chrome.Preflight(p.URL) if err != nil { return } @@ -113,7 +114,7 @@ func (p *Processor) persistPreflight() (err error) { } p.Logger.Debug().Str("url", p.URL.String()).Msg("storing preflight data") - if p.urlid, err = p.Chrome.StorePreflight(p.URL, p.Db, p.response, p.title, p.fn); err != nil { + if p.urlid, err = p.Chrome.StorePreflight(p.URL, p.Db, p.response, p.title, p.technologies, p.fn); err != nil { return } diff --git a/storage/db.go b/storage/db.go index 43931a1..cde4beb 100644 --- a/storage/db.go +++ b/storage/db.go @@ -33,7 +33,7 @@ func (db *Db) Get() (*gorm.DB, error) { } if !db.SkipMigration { - conn.AutoMigrate(&URL{}, &Header{}, &TLS{}, &TLSCertificate{}, &TLSCertificateDNSName{}) + conn.AutoMigrate(&URL{}, &Header{}, &TLS{}, &TLSCertificate{}, &TLSCertificateDNSName{}, &Technologie{}) } return conn, nil diff --git a/storage/models.go b/storage/models.go index fecfacf..799ff79 100644 --- a/storage/models.go +++ b/storage/models.go @@ -21,8 +21,9 @@ type URL struct { Filename string PerceptionHash string - Headers []Header - TLS TLS + Headers []Header + TLS TLS + Technologies []Technologie } // AddHeader adds a new header to a URL @@ -33,6 +34,13 @@ func (url *URL) AddHeader(key string, value string) { }) } +// AddTechnlogies adds a new technologies to a URL +func (url *URL) AddTechnologie(value string) { + url.Technologies = append(url.Technologies, Technologie{ + Value: value, + }) +} + // MarshallCSV returns values as a slice func (url *URL) MarshallCSV() (res []string) { return []string{url.URL, @@ -79,6 +87,14 @@ type Header struct { Value string } +// Technologie contains a technologie +type Technologie struct { + gorm.Model + + URLID uint + Value string +} + // TLS contains TLS information for a URL type TLS struct { gorm.Model diff --git a/web/assets/templates/detail.html b/web/assets/templates/detail.html index 46b1d1d..386e52b 100644 --- a/web/assets/templates/detail.html +++ b/web/assets/templates/detail.html @@ -27,6 +27,11 @@