Add the applications technologies (#104)

chrome/chrome.go

@@ -35,7 +35,7 @@ func NewChrome() *Chrome {
 }
 
 // Preflight will preflight a url
-func (chrome *Chrome) Preflight(url *url.URL) (resp *http.Response, title string, err error) {
+func (chrome *Chrome) Preflight(url *url.URL) (resp *http.Response, title string, technologies []string, err error) {
 	// purposefully ignore bad certs
 	transport := &http.Transport{
 		TLSClientConfig:   &tls.Config{InsecureSkipVerify: true},
@@ -45,7 +45,7 @@ func (chrome *Chrome) Preflight(url *url.URL) (resp *http.Response, title string
 		var erri error
 		proxyURL, erri := url.Parse(chrome.Proxy)
 		if erri != nil {
-			return nil, "", erri
+			return nil, "", nil, erri
 		}
 		transport.Proxy = http.ProxyURL(proxyURL)
 	}
@@ -73,12 +73,13 @@ func (chrome *Chrome) Preflight(url *url.URL) (resp *http.Response, title string
 
 	defer resp.Body.Close()
 	title, _ = GetHTMLTitle(resp.Body)
+	technologies, _ = GetTechnologies(resp)
 
 	return
 }
 
 // StorePreflight will store preflight info to a DB
-func (chrome *Chrome) StorePreflight(url *url.URL, db *gorm.DB, resp *http.Response, title string, filename string) (uint, error) {
+func (chrome *Chrome) StorePreflight(url *url.URL, db *gorm.DB, resp *http.Response, title string, technologies []string, filename string) (uint, error) {
 
 	record := &storage.URL{
 		URL:            url.String(),
@@ -97,6 +98,10 @@ func (chrome *Chrome) StorePreflight(url *url.URL, db *gorm.DB, resp *http.Respo
 		record.AddHeader(k, hv)
 	}
 
+	for _, v := range technologies {
+		record.AddTechnologie(v)
+	}
+
 	// get TLS info, if any
 	if resp.TLS != nil {
 		record.TLS = storage.TLS{

chrome/helpers.go

@@ -2,8 +2,11 @@ package chrome
 
 import (
 	"io"
+	"io/ioutil"
+	"net/http"
 	"strings"
 
+	wappalyzer "github.com/projectdiscovery/wappalyzergo"
 	"golang.org/x/net/html"
 )
 
@@ -44,3 +47,26 @@ func GetHTMLTitle(r io.Reader) (string, bool) {
 
 	return traverse(doc)
 }
+
+func GetTechnologies(resp *http.Response) ([]string, error) {
+
+	//defer resp.Body.Close()
+
+	var technologies []string
+
+	data, err := ioutil.ReadAll(resp.Body)
+
+	if err != nil {
+
+		return technologies, err
+	}
+
+	wappalyzerClient, err := wappalyzer.New()
+	fingerprints := wappalyzerClient.Fingerprint(resp.Header, data)
+
+	for match := range fingerprints {
+		technologies = append(technologies, match)
+	}
+
+	return technologies, nil
+}

cmd/report_serve.go

@@ -113,15 +113,15 @@ func submitHandler(w http.ResponseWriter, r *http.Request) {
 		fn := lib.SafeFileName(url.String())
 		fp := lib.ScreenshotPath(fn, url, options.ScreenshotPath)
 
-		resp, title, err := chrm.Preflight(url)
+		resp, title, technologies, err := chrm.Preflight(url)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
 
 		var rid uint
 		if rsDB != nil {
-			if rid, err = chrm.StorePreflight(url, rsDB, resp, title, fn); err != nil {
+			if rid, err = chrm.StorePreflight(url, rsDB, resp, title, technologies, fn); err != nil {
 				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
@@ -167,6 +167,7 @@ func detailHandler(w http.ResponseWriter, r *http.Request) {
 		Preload("TLS").
 		Preload("TLS.TLSCertificates").
 		Preload("TLS.TLSCertificates.DNSNames").
+		Preload("Technologies").
 		First(&url, id)
 
 	// fmt.Printf("%+v\n", url)

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.10 // indirect
 	github.com/mattn/go-sqlite3 v1.14.6 // indirect
 	github.com/olekukonko/tablewriter v0.0.5
+	github.com/projectdiscovery/wappalyzergo v0.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rs/zerolog v1.20.0
@@ -22,7 +23,7 @@ require (
 	github.com/tomsteele/go-nmap v0.0.0-20191202052157-3507e0b03523
 	github.com/ugorji/go v1.1.4 // indirect
 	github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 // indirect
-	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
+	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
 	golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect
 	gorm.io/driver/sqlite v1.1.4
 	gorm.io/gorm v1.20.12

go.sum

@@ -97,6 +97,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -144,6 +145,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -188,7 +190,9 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
@@ -208,6 +212,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/projectdiscovery/wappalyzergo v0.0.4 h1:JEgo9JzpOc9zdF0RMt3esz4yap4+SQ5WFr81CfPHA84=
+github.com/projectdiscovery/wappalyzergo v0.0.4/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -321,6 +327,8 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -352,6 +360,7 @@ golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210507161434-a76c4d0a0096/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

lib/pagination.go

@@ -75,7 +75,7 @@ func (p *Pagination) Page(data interface{}) (*PaginationPage, error) {
 		offset = (p.CurrPage - 1) * p.Limit
 	}
 
-	if err := db.Limit(p.Limit).Offset(offset).Find(data).Error; err != nil {
+	if err := db.Limit(p.Limit).Offset(offset).Preload("Technologies").Find(data).Error; err != nil {
 		return nil, err
 	}
 

lib/processor.go

@@ -29,8 +29,9 @@ type Processor struct {
 	fn string
 	fp string
 	// preflight response
-	response *http.Response
-	title    string
+	response     *http.Response
+	title        string
+	technologies []string
 	// persistence id
 	urlid uint
 	// screenshot
@@ -88,7 +89,7 @@ func (p *Processor) init() {
 // preflight invokes the Chrome preflight helper
 func (p *Processor) preflight() (err error) {
 	p.Logger.Debug().Str("url", p.URL.String()).Msg("preflighting")
-	p.response, p.title, err = p.Chrome.Preflight(p.URL)
+	p.response, p.title, p.technologies, err = p.Chrome.Preflight(p.URL)
 	if err != nil {
 		return
 	}
@@ -113,7 +114,7 @@ func (p *Processor) persistPreflight() (err error) {
 	}
 
 	p.Logger.Debug().Str("url", p.URL.String()).Msg("storing preflight data")
-	if p.urlid, err = p.Chrome.StorePreflight(p.URL, p.Db, p.response, p.title, p.fn); err != nil {
+	if p.urlid, err = p.Chrome.StorePreflight(p.URL, p.Db, p.response, p.title, p.technologies, p.fn); err != nil {
 		return
 	}
 

storage/db.go

@@ -33,7 +33,7 @@ func (db *Db) Get() (*gorm.DB, error) {
 	}
 
 	if !db.SkipMigration {
-		conn.AutoMigrate(&URL{}, &Header{}, &TLS{}, &TLSCertificate{}, &TLSCertificateDNSName{})
+		conn.AutoMigrate(&URL{}, &Header{}, &TLS{}, &TLSCertificate{}, &TLSCertificateDNSName{}, &Technologie{})
 	}
 
 	return conn, nil

storage/models.go

@@ -21,8 +21,9 @@ type URL struct {
 	Filename       string
 	PerceptionHash string
 
-	Headers []Header
-	TLS     TLS
+	Headers      []Header
+	TLS          TLS
+	Technologies []Technologie
 }
 
 // AddHeader adds a new header to a URL
@@ -33,6 +34,13 @@ func (url *URL) AddHeader(key string, value string) {
 	})
 }
 
+// AddTechnlogies adds a new technologies to a URL
+func (url *URL) AddTechnologie(value string) {
+	url.Technologies = append(url.Technologies, Technologie{
+		Value: value,
+	})
+}
+
 // MarshallCSV returns values as a slice
 func (url *URL) MarshallCSV() (res []string) {
 	return []string{url.URL,
@@ -79,6 +87,14 @@ type Header struct {
 	Value string
 }
 
+// Technologie contains a technologie
+type Technologie struct {
+	gorm.Model
+
+	URLID uint
+	Value string
+}
+
 // TLS contains TLS information for a URL
 type TLS struct {
 	gorm.Model

web/assets/templates/detail.html

@@ -27,6 +27,11 @@ <h2 class="page-title">
           <div class="lh-sm">
             <div>{{ .URL }}</div>
             <div class="text-muted">{{ .Title }}</div>
+            <div>              
+              {{ range .Technologies }}
+              <span class="badge bg-blue">{{ .Value}}</span>
+              {{ end }}
+            </div>
           </div>
           <div class="ml-auto">
             <a href="#" class="text-muted">

web/assets/templates/gallery.html

@@ -71,6 +71,11 @@ <h2 class="page-title">
                 <a href="{{ .URL }}" target="_blank">{{ .URL }}</a>
               </div>
               <div class="text-muted">{{ .Title }}</div>
+              <div>              
+                {{ range .Technologies }}
+                <span class="badge bg-blue">{{ .Value}}</span>
+                {{ end }}
+              </div>
             </div>
             <div class="ml-auto">
               <a href="/details?id={{ .ID }}" class="btn btn-light btn-sm">