user-agent header Issue resolved in headless mode (#868)

* Update README.md

* fix: add user agent handler

* small refactor

---------

Co-authored-by: zy9ard3 <67743789+zy9ard3@users.noreply.github.com>
Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
Co-authored-by: Mzack9999 <mzack9999@protonmail.com>

pkg/engine/hybrid/crawl.go

@@ -237,14 +237,28 @@ func (c *Crawler) addHeadersToPage(page *rod.Page) {
 	if len(c.Headers) == 0 {
 		return
 	}
+
 	var arr []string
+
 	for k, v := range c.Headers {
-		arr = append(arr, k, v)
+		switch {
+		case stringsutil.EqualFoldAny(k, "User-Agent"):
+			userAgentParams := &proto.NetworkSetUserAgentOverride{
+				UserAgent: v,
+			}
+			if err := page.SetUserAgent(userAgentParams); err != nil {
+				gologger.Error().Msgf("headless: could not set user agent: %v", err)
+			}
+		default:
+			arr = append(arr, k, v)
+		}
 	}
-	// ignore cleanup callback
-	_, err := page.SetExtraHeaders(arr)
-	if err != nil {
-		gologger.Error().Msgf("headless: could not set extra headers: %v", err)
+
+	if len(arr) > 0 {
+		_, err := page.SetExtraHeaders(arr)
+		if err != nil {
+			gologger.Error().Msgf("headless: could not set extra headers: %v", err)
+		}
 	}
 }