katana、nuclei、httpx目前都存在流处理bug，已经提交官方处理中 projectdiscovery/httpx#1509 p…

GhostTroops · Jan 1, 2024 · 08255ed · 08255ed
1 parent 8647e7d
commit 08255ed
Show file tree

Hide file tree

Showing 6 changed files with 244 additions and 58 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@ db/51pwnCc/
 config/scan4all_db.db
 ip2gs_51pwn.json
 shell
+config/subdomain.zip
 db/sqlite3.db
 *.dylib
 sqlite3.db

diff --git a/config/config.json b/config/config.json
@@ -6,13 +6,13 @@
       "x": 1,
       "name": "tlsx",
       "cmd": "tlsx -cn -nc -silent -sa -j",
-      "self_do": ".subject_an,.subject_cn,.subject_dn",
+      "self_do": ".subject_an,.subject_cn",
       "next_query_path": {
-        "default":".host,.subject_an,.subject_cn,.subject_dn",
+        "default":".domain,.subject_an,.subject_cn,.host",
         "query_path": {"masscan": ".ip"}
       },
       "result_rm_key": "timestamp,fingerprint_hash,wildcard_certificate,tls_connection",
-      "next": ["httpx","ipgs","masscan","ksubdomain"]
+      "next": ["httpx"]
     },
     {
       "x": 1,
@@ -24,7 +24,7 @@
     {
       "x": 2,
       "name": "ksubdomain",
-      "cmd": "ksubdomain e -b 100m -j -stdin -o -",
+      "cmd": "ksubdomain e -b 100m -j -stdin",
       "next": ["ipgs","masscan"]
     },
     {
@@ -42,13 +42,20 @@
     {
       "x": 4,
       "name": "httpx",
-      "cmd": "httpx -title -websocket -method -server -location -ip  -pipeline -csp-probe -http2 -p 443,80 -nc -silent  -cname -t 64 -json",
+      "result_rm_key": "timestamp,hash,time",
+      "cmd": "httpx -title -websocket -method -server -location -ip  -pipeline -fr -csp-probe -http2 -p 443,80 -nc -silent -td -cname -t 64 -json",
       "next": []
     },
     {
       "x": 5,
       "name": "nuclei",
-      "cmd": "nuclei -nc -silent -j -o - ",
+      "cmd": "nuclei -nc -silent -j",
+      "next": ["gopoc"]
+    },
+    {
+      "x": 6,
+      "name":"katana",
+      "cmd": "katana -nc -silent -j -hl -system-chrome -headless-options '--blink-settings=\"imagesEnabled=false\",--enable-quic=\"imagesEnabled=false\"' -jc -kf all",
       "next": ["gopoc"]
     }
   ]

diff --git a/main.go b/main.go
@@ -4,19 +4,29 @@ import (
 	"github.com/GhostTroops/scan4all/pkg/tools"
 	util "github.com/hktalent/go-utils"
 	"log"
+	"net/http"
 )
 
 func main() {
 	//os.Unsetenv("HTTPS_PROXY")
 	//os.Unsetenv("HTTP_PROXY")
 	util.DoInitAll()
+	go func() {
+		//szTip = "Since you started http://127.0.0.1:6060/debug/pprof/ with -debug, close the program with: control + C"
+		log.Println("debug info: \nopen http://127.0.0.1:6060/debug/pprof/\n\ngo tool pprof -seconds=10 -http=:9999 http://localhost:6060/debug/pprof/heap")
+		http.ListenAndServe(":6060", nil)
+	}()
+
 	var input = make(chan *string)
+	var wg = util.NewSizedWaitGroup(0)
 	util.DoSyncFunc(func() {
-		tools.DoCmds(input, 0)
+		tools.DoCmds(input, 5, &wg)
 	})
-	s := "https://www.paypal.com/"
+	s := "https://www.163.com/"
 	input <- &s
+	// 第一个输入，必须自己关闭
+	close(input)
+	wg.Wait()
 	util.Wg.Wait()
-	log.Println("close input")
 	util.CloseAll()
 }
diff --git a/pkg/common/NewCmd.go b/pkg/common/NewCmd.go
@@ -1,6 +1,7 @@
 package common
 
 import (
+	"bufio"
 	"context"
 	util "github.com/hktalent/go-utils"
 	"io"
@@ -17,6 +18,7 @@ type AsCmd struct {
 	Timeout        time.Duration
 	Wg             *util.SizedWaitGroup
 	InputWriterCbk func(io.WriteCloser)
+	StartFlag      chan bool
 }
 
 func (r *AsCmd) SetTimeout(n time.Duration) *AsCmd {
@@ -54,9 +56,10 @@ func (r *AsCmd) Close() {
 	}
 	if nil != r.Cmd.Process {
 		r.Cmd.Process.Kill()
-		//r.Cmd.Process.Signal(os.Interrupt)
+		r.Cmd.Process.Signal(os.Interrupt)
 	}
 	r.Cmd = nil
+	log.Println(r.Cmdstr, "is closed")
 }
 func (r *AsCmd) Wait() *AsCmd {
 	r.Wg.Wait()
@@ -83,9 +86,8 @@ func (r *AsCmd) Start() *AsCmd {
 			interruptTimer.Stop()
 			killTimer.Stop()
 		}
-
-		if err := r.Cmd.Start(); nil != err {
-			log.Println(err)
+		if err := r.Cmd.Start(); nil == err {
+			log.Println("start", r.Cmdstr)
 		}
 		if err := r.Cmd.Wait(); nil != err {
 			log.Println(err)
@@ -96,35 +98,43 @@ func (r *AsCmd) Start() *AsCmd {
 }
 
 func NewAsCmd(Wg *util.SizedWaitGroup, iw func(io.WriteCloser)) *AsCmd {
-	r := &AsCmd{Wg: Wg, InputWriterCbk: iw}
+	r := &AsCmd{Wg: Wg, InputWriterCbk: iw, StartFlag: make(chan bool, 1)}
 	return r
 }
 
 var re1 = regexp.MustCompile(` +`)
 
+// 所有输出 处理完，关闭 cmd
 func (r *AsCmd) DoCmdOutLine4Cbk(cbk func(*string), arg ...string) *AsCmd {
 	util.WaitFunc4Wg(r.Wg, func() {
+		defer r.Close()
 		var out = make(chan *string)
 		util.WaitFunc4Wg(r.Wg, func() {
 			r.DoCmdOutLine(out, arg...)
 		})
+		var bSt = false
+		// 最后一个 是 nil
 		for i := range out {
 			cbk(i)
+			if !bSt {
+				bSt = true
+				r.StartFlag <- bSt
+			}
 		}
-		r.Close()
 	})
 	return r
 }
 
+// 内部关闭 out
 func (r *AsCmd) DoCmdOutLine(out chan *string, arg ...string) *AsCmd {
 	r.DoCmd(arg...)
 	r1 := r.GetOut4Reader()
 	if nil != r1 {
 		util.WaitFunc4Wg(r.Wg, func() {
+			defer close(out)
 			util.ReadStream4Line(r1, func(s *string) {
 				out <- s
 			})
-			close(out)
 		})
 	}
 	if r.InputWriterCbk != nil {
@@ -170,36 +180,69 @@ func HaveCmd(s ...string) []bool {
 	return a
 }
 
-func DoCmd4Cbk(szCmd string, cbk func(*string), ipt chan *string) {
-	var lk = util.GetLock(szCmd + "_DoCmd4Cbk").Lock()
-	defer lk.Unlock()
+/*
+1、所有的结果处理 cbk 执行结束
+2、延时1秒没有输入，则关闭输入
+*/
+func DoCmd4Cbk(szCmd string, cbk func(*string), ipt chan *string, wg *util.SizedWaitGroup) {
+	//var lk = util.GetLock(szCmd + "_DoCmd4Cbk").Lock()
+	//defer lk.Unlock()
 	var cmdI = GetCmd(szCmd)
 	if nil != cmdI {
 		for x := range ipt {
 			cmdI <- x
 		}
+		// 有 在运行 任务，这里关闭
 		close(ipt)
 		return
 	}
 	RegCmd(szCmd, ipt)
-	var wg = util.NewSizedWaitGroup(5000)
-	cmd := NewAsCmd(&wg, func(wt io.WriteCloser) {
+	cmd := NewAsCmd(wg, func(wt io.WriteCloser) {
 		defer wt.Close()
+		buf := bufio.NewWriter(wt)
 		for x := range ipt {
-			if "" == *x {
+			if "" == *x || util.TestRepeat(szCmd, *x, "WriteCloser") {
 				continue
 			}
-			wt.Write([]byte(*x + "\n"))
+			log.Println("stream input: ", *x, " | ", szCmd)
+			buf.Write([]byte(*x + "\n"))
+			buf.Flush()
 		}
+		log.Println(szCmd, "over input")
 	})
+	var outCbkWg = util.NewSizedWaitGroup(2000)
+	// 必须确保所有回调中调方法都执行完，才能close ipt
 	cmd.DoCmdOutLine4Cbk(func(s *string) {
-		util.WaitFunc4Wg(&wg, func() {
+		util.WaitFunc4Wg(&outCbkWg, func() {
 			cbk(s)
 		})
-		//if nil == s { // time.After(5 * time.Second)
-		//close(ipt)
-		//}
 	}, szCmd)
-	wg.Wait()
-	close(ipt)
+	// 合适的 时机close  ipt
+	go func() {
+		tk := time.NewTicker(128 * time.Millisecond)
+		defer tk.Stop()
+		nCnt := 0
+		bStart := false
+		for {
+			select {
+			case <-cmd.StartFlag:
+				bStart = true
+			case <-tk.C:
+				if bStart && 0 == outCbkWg.WaitLen() {
+					nCnt++
+					if 2 <= nCnt {
+						close(ipt)
+						return
+					}
+				} else {
+					nCnt = 0
+				}
+				//if bStart {
+				//	log.Println("wg.WaitLen: ", outCbkWg.WaitLen())
+				//}
+			}
+		}
+	}()
+	outCbkWg.Wait()
+	//close(ipt)
 }