-
Notifications
You must be signed in to change notification settings - Fork 1
/
item_test.go
106 lines (93 loc) · 2.32 KB
/
item_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
package youcrawl
import (
"os"
"testing"
)
var TestItem DefaultItem
func init() {
TestItem = DefaultItem{Store: map[string]interface{}{}}
}
func TestItem_SetValue(t *testing.T) {
TestItem.SetValue("test1", "test")
}
func TestItem_GetString(t *testing.T) {
TestItem.SetValue("test1", "test")
value, err := TestItem.GetString("test1")
if err != nil {
t.Error("get string not cause error")
}
if value != "test" {
t.Error("get string value not match")
}
TestItem.SetValue("test1", 1)
_, err = TestItem.GetString("test1")
if err != TypeError {
t.Error("must cause type error")
}
_, err = TestItem.GetString("test2")
if err != KeyNotContainError {
t.Error("must cause KeyNotContainError")
}
}
func TestGetInt(t *testing.T) {
TestItem.SetValue("test1", 1)
TestItem.SetValue("test2", "1")
_, err := TestItem.GetInt("test1")
if err != nil {
t.Error(err)
}
_, err = TestItem.GetInt("test2")
if err != TypeError {
t.Error("must cause type error")
}
_, err = TestItem.GetInt("test10")
if err != KeyNotContainError {
t.Error("must cause type error")
}
}
func TestItem_GetFloat64(t *testing.T) {
TestItem.SetValue("test1", 19.999)
TestItem.SetValue("test2", "19.999")
_, err := TestItem.GetFloat64("test1")
if err != nil {
t.Error(err)
}
_, err = TestItem.GetFloat64("test2")
if err != TypeError {
t.Error("must cause type error")
}
_, err = TestItem.GetFloat64("test10")
if err != KeyNotContainError {
t.Error("must cause type error")
}
}
func TestDefaultItem_GetToken(t *testing.T) {
TestItem.SetValue(ItemKeyChannelToken,"thisistoken")
TestItem.GetToken()
}
type WebItems struct {
Label string
}
type WebInfoItem struct {
Title string `json:",omitempty"`
Items []WebItems `json:",omitempty"`
Status string `json:",omitempty"`
URL string `json:",omitempty"`
}
func TestItemStruct(t *testing.T) {
e := NewEngine(&EngineOption{MaxRequest: 5})
addTask := NewTask("http://www.bing.com", WebInfoItem{})
e.AddTasks(&addTask)
e.AddHTMLParser(DefaultTestParser)
e.AddHTMLParser(func(ctx *Context) error {
item := ctx.Item.(WebInfoItem)
doc := ctx.Doc
item.Title = doc.Find("title").Text()
ctx.Item = item
return nil
})
e.AddPipelines(&GlobalStorePipeline{})
e.AddPostProcess(&OutputJsonPostProcess{StorePath: "./output.json"})
e.RunAndWait()
defer os.Remove("./output.json")
}