Seperated removing safewords and regular words
This will help when setting up for splitting the duplicate check into go routinestesting
parent
deca6fcdec
commit
8bb577ee5c
|
|
@ -239,7 +239,6 @@ func removeduplicates(siteBuff *[]siteList, whitelist *[]string) {
|
||||||
c.s = 0
|
c.s = 0
|
||||||
c.w = 0
|
c.w = 0
|
||||||
|
|
||||||
log.Print("Checking for duplicates")
|
|
||||||
var entry []struct {
|
var entry []struct {
|
||||||
r *bool
|
r *bool
|
||||||
s *string
|
s *string
|
||||||
|
|
@ -258,16 +257,22 @@ func removeduplicates(siteBuff *[]siteList, whitelist *[]string) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove safewords and duplicates
|
// Removing safewords
|
||||||
lenEntry := len(entry)
|
log.Print("Checking for safewords")
|
||||||
for i, e := range entry {
|
for i,e := range entry {
|
||||||
for _, w := range safewords {
|
for _,w := range(safewords) {
|
||||||
if *e.s == w {
|
if *e.s == w {
|
||||||
*(entry[i].r) = true
|
*(entry[i].r) = true
|
||||||
c.s++
|
c.s++
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Removing duplicates
|
||||||
|
log.Print("Checking for duplicates")
|
||||||
|
lenEntry := len(entry)
|
||||||
|
for i, e := range entry {
|
||||||
if *(entry[i].r) == true {
|
if *(entry[i].r) == true {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue