Seperated removing safewords and regular words

This will help when setting up for splitting the duplicate check into go routines
testing
Justin Reichardt 2022-08-19 20:20:16 -05:00
parent deca6fcdec
commit 8bb577ee5c
1 changed files with 10 additions and 5 deletions

View File

@ -239,7 +239,6 @@ func removeduplicates(siteBuff *[]siteList, whitelist *[]string) {
c.s = 0 c.s = 0
c.w = 0 c.w = 0
log.Print("Checking for duplicates")
var entry []struct { var entry []struct {
r *bool r *bool
s *string s *string
@ -258,16 +257,22 @@ func removeduplicates(siteBuff *[]siteList, whitelist *[]string) {
} }
} }
// Remove safewords and duplicates // Removing safewords
lenEntry := len(entry) log.Print("Checking for safewords")
for i, e := range entry { for i,e := range entry {
for _, w := range safewords { for _,w := range(safewords) {
if *e.s == w { if *e.s == w {
*(entry[i].r) = true *(entry[i].r) = true
c.s++ c.s++
break break
} }
} }
}
// Removing duplicates
log.Print("Checking for duplicates")
lenEntry := len(entry)
for i, e := range entry {
if *(entry[i].r) == true { if *(entry[i].r) == true {
continue continue
} }