Manages handling fewer entries than cores safely

If the number of entries is fewer than the number of cores then it is being
assumed that you can run this on a single core.
testing
Justin Reichardt 2022-08-20 16:45:33 -05:00
parent 8bc1777b87
commit 7c56c5d5c7
1 changed files with 16 additions and 9 deletions

View File

@ -282,6 +282,9 @@ func removeUnwanted(siteBuff *[]siteList, whitelist *[]string) {
// Removing duplicates
log.Print("Checking for duplicates")
cores := runtime.NumCPU()
lenEntry := len(entry)
if (cores <= lenEntry){
fin := make(chan uint)
for i := cores; i > 0 ; i-- {
go removeDuplicate(entry[i-1:],cores, fin)
@ -290,7 +293,9 @@ func removeUnwanted(siteBuff *[]siteList, whitelist *[]string) {
count :=<-fin
c.d =+ count
}
lenEntry := len(entry)
} else {
removeDuplicate(entry,1, nil)
}
log.Printf("Total: %d\tDuplicates: %d\tSafeWords: %d\tWhitelisted: %d\n", lenEntry, c.d, c.s, c.w)
}
@ -316,8 +321,10 @@ func removeDuplicate (entry []siteEntryPointer, freq int , done chan uint){
}
}
if (done != nil){
done <- cd
}
}
// write2tmp write the siteBuff to the tempfile
func write2tmp(tmpdir string, siteBuff *[]siteList) (err error) {