Manages handling fewer entries than cores safely

If the number of entries is fewer than the number of cores then it is being
assumed that you can run this on a single core.
experimental
Justin Reichardt 2022-08-20 16:45:33 -05:00
parent 5b2557701e
commit bc7bbce83a
1 changed files with 16 additions and 9 deletions

View File

@ -282,6 +282,9 @@ func removeUnwanted(siteBuff *[]siteList, whitelist *[]string) {
// Removing duplicates
log.Print("Checking for duplicates")
cores := runtime.NumCPU()
lenEntry := len(entry)
if (cores <= lenEntry){
fin := make(chan uint)
for i := cores; i > 0 ; i-- {
go removeDuplicate(entry[i-1:],cores, fin)
@ -290,7 +293,9 @@ func removeUnwanted(siteBuff *[]siteList, whitelist *[]string) {
count :=<-fin
c.d =+ count
}
lenEntry := len(entry)
} else {
removeDuplicate(entry,1, nil)
}
log.Printf("Total: %d\tDuplicates: %d\tSafeWords: %d\tWhitelisted: %d\n", lenEntry, c.d, c.s, c.w)
}
@ -316,7 +321,9 @@ func removeDuplicate (entry []siteEntryPointer, freq int , done chan uint){
}
}
if (done != nil){
done <- cd
}
}
// write2tmp write the siteBuff to the tempfile