Manages handling fewer entries than cores safely

If the number of entries is fewer than the number of cores then it is being
assumed that you can run this on a single core.
experimental
Justin Reichardt 2022-08-20 16:45:33 -05:00
parent 5b2557701e
commit bc7bbce83a
1 changed files with 16 additions and 9 deletions

View File

@ -282,16 +282,21 @@ func removeUnwanted(siteBuff *[]siteList, whitelist *[]string) {
// Removing duplicates
log.Print("Checking for duplicates")
cores := runtime.NumCPU()
fin := make(chan uint)
for i := cores; i > 0 ; i-- {
go removeDuplicate(entry[i-1:],cores, fin)
}
for ;cores > 0; cores-- {
count :=<-fin
c.d =+ count
}
lenEntry := len(entry)
if (cores <= lenEntry){
fin := make(chan uint)
for i := cores; i > 0 ; i-- {
go removeDuplicate(entry[i-1:],cores, fin)
}
for ;cores > 0; cores-- {
count :=<-fin
c.d =+ count
}
} else {
removeDuplicate(entry,1, nil)
}
log.Printf("Total: %d\tDuplicates: %d\tSafeWords: %d\tWhitelisted: %d\n", lenEntry, c.d, c.s, c.w)
}
@ -316,7 +321,9 @@ func removeDuplicate (entry []siteEntryPointer, freq int , done chan uint){
}
}
done <- cd
if (done != nil){
done <- cd
}
}
// write2tmp write the siteBuff to the tempfile