Manages handling fewer entries than cores safely
If the number of entries is fewer than the number of cores then it is being assumed that you can run this on a single core.experimental
parent
5b2557701e
commit
bc7bbce83a
|
|
@ -282,6 +282,9 @@ func removeUnwanted(siteBuff *[]siteList, whitelist *[]string) {
|
|||
// Removing duplicates
|
||||
log.Print("Checking for duplicates")
|
||||
cores := runtime.NumCPU()
|
||||
lenEntry := len(entry)
|
||||
|
||||
if (cores <= lenEntry){
|
||||
fin := make(chan uint)
|
||||
for i := cores; i > 0 ; i-- {
|
||||
go removeDuplicate(entry[i-1:],cores, fin)
|
||||
|
|
@ -290,7 +293,9 @@ func removeUnwanted(siteBuff *[]siteList, whitelist *[]string) {
|
|||
count :=<-fin
|
||||
c.d =+ count
|
||||
}
|
||||
lenEntry := len(entry)
|
||||
} else {
|
||||
removeDuplicate(entry,1, nil)
|
||||
}
|
||||
|
||||
log.Printf("Total: %d\tDuplicates: %d\tSafeWords: %d\tWhitelisted: %d\n", lenEntry, c.d, c.s, c.w)
|
||||
}
|
||||
|
|
@ -316,8 +321,10 @@ func removeDuplicate (entry []siteEntryPointer, freq int , done chan uint){
|
|||
}
|
||||
|
||||
}
|
||||
if (done != nil){
|
||||
done <- cd
|
||||
}
|
||||
}
|
||||
|
||||
// write2tmp write the siteBuff to the tempfile
|
||||
func write2tmp(tmpdir string, siteBuff *[]siteList) (err error) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue