justin
/
rhosts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Manages handling fewer entries than cores safely 

If the number of entries is fewer than the number of cores then it is being
assumed that you can run this on a single core.
experimental
Justin Reichardt 2022-08-20 16:45:33 -05:00
parent 5b2557701e
commit bc7bbce83a
1 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/hosts/hosts.go
View File

@ -282,6 +282,9 @@ func removeUnwanted(siteBuff *[]siteList, whitelist *[]string) {
// Removing duplicates // Removing duplicates
log.Print("Checking for duplicates") log.Print("Checking for duplicates")
cores := runtime.NumCPU() cores := runtime.NumCPU()
lenEntry := len(entry)
if (cores <= lenEntry){
fin := make(chan uint) fin := make(chan uint)
for i := cores; i > 0 ; i-- { for i := cores; i > 0 ; i-- {
go removeDuplicate(entry[i-1:],cores, fin) go removeDuplicate(entry[i-1:],cores, fin)
@ -290,7 +293,9 @@ func removeUnwanted(siteBuff *[]siteList, whitelist *[]string) {
count :=<-fin count :=<-fin
c.d =+ count c.d =+ count
} }
lenEntry := len(entry) } else {
removeDuplicate(entry,1, nil)
}
log.Printf("Total: %d\tDuplicates: %d\tSafeWords: %d\tWhitelisted: %d\n", lenEntry, c.d, c.s, c.w) log.Printf("Total: %d\tDuplicates: %d\tSafeWords: %d\tWhitelisted: %d\n", lenEntry, c.d, c.s, c.w)
} }
@ -316,7 +321,9 @@ func removeDuplicate (entry []siteEntryPointer, freq int , done chan uint){
} }
} }
if (done != nil){
done <- cd done <- cd
}
} }
// write2tmp write the siteBuff to the tempfile // write2tmp write the siteBuff to the tempfile