Host File Update

Proxy Server host block list is now available at http://goo.gl/jXiOLp, make sure you do not use this in Windows host file.

http://hosts-file.net/ad_servers.txt
http://jamesisbored.com/iphone/content/hosts.php
http://jansal.googlecode.com/svn/trunk/adblock/hosts
http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext
http://pgl.yoyo.org/as/serverlist.php?hostformat=hosts;showintro=0&mimetype=plaintext
http://someonewhocares.org/hosts/hosts
http://sysctl.org/cameleon/hosts
http://unbelovedhosts.apk.defim.de/hosts
http://winhelp2002.mvps.org/hosts.txt
http://www.malwaredomainlist.com/hostslist/hosts.txt
http://www.montanamenagerie.org/hostsfile/hosts.txt
https://adaway.org/hosts.txt

It’s a good idea to do this script in a directory that is specific to your host file project

#!/bin/bash
clear

echo "Removing old host file downloads - start"
rm host*.txt
rm all.txt
echo "Removing old host file downloads - stop"

echo "Downloading new versions of the host file lists - start"
curl -s -o host1.txt http://goo.gl/A8LzsL
curl -s -o host2.txt http://goo.gl/P1JY6Q
curl -s -o host3.txt http://hosts-file.net/ad_servers.txt
curl -s -o host4.txt http://jamesisbored.com/iphone/content/hosts.php
curl -s -o host5.txt http://jansal.googlecode.com/svn/trunk/adblock/hosts
#curl -s -o host6.txt http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext
#curl -s -o host7.txt http://pgl.yoyo.org/as/serverlist.php?hostformat=hosts;showintro=0&mimetype=plaintext
curl -s -o host8.txt http://someonewhocares.org/hosts/hosts
curl -s -o host9.txt http://sysctl.org/cameleon/hosts
curl -s -o host10.txt http://unbelovedhosts.apk.defim.de/hosts
curl -s -o host11.txt http://winhelp2002.mvps.org/hosts.txt
curl -s -o host12.txt http://www.malwaredomainlist.com/hostslist/hosts.txt
curl -s -o host13.txt http://www.montanamenagerie.org/hostsfile/hosts.txt
curl -s -o host14.txt http://www.mvps.org/winhelp2002/hosts.txt
curl -s -o host15.txt https://adaway.org/hosts.txt
curl -s -o host16.txt https://mirror.cedia.org.ec/malwaredomains/immortal_domains.txt
curl -s -o host17.txt https://mirror.cedia.org.ec/malwaredomains/justdomains
#curl -s -o host18.txt https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext
curl -s -o host19.txt https://raw.githubusercontent.com/lewisje/jansal/master/adblock/hosts
curl -s -o host20.txt http://hosts-file.net/hphosts-partial.asp
echo "Downloading new versions of the host file lists - stop"

echo "Combining all lists into one - start"
cat *.txt > all.txt
echo "Combining all lists into one - stop"

echo "Cleaning the list - step 1"
sed -i '/^#/ d' all.txt
echo "Cleaning the list - step 2"
tr '[:upper:]' '[:lower:]' < all.txt > cleaner.txt
echo "Cleaning the list - step 3"
sed -i -e 's|127.0.0.1||' cleaner.txt
echo "Cleaning the list - step 4"
sed -i -e 's|0.0.0.0||' cleaner.txt
echo "Cleaning the list - step 5"
sed -i -e 's|::1||' cleaner.txt
echo "Cleaning the list - step 6"
sed -i -e 's|fe80::1%lo0||' cleaner.txt
echo "Cleaning the list - step 7"
sed -i -e 's/^[[:space:]]*//' cleaner.txt
echo "Cleaning the list - step 8"
sed -i '/^*{/d' cleaner.txt
echo "Cleaning the list - step 9"
cat cleaner.txt | sort | awk '!a[$0]++' > all.txt
echo "Cleaning the list - step 10"
rm clean*
echo "Cleaning the list - step 11"
sed -i '/^\s*$/d' all.txt
echo "Cleaning the list - step 12"
sed -i -e 's/^/0.0.0.0 /' all.txt

echo "Saving new hosts file"
cp all.txt hosts

echo "Saving new hosts lists file"
cat hosts | sort | sed -e 's|0.0.0.0 ||'  > host

Make sure you chmod the shell script

chmod +x update_hosts.sh

Mac OS X Example – Open Terminal

cd ~/Downloads
curl -O http://winhelp2002.mvps.org/hosts.txt
sudo cp /private/etc/hosts /private/etc/hosts.bk
sudo cp ~/Downloads/hosts.txt /private/etc/hosts

Windows PowerShell first create a text file from the list above or with your own list and save it as C:\Temp\hosts\ServerList.txt

# Directory Parameter
$FileDirectory = "C:\Temp\hosts\lists\"

# If directory doesn't exist create the directory
if((Test-Path $FileDirectory) -eq 0)
    {
        mkdir $FileDirectory;
        # cd $FileDirectory;
    }

    $hostfiltered = ""


clear
$ServerList = (Get-Content "C:\Temp\hosts\ServerList.txt") | Sort-Object

$ServerNumber = 1

write-host $ServerList.count total host lists read from file
foreach ($line in $ServerList)
{
   # write-host $line

    # We assume the file you download is named what you want it to be on your computer
    $FileName = [System.IO.Path]::GetFileName($line)

    $FullFilePath = "$($FileDirectory)Server_$($ServerNumber).txt"

    # write-host     $FullFilePath 
       
    # Give a basic message to the user to let them know what we are doing
    Write-Host "`r`nDownloading '$line' to '$FullFilePath'"
    
    # Download - Start
    $uri = New-Object "System.Uri" "$line" 
    $request = [System.Net.HttpWebRequest]::Create($uri) 
    $request.set_Timeout(15000) #15 second timeout 
    $response = $request.GetResponse() 
    $totalLength = [System.Math]::Floor($response.get_ContentLength()/1024) 
    $responseStream = $response.GetResponseStream() 
    $targetStream = New-Object -TypeName System.IO.FileStream -ArgumentList $FullFilePath, Create 
    $buffer = new-object byte[] 10KB 
    $count = $responseStream.Read($buffer,0,$buffer.length) 
    $downloadedBytes = $count 
    while ($count -gt 0) 
    {         
        [System.Console]::Write("`r`nDownloaded {0}K of {1}K", [System.Math]::Floor($downloadedBytes/1024), $totalLength) 
        $targetStream.Write($buffer, 0, $count) 
        $count = $responseStream.Read($buffer,0,$buffer.length) 
        $downloadedBytes = $downloadedBytes + $count 
    } 
  
    $targetStream.Flush()
    $targetStream.Close() 
    $targetStream.Dispose() 
    $responseStream.Dispose() 
    
    # Give a basic message to the user to let them know we are done
    Write-Host "`r`nDownload complete"
    # Download - Stop   

     $ServerNumber = $ServerNumber + 1
}

cd C:\Temp\hosts\lists

Write-Host "`r`nCombining Lists"

$FullFilePathFiltered = "C:\Temp\hosts\lists\combined_files.txt"
$FullFilePathFiltered1 = "C:\Temp\hosts\lists\combined_files1.txt"

If (Test-Path $FullFilePathFiltered){
    Remove-Item $FullFilePathFiltered
}

If (Test-Path $FullFilePathFiltered1){
    Remove-Item $FullFilePathFiltered1
}

Get-ChildItem -recurse -include "*.txt" | % { Get-Content $_ -ReadCount 0 | Add-Content $FullFilePathFiltered }

Remove-Item C:\Temp\hosts\lists\Server_*

Write-Host "`r`nCombined Lists"

Write-Host "`r`nCleaning File - Remove Localhost"
    (get-content $FullFilePathFiltered) | select-string -pattern 'localhost' -notmatch | Set-Content $FullFilePathFiltered
    (get-content $FullFilePathFiltered) | select-string -pattern 'broadcasthost' -notmatch | Set-Content $FullFilePathFiltered
Write-Host "`r`nCleaning File - Remove Localhost - Done"

Write-Host "`r`nCleaning File - Replace Tabs with space"
cat $FullFilePathFiltered | %{$_ -replace '[\t]',' '}  > $FullFilePathFiltered1
Write-Host "`r`nCleaning File - Replace Tabs with space - Done"

Write-Host "`r`nCleaning File - Replace Double Spaces"
cat $FullFilePathFiltered1 | %{$_ -replace ' {2,}',' '}  > $FullFilePathFiltered
Write-Host "`r`nCleaning File - Replace Double Spaces - Done"

Write-Host "`r`nCleaning File - Remove Empty Lines"
    (Get-Content $FullFilePathFiltered ) | Where { $_ } | Set-Content $FullFilePathFiltered1
Write-Host "`r`nCleaning File - Remove Empty Lines - Done"

Write-Host "`r`nCleaning File - Remove localhost ::1"
    (Get-Content $FullFilePathFiltered1) | Where { $_ -notmatch "^:" } | Set-Content $FullFilePathFiltered
Write-Host "`r`nCleaning File - Remove localhost ::1 - Done"

Write-Host "`r`nCleaning File - Remove comments"
    (Get-Content $FullFilePathFiltered) | Where { $_ -notmatch "^#" } | Set-Content $FullFilePathFiltered
Write-Host "`r`nCleaning File - Remove comments - Done"

Write-Host "`r`nCleaning File - Replace 0.0.0.0 with 127.0.0.1"
cat $FullFilePathFiltered | %{$_ -replace "0.0.0.0","127.0.0.1"}  > $FullFilePathFiltered1
cat $FullFilePathFiltered1 | %{$_ -replace "188.165.196.30","127.0.0.1"}  > $FullFilePathFiltered

cat $FullFilePathFiltered | %{$_ -replace "46.246.119.139","127.0.0.1"}  > $FullFilePathFiltered1


Write-Host "`r`nCleaning File - Replace 0.0.0.0 with 127.0.0.1 - Done"

Write-Host "`r`nCleaning File - Remove Duplicates"
    (get-content $FullFilePathFiltered1) |Sort-Object  | get-unique| out-file $FullFilePathFiltered
Write-Host "`r`nCleaning File - Remove Duplicates - Done"

Clear-Content $FullFilePathFiltered1
Add-Content $FullFilePathFiltered1 (get-content "C:\Temp\hosts\header.txt")
Add-Content $FullFilePathFiltered1 (get-content $FullFilePathFiltered)

Write-Host "`r`nFile Cleaned"

#Only uncomment these if the lists combined is less than a couple MB otherwise it will make your computer nuts.
#Move-Item "C:\Windows\System32\drivers\etc\HOSTS" "C:\Windows\System32\drivers\etc\hosts.old"
#Move-Item $FullFilePathFiltered1 "C:\Windows\System32\drivers\etc\hosts"

Reference: http://winhelp2002.mvps.org/hosts.htm