from sgmllib import SGMLParser
import os, re, time, urllib, random, sys, math
    
from urllib import quote


class AppURLopener(urllib.FancyURLopener):
    def __init__(self, *args):
        self.version = "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)"
        urllib.FancyURLopener.__init__(self, *args)

urllib._urlopener = AppURLopener()

url_quote = quote

def hitCount( query ):

    query = 'http://www.altavista.com/web/results?lang=en&aqb=&sc=on&aqa=%s&aqmode=s&lh=&dt=tmperiod&avkw=aapt&aqs=&rc=dmn&nbq=10&pg=aq&d2=0' % url_quote(query)

    sock = urllib.urlopen(query)
    html = sock.read().lower()

    tag='altavista found'
    p = html.find(tag )
    if p<0:
        tag = 'of <b>'
        p = html.find(tag )
        if p<0:
            return 0
    html = html[p+len(tag):]

    p = html.find('result')
    html = html[:p]

    html = html.replace('.','')
    html = html.replace(',','')

    try:
        return int(html)
    except:
        return 0

def countryList():
    return [ ["Andorra", 507368], ["United Arab Emirates", 485675], ["Afghanistan", 1659914], ["Antigua and Barbuda", 150533], 
             ["Anguilla", 357580], ["Albania", 562171], ["Armenia", 546460], ["Netherlands Antilles", 244385], ["Angola", 539872],
             ["Antarctica", 612907], ["Argentina", 4614932], ["American Samoa", 203345], ["Austria", 4369057],
             ["Australia", 14428914], ["Anguila", 5840], ["Aruba", 570714], ["Azerbaijan", 479812], ["Bosnia", 769785],
             ["Barbados", 827197], ["Bangladesh", 950767], ["Belgium", 3853098], ["Burkina Faso", 353621],
             ["Bulgaria", 1341424], ["Bahrain", 496097], ["Burundi", 365045], ["Benin", 376006], ["Bermuda", 911995],
             ["Brunei", 525944], ["Bolivia", 1282330], ["Brazil", 3353612], ["Bahamas", 1266815], ["Bhutan", 313043],
             ["Bouvet Island", 23080], ["Botswana", 497482], ["Belarus", 915687], ["Belize", 638180], ["Canada", 22947345],
             ["Congo", 743767], ["Central African Republic", 99158], ["Congo Brazzavile", 129], ["Switzerland", 4464939],
             ["Ivory Coast", 159785], ["Cook Islands", 141587], ["Chile", 3273012], ["Cameroon", 365883],
             ["China", 10503149], ["Colombia", 2391704], ["Costa Rica", 2156026], ["Cuba", 2558546], ["Cape Verde", 149038],
             ["Christmas Island", 89484], ["Cyprus", 1206499], ["Czech Republic", 1661565], ["Germany", 13011450],
             ["Djibouti", 228802], ["Denmark", 2821104], ["Dominica", 442528], ["Dominican Republic", 769942],
             ["Algeria", 708433], ["Ecuador", 1503548], ["Estonia", 1104363], ["Egypt", 2919818], ["Western Sahara", 67282],
             ["Eritrea", 326046], ["Spain", 6262644], ["Ethiopia", 572384], ["Finland", 3029327], ["Fiji", 717135],
             ["Falkland Islands", 96205], ["Micronesia", 240281], ["Faroe Islands", 104229], ["France", 17079307],
             ["Gabon", 261195], ["Great Britain (UK*)", 49484], ["Grenada", 635651], ["Georgia", 6911046],
             ["French Guiana", 92279], ["Ghana", 741162], ["Gibraltar", 586670], ["Greenland", 457377],
             ["Gambia", 316286], ["Guinea", 937213], ["Guadeloupe", 390825], ["Equatorial Guinea", 117821], 
             ["Greece", 3314249], ["Guatemala", 1252271], ["Guam", 505723], ["Guinea", 937213], ["Guyana", 361045],
             ["Hong Kong", 4372714], ["Honduras", 932766], ["Croatia", 901307], ["Haiti", 668931], ["Hungary", 2009608],
             ["Indonesia", 2669256], ["Ireland", 6484026], ["Israel", 6124467], ["India", 8285435], ["Iraq", 3962392],
             ["Iran", 1898788], ["Iceland", 927557], ["Italy", 8194116], ["Jamaica", 1503733], ["Jordan", 3596335],
             ["Japan", 11964871], ["Kenya", 1329593], ["Kyrgyzstan", 286245], ["Cambodia", 874170], ["Comoros", 128402],
             ["North Korea", 480082], ["South Korea", 861535], ["Kuwait", 902961], ["Kazakhstan", 545513], 
             ["Laos", 683111], ["Lebanon", 1611341], ["Saint Lucia", 213232], ["Liechtenstein", 476879],
             ["Sri Lanka", 1255410], ["Liberia", 433926], ["Lesotho", 261276], ["Lithuania", 1010669],
             ["Luxembourg", 1509947], ["Latvia", 993776], ["Libya", 379545], ["Morocco", 875836], ["Monaco", 1211341], 
             ["Moldova", 483230], ["Madagascar", 456664], ["Marshall Islands", 114943], ["Macedonia", 588851],
             ["Mali", 618361], ["Myanmar", 612176], ["Mongolia", 418530], ["Macau", 454230], ["Martinique", 408059],
             ["Mauritania", 234854], ["Montserrat", 272426], ["Malta", 1446525], ["Mauritius", 619848],
             ["Maldives", 399717], ["Malawi", 411174], ["Mexico", 12860749], ["Malaysia", 2896572], 
             ["Mozambique", 445068], ["Namibia", 561881], ["New Caledonia", 196439], ["Niger", 379119],
             ["Norfolk Island", 75537], ["Nigeria", 1065105], ["Nicaragua", 942399], ["Netherlands", 4278864],
             ["Norway", 3049784], ["Nepal", 1255808], ["Nauru", 139532], ["Neutral Zone", 27949], 
             ["New Zealand", 5220176], ["Oman", 538416], ["Panama", 1959774], ["Peru", 2534387], ["French Polynesia", 250512],
             ["Papua New Guinea", 357961], ["Philippines", 2321785], ["Pakistan", 2180449], 
             ["Poland", 2836355], ["Puerto Rico", 2244142], ["Portugal", 4164322], ["Paraguay", 1056969], 
             ["Qatar", 435277], ["Reunion", 1958013], ["Romania", 1626421], ["Russia", 5470901], ["Rwanda", 426131],
             ["Saudi Arabia", 1006404], ["Seychelles", 315412], ["Sudan", 636461], ["Sweden", 4009711],
             ["Singapore", 3994066], ["St. Helena", 175232], ["Slovenia", 974837], ["Slovak Republic", 189661],
             ["Sierra Leone", 368211], ["San Marino", 384320], ["Senegal", 740999], ["Somalia", 409418],
             ["Suriname", 236361], ["Sao Tome and Principe", 48854], ["El Salvador", 850296], ["Syria", 898835],
             ["Swaziland", 259978], ["Chad", 1250832], ["Togo", 333450], ["Thailand", 3746028], ["Tajikistan", 200424],
             ["Turkmenistan", 336623], ["Tunisia", 628396], ["Tonga", 351343], ["East Timor", 227194], 
             ["Turkey", 3330430], ["Trinidad and Tobago", 237116], ["Tuvalu", 143394], ["Taiwan", 2969225], 
             ["Tanzania", 703178], ["Ukraine", 1860363], ["Uganda", 676270], ["United Kingdom", 5432441], 
             ["United States", 22451899], ["Uruguay", 1552015], ["Uzbekistan", 496764], ["Vatican City", 102054],
             ["Saint Vincent & the Grenadines", 9584], ["Venezuela", 2375470], ["Virgin Islands (British)", 21933],
             ["Virgin Islands", 769543], ["Vietnam", 3024465], ["Vanuatu", 279065], ["Wallis and Futuna Islands", 8060],
             ["Samoa", 471357], ["Yemen", 431068], ["Mayotte", 96884], ["Yugoslavia", 862796], ["South Africa", 3073302], 
             ["Zambia", 497879], ["Zimbabwe", 995324],
            ]

def saveMatrix( matrix, countries, weights, size, keyw ):
    f = open( 'res_%s.txt' % keyw, 'w' )
    f.write( ','.join( countries[:size] ) +'\n' )
    f.write( ','.join( weights[:size] ) + '\n' )
    for y in range(size):
        f.write( ','.join( [ "%2.8f" % val for val in matrix[y][:size] ] ) +'\n' )
    f.close()


if 1:
    if len(sys.argv)>1:
        keyw = sys.argv[1]
    else:
        keyw = ''
        
    lst = countryList()
    if keyw!='':
        print 'getting counts for', keyw
        for c in lst:
            country = c[0]
            print 'counting ', country
            c[1] = hitCount( '"%s" NEAR "%s"' % (country, keyw) )
    file('scores_'+keyw, 'w').write( `lst` )
    lst.sort( lambda l1,l2: cmp( l2[1], l1[1] ) )
    matrix =[ [0.0]*len(lst) for i in lst]
    countries = [country.strip() for country, score in lst]
    weights = [str(score) for country, score in lst]
    rows=0
    for countryfrom, scorefrom in lst:
        scores = []
        cols=0
        for countryto, scoreto in lst[:rows]:
            print '*',
            searchfor = '"%s" NEAR "%s"' % (countryfrom, countryto)
            if keyw!='': searchfor + ' NEAR "%s"' % keyw
            hits = None
            while hits==None:
                try:
                    hits = hitCount(  searchfor )
                except:
                    hits = None
            score = hits / math.sqrt(scoreto*scorefrom)
            matrix[cols][rows] =  score
            matrix[rows][cols] =  score
            cols+=1
        rows+=1
        saveMatrix( matrix, countries, weights, rows, keyw )
        print ''
        print countryfrom, 'ready'