From 2881a4111a31cdea6f27d88f79bf957e8361d869 Mon Sep 17 00:00:00 2001 From: "simon.kagstrom" Date: Sun, 29 Mar 2009 12:44:35 +0000 Subject: [PATCH] Do country lookup and improve logging. And don't echo always! --- Src/network-broker/ip2country.py | 507 ++++++++++++++++++++++++++++++ Src/network-broker/network-broker | 34 +- 2 files changed, 532 insertions(+), 9 deletions(-) create mode 100644 Src/network-broker/ip2country.py diff --git a/Src/network-broker/ip2country.py b/Src/network-broker/ip2country.py new file mode 100644 index 0000000..5d0d2ee --- /dev/null +++ b/Src/network-broker/ip2country.py @@ -0,0 +1,507 @@ +""" +ip2country - module for looking up the country a given IP address +resides in. + +The method used for this is downloading/caching the APNIC database. + +On initial use, a file of approx 300-400k gets downloaded. But on +successive usage, the table is read from disk (~/.apnicdb) + +Apart from monthly download of the table, this module works +efficiently, since a web hit is not required for each lookup + +For usage info, refer to the demo section at the bottom. +Run this file with python to invoke the simple demo. + +Copyright (c) 2004 by David McNab +Released under the terms of the GNU General Public License (GPL) +You are free to copy, change, redistribute this code, within +the provisions of the GPL. + +No warranty, yada yada - don't blame me if this module causes your +niece to elope on the back of a Harley with a purple alien. +""" + +# CONFIGURATION SECTION +# +# Feel free to tweak these values if you need to + +# FTP URL from which to download APNIC data +apnicUrl = "ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest" + +# pathname for caching APNIC file, '~' may be used +apnicFileDb = "~/.ip2countrydb" + +ipCacheFile = "~/.ip2countryips" + +# set the maximum age of a cached APNIC database in DAYS +# if the cached file is older than this, a new one will +# be downloaded +maxApnicDbAge = 30 + +# +# END OF CONFIGURATION SECTION +import sys, os, time, stat, StringIO, commands, re +class IP2Country: + """ + Looks up IP addresses in APNIC database + + Caches the APNIC database locally, downloading it when + it gets more than a month old (or whatever you set maxApmicDbAge to + """ + apnicUrl = apnicUrl + apnicFileDb = apnicFileDb + ipCacheFile = ipCacheFile + + updateInterval = 86400 * maxApnicDbAge + + countryCodes = { + "AD":"Andorra", + "AE":"United Arab Emirates", + "AF":"Afghanistan", + "AG":"Antigua and Barbuda", + "AI":"Anguilla", + "AL":"Albania", + "AM":"Armenia", + "AN":"Netherlands Antilles", + "AO":"Angola", + "AQ":"Antarctica", + "AR":"Argentina", + "AS":"American Samoa", + "AT":"Austria", + "AU":"Australia", + "AW":"Aruba", + "AZ":"Azerbaijan", + "BA":"Bosnia and Herzegowina", + "BB":"Barbados", + "BD":"Bangladesh", + "BE":"Belgium", + "BF":"Burkina Faso", + "BG":"Bulgaria", + "BH":"Bahrain", + "BI":"Burundi", + "BJ":"Benin", + "BM":"Bermuda", + "BN":"Brunei Darussalam", + "BO":"Bolivia", + "BR":"Brazil", + "BS":"Bahamas", + "BT":"Bhutan", + "BV":"Bouvet Island", + "BW":"Botswana", + "BY":"Belarus", + "BZ":"Belize", + "CA":"Canada", + "CC":"Cocos (Keeling) Islands", + "CD":"Congo, The Democratic Republic of the", + "CF":"Central African Republic", + "CG":"Congo", + "CH":"Switzerland", + "CI":"Cote D'Ivoire", + "CK":"Cook Islands", + "CL":"Chile", + "CM":"Cameroon", + "CN":"China", + "CO":"Colombia", + "CR":"Costa Rica", + "CU":"Cuba", + "CV":"Cape Verde", + "CX":"Christmas Island", + "CY":"Cyprus", + "CZ":"Czech Republic", + "DE":"Germany", + "DJ":"Djibouti", + "DK":"Denmark", + "DM":"Dominica", + "DO":"Dominican Republic", + "DZ":"Algeria", + "EC":"Ecuador", + "EE":"Estonia", + "EG":"Egypt", + "EH":"Western Sahara", + "ER":"Eritrea", + "ES":"Spain", + "ET":"Ethiopia", + "FI":"Finland", + "FJ":"Fiji", + "FK":"Falkland Islands (Malvinas)", + "FM":"Micronesia, Federated States of", + "FO":"Faroe Islands", + "FR":"France", + "FX":"France, Metropolitan", + "GA":"Gabon", + "GB":"United Kingdom", + "GD":"Grenada", + "GE":"Georgia", + "GF":"French Guiana", + "GH":"Ghana", + "GI":"Gibraltar", + "GL":"Greenland", + "GM":"Gambia", + "GN":"Guinea", + "GP":"Guadeloupe", + "GQ":"Equatorial Guinea", + "GR":"Greece", + "GS":"South Georgia and the South Sandwich Islands", + "GT":"Guatemala", + "GU":"Guam", + "GW":"Guinea-Bissau", + "GY":"Guyana", + "HK":"Hong Kong", + "HM":"Heard and Mc Donald Islands", + "HN":"Honduras", + "HR":"Croatia (local name: Hrvatska)", + "HT":"Haiti", + "HU":"Hungary", + "ID":"Indonesia", + "IE":"Ireland", + "IL":"Israel", + "IN":"India", + "IO":"British Indian Ocean Territory", + "IQ":"Iraq", + "IR":"Iran (Islamic Republic of)", + "IS":"Iceland", + "IT":"Italy", + "JM":"Jamaica", + "JO":"Jordan", + "JP":"Japan", + "KE":"Kenya", + "KG":"Kyrgyzstan", + "KH":"Cambodia", + "KI":"Kiribati", + "KM":"Comoros", + "KN":"Saint Kitts and Nevis", + "KP":"Korea, Democratic People's Republic of", + "KR":"Korea, Republic of", + "KW":"Kuwait", + "KY":"Cayman Islands", + "KZ":"Kazakhstan", + "LA":"Lao People's Democratic Republic", + "LB":"Lebanon", + "LC":"Saint Lucia", + "LI":"Liechtenstein", + "LK":"Sri Lanka", + "LR":"Liberia", + "LS":"Lesotho", + "LT":"Lithuania", + "LU":"Luxembourg", + "LV":"Latvia", + "LY":"Libyan Arab Jamahiriya", + "MA":"Morocco", + "MC":"Monaco", + "MD":"Moldova, Republic of", + "MG":"Madagascar", + "MH":"Marshall Islands", + "MK":"Macedonia, The Former Yugoslav republic OF", + "ML":"Mali", + "MM":"Myanmar", + "MN":"Mongolia", + "MO":"Macau", + "MP":"Northern Mariana Islands", + "MQ":"Martinique", + "MR":"Mauritania", + "MS":"Montserrat", + "MT":"Malta", + "MU":"Mauritius", + "MV":"Maldives", + "MW":"Malawi", + "MX":"Mexico", + "MY":"Malaysia", + "MZ":"Mozambique", + "NA":"Namibia", + "NC":"New Caledonia", + "NE":"Niger", + "NF":"Norfolk Island", + "NG":"Nigeria", + "NI":"Nicaragua", + "NL":"Netherlands", + "NO":"Norway", + "NP":"Nepal", + "NR":"Nauru", + "NU":"Niue", + "NZ":"New Zealand", + "OM":"Oman", + "PA":"Panama", + "PE":"Peru", + "PF":"French Polynesia", + "PG":"Papua New Guinea", + "PH":"Philippines", + "PK":"Pakistan", + "PL":"Poland", + "PM":"St. Pierre and Miquelon", + "PN":"Pitcairn", + "PR":"Puerto Rico", + "PT":"Portugal", + "PW":"Palau", + "PY":"Paraguay", + "QA":"Qatar", + "RE":"Reunion", + "RO":"Romania", + "RU":"Russian Federation", + "RW":"Rwanda", + "SA":"Saudi Arabia", + "SB":"Solomon Islands", + "SC":"Seychelles", + "SD":"Sudan", + "SE":"Sweden", + "SG":"Singapore", + "SH":"St. Helena", + "SI":"Slovenia", + "SJ":"Svalbard and Jan Mayen Islands", + "SK":"Slovakia (Slovak Republic)", + "SL":"Sierra Leone", + "SM":"San Marino", + "SN":"Senegal", + "SO":"Somalia", + "SR":"Suriname", + "ST":"Sao Tome and Principe", + "SV":"El Salvador", + "SY":"Syrian Arab Republic", + "SZ":"Swaziland", + "TC":"Turks and Caicos Islands", + "TD":"Chad", + "TF":"French Southern Territories", + "TG":"Togo", + "TH":"Thailand", + "TJ":"Tajikistan", + "TK":"Tokelau", + "TM":"Turkmenistan", + "TN":"Tunisia", + "TO":"Tonga", + "TP":"East Timor", + "TR":"Turkey", + "TT":"Trinidad and Tobago", + "TV":"Tuvalu", + "TW":"Taiwan, Province of China", + "TZ":"Tanzania, United Republic of", + "UA":"Ukraine", + "UG":"Uganda", + "UM":"United States Minor Outlying Islands", + "US":"United States", + "UY":"Uruguay", + "UZ":"Uzbekistan", + "VA":"Holy See (Vatican City State)", + "VC":"Saint Vincent and The Grenadines", + "VE":"Venezuela", + "VG":"Virgin Islands (British)", + "VI":"Virgin Islands (US)", + "VN":"Viet Nam", + "VU":"Vanuatu", + "WF":"Wallis and Futuna Islands", + "WS":"Samoa", + "YE":"Yemen", + "YT":"Mayotte", + "YU":"Yugoslavia", + "ZA":"South Africa", + "ZM":"Zambia", + "ZW":"Zimbabwe", + } + + def __init__(self, **kw): + + self.verbose = kw.get('verbose', False) + + # normalise dbpath if it contains a '~' + if self.apnicFileDb.startswith("~/"): + self.apnicFileDb = os.path.expanduser(self.apnicFileDb) + if self.ipCacheFile.startswith("~/"): + self.ipCacheFile = os.path.expanduser(self.ipCacheFile) + + self.load() + def load(self): + + now = time.time() + + gotLatest = False + if os.path.isfile(self.apnicFileDb): + if os.stat(self.apnicFileDb)[stat.ST_MTIME] - now < self.updateInterval: + self.log("Got latest apnic db, no need to download") + gotLatest = True + + if not gotLatest: + self.download() + + lines = file(self.apnicFileDb).read().split("\n") + + ipv4Recs = [] + self.db = ipTree = {} + for line in lines: + parts = line.split("|") + if len(parts) < 7: + continue + if parts[0] != 'apnic' or parts[2] != 'ipv4': + continue + + # got an alloc + country = parts[1] + ip = parts[3] + + ipbit0, ipbit1, ipbit2, ipbit3 = ip.split(".") + + if not ipTree.has_key(ipbit0): + ipTree0 = ipTree[ipbit0] = {} + else: + ipTree0 = ipTree[ipbit0] + + if not ipTree0.has_key(ipbit1): + ipTree1 = ipTree0[ipbit1] = {} + else: + ipTree1 = ipTree0[ipbit1] + + if not ipTree1.has_key(ipbit2): + ipTree2 = ipTree1[ipbit2] = {} + else: + ipTree2 = ipTree1[ipbit2] + + if not ipTree2.has_key(ipbit3): + ipTree3 = ipTree2[ipbit3] = country + else: + ipTree3 = ipTree2[ipbit3] + + # read in IP address cache + if not os.path.isfile(self.ipCacheFile): + file(self.ipCacheFile, "w") + self.specificIPs = specificIPs = {} + lines = file(self.ipCacheFile).read().strip().split("\n") + #print lines + for line in lines: + if not line: + continue + #print "line: %s" % repr(line) + ip, country = line.split(":") + specificIPs[ip] = country + + self.log("Created apnic lookup tables") + + def lookup(self, ipaddr): + """ + Looks up an IP address, returns tuple (countrycode, country) if IP is + found, or (None, None) if not + """ + bit0, bit1, bit2, bit3 = ipaddr.split(".") + + # consult cached IPs + if self.specificIPs.has_key(ipaddr): + cc = self.specificIPs[ipaddr] + return cc, self.countryCodes.get(cc, '???') + + # screen Class C addresses + if bit0 in ['127', '192', '10']: + return None, None + + # not in cache IPs - consult APNIC database + db = self.db + + if db.has_key(bit0): + db1 = db[bit0] + elif db.has_key("0"): + db1 = db["0"] + else: + return self.lookupWhois(ipaddr) # not found + + if db1.has_key(bit1): + db2 = db1[bit1] + elif db1.has_key("0"): + db2 = db1["0"] + else: + return self.lookupWhois(ipaddr) # not found + + if db2.has_key(bit2): + db3 = db2[bit2] + elif db2.has_key("0"): + db3 = db2["0"] + else: + return self.lookupWhois(ipaddr) # not found + + if db3.has_key(bit3): + cc = db3[bit3] + elif db3.has_key("0"): + cc = db3["0"] + else: + cc = None + + #print repr(cc) + + if not cc: + return self.lookupWhois(ipaddr) + + if cc: + return cc, self.countryCodes.get(cc, "???") + else: + return None, None + + def lookupWhois(self, ipaddr): + """ + Fallback - perform a whois query and extracts the + first 'Country:' line from reply + """ + self.log("Performing whois query for %s" % ipaddr) + cc = None + lines = commands.getoutput("whois %s" % ipaddr).strip().split("\n") + + #print "\n * ".join(lines) + + for line in lines: + if line.lower().startswith("country:"): + cc = line[8:].strip() + + if cc: + # got it - add to memory and file cache + self.specificIPs[ipaddr] = cc + file(self.ipCacheFile, "a").write("%s:%s\n" % (ipaddr, cc)) + + # return it along with country name + return cc, self.countryCodes.get(cc, "???") + else: + return None, None + def download(self): + """ + Downloads the latest apnic database + """ + from ftplib import FTP + + url = self.apnicUrl + + # strip off 'ftp://' prefix if any + if url.startswith("ftp://"): + url = url[6:] + + host, path = url.split("/", 1) + + self.log("Cached DB is old or missing, need a new one") + + self.log("Connecting to apnic db server...") + conn = FTP(host) + self.log("Logging in...") + conn.login() + + s = StringIO.StringIO() + + self.log("Downloading ftp://%s" % path) + conn.retrbinary("RETR /"+path, file(self.apnicFileDb, "wb").write) + self.log("Download complete!") + conn.quit() + self.log("Closing server connection") + def log(self, msg): + if self.verbose: + print "IP2Country:%s" % msg +if __name__ == '__main__': + + import traceback, readline + + # run a demo + print "I2PCountry demo" + + i2pc = IP2Country(verbose=True) + + while True: + print "Enter an IP address, or empty line to quit" + raw = raw_input("> ").strip() + if raw == '': + sys.exit(0) + try: + cc, country = i2pc.lookup(raw) + print "%s => %s (%s)" % (raw, cc, country) + except: + traceback.print_exc() + print "Lookup of IP address %s failed" % repr(raw) diff --git a/Src/network-broker/network-broker b/Src/network-broker/network-broker index 445bb26..8d675fd 100755 --- a/Src/network-broker/network-broker +++ b/Src/network-broker/network-broker @@ -1,6 +1,6 @@ #!/usr/bin/env python -import socket, struct, syslog, time, thread +import socket, struct, syslog, time, thread, ip2country import SocketServer FRODO_NETWORK_MAGIC = 0x1976 @@ -27,8 +27,8 @@ pkt_type_to_str = { def log(pri, msg, echo): syslog.syslog(pri, msg) -# if echo: - print msg + if echo: + print msg def log_error(msg, echo = False): log(syslog.LOG_ERR, msg, echo) @@ -130,9 +130,11 @@ class ConnectToBrokerPacket(Packet): self.key = struct.unpack(">H", data[44:46])[0] self._is_master = struct.unpack(">H", data[46:48])[0] - self.name = struct.unpack(">32s", data[48:48+32])[0] + self.name = struct.unpack("32s", data[48:48+32])[0] self.server_id = struct.unpack(">L", data[80:84])[0] + self.name = self.name[0:self.name.find('\0')] + def get_key(self): return self.key @@ -159,11 +161,11 @@ class ListPeersPacket(Packet): out = struct.pack(">L16sHxx", self.n_peers, "", 0) for peer in self.peers: - out = out + struct.pack(">HH16s16sHH32sL", + out = out + struct.pack(">HH16s16sHH31sBL", 0, peer.public_port, "", peer.public_ip, peer.key, - peer.is_master, peer.name, - peer.id) + peer.is_master, "%s (%s)" % (peer.name, peer.country), + 0, peer.id) return Packet.marshal(self) + out @@ -175,6 +177,15 @@ class Peer: self.addr = addr self.public_ip, self.public_port = self.addr_to_ip_and_port(addr) + + # Lookup which country this guy is from + try: + self.country = srv.ip2c.lookup(self.public_ip)[1] + except ValueError, e: + self.country = None + if self.country == None: + self.country = "Unknown location" + # These will be set by the CONNECT_TO_BROKER packet below self.key = 0 self.name = "" @@ -217,6 +228,9 @@ class Peer: log_info("Sending list of peers (%d) to %s:%d" % (lp.n_peers, self.addr[0], self.addr[1]) ) self.send_packet(lp.marshal()) + else: + log_info("Added game host %s from %s:%d (%s)" % (self.name, self.addr[0], + self.addr[1], self.country)) if pkt.type == SELECT_PEER: peer = self.srv.get_peer_by_id( pkt.get_id() ) @@ -294,7 +308,7 @@ class BrokerPacketHandler(SocketServer.DatagramRequestHandler): class Broker(SocketServer.UDPServer): - def __init__(self, host, req_handler): + def __init__(self, host, req_handler, ip2c): SocketServer.UDPServer.__init__(self, host, req_handler) # Instead of setsockopt( ... REUSEADDR ... ) self.allow_reuse_address = True @@ -302,6 +316,7 @@ class Broker(SocketServer.UDPServer): self.peers_by_id = {} self.id = 0 self.ping_seq = 0 + self.ip2c = ip2c def send_data(self, dst, data): self.socket.sendto(data, dst) @@ -361,8 +376,9 @@ packet_class_by_type = { } if __name__ == "__main__": + ip2c = ip2country.IP2Country(verbose=0) syslog.openlog("frodo") log_info("Starting Frodo network broker", True) - broker = Broker( ("", 46214), BrokerPacketHandler) + broker = Broker( ("", 46214), BrokerPacketHandler, ip2c) thread.start_new_thread(ping_thread_fn, (broker, 5)) broker.serve_forever()