Source code for bgpranking.api

#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
    Entry point of the API
"""

import IPy
#import datetime

import helper_global as h
import constraints as c

__owner_cache = {}

[docs]def get_daily_rank_multiple_asns(asns, date, source): """ Get the ranks of a list of ASNs for one day and one source. :param asns: list of ASNs :param date: date of the rank (format: YYYY-MM-DD) :param source: name of the source for the rank :rtype: list of rank by ASN .. note:: Format of the output: .. code-block:: python [ (asn1, rank), (asn2, rank), ... ] """ string = '|{date}|{source}|rankv{ip_version}'.format(date = date, source = source, ip_version = c.ip_version) to_get = ['{asn}{string}'.format(asn = asn, string = string) for asn in asns] if len(to_get) == 0: return [] return zip(asns, h.__history_db.mget(to_get))
[docs]def get_all_ranks_single_asn(asn, dates_sources, with_details_sources=False): """ Get all the ranks on a timeframe for a single ASN. :param asn: Autonomous System Number :param dates_sources: Dictionaries of the dates and sources .. note:: Format of the dictionary: .. code-block:: python { YYYY-MM-DD: [source1, source2, ...], YYYY-MM-DD: [source1, source2, ...], ... } :param with_details_sources: Returns the details by sources if True :rype: Dictionary .. note:: Format of the output: .. code-block:: python { date1: { 'details': [ (source1, rank), (source2, rank), ... ], 'total': sum_of_ranks } ... } The details key is only present if ``with_details_sources`` is True. """ to_return = {} if not h.asn_exists(asn): return to_return string = '{asn}|{date}|{source}|rankv{ip_version}' p = h.__history_db.pipeline() for date, sources in dates_sources.iteritems(): if len(sources) > 0: p.mget([string.format(asn=asn, date=date, source=source, ip_version = c.ip_version) for source in sources]) ranks = p.execute() i = 0 for date, sources in dates_sources.iteritems(): impacts = h.get_all_weights(date) if len(sources) > 0: to_return[date] = {} sources_ranks = zip(sources, ranks[i]) if with_details_sources: to_return[date].update({'details': sources_ranks}) to_return[date].update({'total' : 0}) for s, r in sources_ranks: if r is not None: to_return[date]['total'] += float(r) * float(impacts[s]) i += 1 return to_return
[docs]def get_all_ranks_all_asns(dates_sources, with_details_sources = False): """ Get all ranks for all the ASNs on a timeframe. :param dates_sources: Dictionaries of the dates and sources .. note:: Format of the dictionary: .. code-block:: python { YYYY-MM-DD: [source1, source2, ...], YYYY-MM-DD: [source1, source2, ...], ... } :param with_details_sources: Also returns the ranks by sources :rype: Dictionary .. note:: Format of the dictionary: .. code-block:: python { YYYY-MM-DD: { asn1 : { 'details': [ (source1, rank), (source2, rank), ... ], 'total': sum_of_ranks } ... } ... } The details key is only present if ``with_details_sources`` is True. """ asns_keys = [] # prepare the keys to get all the asns by date and source for date, sources in dates_sources.iteritems(): asns_keys.extend(['{date}|{source}|asns'.format(date = date, source = source) for source in sources]) # get it p = h.__global_db.pipeline(False) [p.smembers(k) for k in asns_keys] asns_list = p.execute() asns = set.union(*asns_list) to_return = {} for asn in asns: details = get_all_ranks_single_asn(asn, dates_sources, with_details_sources) for date, entries in details.iteritems(): if to_return.get(date) is None: to_return[date] = {} to_return[date][asn] = entries return to_return
[docs]def get_owner(asn, block): """ Get the description of the ASN (usually the owner name). :param asn: Autonomous System Number :param block: IP Block you are looking for :rtype: List .. note:: Format of the list: .. code-block:: python [asn, block, owner] """ global __owner_cache if __owner_cache.get(asn) is None: if not h.asn_exists(asn): return h.unknown_asn timestamps = h.__global_db.smembers(asn) t_keys = [ '{asn}|{t}|ips_block'.format(asn = asn, t = t) for t in timestamps] temp_blocks = h.__global_db.mget(t_keys) __owner_cache[asn] = zip(timestamps, temp_blocks) owner = None for ts, temp_block in __owner_cache[asn]: if temp_block==block: owner = h.__global_db.get('{asn}|{t}|owner'.format(asn = asn, t = ts)) break return [asn, block, owner]
[docs]def get_asn_descs(asn, date = None, sources = None): """ Get all what is available in the database about an ASN for one day :param asn: Autonomous System Number :param date: Date of the information (default: last ranked day) :param sources: List of sources (default: the available sources at ``date``) :rtype: Dictionary .. note:: Format of the dictionary: .. code-block:: python { 'date': date, 'sources': [source1, source2, ...], 'asn': asn, asn: { 'clean_blocks': [ (timestamp, block, descr), ... ], 'old_blocks': [ (timestamp, block, descr), ... ], asn_timestamp: { 'owner': owner_description, 'ip_block': block, 'nb_of_ips': nb, 'sources': [source1, source2, ...] 'rank': subnet_rank }, ... } } """ if not h.asn_exists(asn): # The ASN does not exists in the database return None if date is None: date = h.get_default_date() day_sources = h.daily_sources([date]) if sources is None: sources = list(day_sources) else: sources = list(day_sources.intersection(set(sources))) to_return = {'date': date, 'sources': sources, 'asn': asn, asn: {}} for timestamp in h.__global_db.smembers(asn): # Get the number of IPs found in the database for each subnet asn_timestamp_key = '{asn}|{timestamp}|'.format(asn = asn, timestamp = timestamp) p = h.__global_db.pipeline(False) [p.scard('{asn_ts}{date}|{source}'.format(asn_ts=asn_timestamp_key, date = date, source=source)) for source in sources] ips_by_sources = p.execute() nb_of_ips = sum(ips_by_sources) key_owner = '{asn_ts}owner'.format(asn_ts = asn_timestamp_key) key_block = '{asn_ts}ips_block'.format(asn_ts = asn_timestamp_key) key_clean_set = '{asn}|{date}|clean_set'.format(asn = asn, date = date) owner, ip_block = h.__global_db.mget([key_owner, key_block]) if nb_of_ips > 0: impacts = h.get_all_weights(date) # Compute the local ranking: the ranking if this subnet is the # only one for this AS i = 0 sum_rank = 0 sources_exists = [] for source in sources: sum_rank += float(ips_by_sources[i])*float(impacts[source]) if ips_by_sources[i] != 0: sources_exists.append(source) i += 1 local_rank = sum_rank / IPy.IP(ip_block).len() to_return[asn][timestamp] = {'owner': owner, 'ip_block': ip_block, 'nb_of_ips': nb_of_ips, 'sources': sources_exists, 'rank': local_rank} elif ip_block in h.__history_db.smembers(key_clean_set): if to_return[asn].get('clean_blocks') is None: to_return[asn]['clean_blocks'] = [] to_return[asn]['clean_blocks'].append( (timestamp, ip_block, owner)) else: # Not announced anymore. if to_return[asn].get('old_blocks') is None: to_return[asn]['old_blocks'] = [] to_return[asn]['old_blocks'].append( (timestamp, ip_block, owner)) return to_return
[docs]def get_ips_descs(asn, asn_timestamp, date = None, sources = None): """ Get all what is available in the database about an subnet for one day :param asn: Autonomous System Number :param asn_timestamp: First the subnet has been seen in the db (for this ASN) :param date: Date of the information (default: last ranked day) :param sources: List of sources (default: the available sources at ``date``) :rtype: Dictionary .. note:: Format of the dictionary: .. code-block:: python { 'date': date, 'sources': [source1, source2, ...], 'asn': asn, 'timestamp': asn_timestamp asn_timestamp: { ip: [source1, source2, ...], ... } } """ if date is None: date = h.get_default_date() day_sources = h.daily_sources([date]) if sources is None: sources = list(day_sources) else: sources = list(day_sources.intersection(set(sources))) asn_timestamp_key = '{asn}|{timestamp}|{date}|'.format(asn = asn, timestamp = asn_timestamp, date = date) pipeline = h.__global_db.pipeline(False) [pipeline.smembers('{asn_ts}{source}'.format( asn_ts = asn_timestamp_key, source=source)) for source in sources] ips_by_source = pipeline.execute() i = 0 to_return = {'date': date, 'sources': sources, 'asn': asn, 'timestamp': asn_timestamp, asn_timestamp: {}} for source in sources: ips = ips_by_source[i] for ip_details in ips: ip, timestamp = ip_details.split('|') if to_return[asn_timestamp].get(ip) is None: to_return[asn_timestamp][ip] = [source] else: to_return[asn_timestamp][ip].append(source) i += 1 return to_return
[docs]def get_stats(): """ Return amount of asn and subnets found by source, for all the cached days. :rtype: Dictionary .. note:: Format of the Dictionary .. code-block:: python { date: { 'sources': { source : [nb_asns, nb_subnets], ... }, 'total_asn': total_asn 'total_subnets': total_subnets } ... } """ dates = sorted(h.__history_db_cache.smembers('all_dates')) dates_sources = dict(zip(dates, h.daily_sources(dates))) to_return = {} p = h.__global_db.pipeline(False) for date, sources in dates_sources.iteritems(): to_return[date] = {} for source in sources: current_key = '{date}|{source}|asns'.format(date = date, source = source) p.scard(current_key) p.scard(current_key + '_details') cards = p.execute() i = 0 for date, sources in dates_sources.iteritems(): to_return[date] = {'sources': {}} total_asn = 0 total_subnets = 0 for source in sources: nb_asns = cards[i] nb_subnets = cards[i+1] total_asn += nb_asns total_subnets += nb_subnets to_return[date]['sources'].update( {source: (nb_asns, nb_subnets)}) i = i+2 to_return[date]['total_asn'] = total_asn to_return[date]['total_subnets'] = total_subnets return to_return # Need cached data
[docs]def cache_get_daily_rank(asn, source = 'global', date = None): """ **From the temporary database** Get a single rank. :param asn: Autonomous System Number :param source: Source to use. global is the aggregated view for all the sources :param date: Date of the information (default: last ranked day) :rtype: List .. note:: Format of the list .. code-block:: python [asn, date, source, rank] """ if date is None: date = h.get_default_date() histo_key = '{date}|{source}|rankv{ip_version}'.format( date = date, source = source, ip_version = c.ip_version) return asn, date, source, h.__history_db_cache.zscore(histo_key, asn)
[docs]def cache_get_top_asns(source = 'global', date = None, limit = 50, with_sources = True): """ **From the temporary database** Get worse ASNs. :param source: Source used to rank the ASNs. global is the aggregated view for all the sources :param date: Date of the information (default: last ranked day) :param limit: Number of ASNs to get :param with_sources: Get the list of sources where each ASN has been found. :rtype: Dictionary .. note:: Format of the Dictionary .. code-block:: python { 'source': source, 'date': date, 'top_list': [ ((asn, rank), set([source1, source2, ...])), ... ] } The set of sources is only presetn if ``with_sources`` is True """ if date is None: date = h.get_default_date() histo_key = '{date}|{histo_key}|rankv{ip_version}'.format(date = date, histo_key = source, ip_version = c.ip_version) to_return = {'source': source, 'date': date, 'top_list': []} ranks = h.__history_db_cache.zrevrange(histo_key, 0, limit, True) if ranks is None: return to_return if not with_sources: to_return['top_list'] = ranks else: p = h.__history_db_cache.pipeline(False) [p.smembers('|'.join([date, rank[0]])) for rank in ranks] to_return['top_list'] = zip(ranks, p.execute()) return to_return