logdb.py

Go to the documentation of this file.

## @file logdb.py Routines and classes to access the BDB log file
# To read the documentation generated from this file refer to @ref swarmng.logdb

## @package swarmng.logdb
# Interface to the Log files generated by bdb_writer plugin

from bsddb3.db import *
from logrecord import LogRecord
from struct import pack, unpack
import sys
import os

TIME_EPSILON = sys.float_info.epsilon

## Primary key of the log file database
#
# Most of methods are only for internal use and are not documented.
class PKey:
    ## time field of the associated LogRecord, a floating point value representing time in AU
    time = property
    ## Event ID of the associated LogRecord, integer value that is usually 1.
    event_id = property
    ## Integer ID of the system that the associated LogRecord has a snapshot.
    system_id = property
    
    ## Create a primary key object from time, event_id and system_id respectively.
    def __init__(self, time, event_id, system_id):
        self.time = time
        self.event_id = event_id
        self.system_id = system_id

    @staticmethod
    def decomposeBinary(bin):
        if(len(bin) == 8):
            t, se = unpack('fI', bin)
            s = se % 2**24
            e = se / 2**24
            return (t,e,s)
        else:
            return None

    @staticmethod
    def fromBinary(bin):
        (t,e,s) = PKey.decomposeBinary(bin)
        return PKey(t,e,s)

    @staticmethod
    def compareBinary(s1,s2):
        d1 = PKey.decomposeBinary(s1)
        d2 = PKey.decomposeBinary(s2)
        if(d1 == d2):
            return 0
        elif (d1 < d2):
            return -1
        else:
            return 1

    def __repr__(self):
       m = {'time':self.time, 'sys': self.system_id, 'evt': self.event_id  }
       return m.__repr__();

    def toBinary(self):
        return pack('fI', self.time, self.event_id * 2**24 + self.system_id)

    @staticmethod
    def packSys(s):
        return pack('I',s)
    @staticmethod
    def packTime(t):
        return pack('f',t)
    @staticmethod
    def packEvt(e):
        return pack('B',s)
    @staticmethod
    def unpackSys(bin):
        return unpack('I',bin)[0]
    @staticmethod
    def unpackEvt(bin):
        return unpack('B',bin)[0]
    @staticmethod
    def unpackTime(bin):
        return unpack('f',bin)[0]




def extract_sys(key, data):
    return PKey.fromBinary(key).system_id 

def extract_time(key, data):
    return PKey.fromBinary(key).time

def extract_evt(key, data):
    return PKey.fromBinary(key).event_id

def compare_time(l, r):
    if(len(l) < len(r)):
        return -1
    if(len(l) > len(r)):
        return 1
    elif(len(l) != 4):
        return 0

    lf = PKey.unpackTime(l)
    rf = PKey.unpackTime(r)
    if( lf < rf ):
        return -1
    if( lf > rf ):
        return 1
    else:
        return 0


def compare_sys(l, r):
    if(len(l) < len(r)):
        return -1
    if(len(l) > len(r)):
        return 1
    elif(len(l) != 4):
        return 0

    lf = PKey.unpackSys(l)
    rf = PKey.unpackSys(r)
    if( lf < rf ):
        return -1
    if( lf > rf ):
        return 1
    else:
        return 0

def compare_evt(l, r):
    if(len(l) < len(r)):
        return -1
    if(len(l) > len(r)):
        return 1
    elif(len(l) != 1):
        return 0

    lf = PKey.unpackEvt(l)
    rf = PKey.unpackEvt(r)
    if( lf < rf ):
        return -1
    if( lf > rf ):
        return 1
    else:
        return 0

def iter_cursor(c, mode = DB_NEXT):
    while True:
        n = c.get(mode)
        if n != None :
            yield n
        else:
            c.close()
            raise StopIteration

def iter_secondary_cursor(c, mode = DB_NEXT):
    while True:
        n = c.pget(mode)
        if n != None :
            yield n
        else:
            c.close()
            raise StopIteration

## Interface to BDB log files. open a log file like:
#
# @code{.py}
# >>> db = IndexedLogDB('mydatabase.db')
# >>> records = db.all_records()
# >>> print(next(records))
# >>> print(next(records))
# @endcode
#
# Note that most of the methods below return key-value pairs. The 
# key-value pair is a Python tuple where the first element is of type \ref PKey
# and has time, system_id, event_id properties. the second element is a \ref swarmng.logrecord.LogRecord "LogRecord" 
# object and contains all the information about the planetary system and its attributes.
#
class IndexedLogDB:
    ## Supported version of BDB log files
    #
    fileFormatVersion = "1"
    

    ## Opens a BDB file that contains the primary database and
    #    secondary indices.
    #  @arg @c fn: path to the database file name
    def __init__(self, pathName):
        CACHESIZE = 1024*1024*64 ;
        e = DBEnv()
        e.set_cachesize(0,CACHESIZE,0)
        #print("Trying to initialize environment for `{0}`".format(os.path.dirname(pathName)))
        e.open(os.path.dirname(pathName),DB_INIT_CDB | DB_INIT_MPOOL)
        fn = os.path.basename(pathName);
        #print("Opening `{0}`".format(fn))

        m = DB(e)
        m.open(fn, dbname="metadata", flags=DB_RDONLY)
        
        p = DB(e)
        p.set_bt_compare(PKey.compareBinary)
        p.open(fn, dbname="primary", flags=DB_RDONLY)

        si = DB(e)
        si.set_bt_compare(compare_sys)
        si.set_dup_compare(PKey.compareBinary)
        si.open(fn, dbname="system_idx", flags=DB_RDONLY)

        ti = DB(e)
        ti.set_bt_compare(compare_time)
        ti.set_dup_compare(PKey.compareBinary)
        ti.open(fn, dbname="time_idx", flags=DB_RDONLY)

        ei = DB(e)
        ei.set_bt_compare(compare_evt)
        ei.set_dup_compare(PKey.compareBinary)
        ei.open(fn, dbname="event_idx", flags=DB_RDONLY)

        p.associate(si, extract_sys  )
        p.associate(ti, extract_time )
        p.associate(ei, extract_evt  )
        
        self.primary = p
        self.system_idx = si
        self.time_idx = ti
        self.event_idx = ei
        self.metadata = m
        
        self.validateVersionInfo()
      
    ## Get meta data from the database for the provided `name` string.
    #  
    #  There are two metadata that are always available: `fileFormatVersion` and `swarmngVersion`
    #
    #  Meta data for the log file is a mapping of string -> string.
    #
    #
    #  @arg \c name : string :name of the property we are looking for.
    #  returns : string value for the provided property, None if the property is not found.
    def getMetadata(self,name):
      return self.metadata.get(name)
    
    def validateVersionInfo(self):
      v = self.getMetadata("fileFormatVersion")
      if v != self.fileFormatVersion :
        raise RuntimeError("Mismatching file format version: {0}, required {1}".format(v, self.fileFormatVersion))


    ## Return an iterable for all of the records, the records are
    # sorted by time, then event id then system id.
    #
    # The elements of the returend iterable or key-value pairs.
    #
    def all_records(self):
        c = self.primary.cursor()
        for k,l in iter_cursor(c):
            yield IndexedLogDB.decodeKVP((k,l))

    ## Returns a iterable of times for a time range, it is used for making 
    #    range queries. 
    #
    #  The elements of the returned iterable are floating point values.
    def time_sequence(self, time_range):

        t0, t1 = time_range
        c = self.time_idx.cursor()
        k = PKey.packTime(t0)
        c.set_range(k)
        c.prev();
        for k, p, l in iter_secondary_cursor(c, DB_NEXT_NODUP):
            t = PKey.unpackTime(k)
            if t <= t1 :
                yield t
            else:
                raise StopIteration
    ## Query for all the records that have system id in the system range
    #
    #  Return an iterable of key-value pairs.
    def system_range_records(self, sys_range):
        s0, s1 = sys_range
        c = self.system_idx.cursor()
        k = PKey.packSys(s0)
        c.set_range(k)
        c.prev()
        for k, p, l in iter_secondary_cursor(c):
            s = PKey.unpackSys(k)
            if s <= s1 :
                yield IndexedLogDB.decodeKVP((p,l))
            else:
                raise StopIteration
    ## Query for all the records that have time in the time range
    #
    #  Return an iterable of key-value pairs.
    def time_range_records(self, time_range):
        t0, t1 = time_range
        c = self.time_idx.cursor()
        k = PKey.packTime(t0)
        c.set_range(k)
        c.prev()
        for k, p, l in iter_secondary_cursor(c):
            t = PKey.unpackTime(k)
            if t <= t1 :
                yield IndexedLogDB.decodeKVP((p,l))
            else:
                raise StopIteration

    ## Query for all the records for a system in a given time range.
    #
    #  Return an iterable of key-value pairs.
    def system_at_time(self, sysid, time_range):
        t0, t1 = time_range
        c = self.system_idx.cursor()
        k = PKey.packSys(sysid)
        pk = PKey(t0, 1, sysid).toBinary()
        _, pk, _ = c.pget(k, pk, DB_GET_BOTH_RANGE)
        c.prev()
        for k, pk, l in iter_secondary_cursor(c):
            kk = PKey.fromBinary(pk)
            if t0 <= kk.time <= t1 and kk.system_id == sysid :
                yield IndexedLogDB.decodeKVP((pk,l))
            else:
                raise StopIteration

    ## Return initial coniditions for the system range
    #
    #  Return an iterable of a tuple of system id and LogRecord.
    #
    #  Usage:
    #  @code{.py}
    #  for system_id, lr in d.initial_conditions(Range.interval(10,20)):
    #     print(system_id) # prints integers
    #     print(lr.time)   # lr is a LogRecord object and has a time property
    #  @endcode
    #
    def initial_conditions(self, system_range):
        c = self.system_idx.cursor()

        if system_range.isUniversal() :
            s0 = 0
            s1 = sys.maxint
            r = c.first()
        else:
            s0, s1 = system_range.ulPair()
            k = PKey.packSys(s0)
            r  = c.set_range(k) 

        sysid = s0
        while sysid < s1 :
            if r : 
                ks, l = r
                sysid = PKey.unpackSys(ks)
                yield sysid, LogRecord.from_binary(l)
            else:
                break
            r = c.get(DB_NEXT_NODUP)
    ## Return the final conditions for a range of systems that
    #    is the largest time for which the system has a valid entry
    #
    #  Return an iterable of a tuple of system id and LogRecord.
    #
    #  Usage:
    #  @code{.py}
    #  for system_id, lr in d.final_conditions(Range.interval(10,20)):
    #     print(system_id) # prints integers
    #     print(lr.time)   # lr is a LogRecord object and has a time property
    #  @endcode
    #
    def final_conditions(self,system_range):
        c = self.system_idx.cursor()

        if system_range.isUniversal() :
            s0 = 0
            s1 = sys.maxint
            c.first()
        else:
            s0, s1 = system_range.ulPair()
            k = PKey.packSys(s0)
            c.set_range(k) 

        sysid = s0-1
        while sysid < s1  and  c.next() != None :
            if c.get(DB_NEXT_NODUP) == None :
                ks, l = c.last()
            else:
                ks, l = c.prev()
                
            sysid = PKey.unpackSys(ks)
            yield sysid, LogRecord.from_binary(l)



    @staticmethod
    def decodeKVP(r):
        return (PKey.fromBinary(r[0]),LogRecord.from_binary(r[1]))

    def system_range_for_time_event(self,time,event_id,system_range):
        c = self.primary.cursor()
        s0, s1 = system_range

        k = PKey(time, event_id, s0)
        c.set_range(k.toBinary())
        # We have to check that we are at a valid location
        c.prev()
        for r in iter_cursor(c):
            k,l = IndexedLogDB.decodeKVP(r)
            if s0 <= k.system_id <= s1 and k.time == time and k.event_id == event_id:
                yield (k,l)
            else:
                raise StopIteration

    ## Query the database for with a time range ,system range  and event id range
    #
    #  @arg tr: time range (of type swarmng.range_type.Range)
    #  @arg sr: system range (of type swarmng.range_type.Range)
    #  @arg er: event ID range (of type swarmng.range_type.Range)
    #
    #  Return an iterable of key-value pairs.
    def query(d , tr, sr, er ):

        def filterEventID(q,er):
            for k,l in q:
                if(er.contains(k.event_id)):
                    yield (k,l)    
        
        q0 = d.queryInternal(tr,sr)
        
        if(not er.isUniversal()):
            q = filterEventID(q0,er)
        else:
            q = q0

        return q

    def queryInternal(d, tr, sr):
        if(tr.isUniversal()):
            if(sr.isUniversal()):
                for k,l in d.all_records():
                    yield (k,l)
            else:
                for k,l in d.system_range_records(sr.ulPair()):
                    yield (k,l)
        else:
            if(sr.isUniversal()):
                for k,l in d.time_range_records(tr.ulPair()):
                    yield (k,l)
            else:
                for t in d.time_sequence(tr.ulPair()):
                    for k,l in d.system_range_for_time_event(t,1,sr.ulPair()):
                        yield (k,l)