Welcome, guest | Sign In | My Account | Store | Cart

parse and create fixed size binary data (records / structs) (Python recipe) by Maciej Obarski
ActiveState Code (http://code.activestate.com/recipes/465219/)

Python's struct library is too low-level for direct usage. This recipe (only 40 lines) shows how it can be turned into more developer-friendly tool.

Python, 57 lines
Download
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
from struct import calcsize, pack, unpack

class ConcreteRecord:

    def __init__(self, fields, data=None):
        self._fields = fields
        if data == None: return
        for f in fields:
            name,fmt,start,stop = f
            val = unpack(fmt, data[start:stop])
            if len(val) == 1: val=val[0]
            self.__dict__[name] = val

    def __str__(self):
        lst = []
        for f in self._fields:
            name,fmt,start,stop = f
            val = self.__dict__.get(name)
            if type(val) in (tuple,list):  # U
                lst += [pack(fmt,*val)]    # G
            else:                          # L
                lst += [pack(fmt,val)]     # Y
        return ''.join(lst)


class RecordFactory:

    def __init__(self,record_fmt):
        self.fields = []
        pos = 0
        for field in record_fmt.split():
            if field[0] == "#": continue
            fmt,name = field.split('.')
            size = calcsize(fmt)
            self.fields += [(name,fmt,pos,pos+size)]
            pos += size

    def build(self, data):
        return ConcreteRecord(self.fields, data)

#### EXAMPLE ##################################################################

myrf = RecordFactory("""
    4B.ip
    >H.port
    >I.session_id
""")

r = myrf.build("\x00\x01\x02\x03" + "\x00\x04" + "\xFE\xDC\xBA\x98")
print "ip:        ", r.ip
print "port:      ", r.port
print "session_id:", r.session_id

r.port       = 1029       # equals 0x0405
r.session_id = 101124105  # equals 0x06070809
import binascii
print "record_str:", binascii.hexlify(str(r))

Four lines of code are marked as "UGLY" because I don't have an idea how to write this code as one statement (my intuition tells me that it is possible). I would be grateful for any hints.

Tags: database

2 comments

Dug Song 18 years, 2 months ago  # | flag

dpkt metaclass. check out http://monkey.org/~dugsong/dpkt/ for another implementation of the same idea, using a simple metaclass in dpkt.py.

Mark Andrews 18 years, 1 month ago  # | flag

Using a byte as flags. After creating a ConcreteRecord this class can be used to attach boolean flag names to a byte.

class ByteFlags:

    def __init__(self, flags, data=None):
        self._flags = []
        bit = 1
        for f in flags:
            self._flags += [(f, bit)]
            bit = bit * 2
        if data == None: return
        for f in self._flags:
            name,bit = f
            val = bool(data & bit)
            self.__dict__[name] = val

    def __int__(self):
        val = 0
        for f in self._flags:
            name,bit = f
            if self.__dict__.get(name):
                val |= bit
        return val

    def __str__(self):
        return pack("B",self.__int__())

#### EXAMPLE ####

myrf = RecordFactory("""
    B.flags
    4B.ip
    >H.port
    >I.session_id
""")

r = myrf.build("\x0f" + "\x00\x01\x02\x03" + "\x00\x04" + "\xFE\xDC\xBA\x98")

r.flags= ByteFlags(['bit0','bit1','bit2','bit3','bit4','bit5','bit6','bit7'], r.flags)
r.flags.bit2=False
str(r.flags)
str(r)