Labs

CodernityDB Download

Examples

Here you will find some examples, from different projects / ideas. So they might miss some context information etc.

Example indexes

Custom Hash

It defines index that separates data in two groups >5 and <=5 based on test parameter.

class CustomHashIndex(HashIndex):

    def __init__(self, *args, **kwargs):
        kwargs['key_format'] = 'I'
        kwargs['hash_lim'] = 1
        super(CustomHashIndex, self).__init__(*args, **kwargs)

    def make_key_value(self, data):
        d = data.get('test')
        if d is None:
            return None
        if d > 5:
            k = 1
        else:
            k = 0
        return k, dict(test=d)

    def make_key(self, key):
        return key

MD5 Hash

Nothing more than index shown in Design.

class Md5Index(HashIndex):

    def __init__(self, *args, **kwargs):
        kwargs['key_format'] = '32s'
        super(Md5Index, self).__init__(*args, **kwargs)

    def make_key_value(self, data):
        return md5(data['name']).hexdigest(), None

    def make_key(self, key):
        return md5(key).hexdigest()

With A

It allows to search in database for objects that contains a in their structure.

class WithAIndex(HashIndex):

    def __init__(self, *args, **kwargs):
        kwargs['key_format'] = '32s'
        super(WithAIndex, self).__init__(*args, **kwargs)

    def make_key_value(self, data):
        a_val = data.get("a")
        if a_val:
            if not isinstance(a_val, basestring):
                a_val = str(a_val)
            return md5(a_val).hexdigest(), None
        return None

    def make_key(self, key):
        if not isinstance(key, basestring):
            key = str(key)
        return md5(key).hexdigest()

Simple Tree

That index uses Tree index. It will allow you to search in order for elements that have t in their structure.

class Simple_TreeIndex(TreeBasedIndex):

    def __init__(self, *args, **kwargs):
        kwargs['node_capacity'] = 100
        kwargs['key_format'] = 'I'
        super(Simple_TreeIndex, self).__init__(*args, **kwargs)

    def make_key_value(self, data):
        t_val = data.get('t')
        if t_val is not None:
            return t_val, None
        return None

    def make_key(self, key):
        return key

With Run

This index will allow you to run sum function on database context (you don’t have to retrieve all the data first, and then process it). Strongly recommended on Server version usage.

class WithRun_Index(HashIndex):

    def __init__(self, *args, **kwargs):
        kwargs['key_format'] = 'I'
        super(WithRun_Index, self).__init__(*args, **kwargs)

    def run_sum(self, db_obj, key):
        gen = db_obj.get_many(index_name=self.name, key=key, limit=-1, with_storage=True)
        vals = []
        while True:
            try:
                d = gen.next()
            except StopIteration:
                break
            else:
                vals.append(d.get('x', 0))
        return sum(vals)

    def make_key_value(self, data):
        a_val = data.get("a")
        if a_val is not None:
            out = {'x': data.get('x')}
            return a_val, out
        return None

    def make_key(self, key):
        return key

Example sharded hash

Example sharded index, it will shard records on key into 10 shards. (see Sharding in indexes)

class MySharded(ShardedHashIndex):

    custom_header = """from CodernityDB.sharded_hash import ShardedHashIndex"""

    def __init__(self, *args, **kwargs):
        kwargs['sh_nums'] = 10
        kwargs['key_format'] = 'I'
        kwargs['use_make_keys'] = True
        super(MySharded, self).__init__(*args, **kwargs)

    def make_key_value(self, data):
        return data['x'] % 10, None

    def calculate_shard(self, key):
        return key % self.sh_nums

Example storages

Secure storage

It allows you to crypt storage information with Salsa20 algorithm. To use it you need to have index that will open storage with encryption key.

#!/usr/bin/env python


from CodernityDB.hash_index import UniqueHashIndex
from CodernityDB.storage import Storage
from CodernityDB.database import Database
from hashlib import sha256

import salsa20
import marshal
import os


class Salsa20Storage(Storage):

    def __init__(self, db_path, name, enc_key):
        super(Salsa20Storage, self).__init__(db_path, name)
        self.enc_key = enc_key

    def data_from(self, data):
        iv = data[:8]
        sal = salsa20.Salsa20(self.enc_key, iv, 20)
        s_data = sal.decrypt(data[8:])
        m_data = marshal.loads(s_data)
        return m_data

    def data_to(self, data):
        iv = os.urandom(8)
        m_data = marshal.dumps(data)
        sal = salsa20.Salsa20(self.enc_key, iv, 20)
        s_data = sal.encrypt(m_data)
        return iv + s_data


class EncUniqueHashIndex(UniqueHashIndex):

    __enc_key = 'a' * 32

    custom_header = """
from demo_secure_storage import Salsa20Storage
from hashlib import sha256"""

    def __init__(self, *args, **kwargs):
        super(EncUniqueHashIndex, self).__init__(*args, **kwargs)

    @property
    def enc_key(self):
        return self.__enc_key

    @enc_key.setter
    def enc_key(self, value):
        if len(value) != 32:
            self.__enc_key = sha256(value).digest()
        else:
            self.__enc_key = value
        self.storage.enc_key = self.__enc_key

    def _setup_storage(self):
        if not self.storage:
            self.storage = Salsa20Storage(
                self.db_path, self.name, self.enc_key)

    def _open_storage(self):
        self._setup_storage()
        self.storage.open()

    def _create_storage(self):
        self._setup_storage()
        self.storage.create()


def main():
    db = Database('/tmp/demo_secure')
    key = 'abcdefgh'
    id_ind = EncUniqueHashIndex(db.path, 'id')
    db.set_indexes([id_ind])
    db.create()
    db.id_ind.enc_key = key
    print db.id_ind.storage

    for x in xrange(100):
        db.insert(dict(x=x, data='testing'))

    db.close()
    dbr = Database('/tmp/demo_secure')
    dbr.open()
    dbr.id_ind.enc_key = key

    for curr in dbr.all('id', limit=5):
        print curr


if __name__ == "__main__":
    main()

Example database functions

Join like 1

It will join user with timeline entries. See CodernityDB Demos minitwitt to see more things like this.

def run_timeline(self, db, user, limit):
    u = db.get('user', user)
    it = db.get_many(self.name, user, end=10 ** 11, limit=limit, with_doc=True)
    for curr in it:
        curr['username'] = user
        curr['email'] = u['email']
        curr['pub_date'] = curr['doc']['pub_date']
        curr['text'] = curr['doc']['text']
        del curr['doc']
        yield curr