Here you will find some examples, from different projects / ideas. So they might miss some context information etc.
It defines index that separates data in two groups >5 and <=5 based on test parameter.
class CustomHashIndex(HashIndex):
def __init__(self, *args, **kwargs):
kwargs['key_format'] = 'I'
kwargs['hash_lim'] = 1
super(CustomHashIndex, self).__init__(*args, **kwargs)
def make_key_value(self, data):
d = data.get('test')
if d is None:
return None
if d > 5:
k = 1
else:
k = 0
return k, dict(test=d)
def make_key(self, key):
return key
Nothing more than index shown in Design.
class Md5Index(HashIndex):
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
super(Md5Index, self).__init__(*args, **kwargs)
def make_key_value(self, data):
return md5(data['name']).hexdigest(), None
def make_key(self, key):
return md5(key).hexdigest()
It allows to search in database for objects that contains a in their structure.
class WithAIndex(HashIndex):
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
super(WithAIndex, self).__init__(*args, **kwargs)
def make_key_value(self, data):
a_val = data.get("a")
if a_val:
if not isinstance(a_val, basestring):
a_val = str(a_val)
return md5(a_val).hexdigest(), None
return None
def make_key(self, key):
if not isinstance(key, basestring):
key = str(key)
return md5(key).hexdigest()
That index uses Tree index. It will allow you to search in order for elements that have t in their structure.
class Simple_TreeIndex(TreeBasedIndex):
def __init__(self, *args, **kwargs):
kwargs['node_capacity'] = 100
kwargs['key_format'] = 'I'
super(Simple_TreeIndex, self).__init__(*args, **kwargs)
def make_key_value(self, data):
t_val = data.get('t')
if t_val is not None:
return t_val, None
return None
def make_key(self, key):
return key
This index will allow you to run sum function on database context (you don’t have to retrieve all the data first, and then process it). Strongly recommended on Server version usage.
class WithRun_Index(HashIndex):
def __init__(self, *args, **kwargs):
kwargs['key_format'] = 'I'
super(WithRun_Index, self).__init__(*args, **kwargs)
def run_sum(self, db_obj, key):
gen = db_obj.get_many(index_name=self.name, key=key, limit=-1, with_storage=True)
vals = []
while True:
try:
d = gen.next()
except StopIteration:
break
else:
vals.append(d.get('x', 0))
return sum(vals)
def make_key_value(self, data):
a_val = data.get("a")
if a_val is not None:
out = {'x': data.get('x')}
return a_val, out
return None
def make_key(self, key):
return key
Example sharded index, it will shard records on key into 10 shards. (see Sharding in indexes)
class MySharded(ShardedHashIndex):
custom_header = """from CodernityDB.sharded_hash import ShardedHashIndex"""
def __init__(self, *args, **kwargs):
kwargs['sh_nums'] = 10
kwargs['key_format'] = 'I'
kwargs['use_make_keys'] = True
super(MySharded, self).__init__(*args, **kwargs)
def make_key_value(self, data):
return data['x'] % 10, None
def calculate_shard(self, key):
return key % self.sh_nums
It allows you to crypt storage information with Salsa20 algorithm. To use it you need to have index that will open storage with encryption key.
#!/usr/bin/env python
from CodernityDB.hash_index import UniqueHashIndex
from CodernityDB.storage import Storage
from CodernityDB.database import Database
from hashlib import sha256
import salsa20
import marshal
import os
class Salsa20Storage(Storage):
def __init__(self, db_path, name, enc_key):
super(Salsa20Storage, self).__init__(db_path, name)
self.enc_key = enc_key
def data_from(self, data):
iv = data[:8]
sal = salsa20.Salsa20(self.enc_key, iv, 20)
s_data = sal.decrypt(data[8:])
m_data = marshal.loads(s_data)
return m_data
def data_to(self, data):
iv = os.urandom(8)
m_data = marshal.dumps(data)
sal = salsa20.Salsa20(self.enc_key, iv, 20)
s_data = sal.encrypt(m_data)
return iv + s_data
class EncUniqueHashIndex(UniqueHashIndex):
__enc_key = 'a' * 32
custom_header = """
from demo_secure_storage import Salsa20Storage
from hashlib import sha256"""
def __init__(self, *args, **kwargs):
super(EncUniqueHashIndex, self).__init__(*args, **kwargs)
@property
def enc_key(self):
return self.__enc_key
@enc_key.setter
def enc_key(self, value):
if len(value) != 32:
self.__enc_key = sha256(value).digest()
else:
self.__enc_key = value
self.storage.enc_key = self.__enc_key
def _setup_storage(self):
if not self.storage:
self.storage = Salsa20Storage(
self.db_path, self.name, self.enc_key)
def _open_storage(self):
self._setup_storage()
self.storage.open()
def _create_storage(self):
self._setup_storage()
self.storage.create()
def main():
db = Database('/tmp/demo_secure')
key = 'abcdefgh'
id_ind = EncUniqueHashIndex(db.path, 'id')
db.set_indexes([id_ind])
db.create()
db.id_ind.enc_key = key
print db.id_ind.storage
for x in xrange(100):
db.insert(dict(x=x, data='testing'))
db.close()
dbr = Database('/tmp/demo_secure')
dbr.open()
dbr.id_ind.enc_key = key
for curr in dbr.all('id', limit=5):
print curr
if __name__ == "__main__":
main()
It will join user with timeline entries. See CodernityDB Demos minitwitt to see more things like this.
def run_timeline(self, db, user, limit):
u = db.get('user', user)
it = db.get_many(self.name, user, end=10 ** 11, limit=limit, with_doc=True)
for curr in it:
curr['username'] = user
curr['email'] = u['email']
curr['pub_date'] = curr['doc']['pub_date']
curr['text'] = curr['doc']['text']
del curr['doc']
yield curr