TIP speed up portage with cdb

# Copyright 2004, 2005 Tobias Bell <tobias.bell@web.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA

import portage_db_template
import os
import os.path
import cPickle
import cdb


class _data(object):

    def __init__(self, path, category, uid, gid):
        self.path = path
        self.category = category
        self.uid = uid
        self.gid = gid
        self.addList = {}
        self.delList = []
        self.modified = False
        self.cdbName = os.path.normpath(os.path.join(
            self.path, self.category) + ".cdb")
        self.cdbObject = None

    def __del__(self):
        if self.modified:
            self.realSync()

        self.closeCDB()

    def realSync(self):
        if self.modified:
            self.modified = False
            newDB = cdb.cdbmake(self.cdbName, self.cdbName + ".tmp")
           
            for key, value in iter(self.cdbObject.each, None):
                if key in self.delList:
                    if key in self.addList:
                        newDB.add(key, cPickle.dumps(self.addList[key], cPickle.HIGHEST_PROTOCOL))
                        del self.addList[key]
                elif key in self.addList:                   
                    newDB.add(key, cPickle.dumps(self.addList[key], cPickle.HIGHEST_PROTOCOL))
                    del self.addList[key]
                else:
                    newDB.add(key, value)
               

            self.closeCDB()

            for key, value in self.addList.iteritems():
                newDB.add(key, cPickle.dumps(value, cPickle.HIGHEST_PROTOCOL))
           
            newDB.finish()
            del newDB
           
            self.addList = {}
            self.delList = []

            self.openCDB()

    def openCDB(self):
        prevmask = os.umask(0)
       
        if not os.path.exists(self.path):
            os.makedirs(self.path, 02775)
            os.chown(self.path, self.uid, self.gid)
           
        if not os.path.isfile(self.cdbName):
            maker = cdb.cdbmake(self.cdbName, self.cdbName + ".tmp")
            maker.finish()
            del maker
            os.chown(self.cdbName, self.uid, self.gid)
            os.chmod(self.cdbName, 0664)

        os.umask(prevmask)
           
        self.cdbObject = cdb.init(self.cdbName)

    def closeCDB(self):
        if self.cdbObject:
            self.cdbObject = None


class _dummyData:
    cdbName = ""

    def realSync():
        pass
    realSync = staticmethod(realSync)


_cacheSize = 4
_cache = [_dummyData()] * _cacheSize


class database(portage_db_template.database):   

    def module_init(self):
        self.data = _data(self.path, self.category, self.uid, self.gid)

        for other in _cache:
            if other.cdbName == self.data.cdbName:
                self.data = other
                break
        else:
            self.data.openCDB()
            _cache.insert(0, self.data)           
            _cache.pop().realSync()
           
    def has_key(self, key):
        self.check_key(key)
        retVal = 0

        if self.data.cdbObject.get(key) is not None:
            retVal = 1

        if self.data.modified:
            if key in self.data.delList:
                retVal = 0
            if key in self.data.addList:
                retVal = 1
           
        return retVal

    def keys(self):
        myKeys = self.data.cdbObject.keys()

        if self.data.modified:
            for k in self.data.delList:
                myKeys.remove(k)
            for k in self.data.addList.iterkeys():
                if k not in myKeys:
                    myKeys.append(k)
                   
        return myKeys

    def get_values(self, key):
        values = None
       
        if self.has_key(key):
            if key in self.data.addList:
                values = self.data.addList[key]
            else:
                values = cPickle.loads(self.data.cdbObject.get(key))

        return values
   
    def set_values(self, key, val):
        self.check_key(key)
        self.data.modified = True
        self.data.addList[key] = val

    def del_key(self, key):
        retVal = 0
       
        if self.has_key(key):
            self.data.modified = True
            retVal = 1
            if key in self.data.addList:
                del self.data.addList[key]
            else:
                self.data.delList.append(key)

        return retVal
                   
    def sync(self):
        pass
   
    def close(self):
        pass


if __name__ == "__main__":
    import portage
    uid = os.getuid()
    gid = os.getgid()
    portage_db_template.test_database(database,"/tmp", "sys-apps", portage.auxdbkeys, uid, gid)

# Copyright: 2005 Gentoo Foundation
# Author(s): Brian Harring (ferringb@gentoo.org)
# License: GPL2
# $Id: anydbm.py 1911 2005-08-25 03:44:21Z ferringb $


cdb_module = __import__("cdb")
try:
	import cPickle as pickle
except ImportError:
	import pickle
import copy
import os
import fs_template
from template import reconstruct_eclasses
import cache_errors


class database(fs_template.FsBased):

	autocommits = True
	cleanse_keys = True
	serialize_eclasses = False

	def __init__(self, *args, **config):
		super(database,self).__init__(*args, **config)

		self._db_path = os.path.join(self.location, fs_template.gen_label(self.location, self.label)+".cdb")
		self.__db = None
		try:
			self.__db = cdb_module.init(self._db_path)

		except cdb_module.error:
			try:
				self._ensure_dirs()
				self._ensure_dirs(self._db_path)
				self._ensure_access(self._db_path)
			except (OSError, IOError), e:
				raise cache_errors.InitializationError(self.__class__, e)

			try:
				cm = cdb_module.cdbmake(self._db_path, self._db_path+".tmp")
				cm.finish()
				self._ensure_access(self._db_path)
				self.__db = cdb_module.init(self._db_path)
			except cdb_module.error, e:
				raise cache_errors.InitializationError(self.__class__, e)
		self._adds = {}
		self._dels = {}


	def iteritems(self):
		self.commit()
		return iter(self.__db.each, None)


	def _getitem(self, cpv):
		if cpv in self._adds:
			d = copy.deepcopy(self._adds[cpv])
		else:
			d = pickle.loads(self.__db[cpv])
		return d


	def _setitem(self, cpv, values):
		if cpv in self._dels:
			del self._dels[cpv]
		self._adds[cpv] = values


	def _delitem(self, cpv):
		if cpv in self._adds:
			del self._adds[cpv]
		self._dels[cpv] = True


	def commit(self):
		if not self._adds and not self._dels:
			return
		cm = cdb_module.cdbmake(self._db_path, self._db_path+str(os.getpid()))
		for (key, value) in iter(self.__db.each, None):
			if key in self._dels:
				del self._dels[key]
				continue
			if key in self._adds:
				cm.add(key, pickle.dumps(self._adds.pop(key), pickle.HIGHEST_PROTOCOL))
			else:
				cm.add(key, value)
		for (key, value) in self._adds.iteritems():
			cm.add(key, pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
		cm.finish()
		self._ensure_access(self._db_path)
		self.__db = cdb_module.init(self._db_path)
		self._adds = {}
		self._dels = {}


	def iterkeys(self):
		self.commit()
		return iter(self.__db.keys())


	def has_key(self, cpv):
		return cpv not in self._dels and (cpv in self._adds or cpv in self.__db)


	def __del__(self):
		if getattr(self, "__db", None):
			self.commit()
			self.__db.finish()

portdbapi.auxdbmodule = portage_db_cdb.database
eclass_cache.dbmodule = portage_db_cdb.database

portdbapi.auxdbmodule = cache.cdb.database
eclass_cache.dbmodule = cache.cdb.database

PORTDIR_CACHE_METHOD='cdb'