* hyper dandik transaction support

TODO

@@ -172,7 +172,8 @@ Legend:
 
  * high priority
 
- / transaction stuff for database (eray)
+ + transaction stuff for database (eray)
+   + implement a simple (dandik) version for better reliability
 
  * medium priority
  
@@ -197,10 +198,11 @@ Legend:
    - date type
    - enum type
    - make autoxml dependent on only xmlext and xmlfile interfaces
- ? non-interactive use
 
 5. Post Release
 
+ * high priority
+
  - high-level build commands
    - build command enhancements:
      - aware of repo sources
@@ -215,9 +217,10 @@ Legend:
    - sourceb
  - improve url support: consider adding "other" protocols
    - removable media: media://
-       
+
  * medium priority
 
+ - non-interactive use
  - multi-architecture support (baris, caglar)
    / design decisions
    / extend XML specs to support that

pisi-cli

@@ -23,7 +23,6 @@ import gettext
 __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.ugettext
 
-
 def exit():
     pisi.api.finalize()
     sys.exit(1)

pisi/__init__.py

@@ -12,9 +12,9 @@
 
 # PiSi version
 
-__version__ = "1.0_beta3"
+__version__ = "1.0_beta4"
 
-__all__ = [ 'api', 'packagedb', 'installdb' ]
+__all__ = [ 'api', 'config', 'packagedb', 'installdb' ]
 
 class Exception(Exception):
     """Class of exceptions that must be caught and handled within PISI"""

pisi/api.py

@@ -47,6 +47,7 @@ from pisi.atomicoperations import resurrect_package
 from pisi.metadata import MetaData
 from pisi.files import Files
 import pisi.search
+import pisi.lockeddbshelve as shelve
 
 class Error(pisi.Error):
     pass
@@ -70,6 +71,7 @@ def init(database = True, options = None, ui = None, comar = True):
 
     # initialize repository databases
     if database:
+        shelve.init_dbenv()
         ctx.repodb = pisi.repodb.init()
         ctx.installdb = pisi.installdb.init()
         ctx.filesdb = pisi.files.FilesDB()
@@ -96,6 +98,8 @@ def finalize():
         packagedb.finalize_db()
         pisi.sourcedb.finalize()
         pisi.search.finalize()
+        if ctx.dbenv:
+            ctx.dbenv.close()
         ctx.ui.debug('PISI API finalized')
         ctx.ui.close()
         ctx.initialized = False

pisi/cli/commands.py

@@ -628,9 +628,9 @@ Usage: list-installed
                 ctx.ui.info(unicode(package))
                 ctx.ui.info(unicode(inst_info))
             elif self.options.install_info:
-                ctx.ui.info('%-15s | %s ' % (package.name, inst_info.one_liner()))
+                ctx.ui.info('%-15s  |%s' % (package.name, inst_info.one_liner()))
             else:
-                ctx.ui.info('%15s - %s ' % (package.name, package.summary))
+                ctx.ui.info('%15s - %s' % (package.name, package.summary))
         self.finalize()
 
 class RebuildDb(Command):

pisi/component.py

@@ -97,7 +97,7 @@ class ComponentDB(object):
 
     def list_components(self):
         list = []
-        for (pkg, x) in self.d.iteritems():
+        for (pkg, x) in self.d.items():
             list.append(pkg)
         return list
 

pisi/dbshelve.py

+#!/bin/env python
+#------------------------------------------------------------------------
+#           Copyright (c) 1997-2001 by Total Control Software
+#                         All Rights Reserved
+#------------------------------------------------------------------------
+#
+# Module Name:  dbShelve.py
+#
+# Description:  A reimplementation of the standard shelve.py that
+#               forces the use of cPickle, and DB.
+#
+# Creation Date:    11/3/97 3:39:04PM
+#
+# License:      This is free software.  You may use this software for any
+#               purpose including modification/redistribution, so long as
+#               this header remains intact and that you do not claim any
+#               rights of ownership or authorship of this software.  This
+#               software has been tested, but no warranty is expressed or
+#               implied.
+#
+# 13-Dec-2000:  Updated to be used with the new bsddb3 package.
+#               Added DBShelfCursor class.
+#
+#------------------------------------------------------------------------
+
+"""Manage shelves of pickled objects using bsddb database files for the
+storage.
+"""
+
+#------------------------------------------------------------------------
+
+import cPickle
+import bsddb3.db as db
+import bsddb3.dbobj as dbobj
+
+class DBShelf:
+    """A shelf to hold pickled objects, built upon a bsddb DB object.  It
+    automatically pickles/unpickles data objects going to/from the DB.
+    """
+    def __init__(self, dbenv = None):
+        self.dbenv = dbenv
+        self.db = dbobj.DB(dbenv)
+        self.binary = 1
+
+    # it is better to explicitly close a shelf
+    #def __del__(self):
+    #    self.close()
+
+    def has_key(self, key):
+        return self.db.has_key(key)
+        
+    def clear(self, txn = None):
+        if not txn:
+            txn = self.dbenv.txn_begin()
+            try:
+                for x in self.keys(txn):
+                    del self[x]
+            except db.DBError, e:
+                txn.abort()
+                raise e
+            txn.commit()
+        else:
+            for x in self.keys(txn):
+                del self[x]
+        
+    #def delete(self, txn = None):
+    #    self.db.delete(self, txn
+
+    # another lame pythonic implementation method:
+    #def __getattr__(self, name):
+    #    """Many methods we can just pass through to the DB object.
+    #    (See below)
+    #    """
+    #    print 'aptal bsddb3', name
+    #    return getattr(self.db, name)
+
+    #-----------------------------------
+    # Dictionary access methods
+
+    def __len__(self):
+        return len(self.db)
+        
+    def __getitem__(self, key):
+        txn = self.dbenv.txn_begin()
+        try:
+            data = self.db.get(key)
+        except db.DBError, e:
+            txn.abort()
+            raise e
+        txn.commit()
+        return cPickle.loads(data)
+
+    def __setitem__(self, key, value):
+        # hyperdandik transactions
+        txn = self.dbenv.txn_begin()
+        data = cPickle.dumps(value, self.binary)
+        try:
+            self.db.put(key,data,txn)
+            #print 'put', value
+        except db.DBError, e:
+            txn.abort()
+            raise e
+        txn.commit()
+
+    def __delitem__(self, key):
+        txn = self.dbenv.txn_begin()
+        try:
+            self.db.delete(key, txn)
+        except db.DBError, e:
+            txn.abort()
+            raise e
+        txn.commit()
+
+    def keys(self, txn=None):
+        if txn != None:
+            return self.db.keys(txn)
+        else:
+            return self.db.keys()
+
+    def items(self, txn=None):
+        if txn != None:
+            items = self.db.items(txn)
+        else:
+            items = self.db.items()
+        newitems = []
+
+        for k, v in items:
+            newitems.append( (k, cPickle.loads(v)) )
+        return newitems
+
+    def values(self, txn=None):
+        if txn != None:
+            values = self.db.values(txn)
+        else:
+            values = self.db.values()
+
+        return map(cPickle.loads, values)
+
+    #-----------------------------------
+    # Other methods
+
+    def __append(self, value, txn=None):
+        data = cPickle.dumps(value, self.binary)
+        return self.db.append(data, txn)
+
+    def append(self, value, txn=None):
+        if self.get_type() != db.DB_RECNO:
+            self.append = self.__append
+            return self.append(value, txn=txn)
+        raise db.DBError, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO"
+
+
+    def associate(self, secondaryDB, callback, flags=0):
+        def _shelf_callback(priKey, priData, realCallback=callback):
+            data = cPickle.loads(priData)
+            return realCallback(priKey, data)
+        return self.db.associate(secondaryDB, _shelf_callback, flags)
+
+
+    #def get(self, key, default=None, txn=None, flags=0):
+    def get(self, *args, **kw):
+        # We do it with *args and **kw so if the default value wasn't
+        # given nothing is passed to the extension module.  That way
+        # an exception can be raised if set_get_returns_none is turned
+        # off.
+        data = apply(self.db.get, args, kw)
+        try:
+            return cPickle.loads(data)
+        except (TypeError, cPickle.UnpicklingError):
+            return data  # we may be getting the default value, or None,
+                         # so it doesn't need unpickled.
+
+    def get_both(self, key, value, txn=None, flags=0):
+        data = cPickle.dumps(value, self.binary)
+        data = self.db.get(key, data, txn, flags)
+        return cPickle.loads(data)
+
+    def cursor(self, txn=None, flags=0):
+        c = DBShelfCursor(self.db.cursor(txn, flags))
+        c.binary = self.binary
+        return c
+
+    def put(self, key, value, txn=None, flags=0):
+        data = cPickle.dumps(value, self.binary)
+        return self.db.put(key, data, txn, flags)
+
+    def join(self, cursorList, flags=0):
+        raise NotImplementedError
+
+    #----------------------------------------------
+    # Methods allowed to pass-through to self.db
+    #
+    #    close,  delete, fd, get_byteswapped, get_type, has_key,
+    #    key_range, open, remove, rename, stat, sync,
+    #    upgrade, verify, and all set_* methods.
+
+
+#---------------------------------------------------------------------------
+
+class DBShelfCursor:
+    """
+    """
+    def __init__(self, cursor):
+        self.dbc = cursor
+
+    def __del__(self):
+        self.close()
+
+    def __getattr__(self, name):
+        """Some methods we can just pass through to the cursor object.  (See below)"""
+        return getattr(self.dbc, name)
+
+    #----------------------------------------------
+
+    def dup(self, flags=0):
+        return DBShelfCursor(self.dbc.dup(flags))
+
+    def put(self, key, value, flags=0):
+        data = cPickle.dumps(value, self.binary)
+        return self.dbc.put(key, data, flags)
+
+
+    def get(self, *args):
+        count = len(args)  # a method overloading hack
+        method = getattr(self, 'get_%d' % count)
+        apply(method, args)
+
+    def get_1(self, flags):
+        rec = self.dbc.get(flags)
+        return self._extract(rec)
+
+    def get_2(self, key, flags):
+        rec = self.dbc.get(key, flags)
+        return self._extract(rec)
+
+    def get_3(self, key, value, flags):
+        data = cPickle.dumps(value, self.binary)
+        rec = self.dbc.get(key, flags)
+        return self._extract(rec)
+
+
+    def current(self, flags=0): return self.get_1(flags|db.DB_CURRENT)
+    def first(self, flags=0): return self.get_1(flags|db.DB_FIRST)
+    def last(self, flags=0): return self.get_1(flags|db.DB_LAST)
+    def next(self, flags=0): return self.get_1(flags|db.DB_NEXT)
+    def prev(self, flags=0): return self.get_1(flags|db.DB_PREV)
+    def consume(self, flags=0): return self.get_1(flags|db.DB_CONSUME)
+    def next_dup(self, flags=0): return self.get_1(flags|db.DB_NEXT_DUP)
+    def next_nodup(self, flags=0): return self.get_1(flags|db.DB_NEXT_NODUP)
+    def prev_nodup(self, flags=0): return self.get_1(flags|db.DB_PREV_NODUP)
+
+    def get_both(self, key, value, flags=0):
+        data = cPickle.dumps(value, self.binary)
+        rec = self.dbc.get_both(key, flags)
+        return self._extract(rec)
+
+    def set(self, key, flags=0):
+        rec = self.dbc.set(key, flags)
+        return self._extract(rec)
+
+    def set_range(self, key, flags=0):
+        rec = self.dbc.set_range(key, flags)
+        return self._extract(rec)
+
+    def set_recno(self, recno, flags=0):
+        rec = self.dbc.set_recno(recno, flags)
+        return self._extract(rec)
+
+    set_both = get_both
+
+    def _extract(self, rec):
+        if rec is None:
+            return None
+        else:
+            key, data = rec
+            return key, cPickle.loads(data)
+
+    #----------------------------------------------
+    # Methods allowed to pass-through to self.dbc
+    #
+    # close, count, delete, get_recno, join_item
+
+
+#---------------------------------------------------------------------------

pisi/files.py

@@ -56,7 +56,7 @@ class FilesDB(shelve.LockedDBShelf):
 
     def remove_files(self, files):
         for x in files.list:
-            self.delete(str(x.path))
+            del self[str(x.path)]
 
     def has_file(self, path):
         return self.has_key(str(path))

pisi/installdb.py

@@ -110,14 +110,14 @@ class InstallDB:
 
     def list_installed(self):
         list = []
-        for (pkg, info) in self.d.iteritems():
+        for (pkg, info) in self.d.items():
             if info.state=='i' or info.state=='ip':
                 list.append(pkg)
         return list
 
     def list_pending(self):
         dict = {}
-        for (pkg, x) in self.dp.iteritems():
+        for (pkg, x) in self.dp.items():
             pkginfo = self.d[pkg]
             dict[pkg] = pkginfo
         return dict

pisi/lockeddb.py

+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2005, TUBITAK/UEKAE
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# Please read the COPYING file.
+#
+#
+# Authors:  Eray Ozkural <eray@uludag.org.tr>
+
+import bsddb3.db as db
+import os
+import fcntl
+import types
+
+import gettext
+__trans = gettext.translation('pisi', fallback=True)
+_ = __trans.ugettext
+
+import pisi
+import pisi.context
+
+class Error(pisi.Error):
+    pass
+
+class LockedDB:
+
+    def __init__(self, dbname, mode=0644,
+                 filetype=db.DB_BTREE, dbenv=None):
+        if not dbenv:
+            dbenv = ctx.dbenv
+        filename = os.path.join( pisi.context.config.db_dir(), dbname + '.bdb')
+        if os.access(os.path.dirname(filename), os.W_OK):
+            flags = 'w'
+        elif os.access(filename, os.R_OK):
+            flags = 'r'
+        else:
+            raise Error(_('Cannot attain read or write access to database %s') % dbname)
+        self.open(filename, dbname, filetype, flags, mode)
+
+    def __del__(self):
+        # superclass does something funky, we don't need that
+        pass
+        
+    def open(self, filename, dbname, filetype, flags=db.DB_CREATE, mode=0644):
+        self.filename = filename        
+        self.closed = False
+        #print 'open', filename
+        if type(flags) == type(''):
+            sflag = flags
+            if sflag == 'r':
+                flags = db.DB_RDONLY
+            elif sflag == 'rw':
+                flags = 0
+            elif sflag == 'w':
+                flags =  db.DB_CREATE
+            elif sflag == 'c':
+                flags =  db.DB_CREATE
+            elif sflag == 'n':
+                flags = db.DB_TRUNCATE | db.DB_CREATE
+            else:
+                raise Error, _("Flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags")
+        self.flags = flags
+        pisi.util.check_dir(pisi.context.config.db_dir())
+        if self.flags != db.DB_RDONLY:
+            self.lock()
+        return self.db.open(filename, dbname, filetype, flags, mode)
+
+    def lock(self):
+        self.lockfile = file(self.filename + '.lock', 'w')
+        try:
+            fcntl.flock(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except IOError:
+            raise Error(_("Another instance of PISI is running. Try later!"))
+
+    def close(self):
+        if self.closed:
+            return
+        self.db.close()
+        if self.flags != db.DB_RDONLY:
+            self.unlock()
+        self.closed = True
+
+    def unlock(self):
+        self.lockfile.close()
+        os.unlink(self.filename + '.lock')
+
+    @staticmethod
+    def encodekey(key):
+        '''utility method for dbs that must store unicodes in keys'''
+        if type(key)==types.UnicodeType:
+            return key.encode('utf-8')
+        elif type(key)==types.StringType:
+            return key
+        else:
+            raise Error('Key must be either string or unicode')

pisi/lockeddbshelve.py

@@ -12,8 +12,10 @@
 #
 # Authors:  Eray Ozkural <eray@uludag.org.tr>
 
-import bsddb3.dbshelve as shelve
 import bsddb3.db as db
+import bsddb3.dbobj as dbobj
+#import bsddb3.dbshelve as shelve
+import pisi.dbshelve as shelve
 import os
 import fcntl
 import types
@@ -23,18 +25,33 @@ __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.ugettext
 
 import pisi
-import pisi.context
+import pisi.context as ctx
+from pisi.util import join_path
+
 
 class Error(pisi.Error):
     pass
 
+
+def init_dbenv():
+    ctx.dbenv = dbobj.DBEnv()
+    ctx.dbenv.open(pisi.context.config.db_dir(),
+                   db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_TXN | db.DB_INIT_LOG )
+
+#def open(filename, flags='r', mode = 0644, filetype = db.DB_BTREE):
+#    db = LockedDBShelf(None, mode, filetype, None, True)
+#    db.open(filename, filename, filetype, flags, mode)
+#    return db
+
 class LockedDBShelf(shelve.DBShelf):
     """A simple wrapper to implement locking for bsddb's dbshelf"""
 
     def __init__(self, dbname, mode=0644,
-                 filetype=db.DB_BTREE, dbenv=None):
+                 filetype=db.DB_BTREE, dbenv = None):
+        if dbenv == None:
+            dbenv = ctx.dbenv
         shelve.DBShelf.__init__(self, dbenv)
-        filename = os.path.join( pisi.context.config.db_dir(), dbname + '.bdb')
+        filename = join_path(pisi.context.config.db_dir(), dbname + '.bdb')
         if os.access(os.path.dirname(filename), os.W_OK):
             flags = 'w'
         elif os.access(filename, os.R_OK):
@@ -46,11 +63,10 @@ class LockedDBShelf(shelve.DBShelf):
     def __del__(self):
         # superclass does something funky, we don't need that
         pass
-        
+
     def open(self, filename, dbname, filetype, flags=db.DB_CREATE, mode=0644):
         self.filename = filename        
         self.closed = False
-        #print 'open', filename
         if type(flags) == type(''):
             sflag = flags
             if sflag == 'r':
@@ -65,24 +81,26 @@ class LockedDBShelf(shelve.DBShelf):
                 flags = db.DB_TRUNCATE | db.DB_CREATE
             else:
                 raise Error, _("Flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags")
+        flags |= db.DB_AUTO_COMMIT
         self.flags = flags
-        pisi.util.check_dir(pisi.context.config.db_dir())
-        if self.flags != db.DB_RDONLY:
+        if self.flags & db.DB_RDONLY == 0:
             self.lock()
-        return self.db.open(filename, dbname, filetype, flags, mode)
+        filename = os.path.realpath(filename) # we give absolute path due to dbenv
+        #print 'opening', filename, filetype, flags, mode
+        return self.db.open(filename, None, filetype, flags, mode)
 
     def lock(self):
         self.lockfile = file(self.filename + '.lock', 'w')
         try:
             fcntl.flock(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
         except IOError:
-            raise Error(_("Another instance of PISI is running. Try later!"))
+            raise Error(_("Another instance of PISI is running. Only one instance is allowed to modify the PISI database at a time."))
 
     def close(self):
         if self.closed:
             return
         self.db.close()
-        if self.flags != db.DB_RDONLY:
+        if self.flags & db.DB_RDONLY == 0:
             self.unlock()
         self.closed = True
 

pisi/packagedb.py

@@ -61,7 +61,7 @@ class PackageDB(object):
 
     def list_packages(self):
         list = []
-        for (pkg, x) in self.d.iteritems():
+        for (pkg, x) in self.d.items():
             list.append(pkg)
         return list
 

tests/installdbtests.py

@@ -26,6 +26,8 @@ class InstallDBTestCase(testcase.TestCase):
     def testInstall(self):
         ctx.installdb.purge('installtest')
         ctx.installdb.install('installtest', '0.1', '2', '3')
+        self.assert_(ctx.installdb.is_recorded('installtest'))
+        self.assert_(ctx.installdb.is_installed('installtest'))
 
     def testRemovePurge(self):
         ctx.installdb.install('installtest', '0.1', '2', '3')

tools/cat-db.py

@@ -13,14 +13,15 @@
 
 import sys
 import bsddb3.dbshelve as shelve
+import bsddb3.db as db
 
 sys.path.append('.')
 
 import pisi
 
-d = shelve.open( sys.argv[1], flags='r' )
+d = shelve.open( sys.argv[1], 'r', 0660, filetype = db.DB_BTREE )
 
-for (k, data) in d.iteritems():
-    print k, data
+for key, data in d.items():
+    print key, data
 
 d.close()