Minimum dependency maximum stabilization

ChangeLog

-2018-01-11 Suleyman Poyraz <nipalensisaquila@gmail.com>
+2018-03-11 Suleyman Poyraz <nipalensisaquila@gmail.com>
+    * colorized help output:
+		Colorized command names and now help screen look well.
+		* Minimum dependency, Maximum stabilization:
+		patchfile:
+		patches/applied-patches/minimum-dependency-maximum-stabilization-001.patch
+		If we had one or two dependency we can control inary
+		but there are a lot of runtime dependency. And this is
+		look up bad. I clean runtime dependencies and I write 
+		alternative libraries into inary. I clean runtimes:
+		    * magic: it soo easy if when use ctypes.I import libmagic
+				with ctypes and I use it in inary
+				* plyvel: for basic files database, we musn't a big
+				database library. Shelve is pretty than "Google's"
+				plyvel database library.
+				* ciksemel: I work about this.
+
+
+2018-03-10 Suleyman Poyraz <nipalensisaquila@gmail.com>
+    * Cleaned ınary/api.py:
+		Clean unuseful commands, Moved functions true palaces and
+		writed 2 new module.
+		* Not worked patch delete-runtime-dependency-ciksemel:
+		Not work well minidom. So I returned codes to ciksemel.
+		Should work with minidom detailed.
+
+2018-03-09 Suleyman Poyraz <nipalensisaquila@gmail.com>
+    * Patched delete-runtime-dependency-ciksemel:
+		For compability we shoul use minidom in xml file parsing,
+		we shouldn't use a new runtime dependency. (Please Read CODING)
+
+2018-03-04 Suleyman Poyraz <nipalensisaquila@gmail.com>
+    * Updated RoadMap
+		* Cleaned FIXME tags:
+		Cleaned not understandable Fixme tags
+
+
+2018-03-02 Suleyman Poyraz <nipalensisaquila@gmail.com>
+    * deleted distupdate command:
+		Sulin will be a roolling release distro. So we don't have a 
+		distupdater.
+
+2018-02-11 Suleyman Poyraz <nipalensisaquila@gmail.com>
   * distupdate komutu:
 	Stabilize edildi ve renklendirildi
 	* string -> strutils:
@@ -10,7 +52,7 @@
 	çözüm bulunamadı :(
 
 
-2018-01-07 Suleyman Poyraz <nipalensisaquila@gmail.com>
+2018-02-07 Suleyman Poyraz <nipalensisaquila@gmail.com>
   * System modulleri silindi:
 	Inary içinde onlara ihtiyaç olmadığını farkettim. Scom içine alarak yer
 	kazandım.
@@ -18,7 +60,7 @@
 	Str veri tipi için basit tipte fonksiyonları ve multisplit gibi şeyleri
 	içeren bir modul.
 
-2018-01-04 Suleyman Poyraz <nipalensisaquila@gmail.com>
+2018-02-04 Suleyman Poyraz <nipalensisaquila@gmail.com>
 	* Kozmetik fixler ve düzenlemeler yapıldı(developing branch)
 	* Dokumantasyon taslakları yazıldı.(developing branch)
 	* Merge Developing branch into master on 2018-01-07

inary/analyzer/firmwares.py

@@ -21,11 +21,6 @@ import gettext
 __trans = gettext.translation('inary', fallback=True)
 _ = __trans.gettext
 
-
-FW_PATH = "/lib/firmware"
-INSTALDB = inary.db.installdb.InstallDB()
-COMPONENTDB = inary.db.componentdb.ComponentDB()
-
 class Error(inary.Error):
     pass
 
@@ -46,8 +41,8 @@ def get_firmwares():
 
 def get_firmware_package(firmware):
     try:
-        fw_packages = COMPONENTDB.get_packages("hardware.firmware")
-        unavailable_fw_packages = set(fw_packages).difference(INSTALLDB.list_installed())
+        fw_packages = inary.db.componentdb.ComponentDB().get_packages("hardware.firmware")
+        unavailable_fw_packages = set(fw_packages).difference(inary.db.installdb.InstallDB().list_installed())
         
         if unavailable_fw_packages:
             ctx.ui.info(inary.util.colorize("The following firmwares are not installed:", "yellow"))

inary/analyzer/forensic.py

@@ -78,10 +78,10 @@ def find_corrupted(rootdir, last_changed):
             filepath = '/' + filepath
             if not filepath.startswith(rootdir):
                 continue
-            sys.stdout.write("MISSING %s %s\n" % (package, filepath))
+            sys.stdout.write("MISSING {0} {1}\n".format(package, filepath))
             sys.stdout.flush()
 
-def forensics(rootdir='/',logfile=logfile):
+def forensics(rootdir='/',logfile='logfile'):
     if not rootdir.endswith('/'):
         rootdir += '/'
 

inary/analyzer/magic.py

+
+
+import sys
+import glob
+import os
+import threading
+
+import ctypes
+from ctypes import c_char_p, c_int, c_size_t, c_void_p
+import ctypes.util
+
+libmagic = None
+dll = ctypes.util.find_library('magic') or ctypes.util.find_library('magic1') or ctypes.util.find_library('cygmagic-1')
+
+if dll:
+    libmagic = ctypes.CDLL(dll)
+
+if not libmagic or not libmagic._name:
+    raise ImportError('failed to find libmagic.  Check your installation')
+
+#Magic Flags from libmagic.so
+
+MAGIC_CONTINUE = 0x000020 
+MAGIC_COMPRESS = 0x000004
+MAGIC_NONE = 0x000000 
+MAGIC_MIME = 0x000010
+MAGIC_MIME_ENCODING = 0x000400 
+
+_instances = {}
+
+# Magic function
+magic_t = ctypes.c_void_p
+
+#Error Checking Function
+def errorcheck(result, func, args):
+    if result is None:
+        err = magic_error(args[0])
+        raise Exception(err)
+    elif result is -1:
+        err = magic_error(args[0])
+        raise Exception(err)
+    else:
+        return result
+
+
+magic_file = libmagic.magic_file
+magic_file.restype = c_char_p
+magic_file.argtypes = [magic_t, c_char_p]
+magic_file.errcheck = errorcheck
+
+magic_open = libmagic.magic_open
+magic_open.restype = magic_t
+magic_open.argtypes = [c_int]
+
+magic_close = libmagic.magic_close
+magic_close.restype = None
+magic_close.argtypes = [magic_t]
+
+magic_buffer = libmagic.magic_buffer
+magic_buffer.restype = c_char_p
+magic_buffer.argtypes = [magic_t, c_void_p, c_size_t]
+magic_buffer.errcheck = errorcheck
+
+magic_load = libmagic.magic_load
+magic_load.restype = c_int
+magic_load.argtypes = [magic_t, c_char_p]
+magic_load.errcheck = errorcheck
+
+#Standart Functions
+def _get_wrapper(mime):
+    i = _instances.get(mime)
+    if i is None:
+        i = _instances[mime] = Magic(mime=mime)
+    return i
+
+def file_type(data, mime=False):
+    m = _get_wrapper(mime)
+    return m.get_file_type(data)
+
+# Magic Class
+class Magic:
+
+    def __init__(self, mime=False, magic_file=None, mime_encoding=False,
+                 keep_going=False, uncompress=False):
+
+        self.flags = MAGIC_NONE
+        if mime:
+            self.flags |= MAGIC_MIME
+        if mime_encoding:
+            self.flags |= MAGIC_MIME_ENCODING
+        if keep_going:
+            self.flags |= MAGIC_CONTINUE
+
+        if uncompress:
+            self.flags |= MAGIC_COMPRESS
+
+        self.cookie = magic_open(self.flags)
+        self.lock = threading.Lock()
+
+        magic_load(self.cookie, magic_file)
+
+    def get_file_type(self, data):
+        # If given argument is a file load with magic_file 
+        # If given argument is a buffer load with magic_buffer
+        try:
+            if os.path.isfile(data): 
+                open(data)
+                with self.lock:
+                    return magic_file(self.cookie, data)
+            else: 
+                with self.lock:
+                    if type(data) == str and str != bytes:
+                        buf = data.encode('utf-8', errors='replace')
+                    return magic_buffer(self.cookie, data, len(data))
+        except Exception as err:
+            raise(_("Can Not load file or buffer {}").format(err))
+
+    def __del__(self):
+        if self.cookie and magic_close:
+            magic_close(self.cookie)
+            self.cookie = None
+

inary/analyzer/pkgconfig.py

@@ -15,7 +15,6 @@ import os
 import re
 import sys
 import glob
-import magic
 import shutil
 import fnmatch
 import tempfile
@@ -25,6 +24,8 @@ import subprocess
 
 #Inary functions
 import inary
+import inary.db
+import inary.analyzer.magic
 import inary.context as ctx
 
 #Gettext
@@ -32,11 +33,6 @@ import gettext
 __trans = gettext.translation('inary', fallback=True)
 _ = __trans.gettext
 
-INSTALLDB = inary.db.installdb.InstallDB()
-COMPONENTDB = inary.db.componentdb.ComponentDB()
-CONSTANTS = inary.constants.Constants()
-REPODB = inary.db.repodb.RepoDB()
-FILESDB = inary.db.filesdb.FilesDB()
 
 class Error(inary.Error):
     pass
@@ -44,6 +40,10 @@ class Error(inary.Error):
 class LDD:
     def __init__(self, packages, directory, component, installed_list=True, systembase=False, systemdevel=False):
 
+        self.installdb = inary.db.installdb.InstallDB()
+        self.componentdb = inary.db.componentdb.ComponentDB()
+        self.repodb = inary.db.repodb.RepoDB()
+        self.filesdb = inary.db.filesdb.FilesDB()
         self.systembase = systembase
         self.systemdevel = systemdevel
         self.packages = package
@@ -59,12 +59,12 @@ class LDD:
         # check for components, like system.base, tex.language, etc.
         if self.component:
             for repo in RepoDB.list_repos():
-                if COMPONENTDB.has_component(component):
-                    self.packages.extend(COMPONENTDB.get_packages(component, repo))
+                if self.componentdb.has_component(component):
+                    self.packages.extend(self.componentdb.get_packages(component, repo))
 
         # check for all packages installed on the machine
         if installedlist:
-            self.packages.extend(INSTALLDB.list_installed())
+            self.packages.extend(self.installdb.list_installed())
 
     def find_dependencies_on_pkgconfig(self):
         pkgconfig_list = []
@@ -100,7 +100,7 @@ class LDD:
                 pkgconfig_list.append((result_broken, result_unused, result_undefined, result_lists, result_runpath, package_name))
 
             # Check for a installed package in the system
-            elif package in INSTALLDB.list_installed():
+            elif package in self.installdb.list_installed():
                 if used_inary:
                     raise Error("You've checked for a inary file before\nPlease do not check for a installed package and inary file at the same time")
 
@@ -108,7 +108,7 @@ class LDD:
                     package_name = package
 
                     # Gather runtime dependencies directly from the database of installed packages
-                    package_deps = [dep.name() for dep in INSTALLDB.get_package(package).runtimeDependencies()]
+                    package_deps = [dep.name() for dep in self.installdb.get_package(package).runtimeDependencies()]
                     package_tempdir = False # There is no need of temporary directory, hence we look for files that are installed
 
                     # Same functions in the above part. You can read them
@@ -216,12 +216,12 @@ class LDD:
         # get system.base and system.devel packages
         systembase_packages = []
         systemdevel_packages= []
-        for repo in REPODB.list_repos():
-            for component in COMPONENTDB.list_components(repo):
+        for repo in self.repodb.list_repos():
+            for component in self.componentdb.list_components(repo):
                 if component == "system.base":
-                    systembase_packages.extend(COMPONENTDB.get_packages('system.base', repo))
+                    systembase_packages.extend(self.componentdb.get_packages('system.base', repo))
                 if component == "system.devel":
-                    systemdevel_packages.extend(COMPONENTDB.get_packages('system.devel', repo))
+                    systemdevel_packages.extend(self.componentdb.get_packages('system.devel', repo))
 
         # look for packages that are system.base but are written as dependency
         # mark them with "*"
@@ -293,13 +293,13 @@ class LDD:
             package_pc_files = glob.glob("{}/usr/*/pkgconfig/*.pc".format(package_dir))
         else:
             package_files = set(["/{}".format(file_name.path) \
-                for file_name in INSTALLDB.get_files(package_name).list])
+                for file_name in self.installdb.get_files(package_name).list])
             package_pc_files = set([os.path.realpath("/{}".format(file_name.path)) \
-                    for file_name in INSTALLDB.get_files(package_name).list \
+                    for file_name in self.installdb.get_files(package_name).list \
                     if fnmatch.fnmatch(file_name.path, "*/pkgconfig/*.pc")])
 
         for package_file in package_files:
-            package_file_info = magic.from_file(package_file) #Return file type
+            package_file_info = inary.analyzer.magic.file_type(package_file) #Return file type
             if "LSB shared object" in package_file_info:
                 package_elf_files.append(os.path.realpath(package_file))
             elif "LSB executable" in package_file_info:

inary/api.py

@@ -16,16 +16,25 @@ __trans = gettext.translation('inary', fallback=True)
 _ = __trans.gettext
 
 import inary
+
+#Analyzers
+import inary.analyzer.ctypes_analyzer
+import inary.analyzer.forensic
+import inary.analyzer.pkgconfig
+
 import inary.atomicoperations
 import inary.blacklist
 import inary.config
 import inary.context as ctx
+
+#DataFile Libraries
 import inary.data
 import inary.data.index
 import inary.data.metadata
 import inary.data.pgraph
+
+#DataBase Libraries
 import inary.db.componentdb
-import inary.db.dbhelper
 import inary.db.packagedb
 import inary.db.repodb
 import inary.db.filesdb
@@ -34,8 +43,11 @@ import inary.db.historydb
 import inary.db.sourcedb
 import inary.db.componentdb
 import inary.db.groupdb
+
 import inary.errors
 import inary.file
+
+#Operation Libraries
 import inary.operations.build
 import inary.operations.check
 import inary.operations.emerge
@@ -46,6 +58,7 @@ import inary.operations.helper
 import inary.operations.remove
 import inary.operations.search
 import inary.operations.upgrade
+
 import inary.uri
 import inary.util
 
@@ -128,20 +141,22 @@ def set_options(options):
     ctx.config.set_options(options)
 
 # The following are INARY operations which constitute the INARY API
-from inary.operations.operations import *
+
+#It looking soo bad
+from inary.atomicoperations import *
 
 #Within functions
 from inary.analyzer.conflict import calculate_conflicts
-#from inary.analyzer.firmwares import get_firmware_package
+from inary.analyzer.firmwares import get_firmware_package
 from inary.data.index import index
 from inary.data.pgraph import package_graph
 from inary.fetcher import fetch
-from inary.db.dbhelper import *
 from inary.operations.build import build, build_until
 from inary.operations.helper import calculate_download_sizes, get_package_requirements
 from inary.operations.history import get_takeback_plan
 from inary.operations.info import info
 from inary.operations.install import get_install_order
+from inary.operations.op_wrappers import *
 from inary.operations.remove import get_remove_order
 from inary.operations.upgrade import get_upgrade_order, get_base_upgrade_order
 from inary.operations.search import *

inary/archive.py

@@ -304,6 +304,7 @@ class ArchiveTar(ArchiveBase):
         except OSError:
             pass
 
+        ctx.ui.debug(_("* Target DIR: {}").format(target_dir))
         os.chdir(target_dir)
 
         for tarinfo in self.tar:
@@ -455,8 +456,10 @@ class ArchiveTar(ArchiveBase):
 
 
                 if not os.path.islink(tarinfo.name):
+                    ctx.ui.debug(_("* Chowning {0} ({1}:{2})").format(tarinfo.name, uid, gid))
                     os.chown(tarinfo.name, uid, gid)
                 else:
+                    ctx.ui.debug(_("* LChowning {0} ({1}:{2})").format(tarinfo.name, uid, gid))
                     os.lchown(tarinfo.name, uid, gid)
 
             if callback:

inary/cli/__init__.py

@@ -163,7 +163,7 @@ class CLI(inary.ui.UI):
 
         while True:
             tty.tcflush(sys.stdin.fileno(), 0)
-            prompt = msg + inary.util.colorize(_(' (yes/no)'), 'red')
+            prompt = msg + inary.util.colorize(_(' (yes'), 'green') + '/' + inary.util.colorize(_('no)'), 'red')
             s = input(prompt)
 
             if yes_expr.search(s):

inary/cli/command.py

@@ -167,7 +167,7 @@ class Command(object):
             ('sf' in self.get_name() and not os.access(os.path.join(ctx.config.info_dir(), ctx.const.files_ldb), os.W_OK))):
             raise inary.cli.Error(_("You have to be root for this operation."))
 
-        inary.api.set_scom(self.scom and not ctx.get_option('ignore_comar'))
+        inary.api.set_scom(self.scom and not ctx.get_option('ignore_scom'))
         inary.api.set_userinterface(ui)
         inary.api.set_options(self.options)
 

inary/db/filesdb.py

@@ -46,10 +46,8 @@ class FilesDB(lazydb.LazyDB):
         ctx.ui.info(inary.util.colorize(_('Creating files database...'), 'blue'))
         installdb = inary.db.installdb.InstallDB()
         for pkg in installdb.list_installed():
-            ctx.ui.info(inary.util.colorize(_('  |___* Adding \'{}\' to db... '), 'purple').format(pkg), noln=True)
             files = installdb.get_files(pkg)
             self.add_files(pkg, files)
-            ctx.ui.info(inary.util.colorize(_('OK.'), 'backgroundmagenta'))
         ctx.ui.info(inary.util.colorize(_('Added files database...'), 'blue'))
 
     def get_file(self, path):
@@ -71,12 +69,12 @@ class FilesDB(lazydb.LazyDB):
         return found
 
     def add_files(self, pkg, files):
-
         self.__check_filesdb()
-        ctx.ui.info(inary.util.colorize(_('* Adding \'{}\' to db... '), 'purple').format(pkg), noln=True)
+        ctx.ui.info(inary.util.colorize(_('  |___* Adding \'{}\' to db... '), 'purple').format(pkg), noln=True)
         for f in files.list:
             key=hashlib.md5(f.path.encode('utf-8')).hexdigest()
             self.filesdb[key] = pkg
+        ctx.ui.info(inary.util.colorize(_('OK.'), 'backgroundmagenta'))
 
     def remove_files(self, files):
         for f in files:
@@ -94,7 +92,6 @@ class FilesDB(lazydb.LazyDB):
             self.filesdb.close()
 
     def __check_filesdb(self):
-        ctx.ui.info(_("Checking filesdb..."))
         if isinstance(self.filesdb, shelve.DbfilenameShelf):
             return
 

inary/operations/build.py

@@ -1001,12 +1001,12 @@ class Builder:
     def file_actions(self):
         install_dir = self.pkg_install_dir()
 
-        import magic
+        import inary.analyzer.magic as magic
 
         for root, dirs, files in os.walk(install_dir):
             for fn in files:
                 filepath = util.join_path(root, fn)
-                fileinfo = magic.from_file(filepath)
+                fileinfo = magic.file_type(filepath)
                 strip_debug_action(filepath, fileinfo, install_dir, self.actionGlobals)
                 exclude_special_files(filepath, fileinfo, self.actionGlobals)
 

inary/operations/info.py

@@ -14,6 +14,10 @@ import inary
 import inary.db
 import inary.data
 
+import gettext
+__trans = gettext.translation('inary', fallback=True)
+_ = __trans.gettext
+
 def info(package, installed = False):
     if package.endswith(ctx.const.package_suffix):
         return info_file(package)

inary/scomiface.py

@@ -27,6 +27,7 @@ class Error(inary.Error):
 try:
     import scom
     import dbus
+
 except ImportError:
     raise Error(_("scom-api package is not fully installed"))