refactoring, autocategory, check upload failed

impra/cli.py

@@ -83,9 +83,8 @@ class Cli:
 \n\
 %prog conf [-D|-L| -K,-H {host},-U {user},-X {password},-P {port},\n\
                   -B {boxname}] [-A profileName]\n\
-%prog data [-l |-a {file, label} |-g {label} |-G {id} |-s {pattern} |\n\
+%prog data [-l |-a {file, label} |-g {id} |-s {pattern} | -r {id}]\n\
-                  -r {label} |-R {id}] [-c {catg}, -u {owner}, -b {boxname},\n\
+                  [-c {catg}, -u {owner}]', epilog="""
-                  -o {outputdir}]',epilog="""
 
 conf command Examples:
 
@@ -104,8 +103,8 @@ conf command Examples:
 
 data command Examples:
     
-  List index on a specified box (different from box on active profile)
+  List index
-    imprastorage data -lb boxname
+    imprastorage data -l
 
   Add file
     imprastorage data -a /path/tofile 'my video'
@@ -114,14 +113,11 @@ data command Examples:
   when downloading files)
     imprastorage data -a /path/tofile '2009 - en la playa' -c videos/perso/2009
 
-  Get file 
-    imprastorage data -g '2009 - en la playa'
-
   Get file by id
-    imprastorage data -G 22
+    imprastorage data -g 22
 
   Remove from server a file by id
-    imprastorage data -R 22
+    imprastorage data -g 22
    
   Search files matching pattern :
     imprastorage data -s 'holydays'
@@ -137,20 +133,16 @@ data command Examples:
         # metavar='<ARG1> <ARG2>', nargs=2
         parser.add_option('-v', '--version'       , help='show program\'s version number and exit'                     , action='store_true' , default=False)
         parser.add_option('-q', '--quiet'         , help='don\'t print status messages to stdout'                      , action='store_false', default=True)
-        parser.add_option('-f', '--force'         , help='dont confirm and force action'                               , action='store_true' , default=False)
         parser.add_option('-d', '--debug'         , help='set debug mode'                                              , action='store_true' , default=False)
 
         gpData.add_option('-l', '--list'          , help='list index on imap server'                                   , action='store_true' )
         gpData.add_option('-a', '--add'           , help='add file FILE with specified LABEL on server'                , action='store',       metavar='FILE LABEL ', nargs=2)
-        gpData.add_option('-g', '--get'           , help='get file with specified LABEL from server'                   , action='store',       metavar='LABEL      ')
+        gpData.add_option('-g', '--get'           , help='get file with specified ID from server'                      , action='store',       metavar='ID         ')
-        gpData.add_option('-G', '--get-by-id'     , help='get file with specified ID from server'                      , action='store',       metavar='ID         ')
         gpData.add_option('-s', '--search'        , help='search file with specified PATTERN'                          , action='store',       metavar='PATTERN    ')
-        gpData.add_option('-r', '--remove'        , help='remove FILE with specified LABEL from server'                , action='store',       metavar='LABEL      ')
+        gpData.add_option('-r', '--remove'        , help='remove FILE with specified ID from server'                   , action='store',       metavar='ID         ')
-        gpData.add_option('-R', '--remove-by-id'  , help='remove FILE with specified ID from server'                   , action='store',       metavar='ID         ')
+        gpData.add_option('-c', '--category'      , help='set specified CATEGORY (crit. for opt. -l,-a or -s)'         , action='store',       metavar='CATG       '  , default='')
-        gpData.add_option('-b', '--boxname'       , help='switch boxname on imap server'                               , action='store',       metavar='BOXN       ')
-        gpData.add_option('-c', '--category'      , help='set specified CATEGORY (crit. for opt. -l,-a or -s)'         , action='store',       metavar='CATG       '   , default='none')
         gpData.add_option('-u', '--user'          , help='set specified USER (crit. for opt. -l,-a or -s)'             , action='store',       metavar='OWNER      '  , default='all')
-        gpData.add_option('-o', '--output-dir'    , help='set specified OUTPUT DIR (for opt. -l,-a,-d or -g)'          , action='store',       metavar='DIR        ')
+        #gpData.add_option('-o', '--output-dir'    , help='set specified OUTPUT DIR (for opt. -l,-a,-d or -g)'          , action='store',       metavar='DIR        ')
         parser.add_option_group(gpData)                                                                             
 
         gpConf.add_option('-L', '--list-conf'     , help='list configuration'                                          , action='store_true',  default=False)
@@ -221,7 +213,7 @@ data command Examples:
 
                 o.active_profile = self.ini.get('profile')
                     
-                if not o.list and not o.add and not o.get and not o.get_by_id and not o.search and not o.remove and not o.remove_by_id :
+                if not o.list and not o.add and not o.get and not o.search and not o.remove :
                     parser.error(' no options specified')
                 else :
                     
@@ -247,51 +239,43 @@ you can remove index but all presents files on the box %s will be unrecoverable
                         if o.list :
                             uid     = conf.get('uid' ,'index')
                             date    = conf.get('date','index')
-                            if uid  == None : uid  = 'EMPTY'
+                            account = conf.get('user','imap')
-                            if date == None : date = ''
                             if impst.index != None:
-                                impst.index.print('-'*120+'\n -- INDEX(`'+uid+'`) boxname :`'+impst.rootBox+'` '+date+'\n'+'-'*120)
+                                noData = impst.index.isEmpty()
-                                #encData = impst.index.impraEncrypt(impst.index.toString())
+                                if uid  == None or noData : uid  = 'EMPTY'
-                                #~ dd = """coucou mon joli coeur :*:* je sais que je te saoule avec ça mais bon putain tu va te planter ou merde"""
+                                if date == None or noData : date = ''
-                                #~ dd = """01234567890123456789012345678901234567890123456789#"""
+                                impst.index.print('-'*120+'\n -- ImpraStorage --  [account:'+account+'] [index:`'+uid+'`] [boxname:`'+impst.rootBox+'`]   '+date+'\n'+'-'*120)
-                                #~ 
+
-                                #~ 
+                            status, resp = impst.ih.srv.search(None, '(SUBJECT "%s")' % '584d15abeb71fbd92fa5861970088b32ebb1d2d6650cec6115a28b64877d70f2')
-                                #~ kg = crypt.KeyGen(256)
+                            ids = [m for m in resp[0].split()]
-                                #~ print('-- key --')
+                            for mid in ids :
-                                #~ print(kg.key)
+                                status, resp = impst.ih.srv.fetch(mid,'(UID RFC822.SIZE)')                                
-                                #~ print('-- mark --')
+                                print(mid,status,resp)
-                                #~ print(kg.mark)
-                                #~ km = crypt.Kirmah(kg.key, kg.mark)
-                                #~ encData = km.encrypt(dd,'.index',22)
-                                #~ #print('*'+encData+'*')
-                                #~ decData = km.decrypt(encData,'.index',22)
-                                #~ print('*'+decData+'*')                                
                             
                         elif o.add :
                             impst.addFile(o.add[0],o.add[1],o.user,o.category)
 
                         elif o.get :
-                            impst.getFile(o.get)
+                            ids = []
-
+                            for sid in o.get.split(',') :
-                        elif o.get_by_id :
+                                seq = sid.split('-')
-                            label = impst.index.searchById(o.get_by_id)
+                                if len(seq)==2 : ids.extend(range(int(seq[0]),int(seq[1])+1))
-                            if label !=None :
+                                else: ids.append(sid)
-                                impst.getFile(label)
+                            for sid in ids :                                
-                            else: print(o.get_by_id+' a is not valid id')
+                                key = impst.index.getById(str(sid))
+                                if key !=None : impst.getFile(key)
+                                else: print('-- `%s` is not a valid id --' % sid)
 
                         elif o.search :
-                            label = impst.index.searchByPattern(o.search)
+                            matchIds = impst.index.getByPattern(o.search)
-                            if label==None: 
+                            if matchIds is not None: 
-                                print(' -- no match found for pattern `%s` --' % o.search)
+                                headstr = util.hilite('-'*120+'\n -- SEARCH: `'+o.search+'` -- found '+str(len(matchIds))+' results --\n'+'-'*120)
+                                impst.index.print(headstr,matchIds)
                             else:
-                                impst.index.print('-'*120+'\n -- SEARCH: `'+o.search+'` -- found '+str(len(label))+' results --\n'+'-'*120,label)
+                                print(' -- no match found for pattern `%s` --' % o.search)
                         elif o.remove :
                             print(o.remove)
 
-                        elif o.remove_by_id :
-                            print(o.remove_by_id)
-
-
                     #~ filePath = '/media/Data/dev/big_toph3.jpg' 
                     #~ lab = 'Meuf\'bonne aussi4' 
                     #~ print('\n -- ADD FILE -- ')

impra/core.py

@@ -31,6 +31,7 @@
 
 from base64               import urlsafe_b64encode, b64decode
 from binascii             import b2a_base64, a2b_base64
+from datetime             import datetime, timedelta
 from email.encoders       import encode_base64
 from email.header         import Header
 from email.mime.base      import MIMEBase
@@ -43,13 +44,12 @@ from mmap                 import mmap
 from os                   import remove, urandom, sep
 from os.path              import abspath, dirname, join, realpath, basename, getsize, splitext
 from re                   import split as regsplit, match as regmatch, compile as regcompile, search as regsearch
+from time                 import time
 from impra.imap           import ImapHelper, ImapConfig
-from impra.util           import __CALLER__, RuTime, formatBytes, randomFrom, bstr, quote_escape, stack, run, file_exists, get_file_content, DEBUG, DEBUG_ALL, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, mkdir_p
+from impra.util           import __CALLER__, RuTime, formatBytes, randomFrom, bstr, quote_escape, stack, run, file_exists, get_file_content, DEBUG, DEBUG_ALL, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, mkdir_p, is_binary
 from impra.crypt          import Kirmah, ConfigKey, Noiser, Randomiz, hash_sha256, hash_md5_file 
 
 
-
-
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 # ~~ class FSplitter ~~
 
@@ -60,6 +60,7 @@ class FSplitter :
         """"""
         self.ck         = ck
         self.wkdir      = wkdir
+        self.DIR_CACHE  = join(self.wkdir,'.cache')+sep
         self.DIR_INBOX  = join(self.wkdir,'inbox')+sep
         self.DIR_OUTBOX = join(self.wkdir,'outbox')+sep
         self.DIR_DEPLOY = join(self.wkdir,'deploy')+sep
@@ -111,7 +112,7 @@ class FSplitter :
             
         rt.stop()
         
-    def deployFile(self, hlst, fileName, ext='', dirs=None, fake=False):
+    def deployFile(self, hlst, fileName, ext='', uid='', dirs=None, fake=False):
         """"""
         rt = RuTime(eval(__CALLER__()))
         p = 0
@@ -121,7 +122,14 @@ class FSplitter :
             dirPath = join(self.DIR_DEPLOY,dirs)+sep
             mkdir_p(dirPath)
         else: dirPath = self.DIR_DEPLOY
-        fp = open(dirPath+fileName+ext, 'wb+')
+        
+        filePath = dirPath+fileName
+        if file_exists(filePath+ext):
+            print('\n-- `%s` already exist, deploying file as :\n-- `%s`\n' % (filePath+ext,filePath+'-'+str(uid)+ext))
+            filePath += '-'+str(uid)
+        filePath += ext
+            
+        fp = open(filePath, 'wb+')
         depDir = self.DIR_INBOX
         if fake : depDir = self.DIR_OUTBOX
         while p < hlst['head'][1] :
@@ -129,6 +137,7 @@ class FSplitter :
             p += 1
         fp.close()
         rt.stop()
+        return dirPath+fileName+ext
 
     def _mergePart(self,fp,part,phlst,depDir):
         """"""
@@ -228,6 +237,11 @@ class ImpraIndex:
     """"""
     UID            = 6
     """"""
+    BFLAG          = 7
+    """"""
+    
+    FILE_BINARY    = 'b'
+    FILE_CRYPT     = 'c'
     
     
     def __init__(self, key, mark, encdata='', dicCategory={}, id=0):
@@ -251,29 +265,30 @@ class ImpraIndex:
             if not self.SEP_KEY_INTERN+k in self.dic:
                 self.dic[self.SEP_KEY_INTERN+k] = dicCategory[k]
 
-    def add(self,key, label, count, ext='', usr='', cat='', md5=''):
+    def add(self,key, label, count, ext='', usr='', cat='', md5='', bFlag='b'):
-        """Add an entry to the index with appropriate label, key used by entry
+        """Add an entry to the index
-        to decode data, and parts count
         """
-        if self.search(md5) == None : 
+        if self.get(md5) == None : 
-            self.dic[md5] = (key,label,count,ext,usr,cat, self.id)
+            self.dic[md5] = (key,label,count,ext,usr,cat, self.id, bFlag)
             self.id +=1
         else : 
             print(label+' already exist')
 
     def addUser(self, nameFrom, hashName):
+        """"""
+        if not self.hasUser(hashName):
+            self.dic[self.SEP_KEY_INTERN+'users'][hashName] = nameFrom
+    
+    def hasUser(self, hashName):
         """"""
         if not self.SEP_KEY_INTERN+'users' in self.dic:
             self.dic[self.SEP_KEY_INTERN+'users'] = {}
-        if not hashName in self.dic[self.SEP_KEY_INTERN+'users']:
+        return hashName in self.dic[self.SEP_KEY_INTERN+'users']
-            self.dic[self.SEP_KEY_INTERN+'users'][hashName] = nameFrom
             
     def getUser(self, hashName):
         """"""
         usrName = 'Anonymous'
-        if not str(self.SEP_KEY_INTERN+'users') in self.dic:
+        if self.hasUser(hashName):
-            self.dic[self.SEP_KEY_INTERN+'users'] = {}
-        elif hashName in self.dic[self.SEP_KEY_INTERN+'users']:
             usrName = self.dic[self.SEP_KEY_INTERN+'users'][hashName]
         return usrName
 
@@ -281,12 +296,47 @@ class ImpraIndex:
         """Remove the selected label from the index"""
         self.dic.pop(label, None)
     
-    def search(self,label):
+    def getAutoCatg(self,ext):
-        """Search the corresponding label in the index"""
+        """"""
-        return self.dic.get(label)
+        catg = 'none'
+        if regsearch('\.(jpg|jpeg|gif|png)',ext):
+            catg = 'images'
+        elif regsearch('\.(txt|doc|odt|csv|pdf)',ext):
+            catg = 'doc'
+        elif regsearch('\.(mp4|avi|mpg|mpeg|flv|ogv)',ext):
+            catg = 'films'
+        elif regsearch('\.(mp3|ogg|flac)',ext):
+            catg = 'music'
+        elif regsearch('\.(zip|7z|tar|gz|rar|bz|xz|jar)',ext):
+            catg = 'archives'
+        return catg
         
-    def searchById(self,sid):
+    def isEmpty(self):
-        """Search the corresponding label in the index"""
+        """"""
+        r = [k for i, k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN)]
+        return len(r) == 0
+        
+    def getLabel(self, key):
+        """Get label corresponding to key in the index
+        :Returns: `str`|None label
+        """
+        value = ''
+        row   = self.get(key)
+        if row is not None :
+            value = row[self.LABEL]
+
+    def get(self, key):
+        """Get the corresponding key in the index
+        :Returns: `tuple` row
+        """
+        row = None
+        if key in self.dic : row = self.dic.get(key)
+        return row
+        
+    def getById(self,sid):
+        """Get the corresponding id in the index
+        :Returns: `str`|None key
+        """
         rt = RuTime(eval(__CALLER__(sid)))
         l  = None
         r = [k for i, k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN) and self.dic[k][self.UID] == int(sid)]
@@ -294,15 +344,21 @@ class ImpraIndex:
         rt.stop()
         return l
 
-    def searchByFileHash(self,md5):
+    def getByLabel(self,label):
-        """"""
+        """Get the corresponding label in the index
-        e = None
+        :Returns: `str`|None key
-        if md5 in self.dic:
+        """
-            e = True
+        rt = RuTime(eval(__CALLER__(sid)))
-        return e
+        l  = None
+        r = [k for i, k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN) and self.dic[k][self.LABEL] == int(label)]
+        if len(r)==1: l = r[0]
+        rt.stop()
+        return l
     
-    def searchByPattern(self,pattern):
+    def getByPattern(self,pattern):
-        """"""
+        """Get ids corresponding to label matching the pattern in the index
+        :Returns: `[uid]`|None matchIds
+        """
         rt = RuTime(eval(__CALLER__(pattern)))
         l = None
         r = [ k for i,k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN) and regsearch(pattern,self.dic[k][self.LABEL]) is not None ]
@@ -366,6 +422,8 @@ class ImpraStorage:
     
     def __init__(self, conf, remIndex=False, wkdir=None):
         """"""
+        from impra.util import DEBUG_INFO
+        rt = RuTime(eval(__CALLER__()),DEBUG_INFO)
         if wkdir == None : wkdir = abspath(join(dirname( __file__ ), '..', 'wk'))
         self.wkdir   = wkdir
         self.conf    = conf
@@ -378,6 +436,7 @@ class ImpraStorage:
         self.delids  = []
         if remIndex  : self.removeIndex()
         self.index   = self.getIndex()
+        rt.stop()
 
     def _getIdIndex(self):
         """"""
@@ -408,6 +467,11 @@ class ImpraStorage:
                 encData = str(ms,'utf-8')
         return encData
 
+    def getIndexDefaultCatg(self):
+        """"""
+        usrName = self.conf.get('name','infos')
+        return {'catg':self.conf.get('types','catg'), 'users':{ ('%s' % self.mb.getHashName('all')) : 'all', ('%s' % self.mb.getHashName(usrName)) : usrName}}
+
     def getIndex(self):
         """"""
         from impra.util import DEBUG, DEBUG_LEVEL, DEBUG_WARN, DEBUG_INFO       
@@ -417,20 +481,25 @@ class ImpraStorage:
         uid     = self.conf.get('uid'  ,'index')
         date    = self.conf.get('date ','index')
         nid     = self.conf.get('nid'  ,'index')
-        if nid==None : nid = 0
+        tstamp  = self.conf.get('time' ,'index')
-        self._getIdIndex()
+        if nid is None : nid = 0
-        if self.idx :
+        
+        if tstamp is not None and (datetime.now() - datetime.strptime(tstamp[:-7], '%Y-%m-%d %H:%M:%S')) < timedelta(minutes = 1) :
             # getFromFile
-            if uid != None and int(self.idx) == int(uid) and file_exists(self.pathInd):
+            if uid != None and file_exists(self.pathInd): # int(self.idx) == int(uid) 
+                self.idx = uid
                 encData = get_file_content(self.pathInd)
                 print('--\nindex in cache')
         else :
+            print('refresh index')        
+            self._getIdIndex()
+            if self.idx :
                 encData = self._getCryptIndex()
                 with open(self.pathInd, mode='w', encoding='utf-8') as o:
                     o.write(encData)
-        usrName = self.conf.get('name','infos')
+                self.conf.set('time',str(datetime.now()),'index')
-        usrHash = self.mb.getHashName(usrName)
+        
-        index = ImpraIndex(self.conf.get('key','keys'),self.conf.get('mark','keys'), encData, {'catg':self.conf.get('types','catg'), 'users':{ ('%s' % self.mb.getHashName('all')) : 'all', ('%s' % usrHash) : usrName}}, int(nid))
+        index = ImpraIndex(self.conf.get('key','keys'),self.conf.get('mark','keys'), encData, self.getIndexDefaultCatg(), int(nid))
         rt.stop()
         return index
         
@@ -457,36 +526,81 @@ class ImpraStorage:
         self.conf.set('date',date,'index')
         with open(self.pathInd, mode='w', encoding='utf-8') as o:
             o.write(encData)
-        self.ih.deleteBin()
+        self.conf.set('time',str(datetime.now()),'index')
-        #self.index = self.getIndex()
+        self.clean()
         rt.stop()
     
+    def encryptTextFile(self,path):
+        """"""
+        cdata = self.index.km.subenc(get_file_content(path))
+        with open(self.fsplit.DIR_CACHE+'.~KirmahEnc', mode='w') as o:
+            o.write(cdata)
+        return self.fsplit.DIR_CACHE+'.~KirmahEnc'
+        
+    def decryptTextFile(self,path):
+        """"""
+        data  = self.index.km.subdec(get_file_content(path))
+        with open(path, mode='w') as o:
+            o.write(data)            
+    
     def addFile(self, path, label, usr='all', catg=''):
         """"""
         from impra.util import DEBUG, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, DEBUG_INFO
         rt = RuTime(eval(__CALLER__('"%s","%s","%s"' % (path[:13]+'...',label,usr))),DEBUG_INFO)
 
-        #~ hlst = self.fsplit.addFile(path,label)
-        #~ self.fsplit.deployFile(hlst,True)
         _, ext = splitext(path)
+        
         try:
             md5 = hash_md5_file(path)
             print('--\nmd5sum `%s` %s' % (path,md5))                
-            if not self.index.searchByFileHash(md5) :
+            if not self.index.get(md5) :
+                
+                if catg=='' : catg = self.index.getAutoCatg(ext)
+                
+                bFlag = ImpraIndex.FILE_BINARY
+                if not is_binary(path): 
+                    bFlag = ImpraIndex.FILE_CRYPT
+                    path  = self.encryptTextFile(path)
+                    
                 hlst = self.fsplit.addFile(path,md5)
                 if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE : 
                     print(hlst['head'])
                     for v in hlst['data']:
                         print(v)
-                nameFrom = self.conf.get('name','infos')
-                self.index.addUser(nameFrom, self.mb.getHashName(nameFrom))
                 
+                ownerHash = self.mb.getHashName(usr)
+                self.index.addUser(usr,ownerHash)
+
+                cancel  = False
+                sendIds = []
+                test    = True
                 for row in hlst['data'] :
-                    msg  = self.mb.build(nameFrom,usr,hlst['head'][2],self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
+                    msg  = self.mb.build(self.conf.get('name','infos'),usr,hlst['head'][2],self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
-                    self.ih.send(msg.as_string(), self.rootBox)
+                    mid  = self.ih.send(msg.as_string(), self.rootBox)
+                    if mid is not None :
+                        print('part %s sent as msg %s' % (row[0],mid[1]))
+                        sendIds.append((mid[1],row))
                         remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
+                    else:
+                        print('\n-- error occured when sending part : %s\n-- retrying' % row[0])
+                        mid  = self.ih.send(msg.as_string(), self.rootBox)
+                        if mid is not None :
+                            print('part %s sent as msg %s' % (row[0],mid[1]))
+                            sendIds.append((mid[1],row))
+                            remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
+                        else: 
+                            print('\n-- can\'t send part %s\n-- cancelling ' % row[0])
+                            cancel = True
+                            break
                 
-                self.index.add(hlst['head'][3],label,hlst['head'][1],ext,self.mb.getHashName(usr),catg,md5)
+                if cancel :
+                    for mid, row in sendIds :
+                        self.ih.delete(mid, True)
+                        if file_exists(self.fsplit.DIR_OUTBOX+row[1]+'.ipr') : remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
+                    self.clean()
+                        
+                else :
+                    self.index.add(hlst['head'][3],label,hlst['head'][1],ext,ownerHash,catg,md5,bFlag)
                     self.saveIndex()
                     self.conf.set('nid', str(self.index.id),'index')
             else :
@@ -495,21 +609,18 @@ class ImpraStorage:
             print(e)
         rt.stop()
 
-    def getFile(self,label):
+    def getFile(self,key):
         """"""
         from impra.util import DEBUG, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, DEBUG_INFO
-        
+        row   = self.index.get(key)        
-        rt  = RuTime(eval(__CALLER__('"%s"' % label)),DEBUG_INFO)
+        if row==None : 
-        if label==None : 
+            print('--\n%s not on the server' % key)
-            print('--\n'+str(label)+' unexist')
         else :
-            key = self.index.search(label)
+            rt  = RuTime(eval(__CALLER__('"[%i] %s"' % (row[ImpraIndex.UID],row[ImpraIndex.LABEL]))),DEBUG_INFO)
-            if label!=None and key!=None:
+            ck    = ConfigKey(row[ImpraIndex.HASH])
-                ck    = ConfigKey(key[ImpraIndex.HASH])
+            hlst  = ck.getHashList(key,row[ImpraIndex.PARTS],True)
-                count = int(key[ImpraIndex.PARTS])
-                hlst  = ck.getHashList(label,count,True)
             ids   = self._getIdsBySubject(hlst['head'][2])
-                if len(ids) >= count:
+            if len(ids) >= row[ImpraIndex.PARTS]:
                 status, resp = self.ih.srv.fetch(ids[0],'(BODY[HEADER.FIELDS (TO)])')
                 to = bstr(resp[0][1][4:-4])
                 if to == self.mb.getHashName('all')+'@'+self.mb.DOMAIN_NAME or to == self.mb.getHashName(self.conf.ini.get('name',self.conf.profile+'.infos'))+'@'+self.mb.DOMAIN_NAME :
@@ -519,22 +630,21 @@ class ImpraStorage:
                         print(hlst['head'])
                         for v in hlst['data']:
                             print(v)
-                        self.fsplit.deployFile(hlst, key[ImpraIndex.LABEL],  key[ImpraIndex.EXT], key[ImpraIndex.CATG])
+                    path = self.fsplit.deployFile(hlst, row[ImpraIndex.LABEL],  row[ImpraIndex.EXT], row[ImpraIndex.UID], row[ImpraIndex.CATG])
+                    if row[ImpraIndex.BFLAG] == ImpraIndex.FILE_CRYPT:
+                        self.decryptTextFile(path)
                 else :
-                        #raise Exception(label+' is private')
+                    print('--\n`%s` is private' % row[ImpraIndex.LABEL])
-                        print('--\n'+label+' is private')
             else :
-                    #raise Exception(label+' : invalid count parts '+str(len(ids))+'/'+str(count))
+                print('--\n`%s` invalid count parts %i/%i' %(row[ImpraIndex.LABEL],len(ids),row[ImpraIndex.PARTS]))
-                    print('--\n'+label+' : invalid count parts '+str(len(ids))+'/'+str(count))
+
-            else:
-                #raise Exception(str(label)+' not on the server')
-                print('--\n'+str(label)+' not on the server')
             rt.stop()
 
     def clean(self):
         """"""
         rt = RuTime(eval(__CALLER__()))
-        self.index = self.getIndex()
+        self.ih.deleteBin()
+        if file_exists(self.fsplit.DIR_CACHE+'.~KirmahEnc'):remove(self.fsplit.DIR_CACHE+'.~KirmahEnc')
         rt.stop()
 
 

impra/crypt.py

@@ -26,7 +26,9 @@
 #   You should have received a copy of the GNU General Public License
 #   along with ImpraStorage.  If not, see <http://www.gnu.org/licenses/>.
 
-from impra.util import RuTime, __CALLER__, stack, DEBUG
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# ~~ package crypt ~~
+
 from base64     import urlsafe_b64encode, b64decode
 from binascii   import b2a_base64, a2b_base64
 from hashlib    import sha256, md5
@@ -35,10 +37,12 @@ from random     import choice
 from os         import urandom
 from time       import sleep
 
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+from impra.util import RuTime, __CALLER__, stack, DEBUG
-# ~~ package crypt ~~
 
 
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# ~~ methods ~~
+
 def hash_sha256(data):
     """Get a sha256 hash of str `data`
     :Returns: `str`
@@ -226,17 +230,11 @@ class Kirmah:
         psize   = ceil(len(data)/cpart)
         cp      = 0
         for row in hlst['data']:
-            #~ print(row)
-            #~ print('ns:%i - dataLength:%i - dataEncLength:%i - ne:%i' % (row[2],len(data), len(dataEnc), row[3]))
-            #~ print(data[cp*psize:cp*psize+psize])            
             dataEnc += self.ck.noiser.getNoise(row[2],True)+data[cp*psize:cp*psize+psize]+self.ck.noiser.getNoise(row[3],True)
             cp      += 1
 
         dataEnc = str(b2a_base64(bytes(dataEnc,'utf-8')),'utf-8')
 
-        #~ dataEnc = self.subenc(odata)
-        #~ with open('./.KirmahENC', mode='w') as o:
-            #~ o.write(dataEnc)
         rt.stop()
         return dataEnc
     
@@ -255,27 +253,17 @@ class Kirmah:
         for row in hlst['data']:
             si = ni + row[2]
             ei = si + psize
-            #~ print(row)
-            #~ print('si:%i - ei:%i - datalength:%i - dataDeclength:%i' % (si,ei,len(data),len(dataDec)))
             if cp == cpart-1 :
                 ei = -row[3]                
                 if not si > len(data)+ei : pass
-                    ### to delete #si = len(data)+ei
-                    #~ print('si:%i - ei:%i' % (si,ei))
                 else :
-                    #~ print('si: - ei:%i' % (len(data)+ei-si))
                     dataDec=dataDec[:len(data)+ei-si]
                     break
-            #~ print(data[si:ei])
             dataDec += data[si:ei]
             ni  = ei + row[3]
             cp += 1
         dataDec = self.subdec(dataDec)
   
-        #~ dataDec = self.subdec(data)
-
-        #~ with open('./.KirmahDEC', mode='w') as o:
-            #~ o.write(dataDec)        
         rt.stop()
         return dataDec
 

impra/imap.py

@@ -26,9 +26,9 @@
 #   You should have received a copy of the GNU General Public License
 #   along with ImpraStorage.  If not, see <http://www.gnu.org/licenses/>.
 
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# ~~ package imap ~~
 
+from binascii         import b2a_base64, a2b_base64
+from codecs           import register, StreamReader, StreamWriter
 from email            import message_from_bytes
 from email.header     import decode_header
 from email.message    import Message
@@ -36,10 +36,12 @@ from imaplib          import IMAP4_SSL, Time2Internaldate
 from os.path          import join
 from re               import search, split
 from time             import time
-from impra.util       import __CALLER__, RuTime, bstr, stack
-from binascii         import b2a_base64, a2b_base64
-from codecs           import register, StreamReader, StreamWriter
 
+from impra.util       import __CALLER__, RuTime, bstr, stack
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# ~~ package imap ~~
 
 def _seq_encode(seq,l):
     """"""
@@ -310,7 +312,7 @@ class ImapHelper:
                 #~ print(mid)
                 #status, resp = self.srv.store(mid, '+FLAGS', '\\Deleted')
                 status, resp = self.srv.uid('store', mid, '+FLAGS', '\\Deleted' )
-                print('deleting msg '+str(mid))
+                print('deleting msg %i' % int(mid))
                 if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE:                    
                     print(status)
                     print(resp)
@@ -349,12 +351,15 @@ class ImapHelper:
 
     def send(self, msg, box='INBOX'):
         """"""
-        from impra.util import DEBUG_INFO
+        from impra.util import DEBUG_INFO, DEBUG_LEVEL, DEBUG, DEBUG_NOTICE
-        rt   = RuTime(eval(__CALLER__()),DEBUG_INFO)
+        rt   = RuTime(eval(__CALLER__()))
         mid  = None
         date = Time2Internaldate(time())
         status, resp  = self.srv.append(box, '\Draft', date, bytes(msg,'utf-8'))
         if status==self.OK:
+            if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE:
+                print(status)
+                print(resp)
             mid = str(resp[0],'utf-8')[11:-11].split(' ')
         rt.stop()
         return mid

impra/util.py

@@ -29,17 +29,19 @@
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 # ~~ package util ~~
 
-from hashlib    import sha256
-from math       import log, floor, ceil
-from random     import choice
-from os         import urandom, popen, sep, makedirs
-from os.path    import dirname, realpath, abspath, join
-from time       import time
-from re         import split as regsplit
 from base64     import urlsafe_b64encode
 from inspect    import stack
+from errno      import EEXIST
+from hashlib    import sha256
+from math       import log, floor, ceil
+from os         import urandom, popen, sep, makedirs
+from os.path    import dirname, realpath, abspath, join
+from random     import choice
+from re         import split as regsplit
 from subprocess import PIPE, Popen
 from sys        import stderr, executable as pyexec
+#~ from sys.stdout import isatty
+from time       import time
 
 DEBUG_ALL    = 0
 DEBUG_WARN   = 1
@@ -48,6 +50,9 @@ DEBUG_INFO   = 3
 
 DEBUG       = True
 DEBUG_LEVEL = DEBUG_INFO
+
+COLOR_MODE  = True
+
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 # ~~ methods ~~
 
@@ -75,7 +80,7 @@ def get_file_content(fileName):
     """Get file content of `fileName`
     :Returns: `str`
     """
-    r = open(fileName, "rt")
+    r = open(fileName, 'rt')
     data = r.read()
     r.close()
     return data
@@ -84,7 +89,7 @@ def get_file_binary(fileName):
     """Get file content of `fileName`
     :Returns: `str`
     """
-    r = open(fileName, "rb")
+    r = open(fileName, 'rb')
     data = r.read()
     r.close()
     return data
@@ -102,6 +107,20 @@ def randomFrom(val, sval=0):
     lst = list(range(sval,val))
     return choice(lst)
 
+def is_binary(filename):
+    """Check if given filename is binary."""
+    done = False
+    fp = open(filename, 'rb')
+    try:
+        CHUNKSIZE = 1024
+        while 1:
+            chunk = fp.read(CHUNKSIZE)
+            if b'\0' in chunk:  done = True # found null byte
+            if done or len(chunk) < CHUNKSIZE: break
+    finally:
+        fp.close()
+    return done
+
 def get_file_path(val):
     """"""
     return abspath(dirname(val))+sep
@@ -120,7 +139,7 @@ def mkdir_p(path):
     try:
         makedirs(path)
     except OSError as e: # Python >2.5
-        if e.errno == errno.EEXIST:
+        if e.errno == EEXIST:
             pass
         else: raise
 
@@ -166,16 +185,40 @@ def __CALLER__(args=''):
     :Returns: `str`
     """
     global DEBUG_LEVEL, DEBUG, DEBUG_WARN
-    #~ print(inspect.stack()[1][3])
-    #~ print(print(args))
-    #~ print('-----')
-    #~ print(inspect.stack())
-    #~ print('---------------')
     val = "self.__class__.__name__+'.%s' % stack()[1][3]+'("+quote_escape(args)+") "
     if DEBUG and DEBUG_LEVEL<=DEBUG_WARN : val += "l:'+str(stack()[1][2])"
     else: val += "'"
     return val
 
+def hilite(string, color=32, bold=True):
+    """"""
+    global COLOR_MODE    
+    if COLOR_MODE and True:
+        attr = [color]
+        if bold:
+            attr.append('1')
+        
+        #~ print('\033[1;30mGray like Ghost\033[1;m')
+        #~ print('\033[1;31mRed like Radish\033[1;m')
+        #~ print('\033[1;32mGreen like Grass\033[1;m')
+        #~ print('\033[1;33mYellow like Yolk\033[1;m')
+        #~ print('\033[1;34mBlue like Blood\033[1;m')
+        #~ print('\033[1;35mMagenta like Mimosa\033[1;m')
+        #~ print('\033[1;36mCyan like Caribbean\033[1;m')
+        #~ print('\033[1;37mWhite like Whipped Cream\033[1;m')
+        #~ print('\033[1;38mCrimson like Chianti\033[1;m')
+        #~ print('\033[1;41mHighlighted Red like Radish\033[1;m')
+        #~ print('\033[1;42mHighlighted Green like Grass\033[1;m')
+        #~ print('\033[1;43mHighlighted Brown like Bear\033[1;m')
+        #~ print('\033[1;44mHighlighted Blue like Blood\033[1;m')
+        #~ print('\033[1;45mHighlighted Magenta like Mimosa\033[1;m')
+        #~ print('\033[1;46mHighlighted Cyan like Caribbean\033[1;m')
+        #~ print('\033[1;47mHighlighted Gray like Ghost\033[1;m')
+        #~ print('\033[1;48mHighlighted Crimson like Chianti\033[1;m')        
+        
+        string = '\x1b[%sm%s\x1b[0m' % (';'.join(attr), string)
+        print(string)
+    return string
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 # ~~ class RuTime ~~

imprastorage.py

@@ -32,7 +32,10 @@ from impra.cli  import Cli
 
 
 if __name__ == '__main__':
+    #~ try :
 
         Cli(get_file_path(__file__ ))
+    #~ except Exception as e :
+        #~ print(e)
 
 #python  -O -m compileall impra/*.py

wk/.cache/.gitkeep

@@ -0,0 +1 @@
+# .gitkeep