From 502d3b65c01a6c4f0b78289aab93d6a096bca2b0 Mon Sep 17 00:00:00 2001 From: a-sansara Date: Tue, 18 Sep 2012 19:11:00 +0200 Subject: [PATCH] refactoring, autocategory, check upload failed --- impra/cli.py | 94 ++++++-------- impra/core.py | 296 +++++++++++++++++++++++++++++++-------------- impra/crypt.py | 28 ++--- impra/imap.py | 21 ++-- impra/util.py | 73 ++++++++--- imprastorage.py | 7 +- wk/.cache/.gitkeep | 1 + 7 files changed, 327 insertions(+), 193 deletions(-) create mode 100644 wk/.cache/.gitkeep diff --git a/impra/cli.py b/impra/cli.py index 1cf0dff..4bb76b9 100644 --- a/impra/cli.py +++ b/impra/cli.py @@ -83,9 +83,8 @@ class Cli: \n\ %prog conf [-D|-L| -K,-H {host},-U {user},-X {password},-P {port},\n\ -B {boxname}] [-A profileName]\n\ -%prog data [-l |-a {file, label} |-g {label} |-G {id} |-s {pattern} |\n\ - -r {label} |-R {id}] [-c {catg}, -u {owner}, -b {boxname},\n\ - -o {outputdir}]',epilog=""" +%prog data [-l |-a {file, label} |-g {id} |-s {pattern} | -r {id}]\n\ + [-c {catg}, -u {owner}]', epilog=""" conf command Examples: @@ -104,8 +103,8 @@ conf command Examples: data command Examples: - List index on a specified box (different from box on active profile) - imprastorage data -lb boxname + List index + imprastorage data -l Add file imprastorage data -a /path/tofile 'my video' @@ -114,14 +113,11 @@ data command Examples: when downloading files) imprastorage data -a /path/tofile '2009 - en la playa' -c videos/perso/2009 - Get file - imprastorage data -g '2009 - en la playa' - Get file by id - imprastorage data -G 22 + imprastorage data -g 22 Remove from server a file by id - imprastorage data -R 22 + imprastorage data -g 22 Search files matching pattern : imprastorage data -s 'holydays' @@ -137,20 +133,16 @@ data command Examples: # metavar=' ', nargs=2 parser.add_option('-v', '--version' , help='show program\'s version number and exit' , action='store_true' , default=False) parser.add_option('-q', '--quiet' , help='don\'t print status messages to stdout' , action='store_false', default=True) - parser.add_option('-f', '--force' , help='dont confirm and force action' , action='store_true' , default=False) parser.add_option('-d', '--debug' , help='set debug mode' , action='store_true' , default=False) gpData.add_option('-l', '--list' , help='list index on imap server' , action='store_true' ) gpData.add_option('-a', '--add' , help='add file FILE with specified LABEL on server' , action='store', metavar='FILE LABEL ', nargs=2) - gpData.add_option('-g', '--get' , help='get file with specified LABEL from server' , action='store', metavar='LABEL ') - gpData.add_option('-G', '--get-by-id' , help='get file with specified ID from server' , action='store', metavar='ID ') + gpData.add_option('-g', '--get' , help='get file with specified ID from server' , action='store', metavar='ID ') gpData.add_option('-s', '--search' , help='search file with specified PATTERN' , action='store', metavar='PATTERN ') - gpData.add_option('-r', '--remove' , help='remove FILE with specified LABEL from server' , action='store', metavar='LABEL ') - gpData.add_option('-R', '--remove-by-id' , help='remove FILE with specified ID from server' , action='store', metavar='ID ') - gpData.add_option('-b', '--boxname' , help='switch boxname on imap server' , action='store', metavar='BOXN ') - gpData.add_option('-c', '--category' , help='set specified CATEGORY (crit. for opt. -l,-a or -s)' , action='store', metavar='CATG ' , default='none') + gpData.add_option('-r', '--remove' , help='remove FILE with specified ID from server' , action='store', metavar='ID ') + gpData.add_option('-c', '--category' , help='set specified CATEGORY (crit. for opt. -l,-a or -s)' , action='store', metavar='CATG ' , default='') gpData.add_option('-u', '--user' , help='set specified USER (crit. for opt. -l,-a or -s)' , action='store', metavar='OWNER ' , default='all') - gpData.add_option('-o', '--output-dir' , help='set specified OUTPUT DIR (for opt. -l,-a,-d or -g)' , action='store', metavar='DIR ') + #gpData.add_option('-o', '--output-dir' , help='set specified OUTPUT DIR (for opt. -l,-a,-d or -g)' , action='store', metavar='DIR ') parser.add_option_group(gpData) gpConf.add_option('-L', '--list-conf' , help='list configuration' , action='store_true', default=False) @@ -221,7 +213,7 @@ data command Examples: o.active_profile = self.ini.get('profile') - if not o.list and not o.add and not o.get and not o.get_by_id and not o.search and not o.remove and not o.remove_by_id : + if not o.list and not o.add and not o.get and not o.search and not o.remove : parser.error(' no options specified') else : @@ -245,53 +237,45 @@ you can remove index but all presents files on the box %s will be unrecoverable sys.exit(1) if o.list : - uid = conf.get('uid' ,'index') - date = conf.get('date','index') - if uid == None : uid = 'EMPTY' - if date == None : date = '' + uid = conf.get('uid' ,'index') + date = conf.get('date','index') + account = conf.get('user','imap') if impst.index != None: - impst.index.print('-'*120+'\n -- INDEX(`'+uid+'`) boxname :`'+impst.rootBox+'` '+date+'\n'+'-'*120) - #encData = impst.index.impraEncrypt(impst.index.toString()) - #~ dd = """coucou mon joli coeur :*:* je sais que je te saoule avec ça mais bon putain tu va te planter ou merde""" - #~ dd = """01234567890123456789012345678901234567890123456789#""" - #~ - #~ - #~ kg = crypt.KeyGen(256) - #~ print('-- key --') - #~ print(kg.key) - #~ print('-- mark --') - #~ print(kg.mark) - #~ km = crypt.Kirmah(kg.key, kg.mark) - #~ encData = km.encrypt(dd,'.index',22) - #~ #print('*'+encData+'*') - #~ decData = km.decrypt(encData,'.index',22) - #~ print('*'+decData+'*') - + noData = impst.index.isEmpty() + if uid == None or noData : uid = 'EMPTY' + if date == None or noData : date = '' + impst.index.print('-'*120+'\n -- ImpraStorage -- [account:'+account+'] [index:`'+uid+'`] [boxname:`'+impst.rootBox+'`] '+date+'\n'+'-'*120) + + status, resp = impst.ih.srv.search(None, '(SUBJECT "%s")' % '584d15abeb71fbd92fa5861970088b32ebb1d2d6650cec6115a28b64877d70f2') + ids = [m for m in resp[0].split()] + for mid in ids : + status, resp = impst.ih.srv.fetch(mid,'(UID RFC822.SIZE)') + print(mid,status,resp) + elif o.add : impst.addFile(o.add[0],o.add[1],o.user,o.category) elif o.get : - impst.getFile(o.get) + ids = [] + for sid in o.get.split(',') : + seq = sid.split('-') + if len(seq)==2 : ids.extend(range(int(seq[0]),int(seq[1])+1)) + else: ids.append(sid) + for sid in ids : + key = impst.index.getById(str(sid)) + if key !=None : impst.getFile(key) + else: print('-- `%s` is not a valid id --' % sid) - elif o.get_by_id : - label = impst.index.searchById(o.get_by_id) - if label !=None : - impst.getFile(label) - else: print(o.get_by_id+' a is not valid id') - elif o.search : - label = impst.index.searchByPattern(o.search) - if label==None: - print(' -- no match found for pattern `%s` --' % o.search) + matchIds = impst.index.getByPattern(o.search) + if matchIds is not None: + headstr = util.hilite('-'*120+'\n -- SEARCH: `'+o.search+'` -- found '+str(len(matchIds))+' results --\n'+'-'*120) + impst.index.print(headstr,matchIds) else: - impst.index.print('-'*120+'\n -- SEARCH: `'+o.search+'` -- found '+str(len(label))+' results --\n'+'-'*120,label) + print(' -- no match found for pattern `%s` --' % o.search) elif o.remove : print(o.remove) - elif o.remove_by_id : - print(o.remove_by_id) - - #~ filePath = '/media/Data/dev/big_toph3.jpg' #~ lab = 'Meuf\'bonne aussi4' #~ print('\n -- ADD FILE -- ') diff --git a/impra/core.py b/impra/core.py index f8eeac2..4e8ef65 100644 --- a/impra/core.py +++ b/impra/core.py @@ -31,6 +31,7 @@ from base64 import urlsafe_b64encode, b64decode from binascii import b2a_base64, a2b_base64 +from datetime import datetime, timedelta from email.encoders import encode_base64 from email.header import Header from email.mime.base import MIMEBase @@ -43,13 +44,12 @@ from mmap import mmap from os import remove, urandom, sep from os.path import abspath, dirname, join, realpath, basename, getsize, splitext from re import split as regsplit, match as regmatch, compile as regcompile, search as regsearch +from time import time from impra.imap import ImapHelper, ImapConfig -from impra.util import __CALLER__, RuTime, formatBytes, randomFrom, bstr, quote_escape, stack, run, file_exists, get_file_content, DEBUG, DEBUG_ALL, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, mkdir_p +from impra.util import __CALLER__, RuTime, formatBytes, randomFrom, bstr, quote_escape, stack, run, file_exists, get_file_content, DEBUG, DEBUG_ALL, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, mkdir_p, is_binary from impra.crypt import Kirmah, ConfigKey, Noiser, Randomiz, hash_sha256, hash_md5_file - - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~ class FSplitter ~~ @@ -60,6 +60,7 @@ class FSplitter : """""" self.ck = ck self.wkdir = wkdir + self.DIR_CACHE = join(self.wkdir,'.cache')+sep self.DIR_INBOX = join(self.wkdir,'inbox')+sep self.DIR_OUTBOX = join(self.wkdir,'outbox')+sep self.DIR_DEPLOY = join(self.wkdir,'deploy')+sep @@ -111,7 +112,7 @@ class FSplitter : rt.stop() - def deployFile(self, hlst, fileName, ext='', dirs=None, fake=False): + def deployFile(self, hlst, fileName, ext='', uid='', dirs=None, fake=False): """""" rt = RuTime(eval(__CALLER__())) p = 0 @@ -121,7 +122,14 @@ class FSplitter : dirPath = join(self.DIR_DEPLOY,dirs)+sep mkdir_p(dirPath) else: dirPath = self.DIR_DEPLOY - fp = open(dirPath+fileName+ext, 'wb+') + + filePath = dirPath+fileName + if file_exists(filePath+ext): + print('\n-- `%s` already exist, deploying file as :\n-- `%s`\n' % (filePath+ext,filePath+'-'+str(uid)+ext)) + filePath += '-'+str(uid) + filePath += ext + + fp = open(filePath, 'wb+') depDir = self.DIR_INBOX if fake : depDir = self.DIR_OUTBOX while p < hlst['head'][1] : @@ -129,6 +137,7 @@ class FSplitter : p += 1 fp.close() rt.stop() + return dirPath+fileName+ext def _mergePart(self,fp,part,phlst,depDir): """""" @@ -228,6 +237,11 @@ class ImpraIndex: """""" UID = 6 """""" + BFLAG = 7 + """""" + + FILE_BINARY = 'b' + FILE_CRYPT = 'c' def __init__(self, key, mark, encdata='', dicCategory={}, id=0): @@ -251,58 +265,100 @@ class ImpraIndex: if not self.SEP_KEY_INTERN+k in self.dic: self.dic[self.SEP_KEY_INTERN+k] = dicCategory[k] - def add(self,key, label, count, ext='', usr='', cat='', md5=''): - """Add an entry to the index with appropriate label, key used by entry - to decode data, and parts count + def add(self,key, label, count, ext='', usr='', cat='', md5='', bFlag='b'): + """Add an entry to the index """ - if self.search(md5) == None : - self.dic[md5] = (key,label,count,ext,usr,cat, self.id) + if self.get(md5) == None : + self.dic[md5] = (key,label,count,ext,usr,cat, self.id, bFlag) self.id +=1 else : print(label+' already exist') def addUser(self, nameFrom, hashName): + """""" + if not self.hasUser(hashName): + self.dic[self.SEP_KEY_INTERN+'users'][hashName] = nameFrom + + def hasUser(self, hashName): """""" if not self.SEP_KEY_INTERN+'users' in self.dic: self.dic[self.SEP_KEY_INTERN+'users'] = {} - if not hashName in self.dic[self.SEP_KEY_INTERN+'users']: - self.dic[self.SEP_KEY_INTERN+'users'][hashName] = nameFrom - + return hashName in self.dic[self.SEP_KEY_INTERN+'users'] + def getUser(self, hashName): """""" usrName = 'Anonymous' - if not str(self.SEP_KEY_INTERN+'users') in self.dic: - self.dic[self.SEP_KEY_INTERN+'users'] = {} - elif hashName in self.dic[self.SEP_KEY_INTERN+'users']: + if self.hasUser(hashName): usrName = self.dic[self.SEP_KEY_INTERN+'users'][hashName] return usrName def rem(self,label): """Remove the selected label from the index""" self.dic.pop(label, None) - - def search(self,label): - """Search the corresponding label in the index""" - return self.dic.get(label) + + def getAutoCatg(self,ext): + """""" + catg = 'none' + if regsearch('\.(jpg|jpeg|gif|png)',ext): + catg = 'images' + elif regsearch('\.(txt|doc|odt|csv|pdf)',ext): + catg = 'doc' + elif regsearch('\.(mp4|avi|mpg|mpeg|flv|ogv)',ext): + catg = 'films' + elif regsearch('\.(mp3|ogg|flac)',ext): + catg = 'music' + elif regsearch('\.(zip|7z|tar|gz|rar|bz|xz|jar)',ext): + catg = 'archives' + return catg - def searchById(self,sid): - """Search the corresponding label in the index""" + def isEmpty(self): + """""" + r = [k for i, k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN)] + return len(r) == 0 + + def getLabel(self, key): + """Get label corresponding to key in the index + :Returns: `str`|None label + """ + value = '' + row = self.get(key) + if row is not None : + value = row[self.LABEL] + + def get(self, key): + """Get the corresponding key in the index + :Returns: `tuple` row + """ + row = None + if key in self.dic : row = self.dic.get(key) + return row + + def getById(self,sid): + """Get the corresponding id in the index + :Returns: `str`|None key + """ rt = RuTime(eval(__CALLER__(sid))) l = None r = [k for i, k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN) and self.dic[k][self.UID] == int(sid)] if len(r)==1: l = r[0] rt.stop() return l + + def getByLabel(self,label): + """Get the corresponding label in the index + :Returns: `str`|None key + """ + rt = RuTime(eval(__CALLER__(sid))) + l = None + r = [k for i, k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN) and self.dic[k][self.LABEL] == int(label)] + if len(r)==1: l = r[0] + rt.stop() + return l - def searchByFileHash(self,md5): - """""" - e = None - if md5 in self.dic: - e = True - return e - - def searchByPattern(self,pattern): - """""" + def getByPattern(self,pattern): + """Get ids corresponding to label matching the pattern in the index + :Returns: `[uid]`|None matchIds + """ rt = RuTime(eval(__CALLER__(pattern))) l = None r = [ k for i,k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN) and regsearch(pattern,self.dic[k][self.LABEL]) is not None ] @@ -366,6 +422,8 @@ class ImpraStorage: def __init__(self, conf, remIndex=False, wkdir=None): """""" + from impra.util import DEBUG_INFO + rt = RuTime(eval(__CALLER__()),DEBUG_INFO) if wkdir == None : wkdir = abspath(join(dirname( __file__ ), '..', 'wk')) self.wkdir = wkdir self.conf = conf @@ -378,6 +436,7 @@ class ImpraStorage: self.delids = [] if remIndex : self.removeIndex() self.index = self.getIndex() + rt.stop() def _getIdIndex(self): """""" @@ -408,29 +467,39 @@ class ImpraStorage: encData = str(ms,'utf-8') return encData + def getIndexDefaultCatg(self): + """""" + usrName = self.conf.get('name','infos') + return {'catg':self.conf.get('types','catg'), 'users':{ ('%s' % self.mb.getHashName('all')) : 'all', ('%s' % self.mb.getHashName(usrName)) : usrName}} + def getIndex(self): """""" from impra.util import DEBUG, DEBUG_LEVEL, DEBUG_WARN, DEBUG_INFO rt = RuTime(eval(__CALLER__()),DEBUG_INFO) index = None encData = '' - uid = self.conf.get('uid' ,'index') - date = self.conf.get('date','index') - nid = self.conf.get('nid' ,'index') - if nid==None : nid = 0 - self._getIdIndex() - if self.idx : + uid = self.conf.get('uid' ,'index') + date = self.conf.get('date ','index') + nid = self.conf.get('nid' ,'index') + tstamp = self.conf.get('time' ,'index') + if nid is None : nid = 0 + + if tstamp is not None and (datetime.now() - datetime.strptime(tstamp[:-7], '%Y-%m-%d %H:%M:%S')) < timedelta(minutes = 1) : # getFromFile - if uid != None and int(self.idx) == int(uid) and file_exists(self.pathInd): + if uid != None and file_exists(self.pathInd): # int(self.idx) == int(uid) + self.idx = uid encData = get_file_content(self.pathInd) print('--\nindex in cache') - else: + else : + print('refresh index') + self._getIdIndex() + if self.idx : encData = self._getCryptIndex() with open(self.pathInd, mode='w', encoding='utf-8') as o: o.write(encData) - usrName = self.conf.get('name','infos') - usrHash = self.mb.getHashName(usrName) - index = ImpraIndex(self.conf.get('key','keys'),self.conf.get('mark','keys'), encData, {'catg':self.conf.get('types','catg'), 'users':{ ('%s' % self.mb.getHashName('all')) : 'all', ('%s' % usrHash) : usrName}}, int(nid)) + self.conf.set('time',str(datetime.now()),'index') + + index = ImpraIndex(self.conf.get('key','keys'),self.conf.get('mark','keys'), encData, self.getIndexDefaultCatg(), int(nid)) rt.stop() return index @@ -456,85 +525,126 @@ class ImpraStorage: self.conf.set('uid',ids[1],'index') self.conf.set('date',date,'index') with open(self.pathInd, mode='w', encoding='utf-8') as o: - o.write(encData) - self.ih.deleteBin() - #self.index = self.getIndex() + o.write(encData) + self.conf.set('time',str(datetime.now()),'index') + self.clean() rt.stop() - + + def encryptTextFile(self,path): + """""" + cdata = self.index.km.subenc(get_file_content(path)) + with open(self.fsplit.DIR_CACHE+'.~KirmahEnc', mode='w') as o: + o.write(cdata) + return self.fsplit.DIR_CACHE+'.~KirmahEnc' + + def decryptTextFile(self,path): + """""" + data = self.index.km.subdec(get_file_content(path)) + with open(path, mode='w') as o: + o.write(data) + def addFile(self, path, label, usr='all', catg=''): """""" from impra.util import DEBUG, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, DEBUG_INFO rt = RuTime(eval(__CALLER__('"%s","%s","%s"' % (path[:13]+'...',label,usr))),DEBUG_INFO) - - #~ hlst = self.fsplit.addFile(path,label) - #~ self.fsplit.deployFile(hlst,True) + _, ext = splitext(path) + try: md5 = hash_md5_file(path) - print('--\nmd5sum `%s` %s' % (path,md5)) - if not self.index.searchByFileHash(md5) : + print('--\nmd5sum `%s` %s' % (path,md5)) + if not self.index.get(md5) : + + if catg=='' : catg = self.index.getAutoCatg(ext) + + bFlag = ImpraIndex.FILE_BINARY + if not is_binary(path): + bFlag = ImpraIndex.FILE_CRYPT + path = self.encryptTextFile(path) + hlst = self.fsplit.addFile(path,md5) if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE : print(hlst['head']) for v in hlst['data']: print(v) - nameFrom = self.conf.get('name','infos') - self.index.addUser(nameFrom, self.mb.getHashName(nameFrom)) - - for row in hlst['data'] : - msg = self.mb.build(nameFrom,usr,hlst['head'][2],self.fsplit.DIR_OUTBOX+row[1]+'.ipr') - self.ih.send(msg.as_string(), self.rootBox) - remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr') - self.index.add(hlst['head'][3],label,hlst['head'][1],ext,self.mb.getHashName(usr),catg,md5) - self.saveIndex() - self.conf.set('nid', str(self.index.id),'index') + ownerHash = self.mb.getHashName(usr) + self.index.addUser(usr,ownerHash) + + cancel = False + sendIds = [] + test = True + for row in hlst['data'] : + msg = self.mb.build(self.conf.get('name','infos'),usr,hlst['head'][2],self.fsplit.DIR_OUTBOX+row[1]+'.ipr') + mid = self.ih.send(msg.as_string(), self.rootBox) + if mid is not None : + print('part %s sent as msg %s' % (row[0],mid[1])) + sendIds.append((mid[1],row)) + remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr') + else: + print('\n-- error occured when sending part : %s\n-- retrying' % row[0]) + mid = self.ih.send(msg.as_string(), self.rootBox) + if mid is not None : + print('part %s sent as msg %s' % (row[0],mid[1])) + sendIds.append((mid[1],row)) + remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr') + else: + print('\n-- can\'t send part %s\n-- cancelling ' % row[0]) + cancel = True + break + + if cancel : + for mid, row in sendIds : + self.ih.delete(mid, True) + if file_exists(self.fsplit.DIR_OUTBOX+row[1]+'.ipr') : remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr') + self.clean() + + else : + self.index.add(hlst['head'][3],label,hlst['head'][1],ext,ownerHash,catg,md5,bFlag) + self.saveIndex() + self.conf.set('nid', str(self.index.id),'index') else : print('--\nfile already exist on server as `%s` [id:%i]\n' % (self.index.dic[md5][ImpraIndex.LABEL],self.index.dic[md5][ImpraIndex.UID])) except Exception as e : print(e) rt.stop() - def getFile(self,label): + def getFile(self,key): """""" from impra.util import DEBUG, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, DEBUG_INFO - - rt = RuTime(eval(__CALLER__('"%s"' % label)),DEBUG_INFO) - if label==None : - print('--\n'+str(label)+' unexist') + row = self.index.get(key) + if row==None : + print('--\n%s not on the server' % key) else : - key = self.index.search(label) - if label!=None and key!=None: - ck = ConfigKey(key[ImpraIndex.HASH]) - count = int(key[ImpraIndex.PARTS]) - hlst = ck.getHashList(label,count,True) - ids = self._getIdsBySubject(hlst['head'][2]) - if len(ids) >= count: - status, resp = self.ih.srv.fetch(ids[0],'(BODY[HEADER.FIELDS (TO)])') - to = bstr(resp[0][1][4:-4]) - if to == self.mb.getHashName('all')+'@'+self.mb.DOMAIN_NAME or to == self.mb.getHashName(self.conf.ini.get('name',self.conf.profile+'.infos'))+'@'+self.mb.DOMAIN_NAME : - for mid in ids : - self.ih.downloadAttachment(mid,self.fsplit.DIR_INBOX) - if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE : - print(hlst['head']) - for v in hlst['data']: - print(v) - self.fsplit.deployFile(hlst, key[ImpraIndex.LABEL], key[ImpraIndex.EXT], key[ImpraIndex.CATG]) - else : - #raise Exception(label+' is private') - print('--\n'+label+' is private') + rt = RuTime(eval(__CALLER__('"[%i] %s"' % (row[ImpraIndex.UID],row[ImpraIndex.LABEL]))),DEBUG_INFO) + ck = ConfigKey(row[ImpraIndex.HASH]) + hlst = ck.getHashList(key,row[ImpraIndex.PARTS],True) + ids = self._getIdsBySubject(hlst['head'][2]) + if len(ids) >= row[ImpraIndex.PARTS]: + status, resp = self.ih.srv.fetch(ids[0],'(BODY[HEADER.FIELDS (TO)])') + to = bstr(resp[0][1][4:-4]) + if to == self.mb.getHashName('all')+'@'+self.mb.DOMAIN_NAME or to == self.mb.getHashName(self.conf.ini.get('name',self.conf.profile+'.infos'))+'@'+self.mb.DOMAIN_NAME : + for mid in ids : + self.ih.downloadAttachment(mid,self.fsplit.DIR_INBOX) + if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE : + print(hlst['head']) + for v in hlst['data']: + print(v) + path = self.fsplit.deployFile(hlst, row[ImpraIndex.LABEL], row[ImpraIndex.EXT], row[ImpraIndex.UID], row[ImpraIndex.CATG]) + if row[ImpraIndex.BFLAG] == ImpraIndex.FILE_CRYPT: + self.decryptTextFile(path) else : - #raise Exception(label+' : invalid count parts '+str(len(ids))+'/'+str(count)) - print('--\n'+label+' : invalid count parts '+str(len(ids))+'/'+str(count)) - else: - #raise Exception(str(label)+' not on the server') - print('--\n'+str(label)+' not on the server') - rt.stop() + print('--\n`%s` is private' % row[ImpraIndex.LABEL]) + else : + print('--\n`%s` invalid count parts %i/%i' %(row[ImpraIndex.LABEL],len(ids),row[ImpraIndex.PARTS])) + + rt.stop() def clean(self): """""" rt = RuTime(eval(__CALLER__())) - self.index = self.getIndex() + self.ih.deleteBin() + if file_exists(self.fsplit.DIR_CACHE+'.~KirmahEnc'):remove(self.fsplit.DIR_CACHE+'.~KirmahEnc') rt.stop() diff --git a/impra/crypt.py b/impra/crypt.py index 84c1a6f..87a2b73 100644 --- a/impra/crypt.py +++ b/impra/crypt.py @@ -26,7 +26,9 @@ # You should have received a copy of the GNU General Public License # along with ImpraStorage. If not, see . -from impra.util import RuTime, __CALLER__, stack, DEBUG +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ package crypt ~~ + from base64 import urlsafe_b64encode, b64decode from binascii import b2a_base64, a2b_base64 from hashlib import sha256, md5 @@ -35,10 +37,12 @@ from random import choice from os import urandom from time import sleep -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~ package crypt ~~ +from impra.util import RuTime, __CALLER__, stack, DEBUG +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ methods ~~ + def hash_sha256(data): """Get a sha256 hash of str `data` :Returns: `str` @@ -226,17 +230,11 @@ class Kirmah: psize = ceil(len(data)/cpart) cp = 0 for row in hlst['data']: - #~ print(row) - #~ print('ns:%i - dataLength:%i - dataEncLength:%i - ne:%i' % (row[2],len(data), len(dataEnc), row[3])) - #~ print(data[cp*psize:cp*psize+psize]) dataEnc += self.ck.noiser.getNoise(row[2],True)+data[cp*psize:cp*psize+psize]+self.ck.noiser.getNoise(row[3],True) cp += 1 dataEnc = str(b2a_base64(bytes(dataEnc,'utf-8')),'utf-8') - #~ dataEnc = self.subenc(odata) - #~ with open('./.KirmahENC', mode='w') as o: - #~ o.write(dataEnc) rt.stop() return dataEnc @@ -255,27 +253,17 @@ class Kirmah: for row in hlst['data']: si = ni + row[2] ei = si + psize - #~ print(row) - #~ print('si:%i - ei:%i - datalength:%i - dataDeclength:%i' % (si,ei,len(data),len(dataDec))) if cp == cpart-1 : ei = -row[3] if not si > len(data)+ei : pass - ### to delete #si = len(data)+ei - #~ print('si:%i - ei:%i' % (si,ei)) else : - #~ print('si: - ei:%i' % (len(data)+ei-si)) dataDec=dataDec[:len(data)+ei-si] break - #~ print(data[si:ei]) dataDec += data[si:ei] ni = ei + row[3] cp += 1 dataDec = self.subdec(dataDec) - - #~ dataDec = self.subdec(data) - - #~ with open('./.KirmahDEC', mode='w') as o: - #~ o.write(dataDec) + rt.stop() return dataDec diff --git a/impra/imap.py b/impra/imap.py index d5e5615..6639a2d 100755 --- a/impra/imap.py +++ b/impra/imap.py @@ -26,9 +26,9 @@ # You should have received a copy of the GNU General Public License # along with ImpraStorage. If not, see . -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~ package imap ~~ +from binascii import b2a_base64, a2b_base64 +from codecs import register, StreamReader, StreamWriter from email import message_from_bytes from email.header import decode_header from email.message import Message @@ -36,10 +36,12 @@ from imaplib import IMAP4_SSL, Time2Internaldate from os.path import join from re import search, split from time import time -from impra.util import __CALLER__, RuTime, bstr, stack -from binascii import b2a_base64, a2b_base64 -from codecs import register, StreamReader, StreamWriter +from impra.util import __CALLER__, RuTime, bstr, stack + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ package imap ~~ def _seq_encode(seq,l): """""" @@ -310,7 +312,7 @@ class ImapHelper: #~ print(mid) #status, resp = self.srv.store(mid, '+FLAGS', '\\Deleted') status, resp = self.srv.uid('store', mid, '+FLAGS', '\\Deleted' ) - print('deleting msg '+str(mid)) + print('deleting msg %i' % int(mid)) if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE: print(status) print(resp) @@ -349,12 +351,15 @@ class ImapHelper: def send(self, msg, box='INBOX'): """""" - from impra.util import DEBUG_INFO - rt = RuTime(eval(__CALLER__()),DEBUG_INFO) + from impra.util import DEBUG_INFO, DEBUG_LEVEL, DEBUG, DEBUG_NOTICE + rt = RuTime(eval(__CALLER__())) mid = None date = Time2Internaldate(time()) status, resp = self.srv.append(box, '\Draft', date, bytes(msg,'utf-8')) if status==self.OK: + if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE: + print(status) + print(resp) mid = str(resp[0],'utf-8')[11:-11].split(' ') rt.stop() return mid diff --git a/impra/util.py b/impra/util.py index 0254e92..64d07a3 100755 --- a/impra/util.py +++ b/impra/util.py @@ -29,17 +29,19 @@ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~ package util ~~ -from hashlib import sha256 -from math import log, floor, ceil -from random import choice -from os import urandom, popen, sep, makedirs -from os.path import dirname, realpath, abspath, join -from time import time -from re import split as regsplit from base64 import urlsafe_b64encode from inspect import stack +from errno import EEXIST +from hashlib import sha256 +from math import log, floor, ceil +from os import urandom, popen, sep, makedirs +from os.path import dirname, realpath, abspath, join +from random import choice +from re import split as regsplit from subprocess import PIPE, Popen from sys import stderr, executable as pyexec +#~ from sys.stdout import isatty +from time import time DEBUG_ALL = 0 DEBUG_WARN = 1 @@ -48,6 +50,9 @@ DEBUG_INFO = 3 DEBUG = True DEBUG_LEVEL = DEBUG_INFO + +COLOR_MODE = True + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~ methods ~~ @@ -75,7 +80,7 @@ def get_file_content(fileName): """Get file content of `fileName` :Returns: `str` """ - r = open(fileName, "rt") + r = open(fileName, 'rt') data = r.read() r.close() return data @@ -84,7 +89,7 @@ def get_file_binary(fileName): """Get file content of `fileName` :Returns: `str` """ - r = open(fileName, "rb") + r = open(fileName, 'rb') data = r.read() r.close() return data @@ -102,6 +107,20 @@ def randomFrom(val, sval=0): lst = list(range(sval,val)) return choice(lst) +def is_binary(filename): + """Check if given filename is binary.""" + done = False + fp = open(filename, 'rb') + try: + CHUNKSIZE = 1024 + while 1: + chunk = fp.read(CHUNKSIZE) + if b'\0' in chunk: done = True # found null byte + if done or len(chunk) < CHUNKSIZE: break + finally: + fp.close() + return done + def get_file_path(val): """""" return abspath(dirname(val))+sep @@ -120,7 +139,7 @@ def mkdir_p(path): try: makedirs(path) except OSError as e: # Python >2.5 - if e.errno == errno.EEXIST: + if e.errno == EEXIST: pass else: raise @@ -166,16 +185,40 @@ def __CALLER__(args=''): :Returns: `str` """ global DEBUG_LEVEL, DEBUG, DEBUG_WARN - #~ print(inspect.stack()[1][3]) - #~ print(print(args)) - #~ print('-----') - #~ print(inspect.stack()) - #~ print('---------------') val = "self.__class__.__name__+'.%s' % stack()[1][3]+'("+quote_escape(args)+") " if DEBUG and DEBUG_LEVEL<=DEBUG_WARN : val += "l:'+str(stack()[1][2])" else: val += "'" return val +def hilite(string, color=32, bold=True): + """""" + global COLOR_MODE + if COLOR_MODE and True: + attr = [color] + if bold: + attr.append('1') + + #~ print('\033[1;30mGray like Ghost\033[1;m') + #~ print('\033[1;31mRed like Radish\033[1;m') + #~ print('\033[1;32mGreen like Grass\033[1;m') + #~ print('\033[1;33mYellow like Yolk\033[1;m') + #~ print('\033[1;34mBlue like Blood\033[1;m') + #~ print('\033[1;35mMagenta like Mimosa\033[1;m') + #~ print('\033[1;36mCyan like Caribbean\033[1;m') + #~ print('\033[1;37mWhite like Whipped Cream\033[1;m') + #~ print('\033[1;38mCrimson like Chianti\033[1;m') + #~ print('\033[1;41mHighlighted Red like Radish\033[1;m') + #~ print('\033[1;42mHighlighted Green like Grass\033[1;m') + #~ print('\033[1;43mHighlighted Brown like Bear\033[1;m') + #~ print('\033[1;44mHighlighted Blue like Blood\033[1;m') + #~ print('\033[1;45mHighlighted Magenta like Mimosa\033[1;m') + #~ print('\033[1;46mHighlighted Cyan like Caribbean\033[1;m') + #~ print('\033[1;47mHighlighted Gray like Ghost\033[1;m') + #~ print('\033[1;48mHighlighted Crimson like Chianti\033[1;m') + + string = '\x1b[%sm%s\x1b[0m' % (';'.join(attr), string) + print(string) + return string # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~ class RuTime ~~ diff --git a/imprastorage.py b/imprastorage.py index e6a5ce6..8bcfd2d 100755 --- a/imprastorage.py +++ b/imprastorage.py @@ -32,7 +32,10 @@ from impra.cli import Cli if __name__ == '__main__': - - Cli(get_file_path(__file__ )) + #~ try : + + Cli(get_file_path(__file__ )) + #~ except Exception as e : + #~ print(e) #python -O -m compileall impra/*.py diff --git a/wk/.cache/.gitkeep b/wk/.cache/.gitkeep new file mode 100644 index 0000000..adfc128 --- /dev/null +++ b/wk/.cache/.gitkeep @@ -0,0 +1 @@ +# .gitkeep