refactoring, autocategory, check upload failed
This commit is contained in:
a-sansara
parent
58b93e7eb0
commit
502d3b65c0
88
impra/cli.py
88
impra/cli.py
|
|
@ -83,9 +83,8 @@ class Cli:
|
|||
\n\
|
||||
%prog conf [-D|-L| -K,-H {host},-U {user},-X {password},-P {port},\n\
|
||||
-B {boxname}] [-A profileName]\n\
|
||||
%prog data [-l |-a {file, label} |-g {label} |-G {id} |-s {pattern} |\n\
|
||||
-r {label} |-R {id}] [-c {catg}, -u {owner}, -b {boxname},\n\
|
||||
-o {outputdir}]',epilog="""
|
||||
%prog data [-l |-a {file, label} |-g {id} |-s {pattern} | -r {id}]\n\
|
||||
[-c {catg}, -u {owner}]', epilog="""
|
||||
|
||||
conf command Examples:
|
||||
|
||||
|
|
@ -104,8 +103,8 @@ conf command Examples:
|
|||
|
||||
data command Examples:
|
||||
|
||||
List index on a specified box (different from box on active profile)
|
||||
imprastorage data -lb boxname
|
||||
List index
|
||||
imprastorage data -l
|
||||
|
||||
Add file
|
||||
imprastorage data -a /path/tofile 'my video'
|
||||
|
|
@ -114,14 +113,11 @@ data command Examples:
|
|||
when downloading files)
|
||||
imprastorage data -a /path/tofile '2009 - en la playa' -c videos/perso/2009
|
||||
|
||||
Get file
|
||||
imprastorage data -g '2009 - en la playa'
|
||||
|
||||
Get file by id
|
||||
imprastorage data -G 22
|
||||
imprastorage data -g 22
|
||||
|
||||
Remove from server a file by id
|
||||
imprastorage data -R 22
|
||||
imprastorage data -g 22
|
||||
|
||||
Search files matching pattern :
|
||||
imprastorage data -s 'holydays'
|
||||
|
|
@ -137,20 +133,16 @@ data command Examples:
|
|||
# metavar='<ARG1> <ARG2>', nargs=2
|
||||
parser.add_option('-v', '--version' , help='show program\'s version number and exit' , action='store_true' , default=False)
|
||||
parser.add_option('-q', '--quiet' , help='don\'t print status messages to stdout' , action='store_false', default=True)
|
||||
parser.add_option('-f', '--force' , help='dont confirm and force action' , action='store_true' , default=False)
|
||||
parser.add_option('-d', '--debug' , help='set debug mode' , action='store_true' , default=False)
|
||||
|
||||
gpData.add_option('-l', '--list' , help='list index on imap server' , action='store_true' )
|
||||
gpData.add_option('-a', '--add' , help='add file FILE with specified LABEL on server' , action='store', metavar='FILE LABEL ', nargs=2)
|
||||
gpData.add_option('-g', '--get' , help='get file with specified LABEL from server' , action='store', metavar='LABEL ')
|
||||
gpData.add_option('-G', '--get-by-id' , help='get file with specified ID from server' , action='store', metavar='ID ')
|
||||
gpData.add_option('-g', '--get' , help='get file with specified ID from server' , action='store', metavar='ID ')
|
||||
gpData.add_option('-s', '--search' , help='search file with specified PATTERN' , action='store', metavar='PATTERN ')
|
||||
gpData.add_option('-r', '--remove' , help='remove FILE with specified LABEL from server' , action='store', metavar='LABEL ')
|
||||
gpData.add_option('-R', '--remove-by-id' , help='remove FILE with specified ID from server' , action='store', metavar='ID ')
|
||||
gpData.add_option('-b', '--boxname' , help='switch boxname on imap server' , action='store', metavar='BOXN ')
|
||||
gpData.add_option('-c', '--category' , help='set specified CATEGORY (crit. for opt. -l,-a or -s)' , action='store', metavar='CATG ' , default='none')
|
||||
gpData.add_option('-r', '--remove' , help='remove FILE with specified ID from server' , action='store', metavar='ID ')
|
||||
gpData.add_option('-c', '--category' , help='set specified CATEGORY (crit. for opt. -l,-a or -s)' , action='store', metavar='CATG ' , default='')
|
||||
gpData.add_option('-u', '--user' , help='set specified USER (crit. for opt. -l,-a or -s)' , action='store', metavar='OWNER ' , default='all')
|
||||
gpData.add_option('-o', '--output-dir' , help='set specified OUTPUT DIR (for opt. -l,-a,-d or -g)' , action='store', metavar='DIR ')
|
||||
#gpData.add_option('-o', '--output-dir' , help='set specified OUTPUT DIR (for opt. -l,-a,-d or -g)' , action='store', metavar='DIR ')
|
||||
parser.add_option_group(gpData)
|
||||
|
||||
gpConf.add_option('-L', '--list-conf' , help='list configuration' , action='store_true', default=False)
|
||||
|
|
@ -221,7 +213,7 @@ data command Examples:
|
|||
|
||||
o.active_profile = self.ini.get('profile')
|
||||
|
||||
if not o.list and not o.add and not o.get and not o.get_by_id and not o.search and not o.remove and not o.remove_by_id :
|
||||
if not o.list and not o.add and not o.get and not o.search and not o.remove :
|
||||
parser.error(' no options specified')
|
||||
else :
|
||||
|
||||
|
|
@ -247,51 +239,43 @@ you can remove index but all presents files on the box %s will be unrecoverable
|
|||
if o.list :
|
||||
uid = conf.get('uid' ,'index')
|
||||
date = conf.get('date','index')
|
||||
if uid == None : uid = 'EMPTY'
|
||||
if date == None : date = ''
|
||||
account = conf.get('user','imap')
|
||||
if impst.index != None:
|
||||
impst.index.print('-'*120+'\n -- INDEX(`'+uid+'`) boxname :`'+impst.rootBox+'` '+date+'\n'+'-'*120)
|
||||
#encData = impst.index.impraEncrypt(impst.index.toString())
|
||||
#~ dd = """coucou mon joli coeur :*:* je sais que je te saoule avec ça mais bon putain tu va te planter ou merde"""
|
||||
#~ dd = """01234567890123456789012345678901234567890123456789#"""
|
||||
#~
|
||||
#~
|
||||
#~ kg = crypt.KeyGen(256)
|
||||
#~ print('-- key --')
|
||||
#~ print(kg.key)
|
||||
#~ print('-- mark --')
|
||||
#~ print(kg.mark)
|
||||
#~ km = crypt.Kirmah(kg.key, kg.mark)
|
||||
#~ encData = km.encrypt(dd,'.index',22)
|
||||
#~ #print('*'+encData+'*')
|
||||
#~ decData = km.decrypt(encData,'.index',22)
|
||||
#~ print('*'+decData+'*')
|
||||
noData = impst.index.isEmpty()
|
||||
if uid == None or noData : uid = 'EMPTY'
|
||||
if date == None or noData : date = ''
|
||||
impst.index.print('-'*120+'\n -- ImpraStorage -- [account:'+account+'] [index:`'+uid+'`] [boxname:`'+impst.rootBox+'`] '+date+'\n'+'-'*120)
|
||||
|
||||
status, resp = impst.ih.srv.search(None, '(SUBJECT "%s")' % '584d15abeb71fbd92fa5861970088b32ebb1d2d6650cec6115a28b64877d70f2')
|
||||
ids = [m for m in resp[0].split()]
|
||||
for mid in ids :
|
||||
status, resp = impst.ih.srv.fetch(mid,'(UID RFC822.SIZE)')
|
||||
print(mid,status,resp)
|
||||
|
||||
elif o.add :
|
||||
impst.addFile(o.add[0],o.add[1],o.user,o.category)
|
||||
|
||||
elif o.get :
|
||||
impst.getFile(o.get)
|
||||
|
||||
elif o.get_by_id :
|
||||
label = impst.index.searchById(o.get_by_id)
|
||||
if label !=None :
|
||||
impst.getFile(label)
|
||||
else: print(o.get_by_id+' a is not valid id')
|
||||
ids = []
|
||||
for sid in o.get.split(',') :
|
||||
seq = sid.split('-')
|
||||
if len(seq)==2 : ids.extend(range(int(seq[0]),int(seq[1])+1))
|
||||
else: ids.append(sid)
|
||||
for sid in ids :
|
||||
key = impst.index.getById(str(sid))
|
||||
if key !=None : impst.getFile(key)
|
||||
else: print('-- `%s` is not a valid id --' % sid)
|
||||
|
||||
elif o.search :
|
||||
label = impst.index.searchByPattern(o.search)
|
||||
if label==None:
|
||||
print(' -- no match found for pattern `%s` --' % o.search)
|
||||
matchIds = impst.index.getByPattern(o.search)
|
||||
if matchIds is not None:
|
||||
headstr = util.hilite('-'*120+'\n -- SEARCH: `'+o.search+'` -- found '+str(len(matchIds))+' results --\n'+'-'*120)
|
||||
impst.index.print(headstr,matchIds)
|
||||
else:
|
||||
impst.index.print('-'*120+'\n -- SEARCH: `'+o.search+'` -- found '+str(len(label))+' results --\n'+'-'*120,label)
|
||||
print(' -- no match found for pattern `%s` --' % o.search)
|
||||
elif o.remove :
|
||||
print(o.remove)
|
||||
|
||||
elif o.remove_by_id :
|
||||
print(o.remove_by_id)
|
||||
|
||||
|
||||
#~ filePath = '/media/Data/dev/big_toph3.jpg'
|
||||
#~ lab = 'Meuf\'bonne aussi4'
|
||||
#~ print('\n -- ADD FILE -- ')
|
||||
|
|
|
|||
240
impra/core.py
240
impra/core.py
|
|
@ -31,6 +31,7 @@
|
|||
|
||||
from base64 import urlsafe_b64encode, b64decode
|
||||
from binascii import b2a_base64, a2b_base64
|
||||
from datetime import datetime, timedelta
|
||||
from email.encoders import encode_base64
|
||||
from email.header import Header
|
||||
from email.mime.base import MIMEBase
|
||||
|
|
@ -43,13 +44,12 @@ from mmap import mmap
|
|||
from os import remove, urandom, sep
|
||||
from os.path import abspath, dirname, join, realpath, basename, getsize, splitext
|
||||
from re import split as regsplit, match as regmatch, compile as regcompile, search as regsearch
|
||||
from time import time
|
||||
from impra.imap import ImapHelper, ImapConfig
|
||||
from impra.util import __CALLER__, RuTime, formatBytes, randomFrom, bstr, quote_escape, stack, run, file_exists, get_file_content, DEBUG, DEBUG_ALL, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, mkdir_p
|
||||
from impra.util import __CALLER__, RuTime, formatBytes, randomFrom, bstr, quote_escape, stack, run, file_exists, get_file_content, DEBUG, DEBUG_ALL, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, mkdir_p, is_binary
|
||||
from impra.crypt import Kirmah, ConfigKey, Noiser, Randomiz, hash_sha256, hash_md5_file
|
||||
|
||||
|
||||
|
||||
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# ~~ class FSplitter ~~
|
||||
|
||||
|
|
@ -60,6 +60,7 @@ class FSplitter :
|
|||
""""""
|
||||
self.ck = ck
|
||||
self.wkdir = wkdir
|
||||
self.DIR_CACHE = join(self.wkdir,'.cache')+sep
|
||||
self.DIR_INBOX = join(self.wkdir,'inbox')+sep
|
||||
self.DIR_OUTBOX = join(self.wkdir,'outbox')+sep
|
||||
self.DIR_DEPLOY = join(self.wkdir,'deploy')+sep
|
||||
|
|
@ -111,7 +112,7 @@ class FSplitter :
|
|||
|
||||
rt.stop()
|
||||
|
||||
def deployFile(self, hlst, fileName, ext='', dirs=None, fake=False):
|
||||
def deployFile(self, hlst, fileName, ext='', uid='', dirs=None, fake=False):
|
||||
""""""
|
||||
rt = RuTime(eval(__CALLER__()))
|
||||
p = 0
|
||||
|
|
@ -121,7 +122,14 @@ class FSplitter :
|
|||
dirPath = join(self.DIR_DEPLOY,dirs)+sep
|
||||
mkdir_p(dirPath)
|
||||
else: dirPath = self.DIR_DEPLOY
|
||||
fp = open(dirPath+fileName+ext, 'wb+')
|
||||
|
||||
filePath = dirPath+fileName
|
||||
if file_exists(filePath+ext):
|
||||
print('\n-- `%s` already exist, deploying file as :\n-- `%s`\n' % (filePath+ext,filePath+'-'+str(uid)+ext))
|
||||
filePath += '-'+str(uid)
|
||||
filePath += ext
|
||||
|
||||
fp = open(filePath, 'wb+')
|
||||
depDir = self.DIR_INBOX
|
||||
if fake : depDir = self.DIR_OUTBOX
|
||||
while p < hlst['head'][1] :
|
||||
|
|
@ -129,6 +137,7 @@ class FSplitter :
|
|||
p += 1
|
||||
fp.close()
|
||||
rt.stop()
|
||||
return dirPath+fileName+ext
|
||||
|
||||
def _mergePart(self,fp,part,phlst,depDir):
|
||||
""""""
|
||||
|
|
@ -228,6 +237,11 @@ class ImpraIndex:
|
|||
""""""
|
||||
UID = 6
|
||||
""""""
|
||||
BFLAG = 7
|
||||
""""""
|
||||
|
||||
FILE_BINARY = 'b'
|
||||
FILE_CRYPT = 'c'
|
||||
|
||||
|
||||
def __init__(self, key, mark, encdata='', dicCategory={}, id=0):
|
||||
|
|
@ -251,29 +265,30 @@ class ImpraIndex:
|
|||
if not self.SEP_KEY_INTERN+k in self.dic:
|
||||
self.dic[self.SEP_KEY_INTERN+k] = dicCategory[k]
|
||||
|
||||
def add(self,key, label, count, ext='', usr='', cat='', md5=''):
|
||||
"""Add an entry to the index with appropriate label, key used by entry
|
||||
to decode data, and parts count
|
||||
def add(self,key, label, count, ext='', usr='', cat='', md5='', bFlag='b'):
|
||||
"""Add an entry to the index
|
||||
"""
|
||||
if self.search(md5) == None :
|
||||
self.dic[md5] = (key,label,count,ext,usr,cat, self.id)
|
||||
if self.get(md5) == None :
|
||||
self.dic[md5] = (key,label,count,ext,usr,cat, self.id, bFlag)
|
||||
self.id +=1
|
||||
else :
|
||||
print(label+' already exist')
|
||||
|
||||
def addUser(self, nameFrom, hashName):
|
||||
""""""
|
||||
if not self.hasUser(hashName):
|
||||
self.dic[self.SEP_KEY_INTERN+'users'][hashName] = nameFrom
|
||||
|
||||
def hasUser(self, hashName):
|
||||
""""""
|
||||
if not self.SEP_KEY_INTERN+'users' in self.dic:
|
||||
self.dic[self.SEP_KEY_INTERN+'users'] = {}
|
||||
if not hashName in self.dic[self.SEP_KEY_INTERN+'users']:
|
||||
self.dic[self.SEP_KEY_INTERN+'users'][hashName] = nameFrom
|
||||
return hashName in self.dic[self.SEP_KEY_INTERN+'users']
|
||||
|
||||
def getUser(self, hashName):
|
||||
""""""
|
||||
usrName = 'Anonymous'
|
||||
if not str(self.SEP_KEY_INTERN+'users') in self.dic:
|
||||
self.dic[self.SEP_KEY_INTERN+'users'] = {}
|
||||
elif hashName in self.dic[self.SEP_KEY_INTERN+'users']:
|
||||
if self.hasUser(hashName):
|
||||
usrName = self.dic[self.SEP_KEY_INTERN+'users'][hashName]
|
||||
return usrName
|
||||
|
||||
|
|
@ -281,12 +296,47 @@ class ImpraIndex:
|
|||
"""Remove the selected label from the index"""
|
||||
self.dic.pop(label, None)
|
||||
|
||||
def search(self,label):
|
||||
"""Search the corresponding label in the index"""
|
||||
return self.dic.get(label)
|
||||
def getAutoCatg(self,ext):
|
||||
""""""
|
||||
catg = 'none'
|
||||
if regsearch('\.(jpg|jpeg|gif|png)',ext):
|
||||
catg = 'images'
|
||||
elif regsearch('\.(txt|doc|odt|csv|pdf)',ext):
|
||||
catg = 'doc'
|
||||
elif regsearch('\.(mp4|avi|mpg|mpeg|flv|ogv)',ext):
|
||||
catg = 'films'
|
||||
elif regsearch('\.(mp3|ogg|flac)',ext):
|
||||
catg = 'music'
|
||||
elif regsearch('\.(zip|7z|tar|gz|rar|bz|xz|jar)',ext):
|
||||
catg = 'archives'
|
||||
return catg
|
||||
|
||||
def searchById(self,sid):
|
||||
"""Search the corresponding label in the index"""
|
||||
def isEmpty(self):
|
||||
""""""
|
||||
r = [k for i, k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN)]
|
||||
return len(r) == 0
|
||||
|
||||
def getLabel(self, key):
|
||||
"""Get label corresponding to key in the index
|
||||
:Returns: `str`|None label
|
||||
"""
|
||||
value = ''
|
||||
row = self.get(key)
|
||||
if row is not None :
|
||||
value = row[self.LABEL]
|
||||
|
||||
def get(self, key):
|
||||
"""Get the corresponding key in the index
|
||||
:Returns: `tuple` row
|
||||
"""
|
||||
row = None
|
||||
if key in self.dic : row = self.dic.get(key)
|
||||
return row
|
||||
|
||||
def getById(self,sid):
|
||||
"""Get the corresponding id in the index
|
||||
:Returns: `str`|None key
|
||||
"""
|
||||
rt = RuTime(eval(__CALLER__(sid)))
|
||||
l = None
|
||||
r = [k for i, k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN) and self.dic[k][self.UID] == int(sid)]
|
||||
|
|
@ -294,15 +344,21 @@ class ImpraIndex:
|
|||
rt.stop()
|
||||
return l
|
||||
|
||||
def searchByFileHash(self,md5):
|
||||
""""""
|
||||
e = None
|
||||
if md5 in self.dic:
|
||||
e = True
|
||||
return e
|
||||
def getByLabel(self,label):
|
||||
"""Get the corresponding label in the index
|
||||
:Returns: `str`|None key
|
||||
"""
|
||||
rt = RuTime(eval(__CALLER__(sid)))
|
||||
l = None
|
||||
r = [k for i, k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN) and self.dic[k][self.LABEL] == int(label)]
|
||||
if len(r)==1: l = r[0]
|
||||
rt.stop()
|
||||
return l
|
||||
|
||||
def searchByPattern(self,pattern):
|
||||
""""""
|
||||
def getByPattern(self,pattern):
|
||||
"""Get ids corresponding to label matching the pattern in the index
|
||||
:Returns: `[uid]`|None matchIds
|
||||
"""
|
||||
rt = RuTime(eval(__CALLER__(pattern)))
|
||||
l = None
|
||||
r = [ k for i,k in enumerate(self.dic) if not k.startswith(self.SEP_KEY_INTERN) and regsearch(pattern,self.dic[k][self.LABEL]) is not None ]
|
||||
|
|
@ -366,6 +422,8 @@ class ImpraStorage:
|
|||
|
||||
def __init__(self, conf, remIndex=False, wkdir=None):
|
||||
""""""
|
||||
from impra.util import DEBUG_INFO
|
||||
rt = RuTime(eval(__CALLER__()),DEBUG_INFO)
|
||||
if wkdir == None : wkdir = abspath(join(dirname( __file__ ), '..', 'wk'))
|
||||
self.wkdir = wkdir
|
||||
self.conf = conf
|
||||
|
|
@ -378,6 +436,7 @@ class ImpraStorage:
|
|||
self.delids = []
|
||||
if remIndex : self.removeIndex()
|
||||
self.index = self.getIndex()
|
||||
rt.stop()
|
||||
|
||||
def _getIdIndex(self):
|
||||
""""""
|
||||
|
|
@ -408,6 +467,11 @@ class ImpraStorage:
|
|||
encData = str(ms,'utf-8')
|
||||
return encData
|
||||
|
||||
def getIndexDefaultCatg(self):
|
||||
""""""
|
||||
usrName = self.conf.get('name','infos')
|
||||
return {'catg':self.conf.get('types','catg'), 'users':{ ('%s' % self.mb.getHashName('all')) : 'all', ('%s' % self.mb.getHashName(usrName)) : usrName}}
|
||||
|
||||
def getIndex(self):
|
||||
""""""
|
||||
from impra.util import DEBUG, DEBUG_LEVEL, DEBUG_WARN, DEBUG_INFO
|
||||
|
|
@ -417,20 +481,25 @@ class ImpraStorage:
|
|||
uid = self.conf.get('uid' ,'index')
|
||||
date = self.conf.get('date ','index')
|
||||
nid = self.conf.get('nid' ,'index')
|
||||
if nid==None : nid = 0
|
||||
self._getIdIndex()
|
||||
if self.idx :
|
||||
tstamp = self.conf.get('time' ,'index')
|
||||
if nid is None : nid = 0
|
||||
|
||||
if tstamp is not None and (datetime.now() - datetime.strptime(tstamp[:-7], '%Y-%m-%d %H:%M:%S')) < timedelta(minutes = 1) :
|
||||
# getFromFile
|
||||
if uid != None and int(self.idx) == int(uid) and file_exists(self.pathInd):
|
||||
if uid != None and file_exists(self.pathInd): # int(self.idx) == int(uid)
|
||||
self.idx = uid
|
||||
encData = get_file_content(self.pathInd)
|
||||
print('--\nindex in cache')
|
||||
else :
|
||||
print('refresh index')
|
||||
self._getIdIndex()
|
||||
if self.idx :
|
||||
encData = self._getCryptIndex()
|
||||
with open(self.pathInd, mode='w', encoding='utf-8') as o:
|
||||
o.write(encData)
|
||||
usrName = self.conf.get('name','infos')
|
||||
usrHash = self.mb.getHashName(usrName)
|
||||
index = ImpraIndex(self.conf.get('key','keys'),self.conf.get('mark','keys'), encData, {'catg':self.conf.get('types','catg'), 'users':{ ('%s' % self.mb.getHashName('all')) : 'all', ('%s' % usrHash) : usrName}}, int(nid))
|
||||
self.conf.set('time',str(datetime.now()),'index')
|
||||
|
||||
index = ImpraIndex(self.conf.get('key','keys'),self.conf.get('mark','keys'), encData, self.getIndexDefaultCatg(), int(nid))
|
||||
rt.stop()
|
||||
return index
|
||||
|
||||
|
|
@ -457,36 +526,81 @@ class ImpraStorage:
|
|||
self.conf.set('date',date,'index')
|
||||
with open(self.pathInd, mode='w', encoding='utf-8') as o:
|
||||
o.write(encData)
|
||||
self.ih.deleteBin()
|
||||
#self.index = self.getIndex()
|
||||
self.conf.set('time',str(datetime.now()),'index')
|
||||
self.clean()
|
||||
rt.stop()
|
||||
|
||||
def encryptTextFile(self,path):
|
||||
""""""
|
||||
cdata = self.index.km.subenc(get_file_content(path))
|
||||
with open(self.fsplit.DIR_CACHE+'.~KirmahEnc', mode='w') as o:
|
||||
o.write(cdata)
|
||||
return self.fsplit.DIR_CACHE+'.~KirmahEnc'
|
||||
|
||||
def decryptTextFile(self,path):
|
||||
""""""
|
||||
data = self.index.km.subdec(get_file_content(path))
|
||||
with open(path, mode='w') as o:
|
||||
o.write(data)
|
||||
|
||||
def addFile(self, path, label, usr='all', catg=''):
|
||||
""""""
|
||||
from impra.util import DEBUG, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, DEBUG_INFO
|
||||
rt = RuTime(eval(__CALLER__('"%s","%s","%s"' % (path[:13]+'...',label,usr))),DEBUG_INFO)
|
||||
|
||||
#~ hlst = self.fsplit.addFile(path,label)
|
||||
#~ self.fsplit.deployFile(hlst,True)
|
||||
_, ext = splitext(path)
|
||||
|
||||
try:
|
||||
md5 = hash_md5_file(path)
|
||||
print('--\nmd5sum `%s` %s' % (path,md5))
|
||||
if not self.index.searchByFileHash(md5) :
|
||||
if not self.index.get(md5) :
|
||||
|
||||
if catg=='' : catg = self.index.getAutoCatg(ext)
|
||||
|
||||
bFlag = ImpraIndex.FILE_BINARY
|
||||
if not is_binary(path):
|
||||
bFlag = ImpraIndex.FILE_CRYPT
|
||||
path = self.encryptTextFile(path)
|
||||
|
||||
hlst = self.fsplit.addFile(path,md5)
|
||||
if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE :
|
||||
print(hlst['head'])
|
||||
for v in hlst['data']:
|
||||
print(v)
|
||||
nameFrom = self.conf.get('name','infos')
|
||||
self.index.addUser(nameFrom, self.mb.getHashName(nameFrom))
|
||||
|
||||
ownerHash = self.mb.getHashName(usr)
|
||||
self.index.addUser(usr,ownerHash)
|
||||
|
||||
cancel = False
|
||||
sendIds = []
|
||||
test = True
|
||||
for row in hlst['data'] :
|
||||
msg = self.mb.build(nameFrom,usr,hlst['head'][2],self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
|
||||
self.ih.send(msg.as_string(), self.rootBox)
|
||||
msg = self.mb.build(self.conf.get('name','infos'),usr,hlst['head'][2],self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
|
||||
mid = self.ih.send(msg.as_string(), self.rootBox)
|
||||
if mid is not None :
|
||||
print('part %s sent as msg %s' % (row[0],mid[1]))
|
||||
sendIds.append((mid[1],row))
|
||||
remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
|
||||
else:
|
||||
print('\n-- error occured when sending part : %s\n-- retrying' % row[0])
|
||||
mid = self.ih.send(msg.as_string(), self.rootBox)
|
||||
if mid is not None :
|
||||
print('part %s sent as msg %s' % (row[0],mid[1]))
|
||||
sendIds.append((mid[1],row))
|
||||
remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
|
||||
else:
|
||||
print('\n-- can\'t send part %s\n-- cancelling ' % row[0])
|
||||
cancel = True
|
||||
break
|
||||
|
||||
self.index.add(hlst['head'][3],label,hlst['head'][1],ext,self.mb.getHashName(usr),catg,md5)
|
||||
if cancel :
|
||||
for mid, row in sendIds :
|
||||
self.ih.delete(mid, True)
|
||||
if file_exists(self.fsplit.DIR_OUTBOX+row[1]+'.ipr') : remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr')
|
||||
self.clean()
|
||||
|
||||
else :
|
||||
self.index.add(hlst['head'][3],label,hlst['head'][1],ext,ownerHash,catg,md5,bFlag)
|
||||
self.saveIndex()
|
||||
self.conf.set('nid', str(self.index.id),'index')
|
||||
else :
|
||||
|
|
@ -495,21 +609,18 @@ class ImpraStorage:
|
|||
print(e)
|
||||
rt.stop()
|
||||
|
||||
def getFile(self,label):
|
||||
def getFile(self,key):
|
||||
""""""
|
||||
from impra.util import DEBUG, DEBUG_LEVEL, DEBUG_NOTICE, DEBUG_WARN, DEBUG_INFO
|
||||
|
||||
rt = RuTime(eval(__CALLER__('"%s"' % label)),DEBUG_INFO)
|
||||
if label==None :
|
||||
print('--\n'+str(label)+' unexist')
|
||||
row = self.index.get(key)
|
||||
if row==None :
|
||||
print('--\n%s not on the server' % key)
|
||||
else :
|
||||
key = self.index.search(label)
|
||||
if label!=None and key!=None:
|
||||
ck = ConfigKey(key[ImpraIndex.HASH])
|
||||
count = int(key[ImpraIndex.PARTS])
|
||||
hlst = ck.getHashList(label,count,True)
|
||||
rt = RuTime(eval(__CALLER__('"[%i] %s"' % (row[ImpraIndex.UID],row[ImpraIndex.LABEL]))),DEBUG_INFO)
|
||||
ck = ConfigKey(row[ImpraIndex.HASH])
|
||||
hlst = ck.getHashList(key,row[ImpraIndex.PARTS],True)
|
||||
ids = self._getIdsBySubject(hlst['head'][2])
|
||||
if len(ids) >= count:
|
||||
if len(ids) >= row[ImpraIndex.PARTS]:
|
||||
status, resp = self.ih.srv.fetch(ids[0],'(BODY[HEADER.FIELDS (TO)])')
|
||||
to = bstr(resp[0][1][4:-4])
|
||||
if to == self.mb.getHashName('all')+'@'+self.mb.DOMAIN_NAME or to == self.mb.getHashName(self.conf.ini.get('name',self.conf.profile+'.infos'))+'@'+self.mb.DOMAIN_NAME :
|
||||
|
|
@ -519,22 +630,21 @@ class ImpraStorage:
|
|||
print(hlst['head'])
|
||||
for v in hlst['data']:
|
||||
print(v)
|
||||
self.fsplit.deployFile(hlst, key[ImpraIndex.LABEL], key[ImpraIndex.EXT], key[ImpraIndex.CATG])
|
||||
path = self.fsplit.deployFile(hlst, row[ImpraIndex.LABEL], row[ImpraIndex.EXT], row[ImpraIndex.UID], row[ImpraIndex.CATG])
|
||||
if row[ImpraIndex.BFLAG] == ImpraIndex.FILE_CRYPT:
|
||||
self.decryptTextFile(path)
|
||||
else :
|
||||
#raise Exception(label+' is private')
|
||||
print('--\n'+label+' is private')
|
||||
print('--\n`%s` is private' % row[ImpraIndex.LABEL])
|
||||
else :
|
||||
#raise Exception(label+' : invalid count parts '+str(len(ids))+'/'+str(count))
|
||||
print('--\n'+label+' : invalid count parts '+str(len(ids))+'/'+str(count))
|
||||
else:
|
||||
#raise Exception(str(label)+' not on the server')
|
||||
print('--\n'+str(label)+' not on the server')
|
||||
print('--\n`%s` invalid count parts %i/%i' %(row[ImpraIndex.LABEL],len(ids),row[ImpraIndex.PARTS]))
|
||||
|
||||
rt.stop()
|
||||
|
||||
def clean(self):
|
||||
""""""
|
||||
rt = RuTime(eval(__CALLER__()))
|
||||
self.index = self.getIndex()
|
||||
self.ih.deleteBin()
|
||||
if file_exists(self.fsplit.DIR_CACHE+'.~KirmahEnc'):remove(self.fsplit.DIR_CACHE+'.~KirmahEnc')
|
||||
rt.stop()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,9 @@
|
|||
# You should have received a copy of the GNU General Public License
|
||||
# along with ImpraStorage. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from impra.util import RuTime, __CALLER__, stack, DEBUG
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# ~~ package crypt ~~
|
||||
|
||||
from base64 import urlsafe_b64encode, b64decode
|
||||
from binascii import b2a_base64, a2b_base64
|
||||
from hashlib import sha256, md5
|
||||
|
|
@ -35,10 +37,12 @@ from random import choice
|
|||
from os import urandom
|
||||
from time import sleep
|
||||
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# ~~ package crypt ~~
|
||||
from impra.util import RuTime, __CALLER__, stack, DEBUG
|
||||
|
||||
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# ~~ methods ~~
|
||||
|
||||
def hash_sha256(data):
|
||||
"""Get a sha256 hash of str `data`
|
||||
:Returns: `str`
|
||||
|
|
@ -226,17 +230,11 @@ class Kirmah:
|
|||
psize = ceil(len(data)/cpart)
|
||||
cp = 0
|
||||
for row in hlst['data']:
|
||||
#~ print(row)
|
||||
#~ print('ns:%i - dataLength:%i - dataEncLength:%i - ne:%i' % (row[2],len(data), len(dataEnc), row[3]))
|
||||
#~ print(data[cp*psize:cp*psize+psize])
|
||||
dataEnc += self.ck.noiser.getNoise(row[2],True)+data[cp*psize:cp*psize+psize]+self.ck.noiser.getNoise(row[3],True)
|
||||
cp += 1
|
||||
|
||||
dataEnc = str(b2a_base64(bytes(dataEnc,'utf-8')),'utf-8')
|
||||
|
||||
#~ dataEnc = self.subenc(odata)
|
||||
#~ with open('./.KirmahENC', mode='w') as o:
|
||||
#~ o.write(dataEnc)
|
||||
rt.stop()
|
||||
return dataEnc
|
||||
|
||||
|
|
@ -255,27 +253,17 @@ class Kirmah:
|
|||
for row in hlst['data']:
|
||||
si = ni + row[2]
|
||||
ei = si + psize
|
||||
#~ print(row)
|
||||
#~ print('si:%i - ei:%i - datalength:%i - dataDeclength:%i' % (si,ei,len(data),len(dataDec)))
|
||||
if cp == cpart-1 :
|
||||
ei = -row[3]
|
||||
if not si > len(data)+ei : pass
|
||||
### to delete #si = len(data)+ei
|
||||
#~ print('si:%i - ei:%i' % (si,ei))
|
||||
else :
|
||||
#~ print('si: - ei:%i' % (len(data)+ei-si))
|
||||
dataDec=dataDec[:len(data)+ei-si]
|
||||
break
|
||||
#~ print(data[si:ei])
|
||||
dataDec += data[si:ei]
|
||||
ni = ei + row[3]
|
||||
cp += 1
|
||||
dataDec = self.subdec(dataDec)
|
||||
|
||||
#~ dataDec = self.subdec(data)
|
||||
|
||||
#~ with open('./.KirmahDEC', mode='w') as o:
|
||||
#~ o.write(dataDec)
|
||||
rt.stop()
|
||||
return dataDec
|
||||
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@
|
|||
# You should have received a copy of the GNU General Public License
|
||||
# along with ImpraStorage. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# ~~ package imap ~~
|
||||
|
||||
from binascii import b2a_base64, a2b_base64
|
||||
from codecs import register, StreamReader, StreamWriter
|
||||
from email import message_from_bytes
|
||||
from email.header import decode_header
|
||||
from email.message import Message
|
||||
|
|
@ -36,10 +36,12 @@ from imaplib import IMAP4_SSL, Time2Internaldate
|
|||
from os.path import join
|
||||
from re import search, split
|
||||
from time import time
|
||||
from impra.util import __CALLER__, RuTime, bstr, stack
|
||||
from binascii import b2a_base64, a2b_base64
|
||||
from codecs import register, StreamReader, StreamWriter
|
||||
|
||||
from impra.util import __CALLER__, RuTime, bstr, stack
|
||||
|
||||
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# ~~ package imap ~~
|
||||
|
||||
def _seq_encode(seq,l):
|
||||
""""""
|
||||
|
|
@ -310,7 +312,7 @@ class ImapHelper:
|
|||
#~ print(mid)
|
||||
#status, resp = self.srv.store(mid, '+FLAGS', '\\Deleted')
|
||||
status, resp = self.srv.uid('store', mid, '+FLAGS', '\\Deleted' )
|
||||
print('deleting msg '+str(mid))
|
||||
print('deleting msg %i' % int(mid))
|
||||
if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE:
|
||||
print(status)
|
||||
print(resp)
|
||||
|
|
@ -349,12 +351,15 @@ class ImapHelper:
|
|||
|
||||
def send(self, msg, box='INBOX'):
|
||||
""""""
|
||||
from impra.util import DEBUG_INFO
|
||||
rt = RuTime(eval(__CALLER__()),DEBUG_INFO)
|
||||
from impra.util import DEBUG_INFO, DEBUG_LEVEL, DEBUG, DEBUG_NOTICE
|
||||
rt = RuTime(eval(__CALLER__()))
|
||||
mid = None
|
||||
date = Time2Internaldate(time())
|
||||
status, resp = self.srv.append(box, '\Draft', date, bytes(msg,'utf-8'))
|
||||
if status==self.OK:
|
||||
if DEBUG and DEBUG_LEVEL <= DEBUG_NOTICE:
|
||||
print(status)
|
||||
print(resp)
|
||||
mid = str(resp[0],'utf-8')[11:-11].split(' ')
|
||||
rt.stop()
|
||||
return mid
|
||||
|
|
|
|||
|
|
@ -29,17 +29,19 @@
|
|||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# ~~ package util ~~
|
||||
|
||||
from hashlib import sha256
|
||||
from math import log, floor, ceil
|
||||
from random import choice
|
||||
from os import urandom, popen, sep, makedirs
|
||||
from os.path import dirname, realpath, abspath, join
|
||||
from time import time
|
||||
from re import split as regsplit
|
||||
from base64 import urlsafe_b64encode
|
||||
from inspect import stack
|
||||
from errno import EEXIST
|
||||
from hashlib import sha256
|
||||
from math import log, floor, ceil
|
||||
from os import urandom, popen, sep, makedirs
|
||||
from os.path import dirname, realpath, abspath, join
|
||||
from random import choice
|
||||
from re import split as regsplit
|
||||
from subprocess import PIPE, Popen
|
||||
from sys import stderr, executable as pyexec
|
||||
#~ from sys.stdout import isatty
|
||||
from time import time
|
||||
|
||||
DEBUG_ALL = 0
|
||||
DEBUG_WARN = 1
|
||||
|
|
@ -48,6 +50,9 @@ DEBUG_INFO = 3
|
|||
|
||||
DEBUG = True
|
||||
DEBUG_LEVEL = DEBUG_INFO
|
||||
|
||||
COLOR_MODE = True
|
||||
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# ~~ methods ~~
|
||||
|
||||
|
|
@ -75,7 +80,7 @@ def get_file_content(fileName):
|
|||
"""Get file content of `fileName`
|
||||
:Returns: `str`
|
||||
"""
|
||||
r = open(fileName, "rt")
|
||||
r = open(fileName, 'rt')
|
||||
data = r.read()
|
||||
r.close()
|
||||
return data
|
||||
|
|
@ -84,7 +89,7 @@ def get_file_binary(fileName):
|
|||
"""Get file content of `fileName`
|
||||
:Returns: `str`
|
||||
"""
|
||||
r = open(fileName, "rb")
|
||||
r = open(fileName, 'rb')
|
||||
data = r.read()
|
||||
r.close()
|
||||
return data
|
||||
|
|
@ -102,6 +107,20 @@ def randomFrom(val, sval=0):
|
|||
lst = list(range(sval,val))
|
||||
return choice(lst)
|
||||
|
||||
def is_binary(filename):
|
||||
"""Check if given filename is binary."""
|
||||
done = False
|
||||
fp = open(filename, 'rb')
|
||||
try:
|
||||
CHUNKSIZE = 1024
|
||||
while 1:
|
||||
chunk = fp.read(CHUNKSIZE)
|
||||
if b'\0' in chunk: done = True # found null byte
|
||||
if done or len(chunk) < CHUNKSIZE: break
|
||||
finally:
|
||||
fp.close()
|
||||
return done
|
||||
|
||||
def get_file_path(val):
|
||||
""""""
|
||||
return abspath(dirname(val))+sep
|
||||
|
|
@ -120,7 +139,7 @@ def mkdir_p(path):
|
|||
try:
|
||||
makedirs(path)
|
||||
except OSError as e: # Python >2.5
|
||||
if e.errno == errno.EEXIST:
|
||||
if e.errno == EEXIST:
|
||||
pass
|
||||
else: raise
|
||||
|
||||
|
|
@ -166,16 +185,40 @@ def __CALLER__(args=''):
|
|||
:Returns: `str`
|
||||
"""
|
||||
global DEBUG_LEVEL, DEBUG, DEBUG_WARN
|
||||
#~ print(inspect.stack()[1][3])
|
||||
#~ print(print(args))
|
||||
#~ print('-----')
|
||||
#~ print(inspect.stack())
|
||||
#~ print('---------------')
|
||||
val = "self.__class__.__name__+'.%s' % stack()[1][3]+'("+quote_escape(args)+") "
|
||||
if DEBUG and DEBUG_LEVEL<=DEBUG_WARN : val += "l:'+str(stack()[1][2])"
|
||||
else: val += "'"
|
||||
return val
|
||||
|
||||
def hilite(string, color=32, bold=True):
|
||||
""""""
|
||||
global COLOR_MODE
|
||||
if COLOR_MODE and True:
|
||||
attr = [color]
|
||||
if bold:
|
||||
attr.append('1')
|
||||
|
||||
#~ print('\033[1;30mGray like Ghost\033[1;m')
|
||||
#~ print('\033[1;31mRed like Radish\033[1;m')
|
||||
#~ print('\033[1;32mGreen like Grass\033[1;m')
|
||||
#~ print('\033[1;33mYellow like Yolk\033[1;m')
|
||||
#~ print('\033[1;34mBlue like Blood\033[1;m')
|
||||
#~ print('\033[1;35mMagenta like Mimosa\033[1;m')
|
||||
#~ print('\033[1;36mCyan like Caribbean\033[1;m')
|
||||
#~ print('\033[1;37mWhite like Whipped Cream\033[1;m')
|
||||
#~ print('\033[1;38mCrimson like Chianti\033[1;m')
|
||||
#~ print('\033[1;41mHighlighted Red like Radish\033[1;m')
|
||||
#~ print('\033[1;42mHighlighted Green like Grass\033[1;m')
|
||||
#~ print('\033[1;43mHighlighted Brown like Bear\033[1;m')
|
||||
#~ print('\033[1;44mHighlighted Blue like Blood\033[1;m')
|
||||
#~ print('\033[1;45mHighlighted Magenta like Mimosa\033[1;m')
|
||||
#~ print('\033[1;46mHighlighted Cyan like Caribbean\033[1;m')
|
||||
#~ print('\033[1;47mHighlighted Gray like Ghost\033[1;m')
|
||||
#~ print('\033[1;48mHighlighted Crimson like Chianti\033[1;m')
|
||||
|
||||
string = '\x1b[%sm%s\x1b[0m' % (';'.join(attr), string)
|
||||
print(string)
|
||||
return string
|
||||
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# ~~ class RuTime ~~
|
||||
|
|
|
|||
|
|
@ -32,7 +32,10 @@ from impra.cli import Cli
|
|||
|
||||
|
||||
if __name__ == '__main__':
|
||||
#~ try :
|
||||
|
||||
Cli(get_file_path(__file__ ))
|
||||
#~ except Exception as e :
|
||||
#~ print(e)
|
||||
|
||||
#python -O -m compileall impra/*.py
|
||||
|
|
|
|||
1
wk/.cache/.gitkeep
Normal file
1
wk/.cache/.gitkeep
Normal file
|
|
@ -0,0 +1 @@
|
|||
# .gitkeep
|
||||
Loading…
Reference in New Issue
Block a user