commit ef8cf429e51f18102ae38f70e0d0a8ee8b26afb6 Author: a-Sansara Date: Tue Sep 11 00:12:39 2012 +0200 Initial commit diff --git a/.impra_id_rsa.pub b/.impra_id_rsa.pub new file mode 100644 index 0000000..01142e1 --- /dev/null +++ b/.impra_id_rsa.pub @@ -0,0 +1 @@ +AAAAgKoUbEl0Uwdm4vPLvxFxdSkpm62QHsxV9eUgAbj5F8ctRqfQPCh654MGe0M9w15RqHLuiehsjv7A7r45NAyNUPC1b9bcoAtwMlrFocYvTUtNSFbw0moTQOnYeoulK2Hp7mRjxZ+jDzVu3D/5BMMm40x4KUmljFghGtV8oZd9WNvpAAAAAwEAAQ== \ No newline at end of file diff --git a/impra/__init__.py b/impra/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/impra/cli.py b/impra/cli.py new file mode 100644 index 0000000..e1a0e7c --- /dev/null +++ b/impra/cli.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# software : ImpraStorage # +# version : 0.4 # +# date : 2012 # +# licence : GPLv3.0 # +# author : a-Sansara # +# copyright : pluie.org # +# # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# +# This file is part of ImpraStorage. +# +# ImpraStorage is free software (free as in speech) : you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. +# +# ImpraStorage is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License +# along with ImpraStorage. If not, see . + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ package cli ~~ + +from optparse import OptionParser, OptionGroup +desc="""version : 0.41 copyright : pluie.org +author : a-Sansara license : GNU GPLv3 + +ImpraStorage provided a private imap access to store large files. +Each file stored on the server is split in severals random parts. +Each part also contains random noise data (lenght depends on a crypt key) +to ensure privacy and exclude easy merge without the corresponding key. + +An index of files stored is encrypt (rsa 1024) and regularly updated. +Once decrypt, it permit to perform search on the server and +download each part. + +transfert process is transparent. Just vizualize locally the index of +stored files and simply select files to download or upload. +ImpraStorage automatically launch the parts to download, then merge parts +in the appropriate way to rebuild the original file. Inversely, a file +to upload is splitt -in several parts with addition of noise data), and +ImpraStorage randomly upload each parts then update the index. + +""" + + +class _SimplerOptionParser(OptionParser): + """A simplified OptionParser""" + + def format_description(self, formatter): + return self.description + + def format_epilog(self, formatter): + return self.epilog + +parser = _SimplerOptionParser(prog='imprastorage', usage='\n\n %prog COMMAND [OPTION]...',epilog=""" + +conf command Examples: + + Initialize program and set config on default profile with keys generation : + imprastorage conf -K -H imap.gmail.com -P 993 -U login -X password + + Set config on a new profile with same keys from previous active profile: + imprastorage conf -A profile1 -H myimapserver.net -P 993 -U login \\ + -X password -B boxname + + Load config from a profile (wich become active) : + imprastorage conf -DA profile2 + + List config from profile : + imprastorage conf -LA profile1 + + +data command Examples: + + List index on a specified box (different from box on active profile) + imprastorage data -lb boxname + + Add file + imprastorage data -a /path/tofile 'my video' + + Add file with category (category is also a directory structure recreate + when downloading files) + imprastorage data -a /path/tofile '2009 - en la playa' -c videos/perso/2009 + + Get file + imprastorage data -g '2009 - en la playa' + + Get file by id + imprastorage data -G 22 + + Remove from server a file by id + imprastorage data -R 22 + + Search files matching pattern : + imprastorage data -s 'holydays' + + Search files upload by a particular user on a category : + imprastorage data -s * -c films -u myfriend + +""",description=desc) + +gpData = OptionGroup(parser, '\ndata related Options (command data)\n-----------------------------------') +gpConf = OptionGroup(parser, '\nconf related Options (command conf)\n-----------------------------------') + +# metavar=' ', nargs=2 +parser.add_option('-v', '--version' , help='show program\'s version number and exit' , dest='version' , action='store_true' , default=False) +parser.add_option('-q', '--quiet' , help='don\'t print status messages to stdout' , dest='verbose' , action='store_false', default=True) +parser.add_option('-f', '--force' , help='dont confirm and force action' , dest='force' , action='store_true' , default=False) +parser.add_option('-d', '--debug' , help='set debug mode' , dest='debug' , action='store_true' , default=False) + +gpData.add_option('-l', '--list' , help='list index on imap server' , dest='list_index' , action='store_true' , default=False) +gpData.add_option('-b', '--boxname' , help='switch boxname on imap server' , dest='switch_boxname' , action='store', metavar='BOXN') +gpData.add_option('-a', '--add' , help='add file FILE with specified LABEL on server' , dest='add' , action='store', metavar='FILE LABEL', nargs=2) +gpData.add_option('-g', '--get' , help='get file with specified LABEL from server' , dest='get' , action='store', metavar='LABEL') +gpData.add_option('-G', '--get-by-id' , help='get file with specified ID from server' , dest='get_by_id' , action='store', metavar='ID') +gpData.add_option('-s', '--search' , help='search file with specified PATTERN' , dest='search' , action='store', metavar='PATTERN') +gpData.add_option('-c', '--category' , help='set specified CATEGORY (crit. for opt. -l,-a or -s)' , dest='category' , action='store', metavar='CATG' , default='none') +gpData.add_option('-u', '--user' , help='set specified USER (crit. for opt. -l,-a or -s)' , dest='owner' , action='store', metavar='OWNER' , default='all') +gpData.add_option('-o', '--output-dir' , help='set specified OUTPUT DIR (for opt. -l,-a,-d or -g)' , dest='output' , action='store', metavar='DIR') +gpData.add_option('-r', '--remove' , help='remove FILE with specified LABEL from server' , dest='remove' , action='store', metavar='LABEL') +gpData.add_option('-R', '--remove-by-id' , help='remove FILE with specified ID from server' , dest='remove_by_id' , action='store', metavar='ID') +parser.add_option_group(gpData) + +gpConf.add_option('-L', '--list-conf' , help='list configuration' , dest='list_conf' , action='store') +gpConf.add_option('-A', '--active-profile', help='set active profile' , dest='profile' , action='store', metavar='PROFILE', default='default') +gpConf.add_option('-H', '--set-host' , help='set imap host server' , dest='host' , action='store', metavar='HOST') +gpConf.add_option('-U', '--set-user' , help='set imap user login' , dest='user' , action='store', metavar='USER') +gpConf.add_option('-X', '--set-pass' , help='set imap user password' , dest='password' , action='store', metavar='PASS') +gpConf.add_option('-P', '--set-port' , help='set imap port (default:[%default])' , dest='port' , action='store', metavar='PORT' , default=993) +gpConf.add_option('-B', '--set-boxname' , help='set boxName on imap server (default:[%default])' , dest='boxname' , action='store', metavar='BOXN' , default='__IMPRA') +gpConf.add_option('-K', '--gen-keys' , help='generate new pub/private keys' , dest='generate_keys' , action='store_true', default=False) +gpConf.add_option('-D', '--load-conf' , help='load configuration' , dest='load_conf' , action='store_true', default=False) +parser.add_option_group(gpConf) + +def show_index(): + print('show_index') + +(opts, args) = parser.parse_args() +#~ if not 'toto' in opts.__dict__ : + #~ print("mandatory option is missing\n") + #~ parser.print_help() + #~ exit(-1)*/ +#~ print('--------') +#~ print(opts) +#~ print('--------') +#~ print(args) + diff --git a/impra/core.py b/impra/core.py new file mode 100644 index 0000000..79ee39d --- /dev/null +++ b/impra/core.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# software : ImpraStorage # +# version : 0.4 # +# date : 2012 # +# licence : GPLv3.0 # +# author : a-Sansara # +# copyright : pluie.org # +# # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# +# This file is part of ImpraStorage. +# +# ImpraStorage is free software (free as in speech) : you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. +# +# ImpraStorage is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License +# along with ImpraStorage. If not, see . + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ package core ~~ + +import inspect +from base64 import urlsafe_b64encode +from email.encoders import encode_base64 +from email.header import Header +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import formatdate +from math import ceil, floor +from mmap import mmap +from os import remove, urandom, sep +from os.path import abspath, dirname, join, realpath, basename, getsize, splitext +from re import split as regsplit +from impra.imap import ImapHelper, ImapConfig +from impra.util import __CALLER__, Rsa, RuTime, Noiser, Randomiz, RuTime, hash_sha256, formatBytes, randomFrom, bstr, quote_escape, stack + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class ConfigKey ~~ + +class ConfigKey: + """""" + + def __init__(self, key=None, psize=19710000): + """""" + if key : self.key = bytes(key,'utf-8') + else : self.key = self._build() + self.psize = psize + self.salt = str(self.key[::-4]) + self.noiser = Noiser(self.key) + self.rdmz = Randomiz(1) + + def getHashList(self,name,count,noSorted=False): + """""" + rt = RuTime('getHashList') + rt = RuTime(eval(__CALLER__('"%s",%s,%i' % (name,count,noSorted)))) + self.rdmz.new(count) + dic, lst, hroot = {}, [], hash_sha256(self.salt+name) + for i in range(count) : + self.noiser.build(i) + d = str(i).rjust(2,'0') + # part n°, hash, lns, lne, pos + hpart = hash_sha256(self.salt+name+'.part'+d)[:-3]+str(ord(hroot[i])).rjust(3,'0') + lst.append((d, hpart, self.noiser.lns, self.noiser.lne, self.rdmz.get())) + dic['head'] = (name,count,hroot,self.getKey()) + if not noSorted : + lst = sorted(lst, key=lambda lst: lst[4]) + dic['data'] = lst + rt.stop() + return dic + + def _build(self,l=48): + """""" + return urlsafe_b64encode(urandom(l)) + + def getKey(self): + """""" + return str(self.key,'utf-8') + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class FSplitter ~~ + +class FSplitter : + """""" + + def __init__(self, ck, wkdir='./'): + """""" + self.ck = ck + self.wkdir = wkdir + self.DIR_CACHE = self.wkdir+sep+'cache'+sep + self.DIR_INBOX = self.wkdir+sep+'inbox'+sep + self.DIR_OUTBOX = self.wkdir+sep+'outbox'+sep + self.DIR_DEPLOY = self.wkdir+sep+'deploy'+sep + + def addFile(self, fromPath, label): + """""" + rt = RuTime(eval(__CALLER__())) + fsize = getsize(fromPath) + count = ceil(fsize/self.ck.psize) + minp, maxp = 52, 62 + if fsize < 4800000 : minp, maxp = 8, 12 + elif fsize < 22200000 : minp, maxp = 12, 22 + elif fsize < 48000000 : minp, maxp = 22, 32 + elif fsize < 222000000 : minp, maxp = 32, 42 + if count < minp : count = randomFrom(maxp,minp) + if not count > 62 : + hlst = self._split(fromPath, self.ck.getHashList(label,count, True)) + else : + raise Exception(fromPath+' size exceeds limits (max : '+formatBytes(self.ck.psize*62)+' ['+str(self.ck.psize*64)+' bytes])') + rt.stop() + return hlst + + def _split(self, fromPath, hlst): + """""" + rt = RuTime(eval(__CALLER__())) + f = open(fromPath, 'rb+') + m = mmap(f.fileno(), 0) + p = 0 + psize = ceil(getsize(fromPath)/hlst['head'][1]) + while m.tell() < m.size(): + self._splitPart(m,p,psize,hlst['data'][p]) + p += 1 + m.close() + hlst['data'] = sorted(hlst['data'], key=lambda lst: lst[4]) + rt.stop() + return hlst + + def _splitPart(self,mmap,part,size,phlst): + """""" + rt = RuTime(eval(__CALLER__('mmap,%s,%s,phlist' % (part,size)))) + with open(self.DIR_OUTBOX+phlst[1]+'.ipr', mode='wb') as o: + o.write(self.ck.noiser.getNoise(phlst[2])+mmap.read(size)+self.ck.noiser.getNoise(phlst[3])) + rt.stop() + + def deployFile(self, hlst, ext='', fake=False): + """""" + rt = RuTime(eval(__CALLER__())) + p = 0 + hlst['data'] = sorted(hlst['data'], key=lambda lst: lst[0]) + fp = open(self.DIR_DEPLOY+hlst['head'][0]+ext, 'wb+') + depDir = self.DIR_INBOX + if fake : depDir = self.DIR_OUTBOX + while p < hlst['head'][1] : + self._mergePart(fp,p,hlst['data'][p],depDir) + p += 1 + fp.close() + rt.stop() + + def _mergePart(self,fp,part,phlst,depDir): + """""" + rt = RuTime(eval(__CALLER__('fp,%s,phlist,depDir' % part))) + with open(depDir+phlst[1]+'.ipr', mode='rb') as o: + fp.write(o.read()[phlst[2]:-phlst[3]]) + o.close() + remove(depDir+phlst[1]+'.ipr') + rt.stop() + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class ImpraConf ~~ + +class ImpraConf: + """""" + + SEP_SECTION = '.' + """""" + + def __init__(self, iniFile, profile='default'): + """""" + self.profile = profile + self.ini = iniFile + save = False + if self.ini.isEmpty(): + save = True + rsa = Rsa() + self.set('host','imap.gmail.com','imap') + self.set('port','993','imap') + self.set('user','login','imap') + self.set('pass','**********','imap') + self.set('box' ,'__SMILF','imap') + self.set('pubKey',rsa.pubKey,'keys') + self.set('prvKey',rsa.prvKey,'keys') + self.set('salt' ,'-¤-ImpraStorage-¤-','keys') + if not self.ini.hasSection(self.profile+self.SEP_SECTION+'catg'): + save = True + self.set('users', self.get('name','infos'),'catg') + self.set('types', 'music,films,doc,images,archives,games','catg') + if save : + self.ini.write() + print(self.ini.toString()) + + def get(self, key, section='main', profile=None): + """""" + if profile == None : profile = self.profile + return self.ini.get(key, profile+self.SEP_SECTION+section) + + def set(self, key, value, section='main', profile=None): + """""" + if profile == None : profile = self.profile + return self.ini.set(key, value, profile+self.SEP_SECTION+section) + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class ImpraIndex ~~ + +class ImpraIndex: + """A representation of the index stored on the server""" + + SEP_ITEM = ';' + """Separator used for entry""" + + SEP_TOKEN = '#' + """Separator used for token""" + + SEP_CATEGORY = '¤' + """Separator used for category section""" + + QUOTE_REPL = '-§' + """Char replacement of simple quote String""" + + SEP_KEY_INTERN = '@' + """Separator used for internal key such categories""" + + + def __init__(self, rsa, encdata='', dicCategory={}): + """Initialize the index with rsa and encoded data + + :Parameters: + `rsa` : impra.Rsa + Rsa instance initialized with appropriate private and public + keys to decrypt/encrypt data + `encdata` : str + initial content of the index encrypted with rsa + """ + self.rsa = rsa + self.dic = {} + if encdata =='' : data = encdata + else : data = self.rsa.decrypt(encdata) + data = data.replace(self.QUOTE_REPL, '\'') + ld = regsplit('\n? ?'+self.SEP_CATEGORY+' ?\n?',data) + l = regsplit(self.SEP_ITEM,ld[0]) + for row in l: + d = regsplit(self.SEP_TOKEN,row) + # key : count, hash, ext, usr, cat + if len(d)>4 and d!='': self.dic[d[1]] = d + if len(ld)>1: + l = regsplit(self.SEP_ITEM,ld[1].lstrip('\n')) + for row in l: + d = regsplit(' ?= ?',row,1) + if len(d)> 1 and len(d[0]) > 3 : + self.dic[d[0]] = d[1] + else: + for k in dicCategory : + self.dic[self.SEP_KEY_INTERN+k] = dicCategory[k] + + def add(self,key, label, count, ext='', usr='', cat=''): + """Add an entry to the index with appropriate label, key used by entry + to decode data, and parts count + """ + if self.search(label) == None : + self.dic[label] = (key,label,count,ext,usr,cat) + else : + print(label+' already exist') + + def rem(self,label): + """Remove the selected label from the index""" + self.dic.pop(label, None) + + def search(self,label): + """Search the corresponding label in the index""" + return self.dic.get(label) + + def toString(self): + """Make a string representation of the index as it was store on the server""" + data = cdata = '' + for k in sorted(self.dic): + v = self.dic.get(k) + if k[0]==self.SEP_KEY_INTERN and len(k)>1: + cdata += k+'='+v+self.SEP_ITEM + else : + for i in v: data += str(i)+self.SEP_TOKEN + data = data.rstrip(self.SEP_TOKEN)+self.SEP_ITEM + return data+self.SEP_CATEGORY+'\n'+cdata; + + def encrypt(self): + """""" + return self.rsa.encrypt(self.toString().replace('\'', self.QUOTE_REPL)) + + def print(self): + """Print index content as formated bloc""" + data = self.toString().split(';') + for row in data: + if row.rstrip('\n') != '': print(row) + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class ImpraStorage ~~ + +class ImpraStorage: + """""" + + def __init__(self, rsa, conf, wkdir=None): + """""" + if wkdir == None : wkdir = abspath(join(dirname( __file__ ), '..', 'wk')) + self.wkdir = wkdir + self.conf = conf + self.rootBox = self.conf.get('box','imap') + iconf = ImapConfig(self.conf.get('host','imap'), self.conf.get('port','imap'), self.conf.get('user', 'imap'), self.conf.get('pass', 'imap')) + self.ih = ImapHelper(iconf,self.rootBox) + self.mb = MailBuilder(self.conf.get('salt','keys')) + self.rsa = rsa + self.fsplit = FSplitter(ConfigKey(),self.wkdir) + self.delids = [] + self.index = self.getIndex() + + def _getIdIndex(self): + """""" + mid = None + status, resp = self.ih.srv.search(None, '(SUBJECT "%s")' % self.mb.getHashName('index')) + ids = [m for m in resp[0].split()] + if len(ids) > 0 and int(ids[0]) >= 0 : + mid = ids[len(ids)-1] + for i in ids: + if i != mid : self.delids.append(i) + self.idx = mid + return mid + + def _getIdsBySubject(self,subject): + """""" + status, resp = self.ih.srv.search(None, '(SUBJECT "%s")' % subject) + ids = [m for m in resp[0].split()] + return ids + + def getIndex(self): + """""" + self._getIdIndex() + if self.idx : + msgIndex = self.ih.email(self.idx) + for i in self.delids : self.ih.delete(i) + for part in msgIndex.walk(): + ms = part.get_payload(decode=True) + encData = str(ms,'utf-8') + else : + encData = '' + self.ih.deleteBin() + return ImpraIndex(self.rsa,encData, {'catg':self.conf.get('types','catg')}) + + def saveIndex(self): + """""" + rt = RuTime(eval(__CALLER__())) + if self.idx != None : + self.ih.delete(self.idx) + encData = self.index.encrypt() + msgIndex = self.mb.buildIndex(encData) + print(msgIndex.as_string()) + self.ih.send(msgIndex.as_string(), self.rootBox) + #self.index = self.getIndex() + rt.stop() + + def addFile(self, path, label, usr='all', catg=''): + """""" + rt = RuTime(eval(__CALLER__('"%s","%s","%s"' % (path[:13]+'...',label,usr)))) + + #~ hlst = self.fsplit.addFile(path,label) + #~ self.fsplit.deployFile(hlst,True) + _, ext = splitext(path) + try: + if self.index.search(label)==None : + hlst = self.fsplit.addFile(path,label) + print(hlst['head']) + for v in hlst['data']: + print(v) + nameFrom = self.conf.ini.get('name',self.conf.profile+'.infos') + for row in hlst['data'] : + msg = self.mb.build(nameFrom,usr,hlst['head'][2],self.fsplit.DIR_OUTBOX+row[1]+'.ipr') + self.ih.send(msg.as_string(), self.rootBox) + remove(self.fsplit.DIR_OUTBOX+row[1]+'.ipr') + self.index.add(hlst['head'][3],hlst['head'][0],hlst['head'][1],ext,self.mb.getHashName(usr),catg) + self.saveIndex() + else : + raise Exception(label + ' already exist on server') + except Exception as e : + print(e) + rt.stop() + + def getFile(self,label): + """""" + rt = RuTime(eval(__CALLER__('"%s"' % label))) + key = self.index.search(label) + if key!=None : + ck = ConfigKey(key[0]) + count = int(key[2]) + hlst = ck.getHashList(label,count,True) + ids = self._getIdsBySubject(hlst['head'][2]) + if len(ids) >= count: + status, resp = self.ih.srv.fetch(ids[0],'(BODY[HEADER.FIELDS (TO)])') + to = bstr(resp[0][1][4:-4]) + if to == self.mb.getHashName('all')+'@'+self.mb.DOMAIN_NAME or to == self.mb.getHashName(self.conf.ini.get('name',self.conf.profile+'.infos'))+'@'+self.mb.DOMAIN_NAME : + for mid in ids : + self.ih.downloadAttachment(mid,self.fsplit.DIR_INBOX) + print(hlst['head']) + for v in hlst['data']: + print(v) + self.fsplit.deployFile(hlst, key[3]) + else : + raise Exception(label+' is private') + else : + raise Exception(label+' : invalid count parts '+str(len(ids))+'/'+str(count)) + else: + raise Exception(label+' not on the server') + rt.stop() + + def clean(self): + """""" + rt = RuTime(eval(__CALLER__())) + self.index = self.getIndex() + rt.stop() + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class MailBuilder ~~ + +class MailBuilder: + """A simple mail builder to create mails for ImpraIndex and parts attchments""" + + DOMAIN_NAME = 'impra.storage' + """Domain name used for from and to mail fields""" + + def __init__(self, salt=''): + """""" + self.salt = salt + + def getHashName(self, name): + """Return a simplified hash of specified name + :Returns: `str` + """ + return hash_sha256(self.salt+name)[0:12] + + def build(self, nameFrom, nameTo, subject, filePath): + """Build mail with attachment part + :Returns: 'email.message.Message' + """ + rt = RuTime(eval(__CALLER__('%s' % basename(filePath)))) + msg = MIMEMultipart() + msg['From'] = self.getHashName(nameFrom)+'@'+self.DOMAIN_NAME + msg['To'] = self.getHashName(nameTo)+'@'+self.DOMAIN_NAME + msg['Date'] = formatdate(localtime=True) + msg['Subject'] = Header(subject,'utf-8') + part = MIMEBase('application', 'octet-stream') + part.set_payload(open(filePath, 'rb').read()) + encode_base64(part) + part.add_header('Content-Disposition','attachment; filename="%s"' % basename(filePath)) + msg.attach(part) + rt.stop() + return msg + + def buildIndex(self, data): + """Build mail for ImpraIndex + :Returns: 'email.message.Message' + """ + rt = RuTime(eval(__CALLER__())) + msg = MIMEMultipart() + msg['From'] = self.getHashName('system')+'@'+self.DOMAIN_NAME + msg['To'] = self.getHashName('all')+'@'+self.DOMAIN_NAME + msg['Date'] = formatdate(localtime=True) + msg['Subject'] = Header(self.getHashName('index'),'utf-8') + msg.attach(MIMEText(data,_charset='utf-8')) + rt.stop() + return msg diff --git a/impra/imap.py b/impra/imap.py new file mode 100755 index 0000000..7cee6a1 --- /dev/null +++ b/impra/imap.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# software : ImpraStorage # +# version : 0.4 # +# date : 2012 # +# licence : GPLv3.0 # +# author : a-Sansara # +# copyright : pluie.org # +# # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# +# This file is part of ImpraStorage. +# +# ImpraStorage is free software (free as in speech) : you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. +# +# ImpraStorage is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License +# along with ImpraStorage. If not, see . + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ package imap ~~ + +import inspect +from email import message_from_bytes +from email.header import decode_header +from email.message import Message +from imaplib import IMAP4_SSL, Time2Internaldate +from os.path import join +from re import search, split +from time import time +from impra.util import __CALLER__, RuTime, bstr, stack +from binascii import b2a_base64, a2b_base64 +from codecs import register, StreamReader, StreamWriter + + +def _seq_encode(seq,l): + """""" + if len(seq) > 0 : + l.append('&%s-' % str(b2a_base64(bytes(''.join(seq),'utf-16be')),'utf-8').rstrip('\n=').replace('/', ',')) + elif l: + l.append('-') + +def encode(s): + """""" + l, e, = [], [] + for c in s : + if ord(c) in range(0x20,0x7e): + if e : _seq_encode(e,l) + e = [] + l.append(c) + if c == '&' : l.append('-') + else : + e.append(c) + if e : _seq_encode(e,l) + return ''.join(l) + +def encoder(s): + """""" + e = bytes(encode(s),'utf-8') + return e, len(e) + +def _seq_decode(seq,l): + """""" + d = ''.join(seq[1:]) + pad = 4-(len(d)%4) + l.append(str(a2b_base64(bytes(d.replace(',', '/')+pad*'=','utf-16be')),'utf-16be')) + +def decode(s): + """""" + l, d = [], [] + for c in s: + if c == '&' and not d : d.append('&') + elif c == '-' and d: + if len(d) == 1: l.append('&') + else : _seq_decode(d,l) + d = [] + elif d: d.append(c) + else: l.append(c) + if d: _seq_decode(d,l) + return ''.join(l) + +def decoder(s): + """""" + d = decode(str(s,'utf-8')) + return d, len(d) + +def _codec_imap4utf7(name): + """""" + if name == 'imap4-utf-7': + return (encoder, decoder, Imap4Utf7StreamReader, Imap4Utf7StreamWriter) + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class Imap4Utf7StreamWriter ~~ + +class Imap4Utf7StreamReader(StreamReader): + """""" + + def decode(self, s, errors='strict'): + """""" + return decoder(s) + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class Imap4Utf7StreamWriter ~~ + +class Imap4Utf7StreamWriter(StreamWriter): + """""" + + def decode(self, s, errors='strict'): + """""" + return encoder(s) + + +register(_codec_imap4utf7) + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class ImapConfig ~~ + +class ImapConfig: + """""" + + def __init__(self, host, port, user, pwd): + """""" + self.host = host + self.port = port + self.user = user + self.pwd = pwd + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class ImapHelper ~~ + +class ImapHelper: + """""" + + K_HEAD, K_DATA = 0, 1 + """""" + OK = 'OK' + """""" + KO = 'NO' + """""" + ENCODING = 'utf-8' + """""" + REG_SATUS = r'^"(\w*)" \(([^\(]*)\)' + """""" + NO_SELECT = '\\Noselect' + """""" + CHILDREN = '\\HasChildren' + """""" + NO_CHILDREN = '\\HasNoChildren' + """""" + BOX_BIN = '[Gmail]/Corbeille' + """""" + + def __init__(self, conf, box='INBOX'): + """""" + rt = RuTime(eval(__CALLER__('conf,"'+box+'"'))) + self.srv = IMAP4_SSL(conf.host,conf.port) + self.srv.login(conf.user,conf.pwd) + self.rootBox = box + status, resp = self.srv.select(self.rootBox) + if status == self.KO : + self.createBox(self.rootBox) + self.srv.select(self.rootBox) + rt.stop() + + def status(self,box='INBOX'): + """""" + status, resp = ih.srv.status(box, '(MESSAGES RECENT UIDNEXT UIDVALIDITY UNSEEN)') + if status == 'OK' : + data = search(self.REG_SATUS,bstr(resp[self.K_HEAD])) + l = split(' ',data.group(2)) + dic = {'BOX' : data.group(1)} + for i in range(len(l)): + if i%2 == 0 : dic[l[i]] = int(l[i+1]) + else : dic = {} + return dic + + def countSeen(self, box='INBOX'): + """""" + s = self.status() + return s['MESSAGES']-s['UNSEEN'] + + def countUnseen(self, box='INBOX'): + """""" + return self.status()['UNSEEN'] + + def countMsg(self, box='INBOX'): + """""" + return self.status()['MESSAGES'] + + def _ids(self, box='INBOX', search='ALL', charset=None): + """""" + status, resp = self.srv.search(charset, '(%s)' % search) + return split(' ',bstr(resp[self.K_HEAD])) + + def idsUnseen(self, box='INBOX', charset=None): + """""" + return self._ids(box,'UNSEEN', charset) + + def idsMsg(self, box='INBOX', charset=None): + """""" + return self._ids(box,'ALL', charset) + + def idsSeen(self, box='INBOX', charset=None): + """""" + return self._ids(box,'NOT UNSEEN', charset) + + def listBox(self, box='INBOX', pattern='*'): + """""" + status, resp = self.srv.list(box,pattern) + l = [] + for r in resp : + name = bstr(r).split(' "/" ') + l.append((name[0][1:-1].split(' '),decode(name[1][1:-1]))) + return l + + def createBox(self, box): + """""" + rt = RuTime(eval(__CALLER__(box))) + status, resp = self.srv.create(encode(box)) + rt.stop() + return status==self.OK + + def deleteBox(self, box): + """""" + rt = RuTime(eval(__CALLER__(box))) + status, resp = self.srv.delete(encode(box)) + rt.stop() + return status==self.OK + + def subject(self, mid): + """""" + status, resp = self.srv.fetch(mid, '(body[header.fields (subject)])') + subject = decode_header(str(resp[self.K_HEAD][1][9:-4], 'utf-8'))[0] + s = subject[0] + if subject[1] : + s = str(s,subject[1]) + return s + + def email(self, mid): + """""" + status, resp = self.srv.fetch(mid,'(UID RFC822)') + if status == self.OK : + msg = message_from_bytes(resp[0][1]) + else : + msg = None + return msg + + def deleteBin(self): + """""" + rt = RuTime(eval(__CALLER__())) + self.srv.select(self.BOX_BIN) + ids = self._ids(self.BOX_BIN) + if len(ids) > 0 and ids[0]!='': + for mid in ids : + print('deleting msg '+mid) + status, resp = self.srv.store(mid, '+FLAGS', '\\Deleted') + self.srv.expunge() + self.srv.select(self.rootBox) + rt.stop() + + def delete(self, mid): + """""" + rt = RuTime(eval(__CALLER__('%i' % int(mid)))) + status = None + if int(mid) > 0 : + status, resp = self.srv.store(mid, '+FLAGS', '\\Deleted') + self.srv.expunge() + rt.stop() + return status == self.OK + + def downloadAttachment(self, msg, toDir='./'): + """""" + rt = RuTime(eval(__CALLER__('%i' % int(msg)))) + if not isinstance(msg, Message) : + msg = self.email(msg) + for part in msg.walk(): + filename = part.get_filename() + if filename != None : print(filename) + if part.get_content_maintype() == 'multipart' or not filename : continue + fp = open(join(toDir, filename), 'wb') + #print(part.get_payload(decode=True)[::-1]) + fp.write(part.get_payload(decode=True)) + fp.close() + rt.stop() + + def send(self, msg, box='INBOX'): + """""" + rt = RuTime(eval(__CALLER__())) + self.srv.append(box, '\Draft', Time2Internaldate(time()), bytes(msg,'utf-8')) + rt.stop() + +if __name__ == '__main__': + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + iconf = ImapConfig("imap.gmail.com", 993, 'gpslot.001', '__gpslot#22') + ih = ImapHelper(iconf,'__SMILF') + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + print('\n--------------------------------------------------------------------') + print('-- STATUS DEFAULT BOX --') + print(str(ih.status())) + print('-- STATUS BOX __SMILF --') + print(str(ih.status('__SMILF'))) + print('-- UNSEEN COUNT --') + print(str(ih.countUnseen('__SMILF'))) + print('-- SEEN COUNT --') + print(str(ih.countSeen('__SMILF'))) + print('-- MESSAGE COUNT --') + print(str(ih.countMsg('__SMILF'))) + print('-- UNSEEN IDS --') + print(ih.idsUnseen('__SMILF')) + print('-- MESSAGES IDS --') + print(ih.idsMsg('__SMILF')) + print('-- SEEN IDS --') + lunseen = ih.idsSeen('__SMILF') + print(lunseen) + print('-- LIST BOX --') + lb = ih.listBox('') + print(lb[5][1]) + print('-- SUBJECT ID 1 --') + print(ih.subject(lunseen[0])) + print('-- BODY ID 1 --') + #print(ih.body(lunseen[0])) + print('-- EMAIL ID 1 --') + # 'partial', ('1', 'RFC822', 1, 1024)), + #status, resp = ih.srv.fetch(lunseen[0],'(UID RFC822)') + #status, resp = ih.srv.fetch('4','(UID body[header.fields (from to subject date)])') + #status, resp = ih.srv.fetch(lunseen[1],'(UID RFC822.SIZE)') + #status, resp = ih.srv.fetch(lunseen[1],'(UID RFC822.HEADER)') + #status, resp = ih.srv.fetch(lunseen[1],'(UID BODYSTRUCTURE)') + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + #msg = ih.email(lunseen[0]) + #print(type(msg)) + #print(msg) + #print('-- ATTACHMENT ID 1 --') + #ih.downloadAttachment(lunseen[0]) + + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # ['MIME-Version', 'Received', 'Date', 'Message-ID', 'Subject', 'From', 'To', 'Content-Type'] + print('-- CREATE BOX __SMILF/böx --') + print(ih.createBox("__SMILF/böx")) + print('-- DELETE BOX böx --') + print(ih.deleteBox("böx")) + #~ OK + #~ [b'Success'] + #~ True + #~ NO + #~ [b'[ALREADYEXISTS] Duplicate folder name b\xc3\xb6x (Failure)'] + #~ True + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/impra/util.py b/impra/util.py new file mode 100755 index 0000000..6a7285e --- /dev/null +++ b/impra/util.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# software : ImpraStorage # +# version : 0.4 # +# date : 2012 # +# licence : GPLv3.0 # +# author : a-Sansara # +# copyright : pluie.org # +# # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# +# This file is part of ImpraStorage. +# +# ImpraStorage is free software (free as in speech) : you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. +# +# ImpraStorage is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License +# along with ImpraStorage. If not, see . + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ package util ~~ + +from hashlib import sha256 +from math import log, floor, ceil +from random import choice +from os import urandom, popen +from os.path import dirname, realpath +from time import time +from re import split as regsplit +from base64 import urlsafe_b64encode +from inspect import stack + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ methods ~~ + +def quote_escape(data): + """Escape simple quote + :Returns: `str` + """ + return data.replace('\'', r'\'') + +def get_file_content(fileName): + """Get file content of `fileName` + :Returns: `str` + """ + r = open(fileName, "rt") + data = r.read() + r.close() + return data + +def hash_sha256(data): + """Get a sha256 hash of str `data` + :Returns: `str` + """ + return str(sha256(bytes(data,'utf-8')).hexdigest()) + +def randomFrom(val, sval=0): + """Get a random number from range `sval=0` to `val` + :Returns: `int` + """ + lst = list(range(sval,val)) + return choice(lst) + +def formatBytes(b, p=2): + """Give a human representation of bytes size `b` + :Returns: `str` + """ + units = ['B', 'KB', 'MB', 'GB', 'TB']; + b = max(b,0); + if b == 0 : lb= 0 + else : lb = log(b) + p = floor(lb/log(1024)) + p = min(p, len(units)- 1) + #Uncomment one of the following alternatives + b /= pow(1024,p) + #b /= (1 << (10 * p)) + return str(round(b, p))+' '+units[p] + +def bstr(b,enc='utf-8'): + """""" + return str(b, encoding=enc) + +def __CALLER__(args=''): + """Give basic information of caller method + usage :: + + eval(__CALLER()) + eval(__CALLER('"%s","%s"' % (arg1,arg2))) + + :Returns: `str` + """ + #~ print(inspect.stack()[1][3]) + #~ print(print(args)) + #~ print('-----') + #~ print(inspect.stack()) + #~ print('---------------') + val = "self.__class__.__name__+'.%s' % stack()[1][3]+'("+quote_escape(args)+") l:'+str(stack()[1][2])" + return val + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class Noiser ~~ + +class Noiser: + """""" + + KEY_LEN = 64 + """""" + + def __init__(self, key, part=0): + """""" + if len(key)!=self.KEY_LEN : + raise Exception('Invalid Pass length') + else : + self.key = key + self.build(part) + + def build(self, part): + """""" + if not part < self.KEY_LEN-1 : raise Exception('part exceed limit') + else : + self.part, v = part, 0 + for i in self.key[::-2] : v += i + v = int(ceil(v/4.22)) + self.lns = int(ceil(v/2))-self.key[self.part] + self.lne = int(v-self.lns-self.key[self.part+2]) + + def getNoise(self, l): + """""" + return urandom(l) + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class Randomiz ~~ + +class Randomiz: + """""" + + def __init__(self,count): + """""" + self.lst = list(range(0,count)) + self.count = len(self.lst) + + def new(self,count=None): + """""" + if count : self.count = count + self.__init__(self.count) + + def get(self): + """""" + pos = choice(self.lst) + del self.lst[self.lst.index(pos)] + return pos + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class RuTime ~~ + +class RuTime: + """Give basics time stats""" + + def __init__(self,label): + """Initialize duration with appropriate label""" + self.label = label + self._start() + + def _start(self): + print(' ==> '+self.label) + self.sc = time() + + def stop(self): + """Stop duration and print basics stats duration on console""" + self.ec = time() + self._stats() + + def _stats(self): + print(' <== '+self.label+(' [%.9f s]' % (self.ec - self.sc))+' <¤¤ ') + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class IniFile ~~ + +class IniFile: + """Read a write inifile""" + + def __init__(self,path): + """""" + self.path = path + self.dic = {} + self.read() + + def isEmpty(self): + """""" + return len(self.dic)==0 + + def has(self, key, section='main'): + """""" + d = (key in self.dic[section]) + return d + + def hasSection(self, section): + """""" + d = (section in self.dic) + return d + + def get(self, key, section='main'): + """""" + return self.dic[section][key] + + def set(self, key, val, section='main'): + """""" + v = None + if not section in self.dic: + self.dic[section] = {} + if key in self.dic[section]: + v = self.dic[section].pop(key) + self.dic[section][key] = val + return v + + def rem(self, key, section): + """""" + v = None + if section in self.dic : + if key == '*' : + v = self.dic.pop(section) + elif key in self.dic[section]: + v = self.dic[section].pop(key) + return v + + def write(self,path=None): + """""" + if path == None : path = self.path + content = self.toString() + with open(path, mode='w', encoding='utf-8') as o: + o.write(content) + + def toString(self,path=None): + """""" + if path == None : path = self.path + content = '' + main = '' + for s in self.dic: + if s!='main': + content += '\n['+s+']\n' + for k in sorted(self.dic[s]): + k = k.rstrip(' ') + if s!='main': + content += k+' = '+self.dic[s][k]+'\n' + else : main += k+' = '+self.dic[s][k]+'\n' + return main + content + + def read(self): + """""" + try: + with open(self.path, encoding='utf-8') as o: + csection = 'main' + self.dic[csection] = {} + for l in o: + l = l.rstrip() + d = regsplit(' *= *',l,1) + if len(d)> 1: + self.dic[csection][d[0]] = d[1] + elif len(l)>0 and l[0]=='[': + csection = l.strip('[]') + self.dic[csection] = {} + except IOError : pass + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~ class Rsa ~~ + +class Rsa: + """""" + + def __init__(self, prvKey=None, pubKey=None, dpath='./'): + """""" + self.cpath = dirname(realpath(__file__))+'/../desurveil/scripts/' + self.prvKey = prvKey + self.pubKey = pubKey + self.dpath = dpath + if prvKey == None or pubKey==None : self.key() + + def key(self): + """""" + cmd = self.cpath+"desurveil key -a "+self.dpath+".impra_id_rsa -l "+self.dpath+".impra_id_rsa.pub" + #print(cmd) + try : + with open(self.dpath+'.impra_id_rsa','rt') as f: pass + except IOError as e: + d = popen(cmd).read() + #print(d) + self.prvKey = get_file_content(self.dpath+'.impra_id_rsa') + self.pubKey = get_file_content(self.dpath+'.impra_id_rsa.pub') + #print('pubKey : \n'+self.pubKey) + #print('prvKey : \n'+self.prvKey) + + def encrypt(self,data): + """""" + key = '' + if self.pubKey != None : key = " -CI '"+self.pubKey+"'" + #if self.pubKey != None : key = " -C '"+self.dpath+".impra_id_rsa.pub'" + cmd = self.cpath+"desurveil encrypt -i '"+data+"'"+key + #print(cmd) + return popen(cmd).read() + + def decrypt(self,data): + """""" + key = '' + if self.prvKey != None : key = " -CI '"+self.prvKey+"'" + #if self.prvKey != None : key = " -C '"+self.dpath+".impra_id_rsa'" + cmd = self.cpath+"desurveil decrypt -i '"+data+"'"+key + #print(cmd) + return popen(cmd).read() diff --git a/imprastorage.py b/imprastorage.py new file mode 100755 index 0000000..9ac0ab6 --- /dev/null +++ b/imprastorage.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# software : ImpraStorage # +# version : 0.4 # +# date : 2012 # +# licence : GPLv3.0 # +# author : a-Sansara # +# copyright : pluie.org # +# # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# +# This file is part of ImpraStorage. +# +# ImpraStorage is free software (free as in speech) : you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. +# +# ImpraStorage is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License +# along with ImpraStorage. If not, see . + +from impra.core import ImpraConf, ImpraStorage +from impra.util import IniFile, Rsa, RuTime + +if __name__ == '__main__': + + rt = RuTime(__name__+'()') + conf = ImpraConf(IniFile('./impra.ini')) + rsa = Rsa(conf.ini.get('prvKey',conf.profile+'.keys'),conf.ini.get('pubKey',conf.profile+'.keys')) + impst = ImpraStorage(rsa, conf) + + print('\n -- INDEX DATA -- ') + impst.index.print() + #~ print('-- LIST BOX --') + #~ lb = impst.ih.listBox('/') + #~ print(lb) + + #print('-- DELETE BIN --') + #impst.ih.deleteBin() + + filePath = '/media/Data/dev/big_toph3.jpg' + + lab = 'Meuf\'bonne aussi4' + + print('\n -- ADD FILE -- ') + impst.addFile(filePath,lab,conf.ini.get('name',conf.profile+'.infos'),'images') + + print('\n -- GET FILE -- ') + impst.getFile(lab) + + print('\n -- INDEX DATA -- ') + impst.index.print() + + print('\n -- CLEAN -- ') + impst.clean() + + rt.stop() + +#python -O -m compileall impra/*.py