diff --git a/dist/Android/pmcrypto-sources.jar b/dist/Android/pmcrypto-sources.jar index 92bf958..2b760a5 100644 Binary files a/dist/Android/pmcrypto-sources.jar and b/dist/Android/pmcrypto-sources.jar differ diff --git a/dist/Android/pmcrypto.aar b/dist/Android/pmcrypto.aar index 64faefa..9a5c2bf 100644 Binary files a/dist/Android/pmcrypto.aar and b/dist/Android/pmcrypto.aar differ diff --git a/dist/iOS/Crypto.framework.zip b/dist/iOS/Crypto.framework.zip new file mode 100644 index 0000000..932e885 Binary files /dev/null and b/dist/iOS/Crypto.framework.zip differ diff --git a/dist/iOS/Crypto.framework/Versions/A/Crypto b/dist/iOS/Crypto.framework/Versions/A/Crypto index dea9e19..fa811fe 100644 Binary files a/dist/iOS/Crypto.framework/Versions/A/Crypto and b/dist/iOS/Crypto.framework/Versions/A/Crypto differ diff --git a/dist/iOS/Crypto.framework/Versions/A/Headers/Crypto.objc.h b/dist/iOS/Crypto.framework/Versions/A/Headers/Crypto.objc.h index e231d90..b3b66d0 100644 --- a/dist/iOS/Crypto.framework/Versions/A/Headers/Crypto.objc.h +++ b/dist/iOS/Crypto.framework/Versions/A/Headers/Crypto.objc.h @@ -12,6 +12,7 @@ #include "Armor.objc.h" #include "Models.objc.h" +@class CryptoAttachmentProcessor; @class CryptoPmCrypto; @class CryptoSignatureCollector; @protocol CryptoMIMECallbacks; @@ -25,6 +26,19 @@ - (void)onVerified:(long)verified; @end +/** + * EncryptedSplit when encrypt attachment + */ +@interface CryptoAttachmentProcessor : NSObject { +} +@property(strong, readonly) id _ref; + +- (instancetype)initWithRef:(id)ref; +- (instancetype)init; +- (ModelsEncryptedSplit*)finish:(NSError**)error; +- (void)process:(NSData*)plainData; +@end + /** * PmCrypto structure to manage multiple address keys and user keys Called PGP crypto because it cannot have the same name as the package by gomobile's ridiculous rules. @@ -48,7 +62,7 @@ Called PGP crypto because it cannot have the same name as the package by gomobil - (ModelsDecryptSignedVerify*)decryptMessageVerifyPrivBinKeys:(NSString*)encryptedText verifierKey:(NSString*)verifierKey privateKeys:(NSData*)privateKeys passphrase:(NSString*)passphrase verifyTime:(int64_t)verifyTime error:(NSError**)error; - (NSString*)decryptMessageWithPassword:(NSString*)encrypted password:(NSString*)password error:(NSError**)error; - (ModelsEncryptedSplit*)encryptAttachment:(NSData*)plainData fileName:(NSString*)fileName publicKey:(NSString*)publicKey error:(NSError**)error; -- (ModelsEncryptedSplit*)encryptAttachmentBinKey:(NSData*)plainData fileName:(NSString*)fileName publicKey:(NSData*)publicKey error:(NSError**)error; +- (CryptoAttachmentProcessor*)encryptAttachmentLowMemory:(long)estimatedSize fileName:(NSString*)fileName publicKey:(NSString*)publicKey error:(NSError**)error; - (NSString*)encryptAttachmentWithPassword:(NSData*)plainData password:(NSString*)password error:(NSError**)error; - (NSString*)encryptMessage:(NSString*)plainText publicKey:(NSString*)publicKey privateKey:(NSString*)privateKey passphrase:(NSString*)passphrase trim:(BOOL)trim error:(NSError**)error; - (NSString*)encryptMessageBinKey:(NSString*)plainText publicKey:(NSData*)publicKey privateKey:(NSString*)privateKey passphrase:(NSString*)passphrase trim:(BOOL)trim error:(NSError**)error; diff --git a/glide.lock b/glide.lock index 97d07fb..6f81f53 100644 --- a/glide.lock +++ b/glide.lock @@ -1,22 +1,10 @@ -hash: 7169f4883725d9716bbd62ecee1f32d73e8529b4bb0989173826e986395afe6b -updated: 2018-10-25T13:47:24.679131+02:00 +hash: 7497775ba9a895661860516e3bbf9f367dc7102b014b84d7af225035c0cbfc02 +updated: 2018-10-29T15:51:04.090506+01:00 imports: -- name: github.com/alecthomas/template - version: a0175ee3bccc567396460bf5acd36800cb10c49c - subpackages: - - parse -- name: github.com/alecthomas/units - version: 2efee857e7cfd4f3d0138cc3cbb1b4966962b93a -- name: github.com/dustin/go-humanize - version: 9f541cc9db5d55bce703bd99987c9d5cb8eea45e - name: github.com/konsorten/go-windows-terminal-sequences version: 5c8c8bd35d3832f5d134ae1e1e375b69a4d25242 -- name: github.com/mattn/go-zglob - version: 2ea3427bfa539cca900ca2768d8663ecc8a708c1 - subpackages: - - fastwalk - name: github.com/Sirupsen/logrus - version: 4fabf2fffcecfd47f802869b7b92d75e43c5a095 + version: 566a5f690849162ff53cf98f3c42135389d63f95 - name: golang.org/x/crypto version: 9e4251120d8c43f10024d798bc6dde21d40704a0 repo: https://github.com/ProtonMail/crypto.git @@ -45,12 +33,12 @@ imports: - rsa - ssh/terminal - name: golang.org/x/sys - version: d989b31c87461dc8ab2f1cac6792814e27fadea9 + version: 95b1ffbd15a57cc5abb3f04402b9e8ec0016a52c subpackages: - unix - windows - name: golang.org/x/text - version: 4d1c5fb19474adfe9562c9847ba425e7da817e81 + version: 6f8607282f41687a5ce9b5a0378084237f2f7576 subpackages: - encoding - encoding/charmap @@ -60,16 +48,4 @@ imports: - name: proton/pmmime version: 72b7b1bf2d1b491e1d003632a0fda41410b0bb59 repo: git@gitlab.protontech.ch:ProtonMail/go-pm-mime.git -testImports: -- name: github.com/davecgh/go-spew - version: d8f796af33cc11cb798c1aaeb27a4ebc5099927d - subpackages: - - spew -- name: github.com/pmezard/go-difflib - version: 792786c7400a136282c1664665ae0a8db921c6c2 - subpackages: - - difflib -- name: github.com/stretchr/testify - version: 04af85275a5c7ac09d16bb3b9b2e751ed45154e5 - subpackages: - - assert +testImports: [] diff --git a/glide.yaml b/glide.yaml index a22b341..641b187 100644 --- a/glide.yaml +++ b/glide.yaml @@ -5,3 +5,5 @@ import: repo: https://github.com/ProtonMail/crypto.git - package: proton/pmmime repo: git@gitlab.protontech.ch:ProtonMail/go-pm-mime.git + version: feat/mimevisitor + diff --git a/src/gitlab.com/ProtonMail/go-pm-crypto/armor/armor.go b/src/gitlab.com/ProtonMail/go-pm-crypto/armor/armor.go index 7da9ec9..87fc3c9 100644 --- a/src/gitlab.com/ProtonMail/go-pm-crypto/armor/armor.go +++ b/src/gitlab.com/ProtonMail/go-pm-crypto/armor/armor.go @@ -54,7 +54,7 @@ func SplitArmor(encrypted string) (*models.EncryptedSplit, error) { if err != nil { return nil, err } - split, err := internal.SplitPackets(b.Body, len(encrypted)) + split, err := internal.SplitPackets(b.Body, len(encrypted), -1) if err != nil { return nil, err } diff --git a/src/gitlab.com/ProtonMail/go-pm-crypto/crypto/attachment.go b/src/gitlab.com/ProtonMail/go-pm-crypto/crypto/attachment.go index 979bcca..5e498b6 100644 --- a/src/gitlab.com/ProtonMail/go-pm-crypto/crypto/attachment.go +++ b/src/gitlab.com/ProtonMail/go-pm-crypto/crypto/attachment.go @@ -4,22 +4,50 @@ import ( "bytes" "io" "io/ioutil" - "golang.org/x/crypto/openpgp" "golang.org/x/crypto/openpgp/armor" "golang.org/x/crypto/openpgp/packet" armorUtils "gitlab.com/ProtonMail/go-pm-crypto/armor" "gitlab.com/ProtonMail/go-pm-crypto/internal" "gitlab.com/ProtonMail/go-pm-crypto/models" - "sync" + "sync" + "runtime" ) +//EncryptedSplit when encrypt attachment +type AttachmentProcessor struct { + w *io.WriteCloser + pipe *io.PipeWriter + done sync.WaitGroup + split *models.EncryptedSplit + garbageCollector int + err error +} + +func (ap *AttachmentProcessor) Process(plainData []byte) { + (*ap.w).Write(plainData) +} + +func (ap *AttachmentProcessor) Finish() (*models.EncryptedSplit, error) { + if ap.err != nil { + return nil, ap.err + } + (*ap.w).Close() + (*ap.pipe).Close() + ap.done.Wait() + if ap.garbageCollector > 0 { + runtime.GC() + } + return ap.split, nil +} + //EncryptAttachmentBinKey ... -func (pm *PmCrypto) EncryptAttachmentBinKey(plainData []byte, fileName string, publicKey []byte) (*models.EncryptedSplit, error) { - var wg sync.WaitGroup +func (pm *PmCrypto) encryptAttachment(estimatedSize int, fileName string, publicKey []byte, garbageCollector int) (*AttachmentProcessor, error) { + attachmentProc := &AttachmentProcessor{} // you can also add these one at // a time if you need to - wg.Add(1) + attachmentProc.done.Add(1) + attachmentProc.garbageCollector = garbageCollector pubKeyReader := bytes.NewReader(publicKey) pubKeyEntries, err := openpgp.ReadKeyRing(pubKeyReader) if err != nil { @@ -35,30 +63,27 @@ func (pm *PmCrypto) EncryptAttachmentBinKey(plainData []byte, fileName string, p } reader, writer := io.Pipe() - var encryptErr error - go func() { - defer wg.Done() - var ew io.WriteCloser - ew, encryptErr = openpgp.Encrypt(writer, pubKeyEntries, nil, hints, config) - _, _ = ew.Write(plainData) - plainData = nil - // clear the buffer - ew.Close() - writer.Close() + + go func () { + defer attachmentProc.done.Done() + split, splitError := internal.SplitPackets(reader, estimatedSize, garbageCollector) + if attachmentProc.err != nil { + attachmentProc.err = splitError + } + split.Algo = "aes256" + attachmentProc.split = split }() - split, err := internal.SplitPackets(reader, len(plainData)) - - wg.Wait() - if encryptErr != nil { - return nil, encryptErr + var ew io.WriteCloser + var encryptErr error + ew, encryptErr = openpgp.Encrypt(writer, pubKeyEntries, nil, hints, config) + attachmentProc.w = &ew + attachmentProc.pipe = writer + if attachmentProc.err != nil { + attachmentProc.err = encryptErr } - if err != nil { - return nil, err - } - split.Algo = "aes256" - return split, nil + return attachmentProc, nil } //EncryptAttachment ... @@ -67,7 +92,26 @@ func (pm *PmCrypto) EncryptAttachment(plainData []byte, fileName string, publicK if err != nil { return nil, err } - return pm.EncryptAttachmentBinKey(plainData, fileName, rawPubKey) + ap, err := pm.encryptAttachment(len(plainData), fileName, rawPubKey, -1) + if err != nil { + return nil, err + } + ap.Process(plainData) + split, err := ap.Finish() + if err != nil { + return nil, err + } + return split, nil +} + +//EncryptAttachment ... +func (pm *PmCrypto) EncryptAttachmentLowMemory(estimatedSize int, fileName string, publicKey string) (*AttachmentProcessor, error) { + rawPubKey, err := armorUtils.Unarmor(publicKey) + if err != nil { + return nil, err + } + // Garbage collect every megabyte + return pm.encryptAttachment(estimatedSize, fileName, rawPubKey, 1 << 20) } //DecryptAttachmentBinKey ... diff --git a/src/gitlab.com/ProtonMail/go-pm-crypto/internal/packets.go b/src/gitlab.com/ProtonMail/go-pm-crypto/internal/packets.go index 1f8160f..1327b01 100644 --- a/src/gitlab.com/ProtonMail/go-pm-crypto/internal/packets.go +++ b/src/gitlab.com/ProtonMail/go-pm-crypto/internal/packets.go @@ -5,15 +5,18 @@ import ( "golang.org/x/crypto/openpgp/packet" "gitlab.com/ProtonMail/go-pm-crypto/models" "io" - ) + "runtime" + ) -func SplitPackets(encryptedReader io.Reader, estimatedLength int) (*models.EncryptedSplit, error){ +func SplitPackets(encryptedReader io.Reader, estimatedLength int, garbageCollector int) (*models.EncryptedSplit, error){ var err error + // For info on each, see: https://golang.org/pkg/runtime/#MemStats //kr *KeyRing, r io.Reader) (key *SymmetricKey, symEncryptedData []byte, packets := packet.NewReader(encryptedReader) outSplit := &models.EncryptedSplit{} + gcCounter := 0 // Save encrypted key and signature apart var ek *packet.EncryptedKey @@ -35,9 +38,11 @@ func SplitPackets(encryptedReader io.Reader, estimatedLength int) (*models.Encry case *packet.SymmetricallyEncrypted: // The code below is optimized to not var b bytes.Buffer - // 128 is an estimation of the size difference between input and output, the size difference is most probably + // 2^16 is an estimation of the size difference between input and output, the size difference is most probably // 16 bytes at a maximum though. - b.Grow(128 + estimatedLength) + // We need to avoid triggering a grow from the system as this will allocate too much memory causing problems + // in low-memory environments + b.Grow(1 << 16 + estimatedLength) // empty encoded length + start byte b.Write(make([]byte, 6)) b.WriteByte(byte(1)) @@ -50,6 +55,11 @@ func SplitPackets(encryptedReader io.Reader, estimatedLength int) (*models.Encry } b.Write(block[:n]) actualLength += n + gcCounter += n + if gcCounter > garbageCollector && garbageCollector > 0 { + runtime.GC() + gcCounter = 0 + } } // quick encoding