Add lowmemory functions

dist/iOS/Crypto.framework/Versions/A/Headers/Crypto.objc.h

@@ -12,6 +12,7 @@
 #include "Armor.objc.h"
 #include "Models.objc.h"
 
+@class CryptoAttachmentProcessor;
 @class CryptoPmCrypto;
 @class CryptoSignatureCollector;
 @protocol CryptoMIMECallbacks;
@@ -25,6 +26,19 @@
 - (void)onVerified:(long)verified;
 @end
 
+/**
+ * EncryptedSplit when encrypt attachment
+ */
+@interface CryptoAttachmentProcessor : NSObject <goSeqRefInterface> {
+}
+@property(strong, readonly) id _ref;
+
+- (instancetype)initWithRef:(id)ref;
+- (instancetype)init;
+- (ModelsEncryptedSplit*)finish:(NSError**)error;
+- (void)process:(NSData*)plainData;
+@end
+
 /**
  * PmCrypto structure to manage multiple address keys and user keys
 Called PGP crypto because it cannot have the same name as the package by gomobile's ridiculous rules.
@@ -48,7 +62,7 @@ Called PGP crypto because it cannot have the same name as the package by gomobil
 - (ModelsDecryptSignedVerify*)decryptMessageVerifyPrivBinKeys:(NSString*)encryptedText verifierKey:(NSString*)verifierKey privateKeys:(NSData*)privateKeys passphrase:(NSString*)passphrase verifyTime:(int64_t)verifyTime error:(NSError**)error;
 - (NSString*)decryptMessageWithPassword:(NSString*)encrypted password:(NSString*)password error:(NSError**)error;
 - (ModelsEncryptedSplit*)encryptAttachment:(NSData*)plainData fileName:(NSString*)fileName publicKey:(NSString*)publicKey error:(NSError**)error;
-- (ModelsEncryptedSplit*)encryptAttachmentBinKey:(NSData*)plainData fileName:(NSString*)fileName publicKey:(NSData*)publicKey error:(NSError**)error;
+- (CryptoAttachmentProcessor*)encryptAttachmentLowMemory:(long)estimatedSize fileName:(NSString*)fileName publicKey:(NSString*)publicKey error:(NSError**)error;
 - (NSString*)encryptAttachmentWithPassword:(NSData*)plainData password:(NSString*)password error:(NSError**)error;
 - (NSString*)encryptMessage:(NSString*)plainText publicKey:(NSString*)publicKey privateKey:(NSString*)privateKey passphrase:(NSString*)passphrase trim:(BOOL)trim error:(NSError**)error;
 - (NSString*)encryptMessageBinKey:(NSString*)plainText publicKey:(NSData*)publicKey privateKey:(NSString*)privateKey passphrase:(NSString*)passphrase trim:(BOOL)trim error:(NSError**)error;

glide.lock

@@ -1,22 +1,10 @@
-hash: 7169f4883725d9716bbd62ecee1f32d73e8529b4bb0989173826e986395afe6b
+hash: 7497775ba9a895661860516e3bbf9f367dc7102b014b84d7af225035c0cbfc02
-updated: 2018-10-25T13:47:24.679131+02:00
+updated: 2018-10-29T15:51:04.090506+01:00
 imports:
-- name: github.com/alecthomas/template
-  version: a0175ee3bccc567396460bf5acd36800cb10c49c
-  subpackages:
-  - parse
-- name: github.com/alecthomas/units
-  version: 2efee857e7cfd4f3d0138cc3cbb1b4966962b93a
-- name: github.com/dustin/go-humanize
-  version: 9f541cc9db5d55bce703bd99987c9d5cb8eea45e
 - name: github.com/konsorten/go-windows-terminal-sequences
   version: 5c8c8bd35d3832f5d134ae1e1e375b69a4d25242
-- name: github.com/mattn/go-zglob
-  version: 2ea3427bfa539cca900ca2768d8663ecc8a708c1
-  subpackages:
-  - fastwalk
 - name: github.com/Sirupsen/logrus
-  version: 4fabf2fffcecfd47f802869b7b92d75e43c5a095
+  version: 566a5f690849162ff53cf98f3c42135389d63f95
 - name: golang.org/x/crypto
   version: 9e4251120d8c43f10024d798bc6dde21d40704a0
   repo: https://github.com/ProtonMail/crypto.git
@@ -45,12 +33,12 @@ imports:
   - rsa
   - ssh/terminal
 - name: golang.org/x/sys
-  version: d989b31c87461dc8ab2f1cac6792814e27fadea9
+  version: 95b1ffbd15a57cc5abb3f04402b9e8ec0016a52c
   subpackages:
   - unix
   - windows
 - name: golang.org/x/text
-  version: 4d1c5fb19474adfe9562c9847ba425e7da817e81
+  version: 6f8607282f41687a5ce9b5a0378084237f2f7576
   subpackages:
   - encoding
   - encoding/charmap
@@ -60,16 +48,4 @@ imports:
 - name: proton/pmmime
   version: 72b7b1bf2d1b491e1d003632a0fda41410b0bb59
   repo: git@gitlab.protontech.ch:ProtonMail/go-pm-mime.git
-testImports:
+testImports: []
-- name: github.com/davecgh/go-spew
-  version: d8f796af33cc11cb798c1aaeb27a4ebc5099927d
-  subpackages:
-  - spew
-- name: github.com/pmezard/go-difflib
-  version: 792786c7400a136282c1664665ae0a8db921c6c2
-  subpackages:
-  - difflib
-- name: github.com/stretchr/testify
-  version: 04af85275a5c7ac09d16bb3b9b2e751ed45154e5
-  subpackages:
-  - assert

glide.yaml

@@ -5,3 +5,5 @@ import:
   repo: https://github.com/ProtonMail/crypto.git
 - package: proton/pmmime
   repo: git@gitlab.protontech.ch:ProtonMail/go-pm-mime.git
+  version: feat/mimevisitor
+

src/gitlab.com/ProtonMail/go-pm-crypto/armor/armor.go

@@ -54,7 +54,7 @@ func SplitArmor(encrypted string) (*models.EncryptedSplit, error) {
 	if err != nil {
 		return nil, err
 	}
-	split, err := internal.SplitPackets(b.Body, len(encrypted))
+	split, err := internal.SplitPackets(b.Body, len(encrypted), -1)
 	if err != nil {
 		return nil, err
 	}

src/gitlab.com/ProtonMail/go-pm-crypto/crypto/attachment.go

@@ -4,22 +4,50 @@ import (
 	"bytes"
 	"io"
 	"io/ioutil"
-
 	"golang.org/x/crypto/openpgp"
 	"golang.org/x/crypto/openpgp/armor"
 	"golang.org/x/crypto/openpgp/packet"
 	armorUtils "gitlab.com/ProtonMail/go-pm-crypto/armor"
 	"gitlab.com/ProtonMail/go-pm-crypto/internal"
 	"gitlab.com/ProtonMail/go-pm-crypto/models"
-		"sync"
+	"sync"
+	"runtime"
 	)
 
+//EncryptedSplit when encrypt attachment
+type AttachmentProcessor struct {
+	w *io.WriteCloser
+	pipe *io.PipeWriter
+	done sync.WaitGroup
+	split *models.EncryptedSplit
+	garbageCollector int
+	err error
+}
+
+func (ap *AttachmentProcessor) Process(plainData []byte) {
+	(*ap.w).Write(plainData)
+}
+
+func (ap *AttachmentProcessor) Finish() (*models.EncryptedSplit, error) {
+	if ap.err != nil {
+		return nil, ap.err
+	}
+	(*ap.w).Close()
+	(*ap.pipe).Close()
+	ap.done.Wait()
+	if ap.garbageCollector > 0 {
+		runtime.GC()
+	}
+	return ap.split, nil
+}
+
 //EncryptAttachmentBinKey ...
-func (pm *PmCrypto) EncryptAttachmentBinKey(plainData []byte, fileName string, publicKey []byte) (*models.EncryptedSplit, error) {
+func (pm *PmCrypto) encryptAttachment(estimatedSize int, fileName string, publicKey []byte, garbageCollector int) (*AttachmentProcessor, error) {
-	var wg sync.WaitGroup
+	attachmentProc := &AttachmentProcessor{}
 	// you can also add these one at
 	// a time if you need to
-	wg.Add(1)
+	attachmentProc.done.Add(1)
+	attachmentProc.garbageCollector = garbageCollector
 	pubKeyReader := bytes.NewReader(publicKey)
 	pubKeyEntries, err := openpgp.ReadKeyRing(pubKeyReader)
 	if err != nil {
@@ -35,30 +63,27 @@ func (pm *PmCrypto) EncryptAttachmentBinKey(plainData []byte, fileName string, p
 	}
 
 	reader, writer := io.Pipe()
-	var encryptErr error
+
-	go func() {
+	go func () {
-		defer wg.Done()
+		defer attachmentProc.done.Done()
-		var ew io.WriteCloser
+		split, splitError := internal.SplitPackets(reader, estimatedSize, garbageCollector)
-		ew, encryptErr = openpgp.Encrypt(writer, pubKeyEntries, nil, hints, config)
+		if attachmentProc.err != nil {
-		_, _ = ew.Write(plainData)
+			attachmentProc.err = splitError
-		plainData = nil
+		}
-		// clear the buffer
+		split.Algo = "aes256"
-		ew.Close()
+		attachmentProc.split = split
-		writer.Close()
 	}()
 
-	split, err := internal.SplitPackets(reader, len(plainData))
+	var ew io.WriteCloser
-
+	var encryptErr error
-	wg.Wait()
+	ew, encryptErr = openpgp.Encrypt(writer, pubKeyEntries, nil, hints, config)
-	if encryptErr != nil {
+	attachmentProc.w = &ew
-		return nil, encryptErr
+	attachmentProc.pipe = writer
+	if attachmentProc.err != nil {
+		attachmentProc.err = encryptErr
 	}
 
-	if err != nil {
+	return attachmentProc, nil
-		return nil, err
-	}
-	split.Algo = "aes256"
-	return split, nil
 }
 
 //EncryptAttachment ...
@@ -67,7 +92,26 @@ func (pm *PmCrypto) EncryptAttachment(plainData []byte, fileName string, publicK
 	if err != nil {
 		return nil, err
 	}
-	return pm.EncryptAttachmentBinKey(plainData, fileName, rawPubKey)
+	ap, err := pm.encryptAttachment(len(plainData), fileName, rawPubKey, -1)
+	if err != nil {
+		return nil, err
+	}
+	ap.Process(plainData)
+	split, err := ap.Finish()
+	if err != nil {
+		return nil, err
+	}
+	return split, nil
+}
+
+//EncryptAttachment ...
+func (pm *PmCrypto) EncryptAttachmentLowMemory(estimatedSize int, fileName string, publicKey string) (*AttachmentProcessor, error) {
+	rawPubKey, err := armorUtils.Unarmor(publicKey)
+	if err != nil {
+		return nil, err
+	}
+	// Garbage collect every megabyte
+	return pm.encryptAttachment(estimatedSize, fileName, rawPubKey, 1 << 20)
 }
 
 //DecryptAttachmentBinKey ...

src/gitlab.com/ProtonMail/go-pm-crypto/internal/packets.go

@@ -5,15 +5,18 @@ import (
 	"golang.org/x/crypto/openpgp/packet"
 	"gitlab.com/ProtonMail/go-pm-crypto/models"
 	"io"
-			)
+	"runtime"
+		)
 
-func SplitPackets(encryptedReader io.Reader, estimatedLength int) (*models.EncryptedSplit, error){
+func SplitPackets(encryptedReader io.Reader, estimatedLength int, garbageCollector int) (*models.EncryptedSplit, error){
 	var err error
 
+	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
 	//kr *KeyRing, r io.Reader) (key *SymmetricKey, symEncryptedData []byte,
 	packets := packet.NewReader(encryptedReader)
 
 	outSplit := &models.EncryptedSplit{}
+	gcCounter := 0
 
 	// Save encrypted key and signature apart
 	var ek *packet.EncryptedKey
@@ -35,9 +38,11 @@ func SplitPackets(encryptedReader io.Reader, estimatedLength int) (*models.Encry
 		case *packet.SymmetricallyEncrypted:
 			// The code below is optimized to not
 			var b bytes.Buffer
-			// 128 is an estimation of the size difference between input and output, the size difference is most probably
+			// 2^16 is an estimation of the size difference between input and output, the size difference is most probably
 			// 16 bytes at a maximum though.
-			b.Grow(128 + estimatedLength)
+			// We need to avoid triggering a grow from the system as this will allocate too much memory causing problems
+			// in low-memory environments
+			b.Grow(1 << 16 + estimatedLength)
 			// empty encoded length + start byte
 			b.Write(make([]byte, 6))
 			b.WriteByte(byte(1))
@@ -50,6 +55,11 @@ func SplitPackets(encryptedReader io.Reader, estimatedLength int) (*models.Encry
 				}
 				b.Write(block[:n])
 				actualLength += n
+				gcCounter += n
+				if gcCounter > garbageCollector && garbageCollector > 0 {
+					runtime.GC()
+					gcCounter = 0
+				}
 			}
 
 			// quick encoding