imap: add local DB

imap/database/mailbox.go

@@ -0,0 +1,156 @@
+package database
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+
+	"github.com/boltdb/bolt"
+
+	"github.com/emersion/hydroxide/protonmail"
+)
+
+func serializeUID(uid uint32) []byte {
+	b := make([]byte, 4)
+	binary.BigEndian.PutUint32(b, uid)
+	return b
+}
+
+func unserializeUID(b []byte) uint32 {
+	return binary.BigEndian.Uint32(b)
+}
+
+type Mailbox struct {
+	name string
+	u *User
+}
+
+func (mbox *Mailbox) bucket(tx *bolt.Tx) (*bolt.Bucket, error) {
+	b := tx.Bucket(mailboxesBucket)
+	if b == nil {
+		return nil, errors.New("cannot find mailboxes bucket")
+	}
+	b = tx.Bucket([]byte(mbox.name))
+	if b == nil {
+		return nil, errors.New("cannot find mailbox bucket")
+	}
+	return b, nil
+}
+
+func (mbox *Mailbox) Sync(messages []*protonmail.Message) error {
+	err := mbox.u.db.Update(func(tx *bolt.Tx) error {
+		b, err := mbox.bucket(tx)
+		if err != nil {
+			return err
+		}
+
+		for _, msg := range messages {
+			want := []byte(msg.ID)
+			c := b.Cursor()
+			found := false
+			for k, v := c.First(); k != nil; k, v = c.Next() {
+				if bytes.Equal(v, want) {
+					found = true
+					break
+				}
+			}
+			if found {
+				continue
+			}
+
+			id, _ := b.NextSequence()
+			uid := uint32(id)
+			if err := b.Put(serializeUID(uid), want); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return mbox.u.sync(messages)
+}
+
+func (mbox *Mailbox) UidNext() (uint32, error) {
+	var uid uint32
+	err := mbox.u.db.View(func(tx *bolt.Tx) error {
+		b, err := mbox.bucket(tx)
+		if err != nil {
+			return err
+		}
+
+		uid = uint32(b.Sequence() + 1)
+		return nil
+	})
+	return uid, err
+}
+
+func (mbox *Mailbox) FromUid(uid uint32) (string, error) {
+	var apiID string
+	err := mbox.u.db.View(func(tx *bolt.Tx) error {
+		b, err := mbox.bucket(tx)
+		if err != nil {
+			return err
+		}
+
+		k := serializeUID(uid)
+		v := b.Get(k)
+		if v == nil {
+			return ErrNotFound
+		}
+		apiID = string(v)
+		return nil
+	})
+	return apiID, err
+}
+
+func (mbox *Mailbox) FromSeqNum(seqNum uint32) (string, error) {
+	var apiID string
+	err := mbox.u.db.View(func(tx *bolt.Tx) error {
+		b, err := mbox.bucket(tx)
+		if err != nil {
+			return err
+		}
+
+		c := b.Cursor()
+		var n uint32 = 1
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			if seqNum == n {
+				apiID = string(v)
+				return nil
+			}
+			n++
+		}
+
+		return ErrNotFound
+	})
+	return apiID, err
+}
+
+func (mbox *Mailbox) FromApiID(apiID string) (uint32, uint32, error) {
+	var seqNum, uid uint32
+	err := mbox.u.db.View(func(tx *bolt.Tx) error {
+		b, err := mbox.bucket(tx)
+		if err != nil {
+			return err
+		}
+
+		want := []byte(apiID)
+		c := b.Cursor()
+		var n uint32 = 1
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			if bytes.Equal(v, want) {
+				seqNum = n
+				uid = unserializeUID(k)
+				return nil
+			}
+			n++
+		}
+
+		return ErrNotFound
+	})
+	return seqNum, uid, err
+}

imap/database/user.go

@@ -0,0 +1,87 @@
+package database
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/boltdb/bolt"
+
+	"github.com/emersion/hydroxide/protonmail"
+)
+
+var ErrNotFound = errors.New("message not found in local database")
+
+var (
+	mailboxesBucket = []byte("mailboxes")
+	messagesBucket = []byte("messages")
+)
+
+type User struct {
+	db *bolt.DB
+}
+
+func (u *User) Mailbox(name string) (*Mailbox, error) {
+	err := u.db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists(mailboxesBucket)
+		if err != nil {
+			return err
+		}
+		_, err = b.CreateBucketIfNotExists([]byte(name))
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Mailbox{name, u}, nil
+}
+
+func (u *User) sync(messages []*protonmail.Message) error {
+	return u.db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists(messagesBucket)
+		if err != nil {
+			return err
+		}
+
+		for _, msg := range messages {
+			k := []byte(msg.ID)
+			v, err := json.Marshal(msg)
+			if err != nil{
+				return err
+			}
+			if err := b.Put(k, v); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func (u *User) Message(apiID string) (*protonmail.Message, error) {
+	var msg *protonmail.Message
+	err := u.db.View(func (tx *bolt.Tx) error {
+		b := tx.Bucket(messagesBucket)
+		if b == nil {
+			return ErrNotFound
+		}
+
+		k := []byte(apiID)
+		v := b.Get(k)
+		if v == nil {
+			return ErrNotFound
+		}
+
+		return json.Unmarshal(v, msg)
+	})
+	return msg, err
+}
+
+func Open(path string) (*User, error) {
+	db, err := bolt.Open(path, 0700, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return &User{db}, nil
+}

imap/mailbox.go

@@ -1,9 +1,13 @@
 package imap
 
 import (
+	"strings"
 	"time"
 
 	"github.com/emersion/go-imap"
+
+	"github.com/emersion/hydroxide/imap/database"
+	"github.com/emersion/hydroxide/protonmail"
 )
 
 const delimiter = "/"
@@ -12,6 +16,11 @@ type mailbox struct {
 	name string
 	label string
 	flags []string
+
+	u *user
+	db *database.Mailbox
+
+	total, unread int
 }
 
 func (mbox *mailbox) Name() string {
@@ -35,15 +44,19 @@ func (mbox *mailbox) Status(items []imap.StatusItem) (*imap.MailboxStatus, error
 	for _, name := range items {
 		switch name {
 		case imap.StatusMessages:
-			status.Messages = 0 // TODO
+			status.Messages = uint32(mbox.total)
 		case imap.StatusUidNext:
-			status.UidNext = 1 // TODO
+			uidNext, err := mbox.db.UidNext()
+			if err != nil {
+				return nil, err
+			}
+			status.UidNext = uidNext
 		case imap.StatusUidValidity:
 			status.UidValidity = 1
 		case imap.StatusRecent:
-			status.Recent = 0 // TODO
+			status.Recent = 0
 		case imap.StatusUnseen:
-			status.Unseen = 0 // TODO
+			status.Unseen = uint32(mbox.unread)
 		}
 	}
 
@@ -58,8 +71,207 @@ func (mbox *mailbox) Check() error {
 	return nil
 }
 
-func (mbox *mailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []imap.FetchItem, ch chan<- *imap.Message) error {
+func (mbox *mailbox) sync() error {
-	return errNotYetImplemented // TODO
+	filter := &protonmail.MessageFilter{
+		PageSize: 150,
+		Label: mbox.label,
+		Sort: "ID",
+		Asc: true,
+	}
+
+	total := -1
+	for {
+		offset := filter.PageSize * filter.Page
+		if total >= 0 && offset > total {
+			break
+		}
+
+		var page []*protonmail.Message
+		var err error
+		total, page, err = mbox.u.c.ListMessages(filter)
+		if err != nil {
+			return err
+		}
+
+		if err := mbox.db.Sync(page); err != nil {
+			return err
+		}
+
+		filter.Page++
+	}
+
+	return nil
+}
+
+func getMessageID(id string) string {
+	return id + "@protonmail.com"
+}
+
+func getAddress(addr *protonmail.MessageAddress) *imap.Address {
+	parts := strings.SplitN(addr.Address, "@", 2)
+	if len(parts) < 2 {
+		parts = append(parts, "")
+	}
+
+	return &imap.Address{
+		PersonalName: addr.Name,
+		MailboxName: parts[0],
+		HostName: parts[1],
+	}
+}
+
+func getAddressList(addresses []*protonmail.MessageAddress) []*imap.Address {
+	l := make([]*imap.Address, len(addresses))
+	for i, addr := range addresses {
+		l[i] = getAddress(addr)
+	}
+	return l
+}
+
+func getEnvelope(msg *protonmail.Message) *imap.Envelope {
+	return &imap.Envelope{
+		Date: time.Unix(msg.Time, 0),
+		Subject: msg.Subject,
+		From: []*imap.Address{getAddress(msg.Sender)},
+		Sender: []*imap.Address{getAddress(msg.Sender)},
+		ReplyTo: []*imap.Address{getAddress(msg.ReplyTo)},
+		To: getAddressList(msg.ToList),
+		Cc: getAddressList(msg.CCList),
+		Bcc: getAddressList(msg.BCCList),
+		// TODO: InReplyTo
+		MessageId: getMessageID(msg.ID),
+	}
+}
+
+func hasLabel(msg *protonmail.Message, labelID string) bool {
+	for _, id := range msg.LabelIDs {
+		if labelID == id {
+			return true
+		}
+	}
+	return false
+}
+
+func getFlags(msg *protonmail.Message) []string {
+	var flags []string
+	if msg.IsRead != 0 {
+		flags = append(flags, imap.SeenFlag)
+	}
+	if msg.IsReplied != 0 || msg.IsRepliedAll != 0 {
+		flags = append(flags, imap.AnsweredFlag)
+	}
+	for _, label := range msg.LabelIDs {
+		switch label {
+		case protonmail.LabelStarred:
+			flags = append(flags, imap.FlaggedFlag)
+		case protonmail.LabelDraft:
+			flags = append(flags, imap.DraftFlag)
+		}
+	}
+	// TODO: DeletedFlag
+	return flags
+}
+
+func (mbox *mailbox) getBodyStructure(id string, extended bool) (*imap.BodyStructure, error) {
+	// TODO
+}
+
+func (mbox *mailbox) getBodySection(id string, section *imap.BodySectionName) (imap.Literal, error) {
+	// TODO
+}
+
+func (mbox *mailbox) getMessage(isUid bool, id uint32, items []imap.FetchItem) (*imap.Message, error) {
+	var apiID string
+	var err error
+	if isUid {
+		apiID, err = mbox.db.FromUid(id)
+	} else {
+		apiID, err = mbox.db.FromSeqNum(id)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	seqNum, uid, err := mbox.db.FromApiID(apiID)
+	if err != nil {
+		return nil, err
+	}
+
+	msg, err := mbox.u.db.Message(apiID)
+	if err != nil {
+		return nil, err
+	}
+
+	fetched := imap.NewMessage(seqNum, items)
+	for _, item := range items {
+		switch item {
+		case imap.FetchEnvelope:
+			fetched.Envelope = getEnvelope(msg)
+		case imap.FetchBody, imap.FetchBodyStructure:
+			bs, err := mbox.getBodyStructure(msg.ID, item == imap.FetchBodyStructure)
+			if err != nil {
+				return nil, err
+			}
+			fetched.BodyStructure = bs
+		case imap.FetchFlags:
+			fetched.Flags = getFlags(msg)
+		case imap.FetchInternalDate:
+			fetched.InternalDate = time.Unix(msg.Time, 0)
+		case imap.FetchRFC822Size:
+			fetched.Size = uint32(msg.Size)
+		case imap.FetchUid:
+			fetched.Uid = uid
+		default:
+			section, err := imap.ParseBodySectionName(item)
+			if err != nil {
+				break
+			}
+
+			l, err := mbox.getBodySection(msg.ID, section)
+			if err != nil {
+				return nil, err
+			}
+			fetched.Body[section] = l
+		}
+	}
+
+	return fetched, nil
+}
+
+func (mbox *mailbox) ListMessages(uid bool, seqSet *imap.SeqSet, items []imap.FetchItem, ch chan<- *imap.Message) error {
+	defer close(ch)
+
+	for _, seq := range seqSet.Set {
+		start := seq.Start
+		if start == 0 {
+			start = 1
+		}
+
+		stop := seq.Stop
+		if stop == 0 {
+			if uid {
+				uidNext, err := mbox.db.UidNext()
+				if err != nil {
+					return err
+				}
+				stop = uidNext - 1
+			} else {
+				stop = uint32(mbox.total)
+			}
+		}
+
+		for i := start; i <= stop; i++ {
+			msg, err := mbox.getMessage(uid, i, items)
+			if err != nil {
+				return err
+			}
+			if msg != nil {
+				ch <- msg
+			}
+		}
+	}
+
+	return nil
 }
 
 func (mbox *mailbox) SearchMessages(uid bool, criteria *imap.SearchCriteria) ([]uint32, error) {
@@ -70,11 +282,11 @@ func (mbox *mailbox) CreateMessage(flags []string, date time.Time, body imap.Lit
 	return errNotYetImplemented // TODO
 }
 
-func (mbox *mailbox) UpdateMessagesFlags(uid bool, seqset *imap.SeqSet, operation imap.FlagsOp, flags []string) error {
+func (mbox *mailbox) UpdateMessagesFlags(uid bool, seqSet *imap.SeqSet, operation imap.FlagsOp, flags []string) error {
 	return errNotYetImplemented // TODO
 }
 
-func (mbox *mailbox) CopyMessages(uid bool, seqset *imap.SeqSet, dest string) error {
+func (mbox *mailbox) CopyMessages(uid bool, seqSet *imap.SeqSet, dest string) error {
 	return errNotYetImplemented // TODO
 }
 

imap/user.go

@@ -6,6 +6,7 @@ import (
 	imapbackend "github.com/emersion/go-imap/backend"
 	"github.com/emersion/go-imap-specialuse"
 
+	"github.com/emersion/hydroxide/imap/database"
 	"github.com/emersion/hydroxide/protonmail"
 )
 
@@ -29,6 +30,7 @@ type user struct {
 	u           *protonmail.User
 	privateKeys openpgp.EntityList
 
+	db *database.User
 	mailboxes map[string]*mailbox
 }
 
@@ -40,11 +42,36 @@ func newUser(c *protonmail.Client, u *protonmail.User, privateKeys openpgp.Entit
 		mailboxes: make(map[string]*mailbox),
 	}
 
+	db, err := database.Open(u.Name+".db")
+	if err != nil {
+		return nil, err
+	}
+	uu.db = db
+
 	for _, data := range systemMailboxes {
-		uu.mailboxes[data.name] = &mailbox{
+		mboxDB, err := db.Mailbox(data.label)
+		if err != nil {
+			return nil, err
+		}
+
+		uu.mailboxes[data.label] = &mailbox{
 			name: data.name,
 			label: data.label,
 			flags: data.flags,
+			u: uu,
+			db: mboxDB,
+		}
+	}
+
+	counts, err := c.CountMessages("")
+	if err != nil {
+		return nil, err
+	}
+
+	for _, count := range counts {
+		if mbox, ok := uu.mailboxes[count.LabelID]; ok {
+			mbox.total = count.Total
+			mbox.unread = count.Unread
 		}
 	}
 
@@ -64,11 +91,12 @@ func (u *user) ListMailboxes(subscribed bool) ([]imapbackend.Mailbox, error) {
 }
 
 func (u *user) GetMailbox(name string) (imapbackend.Mailbox, error) {
-	if mbox, ok := u.mailboxes[name]; ok {
+	for _, mbox := range u.mailboxes {
-		return mbox, nil
+		if mbox.name == name {
-	} else {
+			return mbox, nil
-		return nil, imapbackend.ErrNoSuchMailbox
+		}
 	}
+	return nil, imapbackend.ErrNoSuchMailbox
 }
 
 func (u *user) CreateMailbox(name string) error {

protonmail/messages.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 
@@ -132,6 +133,103 @@ func (msg *Message) Encrypt(to []*openpgp.Entity, signed *openpgp.Entity) (plain
 	}, nil
 }
 
+type MessageFilter struct {
+	Page int
+	PageSize int
+	Limit int
+
+	Label string
+	Sort string
+	Asc bool
+	Begin int64
+	End int64
+	Keyword string
+	To string
+	From string
+	Subject string
+	Attachments *bool
+	Starred *bool
+	Unread *bool
+	Conversation string
+	Address string
+	ID []string
+	ExternalID string
+}
+
+func (c *Client) ListMessages(filter *MessageFilter) (total int, messages []*Message, err error) {
+	v := url.Values{}
+	if filter.Page != 0 {
+		v.Set("Page", strconv.Itoa(filter.Page))
+	}
+	if filter.PageSize != 0 {
+		v.Set("PageSize", strconv.Itoa(filter.PageSize))
+	}
+	if filter.Limit != 0 {
+		v.Set("Limit", strconv.Itoa(filter.Limit))
+	}
+	if filter.Label != "" {
+		v.Set("Label", filter.Label)
+	}
+	if filter.Sort != "" {
+		v.Set("Sort", filter.Sort)
+	}
+	if filter.Asc {
+		v.Set("Desc", "0")
+	}
+	if filter.Conversation != "" {
+		v.Set("Conversation", filter.Conversation)
+	}
+	if filter.Address != "" {
+		v.Set("Address", filter.Address)
+	}
+	if filter.ExternalID != "" {
+		v.Set("ExternalID", filter.ExternalID)
+	}
+
+	req, err := c.newRequest(http.MethodGet, "/messages?"+v.Encode(), nil)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	var respData struct {
+		resp
+		Total int
+		Messages []*Message
+	}
+	if err := c.doJSON(req, &respData); err != nil {
+		return 0, nil, err
+	}
+
+	return respData.Total, respData.Messages, nil
+}
+
+type MessageCount struct {
+	LabelID string
+	Total int
+	Unread int
+}
+
+func (c *Client) CountMessages(address string) ([]*MessageCount, error) {
+	v := url.Values{}
+	if address != "" {
+		v.Set("Address", address)
+	}
+	req, err := c.newRequest(http.MethodGet, "/messages/count?"+v.Encode(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var respData struct {
+		resp
+		Counts []*MessageCount
+	}
+	if err := c.doJSON(req, &respData); err != nil {
+		return nil, err
+	}
+
+	return respData.Counts, nil
+}
+
 func (c *Client) GetMessage(id string) (*Message, error) {
 	req, err := c.newRequest(http.MethodGet, "/messages/"+id, nil)
 	if err != nil {