From cfc1b44824bab33df8285e7005c9e5cd7c0593d1 Mon Sep 17 00:00:00 2001 From: emersion Date: Sun, 7 Jan 2018 23:38:13 +0100 Subject: [PATCH] imap: add local DB --- imap/database/mailbox.go | 156 +++++++++++++++++++++++++++ imap/database/user.go | 87 +++++++++++++++ imap/mailbox.go | 228 +++++++++++++++++++++++++++++++++++++-- imap/user.go | 38 ++++++- protonmail/messages.go | 98 +++++++++++++++++ 5 files changed, 594 insertions(+), 13 deletions(-) create mode 100644 imap/database/mailbox.go create mode 100644 imap/database/user.go diff --git a/imap/database/mailbox.go b/imap/database/mailbox.go new file mode 100644 index 0000000..156d461 --- /dev/null +++ b/imap/database/mailbox.go @@ -0,0 +1,156 @@ +package database + +import ( + "bytes" + "encoding/binary" + "errors" + + "github.com/boltdb/bolt" + + "github.com/emersion/hydroxide/protonmail" +) + +func serializeUID(uid uint32) []byte { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uid) + return b +} + +func unserializeUID(b []byte) uint32 { + return binary.BigEndian.Uint32(b) +} + +type Mailbox struct { + name string + u *User +} + +func (mbox *Mailbox) bucket(tx *bolt.Tx) (*bolt.Bucket, error) { + b := tx.Bucket(mailboxesBucket) + if b == nil { + return nil, errors.New("cannot find mailboxes bucket") + } + b = tx.Bucket([]byte(mbox.name)) + if b == nil { + return nil, errors.New("cannot find mailbox bucket") + } + return b, nil +} + +func (mbox *Mailbox) Sync(messages []*protonmail.Message) error { + err := mbox.u.db.Update(func(tx *bolt.Tx) error { + b, err := mbox.bucket(tx) + if err != nil { + return err + } + + for _, msg := range messages { + want := []byte(msg.ID) + c := b.Cursor() + found := false + for k, v := c.First(); k != nil; k, v = c.Next() { + if bytes.Equal(v, want) { + found = true + break + } + } + if found { + continue + } + + id, _ := b.NextSequence() + uid := uint32(id) + if err := b.Put(serializeUID(uid), want); err != nil { + return err + } + } + + return nil + }) + if err != nil { + return err + } + + return mbox.u.sync(messages) +} + +func (mbox *Mailbox) UidNext() (uint32, error) { + var uid uint32 + err := mbox.u.db.View(func(tx *bolt.Tx) error { + b, err := mbox.bucket(tx) + if err != nil { + return err + } + + uid = uint32(b.Sequence() + 1) + return nil + }) + return uid, err +} + +func (mbox *Mailbox) FromUid(uid uint32) (string, error) { + var apiID string + err := mbox.u.db.View(func(tx *bolt.Tx) error { + b, err := mbox.bucket(tx) + if err != nil { + return err + } + + k := serializeUID(uid) + v := b.Get(k) + if v == nil { + return ErrNotFound + } + apiID = string(v) + return nil + }) + return apiID, err +} + +func (mbox *Mailbox) FromSeqNum(seqNum uint32) (string, error) { + var apiID string + err := mbox.u.db.View(func(tx *bolt.Tx) error { + b, err := mbox.bucket(tx) + if err != nil { + return err + } + + c := b.Cursor() + var n uint32 = 1 + for k, v := c.First(); k != nil; k, v = c.Next() { + if seqNum == n { + apiID = string(v) + return nil + } + n++ + } + + return ErrNotFound + }) + return apiID, err +} + +func (mbox *Mailbox) FromApiID(apiID string) (uint32, uint32, error) { + var seqNum, uid uint32 + err := mbox.u.db.View(func(tx *bolt.Tx) error { + b, err := mbox.bucket(tx) + if err != nil { + return err + } + + want := []byte(apiID) + c := b.Cursor() + var n uint32 = 1 + for k, v := c.First(); k != nil; k, v = c.Next() { + if bytes.Equal(v, want) { + seqNum = n + uid = unserializeUID(k) + return nil + } + n++ + } + + return ErrNotFound + }) + return seqNum, uid, err +} diff --git a/imap/database/user.go b/imap/database/user.go new file mode 100644 index 0000000..115f72e --- /dev/null +++ b/imap/database/user.go @@ -0,0 +1,87 @@ +package database + +import ( + "encoding/json" + "errors" + + "github.com/boltdb/bolt" + + "github.com/emersion/hydroxide/protonmail" +) + +var ErrNotFound = errors.New("message not found in local database") + +var ( + mailboxesBucket = []byte("mailboxes") + messagesBucket = []byte("messages") +) + +type User struct { + db *bolt.DB +} + +func (u *User) Mailbox(name string) (*Mailbox, error) { + err := u.db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists(mailboxesBucket) + if err != nil { + return err + } + _, err = b.CreateBucketIfNotExists([]byte(name)) + return err + }) + if err != nil { + return nil, err + } + + return &Mailbox{name, u}, nil +} + +func (u *User) sync(messages []*protonmail.Message) error { + return u.db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists(messagesBucket) + if err != nil { + return err + } + + for _, msg := range messages { + k := []byte(msg.ID) + v, err := json.Marshal(msg) + if err != nil{ + return err + } + if err := b.Put(k, v); err != nil { + return err + } + } + + return nil + }) +} + +func (u *User) Message(apiID string) (*protonmail.Message, error) { + var msg *protonmail.Message + err := u.db.View(func (tx *bolt.Tx) error { + b := tx.Bucket(messagesBucket) + if b == nil { + return ErrNotFound + } + + k := []byte(apiID) + v := b.Get(k) + if v == nil { + return ErrNotFound + } + + return json.Unmarshal(v, msg) + }) + return msg, err +} + +func Open(path string) (*User, error) { + db, err := bolt.Open(path, 0700, nil) + if err != nil { + return nil, err + } + + return &User{db}, nil +} diff --git a/imap/mailbox.go b/imap/mailbox.go index e3ee2f9..8fd3d3e 100644 --- a/imap/mailbox.go +++ b/imap/mailbox.go @@ -1,9 +1,13 @@ package imap import ( + "strings" "time" "github.com/emersion/go-imap" + + "github.com/emersion/hydroxide/imap/database" + "github.com/emersion/hydroxide/protonmail" ) const delimiter = "/" @@ -12,6 +16,11 @@ type mailbox struct { name string label string flags []string + + u *user + db *database.Mailbox + + total, unread int } func (mbox *mailbox) Name() string { @@ -35,15 +44,19 @@ func (mbox *mailbox) Status(items []imap.StatusItem) (*imap.MailboxStatus, error for _, name := range items { switch name { case imap.StatusMessages: - status.Messages = 0 // TODO + status.Messages = uint32(mbox.total) case imap.StatusUidNext: - status.UidNext = 1 // TODO + uidNext, err := mbox.db.UidNext() + if err != nil { + return nil, err + } + status.UidNext = uidNext case imap.StatusUidValidity: status.UidValidity = 1 case imap.StatusRecent: - status.Recent = 0 // TODO + status.Recent = 0 case imap.StatusUnseen: - status.Unseen = 0 // TODO + status.Unseen = uint32(mbox.unread) } } @@ -58,8 +71,207 @@ func (mbox *mailbox) Check() error { return nil } -func (mbox *mailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []imap.FetchItem, ch chan<- *imap.Message) error { - return errNotYetImplemented // TODO +func (mbox *mailbox) sync() error { + filter := &protonmail.MessageFilter{ + PageSize: 150, + Label: mbox.label, + Sort: "ID", + Asc: true, + } + + total := -1 + for { + offset := filter.PageSize * filter.Page + if total >= 0 && offset > total { + break + } + + var page []*protonmail.Message + var err error + total, page, err = mbox.u.c.ListMessages(filter) + if err != nil { + return err + } + + if err := mbox.db.Sync(page); err != nil { + return err + } + + filter.Page++ + } + + return nil +} + +func getMessageID(id string) string { + return id + "@protonmail.com" +} + +func getAddress(addr *protonmail.MessageAddress) *imap.Address { + parts := strings.SplitN(addr.Address, "@", 2) + if len(parts) < 2 { + parts = append(parts, "") + } + + return &imap.Address{ + PersonalName: addr.Name, + MailboxName: parts[0], + HostName: parts[1], + } +} + +func getAddressList(addresses []*protonmail.MessageAddress) []*imap.Address { + l := make([]*imap.Address, len(addresses)) + for i, addr := range addresses { + l[i] = getAddress(addr) + } + return l +} + +func getEnvelope(msg *protonmail.Message) *imap.Envelope { + return &imap.Envelope{ + Date: time.Unix(msg.Time, 0), + Subject: msg.Subject, + From: []*imap.Address{getAddress(msg.Sender)}, + Sender: []*imap.Address{getAddress(msg.Sender)}, + ReplyTo: []*imap.Address{getAddress(msg.ReplyTo)}, + To: getAddressList(msg.ToList), + Cc: getAddressList(msg.CCList), + Bcc: getAddressList(msg.BCCList), + // TODO: InReplyTo + MessageId: getMessageID(msg.ID), + } +} + +func hasLabel(msg *protonmail.Message, labelID string) bool { + for _, id := range msg.LabelIDs { + if labelID == id { + return true + } + } + return false +} + +func getFlags(msg *protonmail.Message) []string { + var flags []string + if msg.IsRead != 0 { + flags = append(flags, imap.SeenFlag) + } + if msg.IsReplied != 0 || msg.IsRepliedAll != 0 { + flags = append(flags, imap.AnsweredFlag) + } + for _, label := range msg.LabelIDs { + switch label { + case protonmail.LabelStarred: + flags = append(flags, imap.FlaggedFlag) + case protonmail.LabelDraft: + flags = append(flags, imap.DraftFlag) + } + } + // TODO: DeletedFlag + return flags +} + +func (mbox *mailbox) getBodyStructure(id string, extended bool) (*imap.BodyStructure, error) { + // TODO +} + +func (mbox *mailbox) getBodySection(id string, section *imap.BodySectionName) (imap.Literal, error) { + // TODO +} + +func (mbox *mailbox) getMessage(isUid bool, id uint32, items []imap.FetchItem) (*imap.Message, error) { + var apiID string + var err error + if isUid { + apiID, err = mbox.db.FromUid(id) + } else { + apiID, err = mbox.db.FromSeqNum(id) + } + if err != nil { + return nil, err + } + + seqNum, uid, err := mbox.db.FromApiID(apiID) + if err != nil { + return nil, err + } + + msg, err := mbox.u.db.Message(apiID) + if err != nil { + return nil, err + } + + fetched := imap.NewMessage(seqNum, items) + for _, item := range items { + switch item { + case imap.FetchEnvelope: + fetched.Envelope = getEnvelope(msg) + case imap.FetchBody, imap.FetchBodyStructure: + bs, err := mbox.getBodyStructure(msg.ID, item == imap.FetchBodyStructure) + if err != nil { + return nil, err + } + fetched.BodyStructure = bs + case imap.FetchFlags: + fetched.Flags = getFlags(msg) + case imap.FetchInternalDate: + fetched.InternalDate = time.Unix(msg.Time, 0) + case imap.FetchRFC822Size: + fetched.Size = uint32(msg.Size) + case imap.FetchUid: + fetched.Uid = uid + default: + section, err := imap.ParseBodySectionName(item) + if err != nil { + break + } + + l, err := mbox.getBodySection(msg.ID, section) + if err != nil { + return nil, err + } + fetched.Body[section] = l + } + } + + return fetched, nil +} + +func (mbox *mailbox) ListMessages(uid bool, seqSet *imap.SeqSet, items []imap.FetchItem, ch chan<- *imap.Message) error { + defer close(ch) + + for _, seq := range seqSet.Set { + start := seq.Start + if start == 0 { + start = 1 + } + + stop := seq.Stop + if stop == 0 { + if uid { + uidNext, err := mbox.db.UidNext() + if err != nil { + return err + } + stop = uidNext - 1 + } else { + stop = uint32(mbox.total) + } + } + + for i := start; i <= stop; i++ { + msg, err := mbox.getMessage(uid, i, items) + if err != nil { + return err + } + if msg != nil { + ch <- msg + } + } + } + + return nil } func (mbox *mailbox) SearchMessages(uid bool, criteria *imap.SearchCriteria) ([]uint32, error) { @@ -70,11 +282,11 @@ func (mbox *mailbox) CreateMessage(flags []string, date time.Time, body imap.Lit return errNotYetImplemented // TODO } -func (mbox *mailbox) UpdateMessagesFlags(uid bool, seqset *imap.SeqSet, operation imap.FlagsOp, flags []string) error { +func (mbox *mailbox) UpdateMessagesFlags(uid bool, seqSet *imap.SeqSet, operation imap.FlagsOp, flags []string) error { return errNotYetImplemented // TODO } -func (mbox *mailbox) CopyMessages(uid bool, seqset *imap.SeqSet, dest string) error { +func (mbox *mailbox) CopyMessages(uid bool, seqSet *imap.SeqSet, dest string) error { return errNotYetImplemented // TODO } diff --git a/imap/user.go b/imap/user.go index 5199565..ab72ded 100644 --- a/imap/user.go +++ b/imap/user.go @@ -6,6 +6,7 @@ import ( imapbackend "github.com/emersion/go-imap/backend" "github.com/emersion/go-imap-specialuse" + "github.com/emersion/hydroxide/imap/database" "github.com/emersion/hydroxide/protonmail" ) @@ -29,6 +30,7 @@ type user struct { u *protonmail.User privateKeys openpgp.EntityList + db *database.User mailboxes map[string]*mailbox } @@ -40,11 +42,36 @@ func newUser(c *protonmail.Client, u *protonmail.User, privateKeys openpgp.Entit mailboxes: make(map[string]*mailbox), } + db, err := database.Open(u.Name+".db") + if err != nil { + return nil, err + } + uu.db = db + for _, data := range systemMailboxes { - uu.mailboxes[data.name] = &mailbox{ + mboxDB, err := db.Mailbox(data.label) + if err != nil { + return nil, err + } + + uu.mailboxes[data.label] = &mailbox{ name: data.name, label: data.label, flags: data.flags, + u: uu, + db: mboxDB, + } + } + + counts, err := c.CountMessages("") + if err != nil { + return nil, err + } + + for _, count := range counts { + if mbox, ok := uu.mailboxes[count.LabelID]; ok { + mbox.total = count.Total + mbox.unread = count.Unread } } @@ -64,11 +91,12 @@ func (u *user) ListMailboxes(subscribed bool) ([]imapbackend.Mailbox, error) { } func (u *user) GetMailbox(name string) (imapbackend.Mailbox, error) { - if mbox, ok := u.mailboxes[name]; ok { - return mbox, nil - } else { - return nil, imapbackend.ErrNoSuchMailbox + for _, mbox := range u.mailboxes { + if mbox.name == name { + return mbox, nil + } } + return nil, imapbackend.ErrNoSuchMailbox } func (u *user) CreateMailbox(name string) error { diff --git a/protonmail/messages.go b/protonmail/messages.go index 0e548e9..22ab98f 100644 --- a/protonmail/messages.go +++ b/protonmail/messages.go @@ -6,6 +6,7 @@ import ( "errors" "io" "net/http" + "net/url" "strconv" "strings" @@ -132,6 +133,103 @@ func (msg *Message) Encrypt(to []*openpgp.Entity, signed *openpgp.Entity) (plain }, nil } +type MessageFilter struct { + Page int + PageSize int + Limit int + + Label string + Sort string + Asc bool + Begin int64 + End int64 + Keyword string + To string + From string + Subject string + Attachments *bool + Starred *bool + Unread *bool + Conversation string + Address string + ID []string + ExternalID string +} + +func (c *Client) ListMessages(filter *MessageFilter) (total int, messages []*Message, err error) { + v := url.Values{} + if filter.Page != 0 { + v.Set("Page", strconv.Itoa(filter.Page)) + } + if filter.PageSize != 0 { + v.Set("PageSize", strconv.Itoa(filter.PageSize)) + } + if filter.Limit != 0 { + v.Set("Limit", strconv.Itoa(filter.Limit)) + } + if filter.Label != "" { + v.Set("Label", filter.Label) + } + if filter.Sort != "" { + v.Set("Sort", filter.Sort) + } + if filter.Asc { + v.Set("Desc", "0") + } + if filter.Conversation != "" { + v.Set("Conversation", filter.Conversation) + } + if filter.Address != "" { + v.Set("Address", filter.Address) + } + if filter.ExternalID != "" { + v.Set("ExternalID", filter.ExternalID) + } + + req, err := c.newRequest(http.MethodGet, "/messages?"+v.Encode(), nil) + if err != nil { + return 0, nil, err + } + + var respData struct { + resp + Total int + Messages []*Message + } + if err := c.doJSON(req, &respData); err != nil { + return 0, nil, err + } + + return respData.Total, respData.Messages, nil +} + +type MessageCount struct { + LabelID string + Total int + Unread int +} + +func (c *Client) CountMessages(address string) ([]*MessageCount, error) { + v := url.Values{} + if address != "" { + v.Set("Address", address) + } + req, err := c.newRequest(http.MethodGet, "/messages/count?"+v.Encode(), nil) + if err != nil { + return nil, err + } + + var respData struct { + resp + Counts []*MessageCount + } + if err := c.doJSON(req, &respData); err != nil { + return nil, err + } + + return respData.Counts, nil +} + func (c *Client) GetMessage(id string) (*Message, error) { req, err := c.newRequest(http.MethodGet, "/messages/"+id, nil) if err != nil {