gmail_imap.py

This is the source code of gmail_imap.py for quickly importing your GMail mail to Outlook Exchange with all your labels.  Open an editor, such as notepad and copy and paste the code below, then save as gmail_imap.py.

# Copyright 2012 Lajos Molnar.
# Licensed under the Creative Commons Attribution-ShareAlike 3.0 license
# See http://creativecommons.org/licenses/by-sa/3.0/

import imaplib, sys, hashlib, base64, pickle, os, getpass, argparse

p = argparse.ArgumentParser('gmail_imap.py', description='helper script to migrate GMail labels to Outlook Categories')
p.add_argument('operation', choices=('import', 'purge', 'apply'))
p.add_argument('-f', '--file', help='db file to import', default='gmail_imported.pickle')
p.add_argument('-r', '--reimport', help='reimport already imported labels', action='store_true')
p.add_argument('-e', '--email', help='gmail account name')
p.add_argument('-p', '--password', '--pwd', help='gmail account name')
p.add_argument('-I', '--inbox', help='label used instead of Inbox', default='Inbox')
p.add_argument('-T', '--trash', help='label used instead of Trash', default='Trash')
p.add_argument('--import-trash', help='also import/purge trash', action='store_true')
p.add_argument('--final', help='purge imported messages', action='store_true')
p.add_argument('--folder', help='Outlook folder where mail was imported to')
p.add_argument('--limit', type=int, help='Maximum number of items to purge at one time')
p.add_argument('--recurse', help='apply labels recursively for messages in all subfolders', action='store_true')
a = p.parse_args()

all_mail, trash = '[Gmail]/All Mail', '[Gmail]/Trash'
# read any existing state
LABELS, labels_done = {}, set()
try:
    with open(a.file, 'rb') as f:
        LABELS = pickle.load(f)
        labels_done = set(v for k, v in LABELS.items() if k != 'MAIL')
except:
    pass

def hashOf(msg):
    m = hashlib.sha1()
    m.update(msg.encode('utf-16')[2:])
    return base64.b64encode(m.digest()).decode('ascii')

if a.operation == 'apply':
    from win32com.client import Dispatch
    print('connecting to Outlook...')
    O = Dispatch('Outlook.Application')
    print('browsing to folder', a.folder, O)
    F = MAPI = O.GetNamespace('MAPI')
    for f in a.folder.split('\\'):
        fs = [f.Name for f in F.Folders]
        F = F.Folders(f)
    print('Cataloging mail...', F.StoreID)
    MAIL = {}
    if a.recurse:
        special = ()
        def walk(F):
            yield F
            for f in F.Folders:
                for f_ in walk(f):
                    yield f_
        folders = list(walk(F))
    else:
        special = ('Inbox', 'Trash')
        folders = [f.Name for f in F.Folders]
        for f in special:
            if f not in folders:
                F.Folders.Add(f)
        folders = [F] + [f for f in F.Folders if f.Name in special]

    for f in folders:
        def cats(i):
            c = set()
            if i.Parent.Name == special:
                c.add(i.Parent.Name)
            if i.Categories:
                c |= set(i.Categories.split(', '))
            return c

        N = f.Items.Count
        print('Applying labels in', f.Name, 'for', N, 'items ...')
        for ix, i in enumerate(f.Items, 1):
            PR_TRANSPORT_MESSAGE_HEADER = "http://schemas.microsoft.com/mapi/proptag/0x007D001E"
            msg = i.PropertyAccessor.GetProperty(PR_TRANSPORT_MESSAGE_HEADER)
            h = hashOf(msg)
            c = set()
            if h in MAIL:
                if (i.EntryID, f.StoreID) != MAIL[h]:
                    # already imported, combine labels
                    old = MAPI.GetItemFromID(*MAIL[h])
                    c = cats(old)
                    if old.UnRead:
                        i.UnRead = True
                    print('already imported', h, 'with', c)
                    print('adding to categories', cats(i))
                    old.Delete()
            elif h in LABELS:
                c = set(c.replace('[Gmail]/', '') for c in LABELS[h] if c != all_mail)
            c -= cats(i)
            if c:
                for cat in c:
                    # apply categories
                    if cat == 'Important':
                        i.Importance = 2
                    elif cat in special and i.Parent.Name != cat:
                        i = i.Move(F.Folders(cat))
                    elif not i.Categories:
                        i.Categories = cat
                    else:
                        i.Categories = i.Categories + ', ' + cat
                i.Save()

            try:
                del LABELS[h]
            except:
                pass
            MAIL[h] = (i.EntryID, f.StoreID)
            if ix % 100 == 0:
                print("{:%}".format(ix / N), end='\r', file=sys.stderr)
                sys.stderr.flush()

    # also save all imported hash
    if LABELS:
        print('WARNING: Could not apply labels for', len(LABELS), 'messages. ')
        print('Please reapply the created', a.file + '.remaining', 'database file later.')
    LABELS['MAIL'] = list(MAIL.keys())

    with open(a.file + '.remaining', 'wb') as f:
        pickle.dump(LABELS, f)
else:
    if a.operation == 'purge' and a.final and 'MAIL' not in LABELS:
        print('For final merge, please specify .remaining file from apply')
        sys.exit(1)

    # connect to IMAP
    M = imaplib.IMAP4_SSL("imap.gmail.com")
    print('logging in...')
    M.login(a.email or getpass.getpass('Email:'), a.password or getpass.getpass())
    
    print('querying labels...')
    typ, data = M.list()
    assert typ == 'OK', typ
    labels = [ eval(d.partition(b') "')[2].partition(b' ')[2], None, None) for d in data ]
    print('found', len(labels), 'labels')
    total_found = 0
    # since all_mail label is critical, verify that it is correct
    assert all_mail in labels or any(l.lower() == all_mail.lower() for l in labels if l != all_mail), "{} not in IMAP folder list".format(all_mail)

    def group(nums, limit=None):
        if limit == None:
            limit = len(nums)
        while nums:
            num_s = num_e = min(nums)
            while num_e + 1 in nums and num_e + 1 < num_s + limit:
                num_e += 1
            yield str(num_s) if num_s == num_e else "%d:%d" % (num_s, num_e)
            nums -= set(range(num_s, num_e + 1))

    def process(M, l, a, nums, LABELS, limit=None, trash=False):
        if l == a.inbox:
            l = 'Inbox'
        elif l == a.trash:
            l = trash

        if limit == None:
            limit = len(nums)
        for num in group(nums, 50):
            print(num, end='\r', file=sys.stderr)
            sys.stderr.flush()

            deleted = set()

            typ, data = M.fetch(num, '(BODY.PEEK[HEADER])')
            assert typ == 'OK', typ
            for d in data:
                if type(d) == type((1,2)):
                    h = hashOf(d[1].decode('ascii'))
                    if a.operation == 'import':
                        try:
                            if l not in LABELS[h]:
                                LABELS[h].append(l)
                        except:
                            LABELS[h] = [l]
                    elif a.operation == 'purge':
                        num = eval(d[0].partition(b' ')[0], None, None)
                        try:
                            if l in LABELS[h]:
                                deleted.add(num)
                                if l == all_mail:
                                    try:
                                        LABELS['TRASH'].append(h)
                                    except:
                                        LABELS['TRASH'] = [h]                                    
                        except:
                            pass

            if a.operation == 'purge':
                limit -= len(deleted)
                for num in group(deleted):
                    print("deleting", num)
                    if trash:
                        M.store(num, '+X-GM-LABELS', '\\Trash')
                    else:
                        M.store(num, '+FLAGS.SILENT', '\\Deleted')
                if limit < 0:
                    break

        if a.operation == 'import':
            print('done', 'total', len(LABELS), 'mail')
            with open(a.file + '.new', 'wb') as f:
                pickle.dump(LABELS, f)
            if sys.platform == 'win32':
                os.unlink(a.file)
            os.rename(a.file + '.new', a.file)
        elif a.operation == 'purge':
            print('expunging...')
            M.expunge()

        M.close()

    def get_nums(M, l):
        print('cataloguing label', l, end='... ')            os.unlink(a.file)

        sys.stdout.flush()
        typ, data = M.select('"' + l + '"')
        if typ == 'NO':
            return set()

        typ, data = M.search('', 'ALL')
        assert typ == 'OK', typ
        nums = set(map(int, data[0].split()))
        print('has', len(nums), 'messages')
        return nums

    for l in labels:
        # don't reimport existing labels
        if a.operation == 'import' and l in labels_done and not a.reimport:
            continue
        elif a.operation == 'purge' and l == all_mail:
            continue

        nums = get_nums(M, l)
        if nums and (l != trash or (a.import_trash and not a.final)):
            total_found += len(nums)
            process(M, l, a, nums, LABELS)
    if a.operation =='purge' and a.final and total_found == 0:
        print('purging imported mail items (total', len(LABELS['MAIL']), ')')
        while True:
            nums = get_nums(M, all_mail)
            if not nums:
                break
            LABELS2 = dict((i, [all_mail]) for i in LABELS['MAIL'])
            process(M, all_mail, a, nums, LABELS2, limit=a.limit, trash=True)
            if not a.limit:
                break

        # remove deleted messages from Trash
        if 'TRASH' in LABELS2:
            while True:
                nums = get_nums(M, trash)
                if not nums:
                    break
                process(M, trash, a, nums, dict((i, [trash]) for i in LABELS2['TRASH']), limit=a.limit)
                if not a.limit:
                    break

    M.logout()
  

Migrating GMail to Exchange (part 3) - Conversion VBA Macro

Once you have configured your Outlook client, the next step is to add the conversion macro to Outlook.

1. Enter Macro editor

Press Alt+F11 to start the Macro editor in Outlook.  A security message may appear to indicate that you are enabling macros.

Select Enable Macros to proceed.

2. Open Outlook.VBA script:

In the Project tool-window, click on Project1 (VbaProject.OTM) => Microsoft Outlook Objects to access ThisOutlookSession.  Double click on ThisOutlookSession to open the script's edit window.

3. Paste macro

Copy and paste the following script into your global Outlook.VBA script (the edit window):

' Copyright 2012 Lajos Molnar except ToBase64String method, which is marked below.
' Licensed under the Creative Commons Attribution-ShareAlike 3.0 license
' See http://creativecommons.org/licenses/by-sa/3.0/

Option Explicit

Const IMAP = "you@domain.com"       ' name of outlook root folder where GMail account is read via IMAP
Const PST = "migrated"              ' name of outlook root folder where mail should be imported to
Const cache_folder = ""    ' name of subfolder inside PST where GMail folders are copied to (or empty)
Const trash_label = "T"    ' alternate name of Trash folder (or empty)

Const date_range = ""               ' date range to import (or empty)

#Const USE_BODY = 0                 ' set to 1 to use whole message body

' =============== HASHING ===============
Function ToBase64String(rabyt)
  'Ref: http://stackoverflow.com/questions/1118947/converting-binary-file-to-base64-string
  With CreateObject("MSXML2.DOMDocument")
    .LoadXML ""
    .DocumentElement.DataType = "bin.base64"
    .DocumentElement.nodeTypedValue = rabyt
    ToBase64String = Replace(.DocumentElement.text, vbLf, "")
  End With
End Function

Function getHash(ByRef sha1, ByRef strToHash As String) As String
    Dim inBytes() As Byte, shaBytes() As Byte, b, b2 As Byte
    Dim r As String
    inBytes() = strToHash
    shaBytes() = sha1.ComputeHash_2(inBytes)
    getHash = ToBase64String(shaBytes)
End Function

Function hashOf(ByRef sha1, i) As String
    Const PR_TRANSPORT_MESSAGE_HEADERS = "http://schemas.microsoft.com/mapi/proptag/0x007D001E"
    Dim olkPA As Outlook.PropertyAccessor
    Set olkPA = i.PropertyAccessor
    Dim body As String
    body = olkPA.GetProperty(PR_TRANSPORT_MESSAGE_HEADERS)
#If USE_BODY Then
    body = body + i.body
    If TypeName(i) = "MailItem" Then body = body + i.HTMLBody
#End If
    hashOf = getHash(sha1, body)
    Set olkPA = Nothing
End Function

Public Sub import_finally_all_mail()
    do_import True
End Sub

Public Sub import_all_labeled_messages()
    do_import False
End Sub

Private Sub do_import(final_import As Boolean)
    Dim dDone As New Scripting.Dictionary
    Dim dIMAP As New Scripting.Dictionary

    Dim sha1 As SHA1CryptoServiceProvider
    Debug.Print "Creating SHA1 service provider..."
    Set sha1 = New SHA1CryptoServiceProvider

    Dim MAPI As Outlook.NameSpace
    Set MAPI = ThisOutlookSession.GetNamespace("MAPI")

    Dim imap_folder As Outlook.MAPIFolder, target As Outlook.MAPIFolder, f As Variant
    Set imap_folder = MAPI.Folders(IMAP)
    Set target = MAPI.Folders(PST)

    Dim i As Variant, i2 As Variant
    Dim items As Outlook.items

    ' catalog imported messages
    Debug.Print "Cataloguing already imported messages... ";

    On Error Resume Next
    target.Folders.Add "Inbox"
    target.Folders.Add "Trash"
    On Error GoTo 0

    Dim target_list, c, cat As String
    target_list = Array(target, target.Folders("Inbox"), target.Folders("Trash"))

    For Each f In target_list
        Debug.Print "("; f.Name; ": "; f.items.Count; "messages) ";
        For Each i In f.items
            Dim h As String
            h = hashOf(sha1, i)

            If dDone.Exists(h) Then
                Debug.Print "***DUPLICATE***"; i.Parent.Name; ":"; i.Subject; "and"; dDone(h).Parent.Name; ":"; dDone(h).Subject
                ' Remove one of the duplicates - combine categories
                If i.Parent.Name = PST Then
                    Set i2 = MAPI.GetItemFromID(dDone(h), target.StoreID)
                    For Each c In Split(i.Categories, "; ")
                        cat = c
                        Set i2 = add_category(i2, cat, target)
                    Next c
                    dDone(h) = i2
                    i.Delete
                Else
                    For Each c In Split(dDone(h).Categories, "; ")
                        cat = c
                        Set i = add_category(i, cat, target)
                    Next c
                    MAPI.GetItemFromID(i.EntryID, target.StoreID).Delete
                    dDone(h) = i.EntryID
                End If
            Else
                Debug.Assert i = MAPI.GetItemFromID(i.EntryID, target.StoreID)
                dDone.Add h, i.EntryID
            End If
            If dDone.Count Mod 1000 = 0 Then Debug.Print dDone.Count; " ";
        Next
    Next
    Debug.Print "done"

    If cache_folder <> "" Then
        import_labels target.Folders(cache_folder), "", dDone, dIMAP, sha1, target, False
    End If

    If final_import Then
        import_labels imap_folder.Folders("[Gmail]").Folders("All Mail"), "", dDone, dIMAP, sha1, target, True
    Else
        import_labels imap_folder, "", dDone, dIMAP, sha1, target, True
    End If
End Sub

Private Function add_category(item, ByVal category As String, target As Outlook.MAPIFolder)
    Set add_category = item
    If category = "" Then Exit Function

    If Left(category, 8) = "[Gmail]/" Then category = Mid(category, 9)
    If category = trash_label Then category = "Trash"

    ' Handle Important separately
    If category = "Important" Then
        item.Importance = olImportanceHigh
        'item.Save
    ElseIf category = "Inbox" Or category = "Trash" Then
        If item.Parent.Name <> category Then
            Set add_category = item.Move(target.Folders(category))
        End If
    ElseIf item.Categories = "" Then
        item.Categories = category
        'item.Save
    ElseIf InStr(", " + item.Categories + ", ", ", " + category + ", ") = 0 Then
        item.Categories = item.Categories + ", " + category
        'item.Save
    End If
End Function

Private Function import_mailitem(i, label As String, _
        dDone As Scripting.Dictionary, dIMAP As Scripting.Dictionary, _
        sha1, target As Outlook.MAPIFolder) As Boolean
    Dim MAPI As Outlook.NameSpace
    Set MAPI = ThisOutlookSession.GetNamespace("MAPI")

    If TypeName(i) = "MailItem" Or TypeName(i) = "AppointmentItem" Or TypeName(i) = "MeetingItem" Then
        Dim i2, d As String

        If dIMAP.Exists(i.EntryID) Then
            d = dIMAP(i.EntryID)
        Else
            d = hashOf(sha1, i)
        End If

        If dDone.Exists(d) Then
            Set i2 = MAPI.GetItemFromID(dDone(d), target.StoreID)
            Debug.Assert i2.Subject = i.Subject
            Debug.Print "["; d; "] "; i2.Subject; " is already imported with categories "; i2.Categories
            Set i2 = add_category(i2, label, target)
            dDone(d) = i2.EntryID
            i2.UnRead = i.UnRead
            i2.Save
            i.Delete
        Else
            Dim UnRead As Boolean
            UnRead = i.UnRead
            Set i2 = i.Move(target)
            Debug.Print "moving ["; d; "] "; i2.Subject; Format(dDone.Count, " (0)")
            Set i2 = add_category(i2, label, target)
            dDone.Add d, i2.EntryID
            i2.UnRead = UnRead
            i2.Save
        End If

        import_mailitem = True
    Else
        Debug.Print "ignoring "; TypeName(i); i.Subject
    End If
End Function

Private Sub import_label(dDone As Scripting.Dictionary, dIMAP As Scripting.Dictionary, _
        folder As Outlook.MAPIFolder, label As String, sha1, _
target As Outlook.MAPIFolder, remote As Boolean)
    Dim i As Variant, items As Outlook.items, N As Integer
    Debug.Print "Importing mail items with label "; label;

    If date_range <> "" Or remote Then
        Dim condition As String
        If remote Then
            Debug.Print " not marked... ";
            condition = "[IMAP Status] = 'Unmarked'"
        End If
        If date_range <> "" Then
            Debug.Print " between "; Replace(date_range, "-", " and "); "... ";
            If condition <> "" Then condition = condition + " And "
            Dim dr
            dr = Split(date_range, "-")
            condition = condition + "[SentOn] >= '" & dr(0) & "' And [SentOn] < '" & dr(1) & "'"
        End If

        Set items = folder.items
        Debug.Print "got items... ";
        Set i = items.Find(condition)
        Debug.Print "searching ...";

        While Not i Is Nothing
            If import_mailitem(i, label, dDone, dIMAP, sha1, target) Then N = N + 1
            Set i = items.FindNext
            Debug.Print label; Format(N, " \#0 ");
        Wend
    Else
        Debug.Print "... "; folder.Name; folder.Parent.Name
        Set items = folder.items
        Debug.Print "got items... ";

        ' local messages get deleted immediately, so we cannot simply loop
        Dim ix
        ix = 1
        While ix <= items.Count
            If import_mailitem(folder.items(ix), label, dDone, dIMAP, sha1, target) Then
                N = N + 1
                If N Mod 100 = 0 Then Debug.Print N;
            Else
                ix = ix + 1
            End If
        Wend
    End If

    Debug.Print "done"

End Sub

Private Sub import_labels(root As Outlook.MAPIFolder, label As String, _
        dDone As Scripting.Dictionary, dIMAP As Scripting.Dictionary, _
        sha1, target As Outlook.MAPIFolder, remote As Boolean)
    Dim f As Variant, folder As Outlook.MAPIFolder
    For Each f In root.Folders
        Set folder = f
        import_labels folder, label + "/" + f.Name, dDone, dIMAP, sha1, target, remote
    Next

    ' We will import All Mail last as it will finally remove the mail.
    ' Also don't import Trash (as it would permanently delete e-mail)
    If label = "" Or label = "/[Gmail]" Or label = "/[Gmail]/All Mail" _
            Then Exit Sub

    import_label dDone, dIMAP, root, Mid(label, 2), sha1, target, remote
End Sub

4. Configure your script

There are a few configuration variables at the top of your script:

IMAP

This is the name of the Outlook root folder where your emails from GMail show up.

PST

This is the name of your target Outlook root folder where you want to migrate your emails to

trash_label

This is the name of your trash label.  If you are importing your Trash with all label information, this is the name of your created label under which you moved the contents of your trash to.  If you are importing your Trash as is, you can enter "Trash" or simply "".

date_range

You have the option to only migrate a portion of your mail, e.g. if you want to only import e-mail from 2011, you would set this to "1/1/2011-1/1/2012".  NOTE: your date format is locale specific, so you need to use your date order.  If you want to migrate all mail, set it to "".  If you are selecting a date range, you should NOT set an IMAP folder size limit.  Otherwise, Outlook may not see all the e-mails from the selected date range.

cache_folder

You might have already copied over your IMAP folders to local folders.  This option allows you to specify the location of the root of the copied folders.  For this script, these have to be a subfolder of your target PST.

NOTE: local migration has not been fully tested.

NOTE 2: do not use this if you have migrated e-mail with categories and now you want to combine these categories.  Instead, simply copy the categorized email into the PST target folder.  Duplicate emails will have their categories merged.  If you do this, pay attention that the Inbox and Trash labels are represented by separate folders, instead of categories.  You need to copy emails with these categories into the respective target sub-folders.

USE_BODY

This script uses the message header of an item to identify it.  I have found this to be unique and it persists across the migration.  If you want to also use the message body, set USE_BODY to 1.   This will slightly slow down the hashing, but it is worth it if you might have messages without message headers.

5. Save your script

Click on the save icon () to save your script.

6. Add the referenced libraries

The script uses two libraries that you have to add to the References.  Select Tools => References.  Look for, and checkmark:

• mscorlib.dll
• Microsoft Scripting Runtime

Now you are ready to start the migration...

Run checkpatch automatically on git commit

This is a script I wrote that hooks into git-commit, and automatically runs checkpatch and “git diff –check” on a commit. If any of these fail, the commit is rejected. (In case you are not already using it, checkpatch looks for common coding mistakes, and it facilitates better kernel code.)

In addition, the script sets file permissions for common files (c, h, makefile, readme) to 0644, and executables (directories, shell scripts) to 0755. This is a common issue when editing files in Windows.

To install, save the script in your kernel GIT repository/ies under the $GIT_DIR/hooks directory (usually .git/hooks) as .git/hooks/pre-commit.

If you want to bypass the check (e.g. if the failure is intended), use the -n flag for git-commit to ignore the checks. Just be sure that you still resolve any failures that were correct, and there really wasn’t a better proper way to write your code without checkpatch errors. Now there is a whole set of discussion about the 80 character limit that sometime forces less understandable code (you can read about the pros/cons and find a patch to checkpatch to make this optional) BTW, Linus prefers in many cases to ignore the 80-char-limit warnings.

But wait! You can do even more. Since you are already modifying files, why not run checkpatch on the whole file (and not just on the lines you are changing)? To see if there are any already existing checkpatch errors in the files that you modified, set up these two bash aliases that will run checkpatch on the whole files that are about to be committed (just run ‘chkp’ - NOTE: this will not work in an empty repository.):

alias gitd='git rev-parse --show-cdup'
alias chkp='`gitd`scripts/checkpatch.pl -f `git diff --name-only HEAD | awk -v i=$(gitd) "{print i\\$0}"`'

Let me know if you have any questions/suggestions.

#!/bin/bash
#
# Run checkpatch.pl and fix permissions on what is about to be
# committed.
#
# Written by Lajos Molnar  based on the sample git
# pre-commit script.

if git-rev-parse --verify HEAD >/dev/null 2>&1
then
  against=HEAD
else
  # Initial commit: diff against an empty tree object
  against=4b825dc642cb6eb9a060e54bf8d69288fbee4904
fi

# Change access modes on committed files - do this first so
# we fix permissions even if there are errors
for i in `git diff-index --cached --name-only $against`
do
  name=${i##*/}
  # set permissions for common kernel files
  if echo $name \
  | grep -Eq "^(README|Kconfig|Kbuild|Makefile|HOWTO|TODO|"\
"\.gitignore|.*\.(c|h|cpp|txt|S|xml|mk))$"
  then
    chmod 0644 $i
    git add $i
  # directories and scripts are executable
  elif [ -d $i ] \
  || (echo $name | grep -Eq "^.*\.(sh|pl|py)$") 
  || (head -1 $i | grep -Eq "^#!")
  then
    chmod 0755 $i
    git add $i
  fi
done

# run simple diff checks
git diff-index --cached $against --check -- || exit $?

# run checkpatch if there is actual code difference
if git diff --cached $against | grep -Eq "^---"
then
  (git diff --cached $against \
  | $GIT_DIR/../scripts/checkpatch.pl --no-signoff -) || exit $?
fi

exit 0