import sys, datetime, os, os.path, fnmatch, shutil, copy, re, subprocess
from copy import deepcopy
from dateutil.rrule import *
from busyfree import *
from parsers import *

import locale, codecs
try:
    # try using the user's default settings.
    locale.setlocale(locale.LC_ALL, '')
except:
    # use the current setting for locale.LC_ALL
    locale.setlocale(locale.LC_ALL, None)

class OL():
    num_lines = 0

    def __init__(self,line=None):
        self.max_width = 74
        if line == None:
            self.line = [None, []]
        else:
            self.line = line
        OL.num_lines += 1

    def set_attr(self, attr):
        self.line[0] = attr

    def add_col(self, pos, wdth, cont):
        self.line[1].append([pos, wdth, cont])

    def str(self, min_attr=-2):
        """docstring for cur"""
        attr = self.line[0]
        lst = self.line[1]

        padding = ''
        s = '' 
        slst = []
        tl = []
        if attr >= min_attr:
            for col in lst[:-1]:
                pos, wdth, cont = col
                padding = ' '*(pos - len(s))
                if cont == None:
                    cont = ''
                else:
                    cont = cont[:wdth]
                s ="%s%s%s" % (s, padding, cont)
            # last column
            col = lst[-1]
            pos, wdth, cont = col
            if attr == 5:
                cont = cont.strip()
                t = cont
                padding = ''
                s = ''
            else:
                padding = ' '*(pos - len(s))
                if cont == None:
                    cont = ''
                if wdth > 0:
                    t = cont[:wdth]
                else:
                    # wrap '0' width strings if necessary
                    t = cont
                    if len(t) >= self.max_width-pos:
                        tlines = text_wrap(t, self.max_width-pos)
                        t = tlines[0]
                        tl = tlines[1:]
            slst.append("%s%s%s" % (s, padding, t))

            if len(tl) > 0:
                for l in tl:
                    slst.append("%s%s" % (' '*pos, l))
        return("\n".join(slst))

    def cur(self, min_attr=-2):
        """docstring for cur"""
        attr = self.line[0]
        lst = self.line[1]

        padding = ''
        s = '' 
        slst = []
        tl = []
        if attr >= min_attr:
            for col in lst[:-1]:
                pos, wdth, cont = col
                padding = ' '*(pos - len(s))
                if cont == None:
                    cont = ''
                else:
                    cont = cont[:wdth]
                s ="%s%s%s" % (s, padding, cont)
            # last column
            col = lst[-1]
            pos, wdth, cont = col
            if attr == 5:
                cont = cont.strip()
                t = cont.strip()
                padding = ''
                s = ''
            else:
                padding = ' '*(pos - len(s))
                if cont == None:
                    cont = ''
                if wdth > 0:
                    t = cont[:wdth]
                else:
                    # wrap '0' width strings if necessary
                    t = cont
                    if len(t) >= self.max_width-pos:
                        tlines = text_wrap(t, self.max_width-pos)
                        t = tlines[0]
                        tl = tlines[1:]
            slst.append("%s%s%s%s%s" % (attrs[attr], s, padding, t, all_off))

            if len(tl) > 0:
                for l in tl:
                    slst.append("%s%s%s%s" % (' '*pos, attrs[attr],
                        l, all_off))
        return("\n".join(slst))

    def htm(self, min_attr=-2, prnt=False):
        """docstring for html table row"""
        # there should be 4 columns
        hash = {}
        attr = self.line[0]
        hash['beg_font'] = ''
        hash['end_font'] = ''
        lst = self.line[1]
        num_cols = len(lst)
        empty_cols = 4 - len(lst)
        retval = ''
        if attr >= min_attr:
            if prnt:
                hash['color'] = 'color = "%s"' % print_color_attrs[attr]
            else:
                hash['color'] = 'color = "%s"' % main_color_attrs[attr]
            if attr == 0:
                hash['beg_font'] = '<b>'
                hash['end_font'] = '</b>'
            elif attr == -2:
                hash['color'] += ' size="-1"'
                hash['beg_font'] = '<i>'
                hash['end_font'] = '</i>'
            tmp = []
            if num_cols > 1:
                if empty_cols == 1:
                    tmp.append("""\
<td width="5%">
    &nbsp;
</td>""")
                elif empty_cols > 1:
                    tmp.append("""\
<td colspan="%s">
    &nbsp;
</td>""" % empty_cols)
                for col in lst[:-1]:
                    pos, wdth, cont = col
                    hash['align'] = ''
                    if cont == None:
                        hash['cont'] = '&nbsp;'
                    else:
                        cont = cont.strip()
                        if cont != '':
                            hash['cont'] = cont
                            cwdth = len(cont)
                            hash['cont'] = cont[:wdth]
                        else:
                            hash['cont'] = '&nbsp;'
                    # if ':' in cont or 'm' in cont or ' ' in cont:
                    if ':' in cont or ' ' in cont:
                        hash['align'] = 'align="right"'
                    else:
                        hash['align'] = 'align="center"'
                    tmp.append("""\
<td %(align)s valign="top">
    <font %(color)s>%(cont)s</font>
</td>""" % hash)
            # last or only column
            col = lst[-1]
            pos, wdth, cont = col
            if cont == None:
                hash['cont'] = '&nbsp;'
            else:
                cont = cont.strip()
                if cont == '':
                    hash['cont'] = '&nbsp;'
                else:
                    hash['cont'] = cont
            if pos > 0:
                # last column
                if num_cols == 1:
                    tmp.append("""\
<td colspan="%s">
    &nbsp;
</td>""" % empty_cols)
                tmp.append("""\
<td valign="top" width="75%%">
    <font %(color)s>%(beg_font)s%(cont)s%(end_font)s</font>
</td>""" % hash)
            else:
                tmp.append("""\
<td valign="top" align="left" colspan="4">
    <font %(color)s>%(beg_font)s%(cont)s%(end_font)s</font>
</td>""" % hash)

            retval = """<tr>%s</tr>""" % "\n".join(tmp)
        return(retval)

    def pre(self, min_attr=-2, prnt=False):
        """docstring for cur"""
        hash = {}
        attr = self.line[0]
        hash['beg_font'] = ''
        hash['end_font'] = ''
        if prnt:
            hash['color'] = 'color = "%s"' % print_color_attrs[attr]
        else:
            hash['color'] = 'color = "%s"' % main_color_attrs[attr]
        if attr == 0:
            hash['beg_font'] = '<b>'
            hash['end_font'] = '</b>'
        elif attr == -2:
            hash['color'] += ' size="-1"'
            hash['beg_font'] = '<i>'
            hash['end_font'] = '</i>'
        before = "<font %(color)s>%(beg_font)s" % hash
        after = "%(end_font)s</font>" % hash
        attr = self.line[0]
        lst = self.line[1]

        padding = ''
        s = '' 
        slst = []
        tl = []
        if attr >= min_attr:
            for col in lst[:-1]:
                pos, wdth, cont = col
                padding = ' '*(pos - len(s))
                if cont == None:
                    cont = ''
                else:
                    cont = cont[:wdth]
                s ="%s%s%s" % (s, padding, cont)
            # last column
            col = lst[-1]
            pos, wdth, cont = col
            if attr == 5:
                cont = cont.strip()
                t = cont
                padding = ''
                s = ''
            else:
                padding = ' '*(pos - len(s))
                if cont == None:
                    cont = ''
                if wdth > 0:
                    t = cont[:wdth]
                else:
                    # wrap '0' width strings if necessary
                    t = cont
                    if len(t) >= self.max_width-pos:
                        tlines = text_wrap(t, self.max_width-pos)
                        t = tlines[0]
                        tl = tlines[1:]
            slst.append("%s%s%s%s%s" % (before, s, padding, t, after))

            if len(tl) > 0:
                for l in tl:
                    slst.append("%s%s%s%s" % (before,' '*pos, l, after))
        return("\n".join(slst))

class ETMData:
    def _init__(self):
        self.alerts = []
        self.loadData()
        if self.changed:
            self.loadData()
            
    def getmtime(self):
        m = 0
        c = 0
        for path, subdirs, names in os.walk(etmdata):
            for name in names:
                if fnmatch.fnmatch(name, '[!.]*.txt'):
                    file = os.path.join(path,name)
                    m = max(m, os.path.getmtime(file))
                    c += 1
        return(m+c)

    def loadData(self):
        """Walk the etmdata directory adding files, lines and linenumbers to
        hashes. projects[shortfile] will contain the project line hash,
        idnum_hash[idnum] will contain the file and linenumber of the item
        and items_list will contain a list of the (checked) hashes for
        the items (events, tasks and actions)."""
        self.messages = []
        self.load_errors = []
        self.changed = False        # true if a file has been updated
        self.projects = {}          # project shortfile -> project line hash
        self.context_groups = {}    # context -> list of itemnum (list view)
        self.keyword_groups = {}    # keyword -> list of itemnum (list view)
        self.project_groups = {}    # project -> list of itemnum (list view)
        self.date_groups = {}       # date -> list of itemnum
        self.item_list = []         # list of item hashes (idexed by itemnum)
        self.idnum_hash = {}        # idnum -> file, linenumber
        self.itemnum_idnum = {}     # itemnum -> idnum
        self.groups = {}            # for grouped task prerequisites 
        self.cur_prereq = []
        self.cur_level = 0
        self.cur_id = None
        self.find = None
        self.context = 'none'
        self.keyword = 'none'
        self.contexts = set([])
        self.keywords = set([])
        self.repeats = set([])
        self.project = 'none'
        self.title = ''
        self.pattern = ''
        self.item_hash = {} 
        self.agenda_hash = {}       # hash of lists of item hashes; keys for
                                    # 'next', 'recent', 'available'
        self.time_hash = {}         # for time reports
        self.sched_hash = {}        # for busy/free
        self.alerts = []
        self.finished = set([])
        self.activeview = 'agenda'
        self.options = {}
        self.dates = {}             # dates of prereqs keyed by idnum
        self.options['agenda'] = {} 
        idnum = 0
        itemnum = 0
        self.today = get_today()
        self.check_rotating_files()
        self.show_idnums = False
        all_msgs = []
        for path, subdirs, names in os.walk(etmdata):
            for name in names:
                if fnmatch.fnmatch(name, '[!.]*.txt'):
                    file_msgs = []
                    new_items = []
                    file = os.path.join(path,name)
                    # use the relative path as the key
                    shortfile = file[len(etmdata)+1:]
                    lines = self.get_filelines(file)
                    if not lines:
                        continue
                    # process the first line using the global dflts
                    p_line = lines[0].strip()
                    m1, p_hash = self.line2hash(dflts, p_line, False)
                    if len(m1) == 0:
                        self.projects[shortfile] = p_hash
                    else:
                        p_hash = {}

                    m2, p_hash = self.check_hash(p_hash, False)
                    if len(m1+m2) > 0:
                        file_msgs.insert(0,"Errors in %s" % shortfile)
                        file_msgs.extend(m1+m2)
                        file_msgs.append("  line 1: %s" % p_line)
                        all_msgs.extend(file_msgs)
                        continue
                    cur_prereq = []
                    groups = {}
                    cur_level = 0
                    cur_id = None
                    for l in range(1, len(lines)):
                        line_msgs = []
                        if not lines[l].strip() or \
                                comment_regex.match(lines[l]):
                            continue
                        i_list = []
                        # we have an event, task or action
                        idnum += 1
                        i_line = lines[l].strip()

                        m3 = []

                        m1, i_hash = self.line2hash(p_hash, i_line)
                        m2, i_hash = self.check_hash(i_hash)
                        linenum = l + 1
                        i_hash['idnum'] = num2alpha(idnum)

                        if len(m1+m2) > 0:
                            line_msgs.extend(m1+m2)
                            file_msgs.extend(line_msgs)
                            continue
                        else:
                            i_list = []
                            i_list.append(i_hash) 
                            new_items.extend(i_list)
                        if len(m1+m2+m3) > 0:
                            line_msgs.extend(m1+m2+m3)
                            line_msgs.insert(0, "  i_line %s: %s" % 
                                    (shortfile, i_line))
                            file_msgs.extend(line_msgs)
                            continue
                        else:
                            key = (file, linenum)
                            self.idnum_hash[i_hash['idnum']] = key
                        if i_hash['type'] == 't':
                            # we have a task
                            (cur_prereq, cur_id, cur_level,
                                 groups, i_hash) = self.add_prereq(
                                 cur_prereq,
                                 cur_id,
                                 cur_level,
                                 groups,
                                 i_hash,
                                 )
                            # if has(i_hash, 'd') and not has(i_hash, 'f'):
                            if has(i_hash, 'd'):
                                self.dates.setdefault(i_hash['idnum'], 
                                    []).append(i_hash['d'])
                            if has(i_hash,'f'):
                                self.finished.add(i_hash['idnum'])
                                available = False
                            else:
                                available = True
                            if has(i_hash, 'prereq'):
                                for id in i_hash['prereq']:
                                    # if not id in self.finished:
                                    if not id in self.finished:
                                        if has(i_hash, 'd'):
                                            if has(self.dates, id) and \
                                            self.dates[id][-1]<=i_hash['d']:
                                                available = False
                                                break
                                        else:
                                            available = False
                                            break 
                                if not has(i_hash, 'd'):
                                    ldate = None
                                    for id in i_hash['prereq']:
                                        if id in self.dates:
                                            ldate = self.dates[id][-1]
                                    if ldate:
                                        i_hash['d'] = ldate
                            i_hash['available'] = available
                    if len(file_msgs) > 0:
                        file_msgs.insert(0,"Errors in %s" % shortfile)
                        all_msgs.extend(file_msgs)
                    self.item_list.extend(new_items)
        self.load_errors = all_msgs
        return(all_msgs)

    def add_prereq(self, cur_prereq, cur_id, cur_level, groups, hash):
        """determine prereq's based on periods or +/-"""
        pr = hash['chrcode']
        new_id = hash['idnum']
        if pr[0] in ['-', '+']:
            # we're using plus and minus for tasks
            hash['prereq'] = copy.deepcopy(cur_prereq)
            if pr[0] == '-':
                cur_prereq.append(new_id) 
            return(cur_prereq, cur_id, cur_level, groups, hash)
        elif pr[0] in ['.', '_']:
            # we're using periods and underscores for tasks
            level = len(pr) - 1
            if pr[0] == '_':
                groups.setdefault(level,[]).append('%s' % new_id)
            if level >= cur_level + 1:
                if (groups.has_key(cur_level)):
                    cur_prereq.append(groups[cur_level])
                    groups[cur_level] = []
                cur_prereq.append(['%s' % cur_id])
            elif level < cur_level:
                cur_prereq = cur_prereq[:-(cur_level-level)-1]
            prereq = []
            for g in cur_prereq:
                for i in g:
                    if i not in prereq:
                        prereq.append(i)
            hash['prereq'] = prereq
            return(cur_prereq, new_id, level, groups, hash)

    def check_rotating_files(self):
        self.current_hash = {}
        lastyear = str(int(get_today().strftime("%Y"))-1)
        thismonth = get_today().strftime("%m")
        # task, event, done and action are set in etmrc
        for t in [task, event, done, action, note]:
            cur = "%s_%s.txt" % (thismonth, t)
            self.current_hash[t] = cur
            curfile = os.path.join(etmdata, cur)
            bak = ".%s-%s_%s" % (lastyear, thismonth, t)
            bakfile =  os.path.join(etmdata, bak)
            if not os.path.isfile(curfile):
                fo = open(curfile, 'w')
                fo.write("%s-%s\n" %
                    (t, thismonth))
                fo.close()
            if not os.path.isfile(bakfile):
                os.rename(curfile, bakfile)
                fo = open(curfile, 'w')
                fo.write("%s-%s\n" %
                    (t, thismonth))
                fo.close()
        if self.new_version():
            actn = "~ installed etm version %s @p 1 @d %s\n" % (
                version, get_today().strftime(date_fmt))
            cur_actn = "%s_%s.txt" % (thismonth, action)
            actn_file = os.path.join(etmdata, cur_actn)
            self.addline2file(actn, actn_file)
                
    def new_version(self):
        v_file = os.path.join(etmdir, 'etm_version.txt')
        if os.path.isfile(v_file):
            fo = open(v_file, 'r')
            try:
                current_version = fo.readline().strip()
            except:
                current_version = 0
            fo.close()
        else:
            current_version = 0
        if 'x' not in version and current_version != version:
            fo = open(v_file, 'w')
            fo.write("%s" % version)
            fo.close()
            return(True)
        else:
            return(False)

    def new_project(self, rel_path, project_line):
        subdir, name = os.path.split(rel_path)
        fullname = "%s.txt" % (name)
        fullpath = os.path.join(etmdata, subdir)
        if not os.path.isdir(fullpath):
            os.makedirs(fullpath)
        filename = os.path.join(fullpath, fullname)
        if os.path.isfile(filename):
            return([False, "Error: file '%s' already exists" % filename])
        fo = open(filename, 'w')
        fo.write("%s\n" % project_line)
        fo.close()
        self.logaction(filename, "created '%s'" % filename)
        return([True, "created '%s'" % filename])

    def mark_complete(self, idnum, date=None):
        file, linenum = self.idnum_hash[idnum]
        old_line = self.getlinefromfile(linenum, file)

        m1, task = self.line2hash({}, old_line)
        m2, task = self.check_hash(task)
        if 'type' not in task or task['type'] != 't':
            msg = "only tasks can be completed"
            return(False, msg)
        if has(task, 'f'):
            msg = "only unfinished tasks can be marked complete"
            return(False, msg)
        if date:
            try:
                fdate = parse_date(date)
                task['f'] = fdate.strftime(date_fmt)
                act = "marked %s completed on %s" % (idnum, task['f'])

            except:
                return(False, "Could not parse '%s'" % date)
        else:
            task['f'] = get_today().strftime(date_fmt)
            act = "marked '%s' completed on '%s'" % (task['t'], task['f'])

        new_line = "%s %s" % (task['chrcode'], task['t'])
        for key in task_keys:
            if has(task, key):
                new_line += " @%s %s" % (key, task[key])
        self.replacelineinfile(linenum, new_line, file)
        return(True, act)

    def mark_incomplete(self, idnum):
        file, linenum = self.idnum_hash[idnum]
        old_line = self.getlinefromfile(linenum, file)

        m1, task = self.line2hash({}, old_line)
        m2, task = self.check_hash(task)
        if 'type' not in task or task['type'] != 't':
            msg = "only tasks can be completed"
            return(False, msg)
        if not has(task, 'f'):
            msg = "only finished tasks can be marked incomplete"
            return(False, msg)
        act = "removed finish date '%s' from '%s'" % (task['f'], task['t'])
        del task['f']

        new_line = "%s %s" % (task['chrcode'], task['t'])
        for key in task_keys:
            if has(task, key):
                new_line += " @%s %s" % (key, task[key])
        self.replacelineinfile(linenum, new_line, file)
        return(True, act)

    def update_task(self, task, nextdue):
        idnum = task['idnum']
        file, linenum = self.idnum_hash[idnum]
        self.backup(file)
        title = "%s %s" % (task['chrcode'], task['t'])
        orig_line = ''
        for c in task_keys:
            if has(task,c):
                orig_line += " @%s %s" % (c, task[c])
        rep_line = ''
        if has(task,'f'):
            # don't put any of the repeat keys in the 'done' task
            for c in no_repeat_keys:
                if has(task,c):
                    rep_line += " @%s %s" % (c, task[c])
            curfile = os.path.join(etmdata, self.current_hash[done])
            retval = self.addline2file(title+rep_line, curfile)
            task['f'] = task['sf'] = None
        if parse(nextdue) != parse(task['d']):
            task['d'] = format_date(nextdue, date_fmt)
        new_line = ''
        for c in task_keys:
            if has(task,c):
                new_line += " @%s %s" % (c, task[c])
        if new_line != orig_line:
            self.replacelineinfile(linenum, title+new_line, file)

    def get_rrule(self, hash):
        """Return the rrule for the repeated hash."""
        # The duedate for repeated tasks will be reset each time the task is
        # finished so the duedate should be for the next unfinished
        # repetition. For events (e.g. birthdays, faculty meetings, etc), the
        # duedate should be for the first event.
        mr = []
        first_d = None
        hash['rr'] = None
        try:
            frequency = frequency_names[hash['r']]
        except:
            mr.append("    unrecognized frequency '@r %s'" % hash['r'])
            frequency = ''
        until = hash.setdefault('u', None)
        include = hash.setdefault('l', None)
        exclude = hash.setdefault('x', None)
        if has(hash, 'd'):
            duedate = parse(hash['d'])

        if has(hash, 'w'):
            # fix three character weekday abbreviations
            weekdays = hash['w'].upper()
            for x in ['MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT', 'SUN']:
                weekdays = re.sub(x, x[:2], weekdays)
            hash['w'] = weekdays

        rule = [frequency]
        if 'i' in hash and int(hash['i']) != 1:
            rule.append("interval=%s" % hash['i'])
        # XXX not for list repetitions
        if hash['r'] != 'l':
            try:
                dtstart = map(int, duedate.strftime("%Y,%m,%d").split(','))
                dtstart = ",".join(map(str, dtstart))
            except:
                mr.append("    parse '@d %s' failed" % (
                    hash['d']))

            rule.append("dtstart=datetime.datetime(%s)" % dtstart)
            for key in by_names.keys():
                if key in hash:
                    rule.append("by%s=%s" % (by_names[key],
                        hash[key]))
            if until != None:
                try:
                    until = parse(until)
                    until = map(int, until.strftime("%Y,%m,%d").split(
                        ','))
                    until = ",".join(map(str, until))
                    rule.append("until=datetime.datetime(%s)" % until)

                except:
                    mr.append("    parse '@u %s' failed" % (
                    hash['u']))

            rule = ', '.join(rule)
        includedates = []
        if include != None:
            m = parens_regex.match(include)
            if m:
                include = m.group(1)
            indates = include.split(',')
            for date in indates:
                try:
                    dt = parse(date)
                    dt = map(int, dt.strftime("%Y,%m,%d").split(','))
                    dt = ",".join(map(str, dt))
                    includedates.append("%s" % dt)
                except:
                    mr.append("    parse '@l %s' failed" %(
                    date))

            if (hash['r'] == 'l' and ('d' not in hash or
                hash['d'] == None)):
                hash['d'] = indates[0]
                duedate = parse(hash['d'])
        excludedates = []
        if exclude != None:
            m = parens_regex.match(exclude)
            if m:
                exclude = m.group(1)
            exdates = exclude.split(',')
            for date in exdates:
                try:
                    dt = parse(date)
                    dt = map(int, dt.strftime("%Y,%m,%d").split(','))
                    dt = ",".join(map(str, dt))
                    excludedates.append("%s" % dt)
                except:
                    mr.append("    parse '@x %s' failed" % (
                    date))

        try:
            rr = rruleset()
            if hash['r'] != 'l':
                eval("rr.rrule(rrule(%s))" % rule)
        except:
            mr.append("    could not parse '%s'" % rule)

        if includedates:
            for dt in includedates:
                try:
                    eval("rr.rdate(datetime.datetime(%s))" % dt)
                except:
                    mr.append("    could not parse '@l %s'" % hash['l'])
                    return(False)
        if excludedates:
            for dt in excludedates:
                try:
                    eval("rr.exdate(datetime.datetime(%s))" % dt)
                except:
                    mr.append("    could not parse '@x %s'" % hash['x'])
                    print(mr)
        if len(mr) > 0:
            hash['rr'] = None
        else:
            hash['rr'] = rr
        return(mr)

    def get_next(self, hash):
        """Return formated date of the next due date for the repeated task."""
        time = datetime.time(0,0,0,0)
        # today = datetime.datetime.combine(get_today(), time)
        today = datetime.datetime.combine(get_today(), time)
        if has(hash, 'd'):
            due = datetime.datetime.combine(parse(hash['d']), time)
        else:
            due = today
        if has(hash, 'o'):
            overdue = hash['o']
        else:
            overdue = 'k'
        finished = due
        if has(hash,'f'):
            finished = datetime.datetime.combine(parse(hash['f']), time)
        elif hash['type'] in ['e', 'a'] and due.date() < get_today():
            finished = due

        if has(hash, 'rr'):
            r = hash['rr']
            if overdue == 'r':
                first = r.after(finished, inc=True)
                second = r.after(first)
                retval = second
            elif overdue == 's':
                try:
                    retval = r.after(today, inc=True)
                except:
                    retval = ''
            elif overdue == 'k':
                retval = r.after(due)
        else:
            if not has(hash, 'l'):
                return('')
            if overdue == 'r':
                d = finished
                n = 1
            elif overdue == 's':
                d = today
                n = 0
                if finished and finished == today:
                    n = 1
            else: # overdue == 'k'
                d = due
                n = 1
            m = parens_regex.match(hash['l'])
            if m:
                parens = True
                items = m.group(1)
            else:
                parens = False
                items = hash['l']
            lst = items.split(',')
            l = []
            for i in range(len(lst)):
                if parse(lst[i]) < d:
                    continue
                else:
                    l = lst[i:]
                    break
            if len(l) > n:
                retval = parse(l[n])
            else:
                retval = ''

        if retval:
            if retval.date() <= due.date():
                retval = ''
            else:
                retval = retval.strftime(date_fmt)
        else:
            retval = ''
        return retval

    def get_reps(self, hash, startdate, stopdate):
        """Check hash with rrule and if startdate is not None,
        return items between startdate and stopdate, else return the first
        item on or after startdate."""
        mr = []
        time = datetime.time(0,0,0,0)
        startdate = parse_date(startdate)
        startdt = datetime.datetime.combine(startdate, time)
        stopdate = parse_date(stopdate)
        stopdt = datetime.datetime.combine(stopdate, time)
        if has(hash, 'd'):
            duedate = parse_date(hash['d'])
        else:
            duedate = get_today()
        duedt = datetime.datetime.combine(duedate, time)

        if has(hash,'b'):
            # extend the stopdate enough to catch begins
            try:
                stopdt = stopdt + int(hash['b']) * oneday
            except:
                mr.append(sys.exc_info())

        i_list = []
        if has(hash, 'rr'):
            r = hash['rr']
            # get the repetitions
            i_list = []
            try:
                reps = r.between(duedt, stopdt, inc=True)
            except:
                mr.append("    could not parse 'r.between(%s, %s, inc=True)'"
                    % (duedt, stopdt))
                mr.append(sys.exc_info())
                print(mr)
        elif has(hash, 'r') and hash['r'] == 'l':
            m = parens_regex.match(hash['l'])
            if m:
                parens = True
                items = m.group(1)
            else:
                parens = False
                items = hash['l']
            lst = items.split(',')
            reps = []
            for d in lst:
                try:
                    dt = parse(d)
                    if dt >= duedt and dt <= stopdt:
                        reps.append(dt)
                except:
                    mr.append("    could not parse date '%s' in @%s" % 
                            (d.strip(), 'l'))
        else:
            return(mr, [])

        first = True
        for rep in reps:
            first = False
            hash_copy = copy.deepcopy(hash)
            hash_copy['d'] = rep.strftime(date_fmt)
            hash_copy['sd'] = rep.strftime(weekday_fmt)
            if hash_copy['type'] == 't' and not has(hash_copy, 'f'):
                numdays = -(get_today() - rep.date()).days
                if abs(numdays) > 999:
                    numdays = "***"
                elif numdays > 0:
                    numdays = "+%s" % numdays
                hash_copy['D'] = numdays
            i_list.append(hash_copy)

        return mr, i_list

    def filter(self, hash, options):
        """Check whether or not task satisfies the search criteria in
        options and add pattern to title."""
        self.pattern = []
        # set self.pattern first
        if has(options,'find'):
            self.pattern.append("t|n~'%s'" % (options['find']))
        if has(options,'context'):
            self.pattern.append("c~'%s'" % (options['context']))
        if has(options,'keyword'):
            self.pattern.append("k~'%s'" % (options['keyword']))
        if has(options,'project'):
            self.pattern.append("j~'%s'" % (options['project']))

        # then check for matches and bail at the first failure
        if has(options,'find'):
            find_regex = re.compile(r'%s' % options['find'].decode(encoding), 
                    re.IGNORECASE | re.LOCALE)
            if not (find_regex.search(hash['t']) or 
                    (has(hash,'n') and find_regex.search(hash['n']))):
                return False
        if has(options,'context'):
            context_regex = re.compile(r'%s' % 
                options['context'].decode(encoding), re.IGNORECASE)
            if not (has(hash, 'c') and context_regex.search(hash['c'])):
                return False
        if has(options,'keyword'):
            keyword_regex = re.compile(r'%s' %
                options['keyword'].decode(encoding), re.IGNORECASE)
            if not (has(hash, 'k') and keyword_regex.search(hash['k'])):
                return False
        if has(options,'project'):
            project_regex = re.compile(r'%s' %
                options['project'].decode(encoding), re.IGNORECASE)
            if not project_regex.search(hash['j']):
                return False
        return True

    def prepare_view(self, options):
        """Return a title and a hash (groups) of lists of items (hashes)
        satisfying options."""
        self.select_items(options)
        view = options['view'] # a, l, b, or r
        h = ''
        lst = [OL()]
        cols = 78
        if view == 'b':
            (hdr, lst, cols) = self.prepare_busy(options)
        elif view == 'r':
            (hdr, lst) = self.prepare_reck(options)
        elif view == 'a':
            (hdr, lst) = self.prepare_agenda(options)
        elif view == 'l':
            (hdr, lst) = self.prepare_list(options)
        hdr = hdr.strip()
        if len(lst) == 0:
            lst = [OL([0, [[0, 0, 'no matching items were found' ] ]])]
        if has(options, 'export'):
            if has_icalendar:
                self.make_calendar(options['export'])
                txt = 'Items listed below were exported to %s.ics.' % options['export']
                lst.insert(0, OL([1, [[0, 0, txt ] ]]))
                txt = 'Items listed above were exported to %s.ics.' % options['export']
                lst.append(OL([1, [[0, 0, txt ] ]]))
            else:
                txt = 'Export failed. Could not find the required icalendar module.'
                lst = [OL([1, [[0, 0, txt ] ]])]
        w = (cols - len(hdr))/2
        h = OL([5, [[0, 0, "%s %s %s" % ('_'*w, hdr, '_'*w) ] ]])
        H = OL([5, [[0, 0, hdr] ]])
        sig = "etm %s" % version
        l = cols - 4 - len(sig)
        f = OL([5, [[0, 0, "%s %s %s" % ('_'*3, sig, '_'*l) ] ]])
        return(H, h, lst, f)

    def select_items(self, options):
        """Populate agenda_hash, item_hash and the group hashes with items
        satisfying startdate/today and stopdate."""
        # assumes options['begin'], ... are set
        if self.changed:
            self.loadData()
            info = self.process_items()
        startdate = options['begin'].strftime(date_fmt)
        stopdate = options['end'].strftime(date_fmt)
        self.item_hash = {}
        self.xlst = []
        self.agenda_hash = {}
        self.project_groups = {}
        self.date_groups = {}
        self.context_groups = {}
        self.keyword_groups = {}
        self.sched_hash = {}
        self.time_hash = {}
        todaysf = get_today().strftime(date_fmt)
        xlst = []
        for itemnum in range(len(self.item_list)):
            hash = self.item_list[itemnum]
            if not self.filter(hash, options):
                # go to the next item
                continue
            if has(options, 'show'):
                if hash['type'] == 't':
                    # check tasks
                    if 'f' not in options['show'] and has(hash, 'f'):
                        continue
                elif not hash['type'] in options['show'].lower():
                    # check events and actions
                    continue
            # add rrule reps if necessary
            i_list = []
            if has(hash, 'r'):
                mr, reps = self.get_reps(hash, startdate, stopdate)
                if len(mr) > 0:
                    self.messages.extend(mr)
                else:
                    i_list = reps
            else:
                i_list = [hash]
            for i_hash in i_list:

                add2group = None
                if has(i_hash, 'd'):
                    itemdate = i_hash['d']
                    if i_hash['type'] == 't':
                        if has(i_hash, 'f'):
                            if i_hash['f'] >= startdate and \
                                i_hash['f'] <= stopdate:
                                # if the task was finished in the
                                # period, add it to the item hash for
                                # date it was finished.
                                self.agenda_hash.setdefault(i_hash['f'], 
                                    []).append(i_hash)
                                self.item_hash.setdefault(i_hash['f'], 
                                    []).append(i_hash)
                                add2group = i_hash['f']
                        else:                                
                            # unfinished task with date
                            if (has(i_hash, 'available') 
                                and i_hash['available']):
                                    waitingfor = False
                            else:
                                waitingfor = True
                                if (has(options, 'show') and 
                                    'w' not in options['show']):
                                    continue
                            
                            beginby = None
                            if (has(options,'show') and has(i_hash, 'B')
                                and 'b' in options['show']):
                                if i_hash['B'] <= todaysf and \
                                    options['view'] == 'a':
                                    beginby = todaysf
                                elif i_hash['B'] <= stopdate and \
                                    i_hash['B'] >= startdate:
                                    beginby = i_hash['B']
                                
                            if beginby:
                                if beginby <= itemdate:
                                    add2group = beginby
                                    self.agenda_hash.setdefault(beginby, 
                                        []).append(i_hash)
                                    if waitingfor:
                                        self.agenda_hash.setdefault('w', 
                                            []).append(i_hash)
                                    else:
                                        self.agenda_hash.setdefault('b', 
                                            []).append(i_hash)
                                
                            if (has(options, 'show') 
                                and 't' not in options['show'] 
                                and not beginby
                                and not waitingfor):
                                continue
                            if itemdate >= startdate and itemdate <= stopdate:
                                # such tasks belong in list view
                                self.item_hash.setdefault(itemdate, 
                                        []).append(i_hash)
                                if not beginby:
                                    self.agenda_hash.setdefault(itemdate, 
                                            []).append(i_hash)
                                if waitingfor:
                                    self.agenda_hash.setdefault('w', 
                                        []).append(i_hash)                                    
                                add2group = itemdate
                            if itemdate < todaysf and options['view'] == 'a':
                                # only add pastdue to agenda view
                                self.agenda_hash.setdefault(todaysf,
                                    []).append(i_hash)
                                add2group = todaysf

                    elif i_hash['type'] in ['e', 'a', 'n']:
                        if  itemdate <= stopdate and itemdate >= startdate:
                            self.item_hash.setdefault(itemdate, 
                                        []).append(i_hash)
                            add2group = itemdate
                        if itemdate == todaysf:
                            self.agenda_hash.setdefault(todaysf,
                                    []).append(i_hash)
                        elif itemdate > todaysf and itemdate <= stopdate:
                            self.agenda_hash.setdefault(itemdate, 
                                    []).append(i_hash)

                else: # no date, must be a task
                    if has(options, 'show'):
                        if has(i_hash, 'f'):
                            if 'f' not in options['show']:
                                continue
                            elif i_hash['f'] < startdate or \
                                i_hash['f'] > stopdate:
                                continue
                            else:
                                self.agenda_hash.setdefault(i_hash['f'], 
                                    []).append(i_hash)
                                self.item_hash.setdefault(i_hash['f'], 
                                    []).append(i_hash)
                                add2group = i_hash['f']
                        elif (has(i_hash,'available') and \
                            i_hash['available']):
                            if 't' not in options['show']:
                                continue 
                            else:
                                add2group = '%s' % NoDueDate
                                self.agenda_hash.setdefault('n', 
                                        []).append(i_hash)
                        else:
                            if 'w' not in options['show']:
                                continue
                            else:
                                self.agenda_hash.setdefault('w', 
                                        []).append(i_hash)
                                add2group = '%s' % NoDueDate
                                self.agenda_hash.setdefault('n', 
                                        []).append(i_hash)

                if add2group:
                    # for time reports
                    if options['view'] == 'r':
                        self.update_times(i_hash, options)
                    # for busy/free
                    elif options['view'] == 'b':
                        if has(i_hash, 'd') and has(i_hash, 'S') and \
                            has(i_hash, 'E'):
                            self.sched_hash.setdefault(i_hash['d'], 
                            []).append([i_hash['S'], i_hash['E']])

                    m = year_regex.search(i_hash['t'])
                    if m:
                        startyear = m.group(1)
                        endyear = parse(add2group).strftime("%Y")
                        y = year2string(startyear, endyear)
                        i_hash['t'] = year_regex.sub(y, i_hash['t'])

                    self.date_groups.setdefault(
                                add2group, []).append(i_hash)

                    self.project_groups.setdefault(
                        i_hash['j'], []).append(i_hash)

                    if has(i_hash, 'c'):
                        self.context_groups.setdefault(
                            i_hash['c'], []).append(i_hash)
                    else:
                        self.context_groups.setdefault(
                            'none', []).append(i_hash)
                    if has(i_hash, 'k'):
                        m = parens_regex.match(i_hash['k'])
                        if m:
                            words = m.group(1)
                            keywords = words.split(',')
                        else:
                            keywords = [i_hash['k']]
                        for word in keywords:
                            self.keyword_groups.setdefault(
                                word, []).append(i_hash)
                    else:
                        self.keyword_groups.setdefault(
                                'none', []).append(i_hash)
        # add days with no items to the busy/free hash
        dt = parse_date(options['begin'])
        while dt <= parse_date(options['end']):
            dts = format_date(dt, date_fmt)
            if dts not in self.sched_hash:
                self.sched_hash[dts] = []
            dt = dt + oneday
        self.process_alerts()

    def process_alerts(self):
        self.alerts = []
        # make sure we have all the relevant non-list repetitions
        todaysf = get_today().strftime(date_fmt)
        if has(self.item_hash, todaysf):
            for i_hash in self.item_hash[todaysf]:
                if i_hash['type'] == 'e': 
                    if has(i_hash, 'a'):
                        # update alerts
                        alrts = i_hash['a']
                        m = parens_regex.match(alrts)
                        if m:
                            alrts = m.group(1)
                        a =(i_hash['t'],
                        parse(i_hash['s']).strftime(datetime_fmt),
                            map(int, alrts.split(',')))
                        self.alerts.append(a)

    def process_items(self):
        """Update item hashes taking account of do_next, repetitions and
        completions. 
        """
        # item_list now contains all non repeating items plus list type
        # repetitions. Non list type repetitions now contain an rrule which
        # will be processed by select_items using start and stop dates.
        current_project = None
        info = []
        current_idnum = ""
        for itemnum in range(len(self.item_list)):
            i_hash = self.item_list[itemnum]
            i_hash['itemnum'] = itemnum
            idnum = i_hash['idnum']
            file, num = self.idnum_hash[i_hash['idnum']]
            i_hash['D'] = ''
            i_hash['S'] = ''
            i_hash['E'] = ''

            self.itemnum_idnum[itemnum] = idnum

            # for selecting from prior contexts or keywords
            this_project = i_hash['j']
            if has(i_hash, 'c'):
                self.contexts.add(i_hash['c'])
            if has(i_hash, 'k'):
                m = parens_regex.match(i_hash['k'])
                if m:
                    keywords = m.group(1).split(',')
                else:
                    keywords = i_hash['k'].split(',')
                words = [x.strip() for x in keywords 
                    if x != '' and x != '[]' ]
                for word in words:
                    self.keywords.add(word)
            if has(i_hash, 'r'):
                temp = [i_hash['r']]
                for key in repeat_keys:
                    if has(i_hash, key):
                        temp.append("@%s %s" % (key, i_hash[key]))
                self.repeats.add(" ".join(temp))

            if has(i_hash, 'r') and i_hash['r'] == 'l' and \
                    not has(i_hash, 'd'):
                # set the date for repeating, list-only events to the
                # first date from the list
                m = parens_regex.match(i_hash['l'])
                if m:
                    parens = True
                    items = m.group(1)
                else:
                    parens = False
                    items = i_hash['l']
                lst = items.split(',')
                i_hash['d'] = parse(lst[0]).strftime(date_fmt)

            if has(i_hash, 'd'):
                itemdate = parse_date(i_hash['d'])
                i_hash['d'] = itemdate.strftime(date_fmt)
                itemdatefmt = i_hash['d']
                i_hash['sd'] = format_date(itemdate, weekday_fmt)

                # events
                if i_hash['type'] == 'e': 
                    if has(i_hash, 's'):
                        startdt = parse(i_hash['s'])
                        if use_ampm:
                            i_hash['s'] = startdt.strftime(timefmt)[:-1].lower()
                            i_hash['s'] = leadingzero.sub('', i_hash['s'])
                        else:
                            i_hash['s'] = startdt.strftime(timefmt)
                        i_hash['S'] = startdt.strftime("%H:%M")
                    else:
                        i_hash['s'] = ''
                        i_hash['S'] = ''
                    if has(i_hash, 'e'):
                        enddt = parse(i_hash['e'])
                        i_hash['e'] = enddt.strftime(timefmt)[:-1].lower()
                        if use_ampm:
                            i_hash['e'] = enddt.strftime(timefmt)[:-1].lower()
                            i_hash['e'] = leadingzero.sub('', i_hash['e'])
                        else:
                            i_hash['e'] = enddt.strftime(timefmt)
                        i_hash['E'] = enddt.strftime("%H:%M")
                    else:
                        i_hash['e'] = ''
                        i_hash['E'] = ''

                # notes 
                elif i_hash['type'] == 'n': 
                    if has(i_hash, 's'):
                        startdt = parse(i_hash['s'])
                        if use_ampm:
                            i_hash['s'] = \
                                startdt.strftime(timefmt)[:-1].lower()
                            i_hash['s'] = leadingzero.sub('', i_hash['s'])
                        else:
                            i_hash['s'] = startdt.strftime(timefmt)
                        i_hash['S'] = startdt.strftime("%H:%M")
                    else:
                        i_hash['s'] = datetime.datetime.now().strftime(timefmt)
                        i_hash['S'] = datetime.datetime.now().strftime("%H:%M")

                # tasks
                elif i_hash['type'] == 't':
                    i_hash['s'] = ''
                    i_hash['S'] = ''
                    i_hash['e'] = ''
                    i_hash['E'] = ''
                    nextdue = i_hash['d']

                    if this_project != current_project:
                        current_project = this_project
                        current_idnum = i_hash['idnum']
                        current_duedate = i_hash['d']
                        different_task = True
                    elif i_hash['idnum'] != current_idnum:
                        current_idnum = i_hash['idnum']
                        different_task = True
                    else:
                        different_task = False

                    ### deal with finished tasks ###
                    finished = has(i_hash, 'f')
                    
                    if finished:
                        if has(i_hash,'r'):
                            nextdue = self.get_next(i_hash)
                            if nextdue:
                                old_f = i_hash['f']
                                self.update_task(i_hash, nextdue)
                                file, num = self.idnum_hash[i_hash['idnum']]
                                info.extend([
                    "  reset due date for task finished %s to %s for %s"
                                        % (old_f, nextdue, idnum),
                    "    %2s: %s" % (num, i_hash['t'][:70])
                                        ])
                                finished = False
                            else:
                                i_hash['D'] = 'X'
                                finishdate = parse(i_hash['f']).date()
                                i_hash['sf'] = finishdate.strftime(
                                        weekday_fmt)
                                i_hash['d'] = i_hash['f']
                        else: # no r
                            i_hash['D'] = 'X'
                            finishdate = parse(i_hash['f']).date()
                            i_hash['sf'] = finishdate.strftime(
                                        weekday_fmt)
                            # finishdate is the relevant date for this task
                            i_hash['d'] = i_hash['f']
                            i_hash['sd'] = i_hash['sf']
                    else: # unfinished task
                        if has(i_hash,'r'):
                            # handle overdue 's' (skip) tasks
                            if 'o' in i_hash and i_hash['o'] == 's' \
                                    and itemdate < get_today():
                                nextdue = self.get_next(i_hash)
                                nd = parse(nextdue).date()
                                if nd >= get_today():
                                    if nd > itemdate:
                                        if different_task or i_hash['r']:
                                            # first rep for this idnum
                                            self.update_task(
                                                    i_hash, nextdue)
                                            info.extend([
                        "  skipped due date of %s and reset to %s for %s"
                                        % (current_duedate, nextdue, file),
                                        "    %2s: %s" % (num,
                                            i_hash['t'][:70])
                                        ])

                                        i_hash['f'] = None
                                        i_hash['sf'] = None
                                        finished = False
                                        i_hash['D'] = '!'
                        else: # not repeating
                            nextdue = itemdate
                        numdays = -(get_today() -
                            parse_date(nextdue)).days
                        if abs(numdays) > 999:
                            numdays = "***"
                        elif numdays > 0:
                            numdays = "+%s" % numdays
                        elif numdays == 0:
                            numdays = "0"
                        i_hash['D'] = numdays

                        if has(i_hash,'b'):
                            try:
                                begindate = itemdate - int(i_hash['b'])*oneday
                                i_hash['B'] = begindate.strftime(date_fmt)
                                i_hash['sb'] = format_date(begindate,
                                        weekday_fmt)
                            except:
                                begindate = None
                                i_hash['b'] = None
                                i_hash['B'] = None
                                i_hash['sb'] = ''
                        else: # we have 'b'
                            # make sure i_hash has the key for sorting
                            begindate = None
                            i_hash['b'] = None
                            i_hash['B'] = None
                            i_hash['sb'] = 'none'

                # actions
                elif i_hash['type'] == 'a':
                    i_hash['s'] = ''
                    i_hash['S'] = ''
                    i_hash['e'] = ''
                    i_hash['E'] = ''
            else: # no 'd'
                # begin no date stuff for tasks, events and actions
                # make sure task has the key for sorting
                if i_hash['type'] == 'n': 
                    itemdate = get_today()
                    i_hash['d'] = itemdate.strftime(date_fmt)
                    itemdatefmt = i_hash['d']
                    i_hash['sd'] = format_date(itemdate, weekday_fmt)
               
                else:
                    itemdate = None
                    itemdatefmt = None
                    i_hash['d'] = ''
                    i_hash['sd'] = None
                    i_hash['D'] = ''

                    i_hash['s'] = ''
                    i_hash['S'] = ''
                    i_hash['e'] = ''
                    i_hash['E'] = ''

                if i_hash['type'] == 't':
                    next =True
                    if this_project != current_project:
                        # make the first task in a project available
                        current_project = this_project
                        different_task = True
                        current_idnum = i_hash['idnum']
                    finished = has(i_hash, 'f')
                    if finished:
                        i_hash['D'] = 'X'
                    if not finished:
                        i_hash['f'] = None
                        i_hash['sf'] = None
                        i_hash['D'] = ''
                        if next:
                            self.agenda_hash.setdefault('n', 
                                    []).append(i_hash)

                        # make sure task has the key for sorting
                        # handle overdue 's' (skip) tasks
                        if 'o' in i_hash and i_hash['o'] == 's' \
                                and 'd' in i_hash \
                                and parse_date(i_hash['d']) < get_today() \
                                and 'r' in i_hash and i_hash['r'] != None:
                            nextdue = self.get_next(i_hash)
                            nd = parse_date(nextdue)
                            if nd >= get_today():
                                if nd > parse_date(i_hash['d']).date():
                                    i_hash['d'] = format_date(nextdue,
                                            date_fmt)
                                    if different_task:
                                        self.update_task(
                                                i_hash, nextdue)
                                        file = etmdata_regex.sub('',
                                                i_hash['file'])
                                        info.extend([
                        "  skipped due date of %s and reset to %s for %s"
                                    % (current_duedate, nextdue, file),
                        "    %2s: %s" % (i_hash['line'], i_hash['t'][:70])
                                    ])
        return(info)

    def prepare_item(self, options, i_hash):
        """"""
        # grouped by date:  
        #       events:   s, e, (idnum?) title
        #       notes:    s,    (idnum?) title
        #       tasks:       D, (idnum?) title
        #       actions:     p, (idnum?) title
        # grouped by project, etc
        #       events:  sd, s, (idnum?) title
        #       notes:   sd, s, (idnum?) title
        #       tasks:   sd, D, (idnum?) title
        #       actions: sd, p, (idnum?) title
        # where d is, eg., 13 Oct, and thus needs at most 6 chars (as do
        # s and e)
        try:
            D  = int(i_hash['D'])
            if D < 0:
                attr = 1
            elif D == 0:
                attr = 2
            elif D <= agenda_dflt:
                attr = 3
            elif D <= next_dflt:
                attr = 4
            else:
                attr = -1
        except:
            attr = -1
        if has(self.agenda_hash, 'w') and i_hash in self.agenda_hash['w']:
            attr = 7
        elif has(self.agenda_hash, 'b') and i_hash in self.agenda_hash['b']:
            attr = 6
        elif i_hash['type'] == 'n':
            attr = 8
        if self.show_idnums:
            if options['groupby'] == 'd':
                second = 18
                if i_hash['type'] == 't':
                    l1 = OL([attr, [(9, 4, '%4s' % i_hash['D']), 
                        (17, 0, "%s) %s" % (i_hash['idnum'], i_hash['t']))]])
                elif i_hash['type'] == 'e':
                    l1 = OL([attr, [(2, 6, "%6s" % i_hash['s']), 
                        (9, 6, "%6s" % i_hash['e']), 
                        (17, 0, "%s) %s" % (i_hash['idnum'], i_hash['t']))]])
                elif i_hash['type'] == 'n':
                    l1 = OL([attr, [
                        # (2, 6, "%6s" % i_hash['s']), 
                        (2, 6, ""), 
                        (9, 6, "   N"), 
                        (17, 0, "%s) %s" % (i_hash['idnum'], i_hash['t']))]])
                elif i_hash['type'] == 'a':
                    l1 = OL([attr, [(9, 6, '%*sm' % (len(str(i_hash['p']))+3, i_hash['p'])), 
                        (17, 0, "%s) %s" % (i_hash['idnum'], i_hash['t']))]])
            else:
                if i_hash['sd']:
                    date = i_hash['sd'][4:]
                else:
                    date = ''
                if i_hash['type'] == 't':
                    l1 = OL([attr, [(2, 6, date), 
                        (9, 4, '%4s' % i_hash['D']), 
                        (17, 0, "%s) %s" % (i_hash['idnum'], i_hash['t']))]])
                elif i_hash['type'] == 'e':
                    l1 = OL([attr, [(2, 6, date), 
                        (9, 6, "%6s" % i_hash['s']), 
                        (17, 0, "%s) %s" % (i_hash['idnum'], i_hash['t']))]])
                elif i_hash['type'] == 'n':
                    l1 = OL([attr, [(2, 6, date), 
                        (9, 6, "   N"), 
                        # (9, 6, ""), 
                        (17, 0, "%s) %s" % (i_hash['idnum'], i_hash['t']))]])
                elif i_hash['type'] == 'a':
                    l1 = OL([attr, [(2, 6, date), 
                        (9, 6, '%*sm' % (len(str(i_hash['p']))+3, i_hash['p'])), 
                        (17, 0, "%s) %s" % (i_hash['idnum'], i_hash['t']))]])
        else:
            if options['groupby'] == 'd':
                second = 18
                if i_hash['type'] == 't':
                    l1 = OL([attr, [(9, 4, '%4s' % i_hash['D']), 
                        (17, 0, "%s" % (i_hash['t']))]])
                elif i_hash['type'] == 'e':
                    l1 = OL([attr, [(2, 6, "%6s" % i_hash['s']), 
                        (9, 6, "%6s" % i_hash['e']), 
                        (17, 0, "%s" % (i_hash['t']))]])
                elif i_hash['type'] == 'n':
                    l1 = OL([attr, [
                        # (2, 6, "%6s" % i_hash['s']), 
                        (2, 6, ""), 
                        (9, 6, "   N"), 
                        (17, 0, "%s" % (i_hash['t']))]])
                elif i_hash['type'] == 'a':
                    l1 = OL([attr, [(9, 6, '%*sm' % (len(str(i_hash['p']))+3, i_hash['p'])), (17, 0, "%s" % (i_hash['t']))]])
            else:
                if i_hash['sd']:
                    date = i_hash['sd'][4:]
                else:
                    date = ''
                if i_hash['type'] == 't':
                    l1 = OL([attr, [(2, 6, date), 
                        (9, 4, '%4s' % i_hash['D']), 
                        (17, 0, i_hash['t'])]])
                elif i_hash['type'] == 'e':
                    l1 = OL([attr, [(2, 6, date), 
                        (9, 6, "%6s" % i_hash['s']), 
                        (17, 0, i_hash['t'])]])
                elif i_hash['type'] == 'n':
                    l1 = OL([attr, [(2, 6, date), 
                        # (9, 6, "%6s" % i_hash['s']), 
                        (9, 6, "   N"), 
                        (17, 0, i_hash['t'])]])
                elif i_hash['type'] == 'a':
                    l1 = OL([attr, [(2, 6, date), 
                        (9, 6, '%*sm' % (len(str(i_hash['p']))+3, i_hash['p'])), 
                        (17, 0, i_hash['t'])]])
        temp = []
        l2 = OL()
        keys = g_keys[options['groupby']]
        for key in keys:
            if has(i_hash, key):
                temp.append("@%s %s" % (key, str(i_hash[key]).strip()))
        if len(temp) > 0:
            tempstr = " ".join(temp).strip()
            #  if self.show_idnums:
            if True:
                l2 = OL([-2, [[19, 0, 
                    "%s" % (tempstr)]]])
            else:
                l2 = OL([-2, [[19, 0, 
                    "%s) %s" % (i_hash['idnum'], tempstr)]]])
        return([l1, l2])


    def prepare_list(self, options):
        groupby = options['groupby']
        if groupby == 'd':
            group_hash = self.date_groups
        elif groupby == 'j':
            group_hash = self.project_groups
        elif groupby == 'c':
            group_hash = self.context_groups
        elif groupby == 'k':
            group_hash = self.keyword_groups
        groups = group_hash.keys()
        groups.sort()
        pattern = []
        if has(options,'show') and options['show'] != list_show:
            pattern.append("showing '%s'" % options['show'])
        if has(options,'context'):
            pattern.append("c~'%s'" % (options['context']))
        if has(options,'keyword'):
            pattern.append("k~'%s'" % (options['keyword']))
        if has(options,'project'):
            pattern.append("j~'%s'" % (options['project']))
        if has(options,'find'):
            pattern.append("t|n~'%s'" % (options['find']))
        if len(pattern) > 0:
            pattern =  "(%s): " % (", ".join(pattern))
        else:
            pattern = ""
        pattern = "%s%s ~ %s" % (pattern, format_date(options['begin'],
            date_fmt),
            format_date(options['end'], date_fmt))
        hdr = "%s: %s" % (List, pattern)
        lst = []
        for group in groups:
            if groupby == 'd':
                g = OL([0, [[0, 0, format_date(group, weekday_fmt)]]])
            else:
                g = OL([0, [[0, 0, group]]])
            lst.append(g)
            for item in sort_listofhashes_by_field(list(group_hash[group]),
                'type', 'd'):
                l =self.prepare_item(options, item)
                if len(l) > 0:
                    lst.extend(l)
        return(hdr, lst)

    def prepare_agenda(self, options):
        # from list of (field names, pos, width) format line
        #  lfmt = [(3, 6, 's'), (10, 6, 'e'), (17, 3, 'D'), (21, 0, 't')]
        groupby = options['groupby']
        if groupby == 'd':
            group_hash = self.agenda_hash
        elif groupby == 'j':
            group_hash = self.project_groups
        elif groupby == 'c':
            group_hash = self.context_groups
        elif groupby == 'k':
            group_hash = self.keyword_groups
        groups = group_hash.keys()
        groups.sort()
        pattern = []
        if has(options,'show') and options['show'] != agenda_show:
            pattern.append("showing '%s'" % options['show'])
        if has(options,'context'):
            pattern.append("c~'%s'" % (options['context']))
        if has(options,'keyword'):
            pattern.append("k~'%s'" % (options['keyword']))
        if has(options,'project'):
            pattern.append("j~'%s'" % (options['project']))
        if has(options,'find'):
            pattern.append("t|n~'%s'" % (options['find']))
        if len(pattern) > 0:
            pattern =  "(%s): " % (", ".join(pattern))
        else:
            pattern = ""
        hdr = "%s: %s" % (Agenda, pattern)
        lst = []
        hdr = "%s: %s%s ~ %s" % (Agenda, pattern, 
            format_date(get_today(), date_fmt),
            format_date(options['end'], date_fmt))
        for group in groups:
            if groupby == 'd':
                if group == 'n':
                    grp =  "%s" % NoDueDate
                elif group in ['b', 'w']:
                    continue
                else:
                    grp = format_date(group,weekday_fmt)
            else:
                if group in ['b', 'w']:
                    continue
                else:
                    grp = group
            g = OL([0, [[0, 0, grp]]])
            lst.append(g)
            for item in sort_listofhashes_by_field(list(group_hash[group]),
                'type', 'd'):
                l =self.prepare_item(options, item)
                if len(l) > 0:
                    lst.extend(l)
        return(hdr, lst)


    def prepare_busy(self, options):
        """docstring for prepare_busy"""
        pattern = []
        if has(options,'context'):
            pattern.append("c~'%s'" % (options['context']))
        if has(options,'keyword'):
            pattern.append("k~'%s'" % (options['keyword']))
        if has(options,'project'):
            pattern.append("j~'%s'" % (options['project']))
        if has(options,'find'):
            pattern.append("t|n~'%s'" % (options['find']))
        if len(pattern) > 0:
            pattern =  "(%s): " % (", ".join(pattern))
        else:
            pattern = ""
        sched_data = []
        keys = self.sched_hash.keys()
        keys.sort()
        for key in keys:
            evnts = self.sched_hash[key]
            evnts.sort()
            sched_data.append([key, evnts])
        bb,bt,fb,ft,cols = data2busyfree(options['opening'],
                options['closing'], options['minimum'], 
                options['slack'], options['slotsize'], sched_data, True)
        ret = []
        tl = []
        if 'b' in options['include'] or 'B' in options['include']:
            tl.append('busy')
            if 'b' in options['include']:
                ret.append(OL([0, [[0, cols,
            "Busy periods between %s and %s"
            % (options['opening'][:6].lower(), 
                options['closing'][:6].lower())]]]))
                for l in bb:
                    ret.append(OL([-1, [[2, cols-1, l]]]))
                ret.append(OL([0, [[0, cols, ""]]]))
            if 'B' in options['include']:
                ret.append(OL([0, [[0, cols, "Busy periods (anytime)"]]]))
                for l in bt:
                    ret.append(OL([-1, [[2, cols-1, l]]]))
                ret.append(OL([0, [[0, cols, ""]]]))
        if 'f' in options['include'] or 'F' in options['include']:
            tl.append('free')
            if 'f' in options['include']:
                ret.append(OL([0, [[0, cols,
                "Free periods between %s and %s (%sm minimum; %sm buffer)"
                % (options['opening'][:6].lower(),
                    options['closing'][:6].lower(),
                    options['minimum'],
                    options['slack'])]]]))
                for l in fb:
                    ret.append(OL([-1, [[2, cols-1, l]]]))
                ret.append(OL([0, [[0, cols, ""]]]))
            if 'F' in options['include']:
                ret.append(OL([0, [[0, cols,
                "Free periods between %s and %s (%sm minimum; %sm buffer)"
                % (options['opening'][:6].lower(), 
                    options['closing'][:6].lower(), 
                    options['minimum'],
                    options['slack'])]]]))
                for l in ft:
                    ret.append(OL([-1, [[2, cols-1, l]]]))
        period = "%s ~ %s" % (options['begin'].strftime(date_fmt),
                options['end'].strftime(date_fmt))
        h = '%s periods: %s%s' % ('/'.join(tl), pattern, period)
        return(h, ret, cols)

    def prepare_reck(self, options):
        """docstring"""
        l = []
        if options['itemize'] == None:
            itemize = 0
        else:
            itemize = 1
        pattern = ["CDKI~%s%s%s%s" % (
            options['c_position'], 
            options['d_position'],
            options['k_level'],
            itemize) ]
        if has(options,'context'):
            pattern.append("c~'%s'" % (options['context']))
        if has(options,'keyword'):
            pattern.append("k~'%s'" % (options['keyword']))
        if has(options,'project'):
            pattern.append("j~'%s'" % (options['project']))
        if has(options,'find'):
            pattern.append("t|n~'%s'" % (options['find']))
        if len(pattern) > 0:
            pattern =  "(%s) " % (", ".join(pattern))
        else:
            pattern = ""
        pattern = "%s%s ~ %s" % (pattern, format_date(options['begin'],
            date_fmt), format_date(options['end'], date_fmt))
        cols = 79
        if self.time_hash:
            hash = copy.deepcopy(self.time_hash)
            total = m2h(hash[()])
            del hash[()]
            h = "%s %s [%s]" % (Reckoning, pattern, total)
        else:
            total = 0
            h = "%s %s" % (Reckoning, pattern)
            text = "no relevant entries were found"
            l.append(OL([0,
                    [
                        [0, cols-1, "%s" % text],
                    ]
                ]))
            return(h, l)
        if not hash:
            # we only had the one key, ()
            text = "total time: %s" % total
            l.append(OL([0,
                    [
                        [0, cols-1, text],
                    ]
                ]))
            return h, l
        # we have at least one entry
        keys = hash.keys()
        keys.sort()
        indent = 8 
        m_space = 7
        first = True
        len_first = 0
        for key in keys:
            # key is a tuple, we only want the last component and the hash
            # value for the key which will be the number of minutes
            item = key[-1]
            if first:
                len_first = len(key)
                first = False

            if not hash[key]:
                continue
            i = len(key) - len_first
            m_pos = i*indent + 1
            t_pos = m_pos + m_space + 1
            tmp = m2h(hash[key])
            attr = -1
            #  the following requires python >= 2.6
            #  m_text = "{0:<{width}}".format(tmp, width=m_space)
            m_text = "%*s)"% (m_space-1, tmp[:m_space])
            l.append(OL(
                [attr,
                    [
                        [m_pos, m_space, m_text],
                        #  [t_pos, cols-t_pos-2, item]
                        # use width 0 to wrap if necessary
                        [t_pos, 0, item]
                    ]
                ]))
        return(h, l)

    def make_calendar(self, fname):
        if not has_icalendar:
            return(False)
        cal = Calendar()
        cal.add('prodid', '-//etm %s//dgraham.us//' % version)
        cal.add('version', '2.0')
        for date in self.item_hash.keys():
            for task in self.item_hash[date]:
                if task['type'] == 'e':
                    item = Event()
                elif task['type'] == 't':
                    item = Todo()
                elif task['type'] in ['a', 'n']:
                    item = Journal()
                if has(task, 't'):
                    item.add('summary', task['t'])
                if 'c' in task and task['c']:
                    item.add('location', task['c'])
                if 'k' in task and task['k']:
                    m = parens_regex.match(task['k'])
                    if m:
                        keywords = m.group(1).split(',')
                    else:
                        keywords = task['k'].split(',')
                    words = ";".join([x.strip() for x in keywords])
                    item.add('categories', "%s" % words)
                if 'n' in task and task['n']:
                    # maybe this should be comment
                    item.add('description', task['n'])
                if 'd' in task and task['d']:
                    if task['type'] == 't':
                        item.add('due', l2u(task['d']))
                    else: # event or action entry
                        allday = True
                        if 's' in task and task['s']:
                            item.add('dtstart', l2u(task['d'], task['s']))
                            allday = False
                        if 'e' in task and task['e']:
                            item.add('dtend', l2u(task['d'], task['e']))
                            allday = False
                        if 'p' in task and task['p']:
                            item.add('comment', "%s minutes" % task['p'])
                        if allday:
                            item.add('dtstart', l2u(task['d']))
                            thisday = parse_date(task['d'])
                            nextday = thisday + oneday
                            item.add('dtend', l2u(nextday))
                if 'f' in task and task['f']:
                    item.add('completed', l2u(task['f']))

                item['uid'] = 'etm%s' % task['idnum']
                cal.add_component(item)

        (name, ext) = os.path.splitext(fname)
        pname = os.path.join(etmical, "%s.ics" % name)
        fo = open(pname, 'wb')
        fo.write(cal.as_string())
        fo.close()
        return(True)

    def makefilewithline(self, line, file):
        fo = codecs.open(file, 'w', encoding)
        l = [line]
        fo.writelines(l)
        fo.close()
        self.changed = True
        self.logaction(file, "\n >: %s" % line.rstrip())

    def addline2file(self, line, file):
        fo = codecs.open(file, 'r', encoding)
        lines = fo.readlines()
        fo.close()
        self.changed = True
        l = ["%s\n" % x.rstrip() for x in lines]
        l.append("%s\n" % line.rstrip())
        fo = codecs.open(file, 'w', encoding)
        fo.writelines(l)
        fo.close()
        self.logaction(file, "\n +: %s" % line.rstrip())

    def replacelineinfile(self, num, line, file):
        self.backup(file)
        fo = codecs.open(file, 'r', encoding)
        lines = fo.readlines()
        fo.close()
        # leave any blank lines here to keep line numbers correct
        l = ["%s\n" % x.rstrip() for x in lines]
        orig_line = l[num - 1].strip()
        l[num - 1] = "%s\n" % line.strip()
        fo = codecs.open(file, 'w', encoding)
        fo.writelines(l)
        fo.close()
        self.changed = True
        self.logaction(file, "\n -: %s\n +: %s" % 
                (orig_line, line))

    def deletelineinfile(self, num, file):
        self.backup(file)
        fo = codecs.open(file, 'r', encoding)
        lines = fo.readlines()
        fo.close()
        l = ["%s\n" % x.rstrip() for x in lines]
        orig_line = l.pop(num - 1)
        fo = codecs.open(file, 'w', encoding)
        fo.writelines(l)
        fo.close()
        self.changed = True
        self.logaction(file, "\n -: %s" % orig_line)

    def getlinefromfile(self, line, file):
        fo = codecs.open(file, 'r', encoding)
        lines = fo.readlines()
        fo.close()
        if line <= len(lines):
            return(lines[line-1].rstrip())
        else:
            return None

    def backup(self, file):
        pathname, ext = os.path.splitext(file)
        dir, name = os.path.split(pathname)
        bak = os.path.join(dir, ".%s.bk1" % name)
        if os.path.exists(bak):
            m_secs = os.path.getmtime(bak)
            m_date = datetime.date.fromtimestamp(m_secs)
        else:
            m_date = None
        # only backup if there is no backup or it is at least 
        # one day old
        if not m_date or m_date < get_today():
            for i in range(1, numbaks):
                baknum = numbaks - i
                nextnum = baknum + 1
                bakname = os.path.join(dir,".%s.bk%d" % (name, baknum))
                if os.path.exists(bakname):
                    nextname = os.path.join(dir,".%s.bk%d" % (name,
                     nextnum))
                    shutil.move(bakname, nextname)
            shutil.copy(file, bak)

    def logaction(self, file, logentry):
        pathname, ext = os.path.splitext(file)
        dir, name = os.path.split(pathname)
        logfile = os.path.join(dir, ".%s.log" % name)
        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        fo = codecs.open(logfile, 'a', encoding)
        #  fo = open(logfile, 'a')
        fo.write("%s: %s\n" % (now, logentry))
        fo.close()


    def get_filelines(self, file):
        """Get lines from file."""
        try:
            fo = codecs.open(file,'r', encoding)
            lines = fo.readlines()
            fo.close()
        except:
            print("Could not open %s for reading" % file)
            lines = []
        return([x for x in lines])

    def line2hash(self, dflts, line, item=True):
        """Try to parse etmdata file line by splitting on @. Return hash
        is ok, report error otherwise."""
        hash = copy.deepcopy(dflts)
        parts = []
        name = ""
        ok = False
        while (range_regex.match(line)):
            m = range_regex.match(line)
            r = eval(m.group(2))
            line = "%s%s%s" % (m.group(1), r, m.group(3))
        if item: # this is a task, event or action entry line
            m = item_regex.match(line)
            if m:
                msg = []
                hash['chrcode'] = m.group(1)
                if m.group(1)[0] in ['+', '-', '.', '_']:
                    hash['type'] = 't'
                elif m.group(1)[0] == '*':
                    hash['type'] = 'e'
                elif m.group(1)[0] == '~':
                    hash['type'] = 'a'
                elif m.group(1)[0] == '!':
                    hash['type'] = 'n'
                parts = m.group(2).split(' @')
                name = parts.pop(0).strip()
                hash['t'] = name
                ok = True
        else: # item = False; this is a project line
            m = project_regex.match(line)
            if m:
                parts = line.split(' @')
                name = parts.pop(0).strip()
                hash['chrcode'] = ''
                hash['type'] = 'j'
                hash['j'] = name
                ok = True
        if len(parts) > 0:
            for part in parts:
                m = part_regex.match(part)
                if m:
                    k = m.group(1)
                    v = m.group(2)
                    hash[k.strip()] = v.strip()
                else:
                    ok = False
        if ok:
            return([], hash)
        elif item:
            return ['    could not parse entry line'], hash
        else:
            return ['    could not parse project line'], hash

    def add_time(self, key, minutes):
        self.time_hash.setdefault(key, 0)
        self.time_hash[key] += minutes

    def update_times(self, hash, options):
        #  increment is set in etmrc
        #  increment, c_position, d_position, k_level, itemized 
        #  hash must be event with @d and @s or action entry with @j
        itemize = options['itemize']
        c_position = options['c_position']
        d_position = options['d_position']
        k_level = options['k_level']
        minutes = 0
        if has(hash,'e'):
            sdt = "%s %s" % (hash['d'], hash['s'])
            edt = "%s %s" % (hash['d'], hash['e'])
            start = parse(sdt)
            end = parse(edt)
            td = (end - start).seconds
            minutes = td/60
            if td%60 > 0:
                minutes += 1
        elif has(hash,'p'):
            minutes = int(hash['p'])
        else:
            return
        i = minutes/increment
        if minutes%increment > 0:
            i += 1
        minutes = i*increment
        # key depends on c_position, d_position, k_level and itemized
        context = ""
        date = ""
        if itemize and has(hash, 't'):
            title = hash['t']
        else:
            title = ""
        if c_position > 0:
            if has(hash, 'c'):
                context = hash['c']
            else:
                context = 'no context'
        if d_position > 0:
            if has(hash, 'd'):
                date = hash['d']
            else:
                date = 'no date'
        if k_level > 0:
            self.add_time(tuple(), minutes)
            if not has(hash,'k'):
                words = ['no keywords']
            else:
                m = parens_regex.match(hash['k'])
                if m:
                    keywords = m.group(1)
                else:
                    keywords = hash['k']
                words = keywords.split(',')
            for word in words:
                parts = word.split(':')[:k_level]
                k = parts
                if d_position:
                    k.insert(min(len(k), d_position-1), date)
                if c_position:
                    k.insert(min(len(k), c_position-1), context)
                tmp = []
                for part in k:
                    tmp.append(part.strip())
                    self.add_time(tuple(tmp), minutes)
                if itemize:
                    tmp.append(title)
                    self.add_time(tuple(tmp), minutes)
        elif k_level == 0:
            self.add_time(tuple(), minutes)
            k = []
            tmp = []
            if d_position > 0:
                k.insert(min(len(k), d_position-1), date)
            if c_position > 0:
                k.insert(min(len(k), c_position-1), context)
            if len(k) > 0:
                for part in k:
                    tmp.append(part)
                    self.add_time(tuple(tmp), minutes)
            if itemize:
                tmp.append(title)
                self.add_time(tuple(tmp), minutes)

    def check_hash(self, hash, item=True):
        """Check hash returned by line2hash for presence of required
        fields and unknown fields and for the required format of the
        entries, e.g., that the content of a date field can be parsed as a
        date."""
        msgs = []
        if item:
            req = set([])
            if not has(hash, 't'):
                msgs.append("    missing title")
            if not has(hash, 'type'):
                keys = set(all_keys)
                msgs.append("    missing 'type' key - limited extra or missing check")
            else:
                t = hash['type']
                if  t == 'e':
                    keys = set(event_keys)
                    if 'r' not in hash or hash['r'] != 'l':
                        req.add('d')
                    if 'a' in hash or 'e' in hash:
                        req.add('s')
                    ts = 'event'
                elif t == 't':
                    keys = set(task_keys)
                    if 'r' in hash:
                        req.add('d')
                    ts = 'task'
                elif t == 'a':
                    keys = set(action_keys)
                    req.update(['d', 'p'])
                    ts = 'action'
                elif t == 'n':
                    keys = set(note_keys)
                    if not has(hash, 'd'):
                        hash['d'] = datetime.date.today().strftime(date_fmt)
                    if not has(hash, 's'):
                        hash['s'] = datetime.datetime.now().strftime(timefmt)
                    req.update(['d', 's'])
                    ts = 'note'

        else: # project
            req = set([])
            keys = set(project_keys)
            if not has(hash, 'j'):
                msgs.append("    missing project name")
        if 'r' in hash and hash['r'] == 'l':
            req.add('l')

        for k in repeat_keys:
            if k in hash:
                req.add('r')
                break
        h_keys = set([x for x in hash.keys() if x not in special_keys])
        # keys we should not have but do
        extra = list(h_keys.difference(keys))
        if len(extra) > 0:
            extra.sort()
            msgs.append("    unrecognized keys: %s" % ", ".join(extra))
        # keys we should have but don't
        missing = list(req.difference(h_keys))
        if len(missing) > 0:
            missing.sort()
            msgs.append("    missing but required keys: %s" % 
                    ", ".join(missing))

        # restrict checks to the keys we should have
        h_keys.intersection_update(keys)


        # dates
        for key in list(h_keys.intersection(set(date_keys))):
            try:
                hash[key] = parse_date(hash[key]).strftime(date_fmt)
            except:
                msgs.append("    could not parse date '%s' in @%s" %
                    (hash[key].strip(), key))


        # list of dates
        for key in list(h_keys.intersection(set(list_date_keys))):
            m = parens_regex.match(hash[key])
            if m:
                parens = True
                items = m.group(1)
            else:
                parens = False
                items = hash[key]
            lst = items.split(',')
            dts = []
            for d in lst:
                try:
                    dts.append(parse_date(d).strftime(date_fmt))
                except:
                    msgs.append("    could not parse date '%s' in @%s" % 
                            (d.strip(), key))
            if parens:
                hash[key] =  "(%s)" % ", ".join(dts)
            else:
                hash[key] = ", ".join(dts)

        # time
        has_s = False
        if has(hash, 's'):
            try:
                stime = parse(hash['s'])
                hash['s'] = stime.strftime(timefmt)
                if has(hash, 'e'):
                    try:
                        etime = stime + int(hash['e'])*oneminute
                        hash['e'] = etime.strftime(timefmt)
                    except: # must be a string
                        try:
                            etime = parse(hash['e'])
                            hash['e'] = etime.strftime(timefmt)
                            # ending times should be later than starting times                        
                            if etime <= stime:
                                msgs.append(
                                "    @e (%s) should be later than @s (%s)" 
                                % (etime.strftime(timefmt), stime.strftime(timefmt)))
                        except:
                            msgs.append(
                            "    Could not parse time '@e %s'" % hash['e']) 
                            del hash['e']
                else:
                    # set the missing e equal to s plus extent
                    etime = stime + extent*oneminute
                    hash['e'] = etime.strftime(timefmt)
            except:
                msgs.append("    Could not parse time '@s %s'" % hash['s'])
                del hash['s']

        # integer
        for key in list(h_keys.intersection(set(integer_keys))):
            try:
                hash[key] = int(hash[key])
            except:
                msgs.append("    could not convert '%s' in @%s to integer" % (hash[key].strip(), key))
                del hash[key]

        # list of integers
        for key in list(h_keys.intersection(set(list_integer_keys))):
            m = parens_regex.match(hash[key])
            if m:
                parens = True
                items = m.group(1)
            else:
                parens = False
                items = hash[key]
            lst = items.split(',')
            dts = []
            for d in lst:
                try:
                    d = int(d)
                    if type(d) == int:
                        dts.append(str(d))
                except:
                    msgs.append("    could not convert '%s' in @%s to integer"
                     % (d.strip(), key))
            if parens:
                hash[key] = "(%s)" % ", ".join(dts)
            else:
                hash[key] = ", ".join(dts)

        # add non list repetitions. The list repetitions don't use an rrule.
        if has(hash, 'r') and hash['r'] != 'l':
            # set hash['rr'] equal to the rrule
            if has(hash, 'f') and has(hash, 'o') and hash['o'] == 'r':
                # we need to reset the starting date for the rrule when
                # restarting a finished task
                hash['d'] = format_date(hash['f'], date_fmt)

            mr = self.get_rrule(hash)
            msgs.extend(mr)
        return(msgs, hash)


def main():
    global attrs, all_off
    cmd = cmdlc = 'a'
    args = []
    if len(sys.argv) > 1:
        args = sys.argv[1:]
        cmd = args.pop(0)
        if len(cmd) == 2 and cmd[0] == '-':
            # drop the minus sign
            cmd = cmd[1:]
        cmdlc = cmd.lower()
    if cmd == 'a':
        startdate = get_today()
        stopdate = get_today() + (int(agenda)-1)*oneday
    else:
        startdate = get_today() - int(recent)*oneday
        stopdate = get_today() + (int(next)-int(recent)-1)*oneday
    if cmdlc == 'h':
        print(help)
    elif cmdlc == 'c':
        print(date_calculator(" ".join(args)))
    elif cmdlc == 'n':
        print(newer())
    elif cmdlc not in ['l', 'b', 'r', 'a']:
        print("Unrecognized command: '%s'" % cmd)
        print("Use 'h' for help.")
    else:
        for parser in [aparser, lparser, bparser, rparser]:
            parser.add_option("-t", action = "store_true",
            dest='text_output', 
            help = """Produce text (ascii) output if true, otherwise produce curses output.  Default: False.""")
        (options, err) = parse_args(cmd, args, startdate, stopdate)
        if len(err) > 0:
            for e in err:
                print(e)
            sys.exit()
        etmdata = ETMData()
        etmdata.loadData()
        if etmdata.load_errors:
            for msg in etmdata.load_errors:
                print(msg)
        if etmdata.changed:
            etmdata.loadData()
        info = etmdata.process_items()
        if info:
            for msg in info:
                print(msg)
        if options['verbose_toggle']:
            minattr = -2
        else:
            minattr = -1
        hdr, hd, lst, ft = etmdata.prepare_view(options)
        if options['text_output']:
            attrs = {}
            all_off = ''
            if cmdlc == 'a':
                print(hd.str().encode(encoding))
                for l in lst:
                    s = l.str(minattr)
                    if s:
                        print(s.encode(encoding))
                print(ft.str().encode(encoding))
            elif cmdlc == 'l':
                print(hd.str())
                for l in lst:
                    if l:
                        s = l.str(minattr)
                        if s:
                            print(s.encode(encoding))
                print(ft.str().encode(encoding))
            elif cmdlc == 'r':
                print(hd.str())
                for l in lst:
                    print(l.str().encode(encoding))
                print(ft.str().encode(encoding))
            elif cmdlc == 'b':
                print(hd.str())
                for l in lst:
                    print(l.str().encode(encoding))
                print(ft.str().encode(encoding))
        else:
            all_off, attrs = get_attrs()
            if cmdlc == 'a':
                print(hd.cur())
                for l in lst:
                    s = l.cur(minattr)
                    if s:
                        print(s.strip())
                print("%s" % ft.cur())
            elif cmdlc == 'l':
                print(hd.cur())
                for l in lst:
                    try:
                        s = l.cur(minattr)
                        if s:
                            print(s)
                    except:
                        print(type(l),l)
                print(ft.cur())
            elif cmdlc == 'r':
                print(hd.cur())
                for l in lst:
                    print(l.cur())
                print(ft.cur())
            elif cmdlc == 'b':
                print(hd.cur())
                for l in lst:
                    print(l.cur())
                print(ft.cur())

def main_loop():
    global msg, attrs, all_off
    if sys.platform == 'darwin' or 'linux' in sys.platform:
        clear = "clear"
    else:
        clear = "cls"
    # for parser in [aparser, lparser, bparser, rparser]:
    #     parser.add_option("-t", action = "store_true",
    #     dest='text_output', 
    #     help = """Produce text (ascii) output if true, otherwise produce curses output.  Default: False.""")
    etmdata = ETMData()
    etmdata.loadData()
    if etmdata.load_errors:
        for msg in etmdata.load_errors:
            print(msg)
    if etmdata.changed:
        etmdata.loadData()
    lastmtime = etmdata.getmtime()
    etmdata.process_items()
    cmd = cmdlc = 'a'
    args = []
    loop = True
    os.system(clear)
    all_off, attrs = get_attrs()
    while loop:
        clear_msg()
        today = get_today()
        sys.stdout.write("etm command (enter 'h' for help or '' to exit): ")
        response = sys.stdin.readline().strip()
        if not response:
            loop = False
            break
        args = response.split()
        cmd = args.pop(0)
        cmdlc = cmd.lower()
        if cmdlc == 'a':
            startdate = get_today()
            stopdate = get_today() + (int(agenda)-1)*oneday
        else:
            startdate = get_today() - int(recent)*oneday
            stopdate = get_today() + (int(next)-int(recent)-1)*oneday
        if cmdlc == 'h':
            clear_msg()
            os.system(clear)
            print(help)
        elif cmdlc == 'c':
            os.system(clear)
            print(date_calculator(" ".join(args)))
        elif cmdlc == 'n':
            os.system(clear)
            print(newer())
        elif cmdlc not in ['l', 'b', 'r', 'a']:
            os.system(clear)
            print("Unrecognized command: '%s'" % cmd)
            print("Use 'h' for help.")
            continue
        else:
            try:
                os.system(clear)
                (options, err) = parse_args(cmd, args, startdate, stopdate)
                if len(err) > 0:
                    for e in err:
                        print(e)
                    continue
            except:
                print("could not parse '%s'" %  response)
                continue
            l = etmdata.getmtime()
            if l > lastmtime or today != get_today():
                etmdata = ETMData()
                etmdata.loadData()
                if etmdata.load_errors:
                    for msg in etmdata.load_errors:
                        print(msg)
                if etmdata.changed:
                    etmdata.loadData()
                lastmtime = etmdata.getmtime()
                etmdata.process_items()

            if options['verbose_toggle']:
                minattr = -2
            else:
                minattr = -1
            z, hd, lst, ft = etmdata.prepare_view(options)
            os.system(clear)
            if cmdlc == 'a':
                print(hd.cur())
                for l in lst:
                    s = l.cur(minattr)
                    if s:
                        try:
                            print(s.strip())
                        except:
                            print(l)
                print("%s" % ft.cur())
            elif cmdlc == 'l':
                print(hd.cur())
                for l in lst:
                    try:
                        s = l.cur(minattr)
                        if s:
                            print(s)
                    except:
                        print(type(l),l)
                print(ft.cur())
            elif cmdlc == 'r':
                print(hd.cur())
                for l in lst:
                    print(l.cur())
                print(ft.cur())
            elif cmdlc == 'b':
                print(hd.cur())
                for l in lst:
                    print(l.cur())
                print(ft.cur())


if __name__ == '__main__':
    main()