Output a log message every x% of a long computation

import logging

logging.basicConfig(level=logging.INFO)


def time_to_log(data, percent=10):
    percent = percent if percent > 1 else percent * 100
    data_size = len(data)
    log_every = data_size * percent / 100
    return float(data_size), lambda i: not i % log_every


data = range(100)
data_size, is_time_to_log = time_to_log(data, 0.2)

for i, d in enumerate(data):
    if is_time_to_log(i):
        logging.info("processing... {:.2%}".format(i / data_size))
    pass  # do whatever is needed
logging.info("processing... 100.00%")

Python Puzzle #2

Here is the challenge #2. This one deserves more explanations.
Your team has to maintain a legacy project, and for various historical reasons, the project has not enough documentation. Features delivery is always a higher priority than the documentation update.
So, you decide it's time that every team member commits to read and improve the documentation.
The plan is to spend an hour every Monday, Wednesday and Friday to this task, each team member in turn.

you want a function, that given a start date and a number of weeks, will assign the team member to this maintenance.

import unittest
import datetime


team_members = [
    "Dilbert",
    "Wally",
    "Alice",
    "Pointy haired Boss",
    "Dogbert",
    "Asok",
    "Ted",
    "Loud Howard",
    "Carol",
    "Ratbert",
    "Catbert",
    "Dilmom",
    "Phil, prince of insufficient light",
    "Bob the dinosaur",
    "Garbageman",
    "Tina",
    "Accountig Troll",
    "Elbonian guy",
    "Mordac",
    "Topper",
    "Liz",
]


def build_rota(start_date, weeks=4):
    pass


class TestDottedToDict(unittest.TestCase):
    def setUp(self):
        self.expected = [(datetime.date(2014, 12, 26), 'Dilbert'),
                         (datetime.date(2014, 12, 29), 'Wally'),
                         (datetime.date(2014, 12, 31), 'Alice'),
                         (datetime.date(2015, 1, 2), 'Pointy haired Boss'),
                         (datetime.date(2015, 1, 5), 'Dogbert'),
                         (datetime.date(2015, 1, 7), 'Asok'),
                         (datetime.date(2015, 1, 9), 'Ted'),
                         (datetime.date(2015, 1, 12), 'Loud Howard'),
                         (datetime.date(2015, 1, 14), 'Carol'),
                         (datetime.date(2015, 1, 16), 'Ratbert'),
                         (datetime.date(2015, 1, 19), 'Catbert'),
                         (datetime.date(2015, 1, 21), 'Dilmom'),
                         (datetime.date(2015, 1, 23), 'Phil, prince of insufficient light'),
                         (datetime.date(2015, 1, 26), 'Bob the dinosaur'),
                         (datetime.date(2015, 1, 28), 'Garbageman'),
                         (datetime.date(2015, 1, 30), 'Tina'),
                         (datetime.date(2015, 2, 2), 'Accountig Troll'),
                         (datetime.date(2015, 2, 4), 'Elbonian guy'),
                         (datetime.date(2015, 2, 6), 'Mordac'),
                         (datetime.date(2015, 2, 9), 'Topper'),
                         (datetime.date(2015, 2, 11), 'Liz'),
                         (datetime.date(2015, 2, 13), 'Dilbert'),
                         (datetime.date(2015, 2, 16), 'Wally'),
                         (datetime.date(2015, 2, 18), 'Alice'),
                         (datetime.date(2015, 2, 20), 'Pointy haired Boss'),
                         (datetime.date(2015, 2, 23), 'Dogbert'),
                         (datetime.date(2015, 2, 25), 'Asok'),
                         (datetime.date(2015, 2, 27), 'Ted'),
                         (datetime.date(2015, 3, 2), 'Loud Howard'),
                         (datetime.date(2015, 3, 4), 'Carol'),
                         (datetime.date(2015, 3, 6), 'Ratbert'),
                         (datetime.date(2015, 3, 9), 'Catbert'),
                         (datetime.date(2015, 3, 11), 'Dilmom'),
                         (datetime.date(2015, 3, 13), 'Phil, prince of insufficient light'),
                         (datetime.date(2015, 3, 16), 'Bob the dinosaur'),
                         (datetime.date(2015, 3, 18), 'Garbageman'),
                         (datetime.date(2015, 3, 20), 'Tina'),
                         (datetime.date(2015, 3, 23), 'Accountig Troll'),
                         (datetime.date(2015, 3, 25), 'Elbonian guy'),
                         (datetime.date(2015, 3, 27), 'Mordac'),
                         (datetime.date(2015, 3, 30), 'Topper'),
                         (datetime.date(2015, 4, 1), 'Liz'),
                         (datetime.date(2015, 4, 3), 'Dilbert'),
                         (datetime.date(2015, 4, 6), 'Wally'),
                         (datetime.date(2015, 4, 8), 'Alice')]

    def test_rota(self):
        got = list(build_rota(datetime.date(2014, 12, 25), weeks=15))
        self.assertListEqual(got, self.expected)

Have fun!

Python Puzzle #1

I plan to publish a serie of Python puzzle, or challenges.
Mostly around data manipulation, they can be a great way to improve your python skills.
There is many ways to solve this, you can chose your style from:

• best performance 
• best readability 
• least lines of code 
• most obfuscated 

the goal is to pass the unit test(s).

import unittest


def dotted_to_dict(incoming):
    pass


class TestDottedToDict(unittest.TestCase):
    def setUp(self):
        self.dotted = [{'a': 1, 'b.e.f': 2, 'b.e.g': 9, 'c.a': 5, 'c.d': 10},
                       {'a': 6, 'b.f.f': 8, 'b.f.g': 0, 'c.a': None, 'c.d': 15},
        ]

        self.expected = [{'a': 1, 'b': {'e': {'f': 2, 'g': 9}}, 'c': {'a': 5, 'd': 10}},
                         {'a': 6, 'b': {'f': {'f': 8, 'g': 0}}, 'c': {'a': None, 'd': 15}},
        ]

    def testClasses(self):
        """
        """
        got = dotted_to_dict(self.dotted)
        self.assertEquals(got, self.expected)

Please comment if you feel the question is not clear enough.
Have fun!

I've just released a small utility to process python source files
https://pypi.python.org/pypi/sourceparse
source and doc available at
https://github.com/nlaurance/sourceparse

I use this to inject code pieces to custom checkers, I hope you'll find it usefull

Opimize sqlalchemy queries with a With statement

More often than not, I need SQLAlchemy to issue as little SQL statement as possible. So I gave a try toward joinedload_all.
What I wanted was a nice way to hide this behind a with statement. It's a nice way to move the cumbersome extra options out of the way, but then I realized the inspection of the class could do the job for me.
This is what I got so far.

from sqlalchemy.orm import RelationshipProperty
from sqlalchemy.orm import joinedload_all

class LoadAll(object):
    """ builds a query with all the joined load
    """

    def __init__(self, klass):
        self.klass = klass

    def explore_relations(self):

        def follow(mapper, visited=None):
            """
            builds a tree of relations of the given mapper
            """
            result = {}
            visited = visited or []
            if mapper is not None and mapper not in visited:
                visited.append(mapper)
                for prop in mapper.iterate_properties:
                    if isinstance(prop, RelationshipProperty):
                        # next level
                        prop_arg = prop.argument
                        prop_arg = prop_arg() if callable(prop_arg) else prop_arg
                        prop_map = getattr(prop_arg, '__mapper__', None)
                        if prop_map is not None:
                            result[prop.key] = follow(prop_map, visited)
            return result
        return follow(self.klass.__mapper__)

    def paths(self, tree, cur=[]):
        """
        :param tree: a tree as nested dicts
        :param cur: current path, used to hold the result
        :return: all paths from root to leaves
        """
        if not tree:  # previous was a leaf, path to us is a solution
            yield cur
        else:  # we're a node
            for node, subtree in tree.iteritems():
                for path in self.paths(subtree, cur+[node,]):
                    yield path

    def __enter__(self):
        q = my_session.query(self.klass)
        tree = self.explore_relations()
        all_clauses = self.paths(tree)
        for join_clauses in all_clauses:
            q = q.options(joinedload_all(*join_clauses))
        return q

    def __exit__(self, exc_type, exc_val, exc_tb):
        pass

If given a setup like this

Employee N--1 Department N--1 Office N--1 Currency
                                    +N--1 Country 

in SQLAlchemy

from sqlalchemy import Column, DateTime, String, Integer, ForeignKey, func
from sqlalchemy.orm import relationship, backref
from sqlalchemy.ext.declarative import declarative_base

from sqlalchemy import create_engine
engine = create_engine('sqlite:///:memory:', echo=True)

from sqlalchemy.orm import sessionmaker
session = sessionmaker()
session.configure(bind=engine)
my_session = session()

Base = declarative_base()

class Country(Base):
    __tablename__ = 'country'
    id = Column(Integer, primary_key=True)
    name = Column(String, nullable=False)

class Currency(Base):
    __tablename__ = 'currency'
    id = Column(Integer, primary_key=True)
    name = Column(String, nullable=False)

class Office(Base):
    __tablename__ = 'office'
    id = Column(Integer, primary_key=True)
    name = Column(String, nullable=False)
    country_id = Column(Integer, ForeignKey('country.id'))
    country = relationship("Country", backref="office")
    currency_id = Column(Integer, ForeignKey('currency.id'))
    currency = relationship("Currency", backref="office")

class Department(Base):
    __tablename__ = 'department'
    id = Column(Integer, primary_key=True)
    name = Column(String, nullable=False)
    office_id = Column(Integer, ForeignKey('office.id'))
    office = relationship("Office", backref="dpts")


class Employee(Base):
    __tablename__ = 'employee'
    id = Column(Integer, primary_key=True)
    name = Column(String)
    hired_on = Column(DateTime, default=func.now())
    department_id = Column(Integer, ForeignKey('department.id'))
    department = relationship(
        "Department",
        backref=backref('employees',
                         uselist=True,
                         cascade='delete,all'))


Base.metadata.create_all(engine)

c1 = Currency(name="euro")
c2 = Currency(name="sterling")
d1 = Department(name="dpt1", office=Office(name="Dublin", country=Country(name="IE"), currency=c1))
d2 = Department(name="dpt2", office=Office(name="London", country=Country(name="UK"), currency=c2))
e1 = Employee(name="dilbert", department=d1)
e2 = Employee(name="wally", department=d2)
e3 = Employee(name="alice", department=d2)
my_session.add(e1)
my_session.add(e2)
my_session.add(e3)

my_session.commit()

If we then use it like this

with LoadAll(Employee) as myq:
    for e in myq.all():
        print e.name, e.department.name, e.department.office.name, \
              e.department.office.country.name, e.department.office.currency.name

sqlalchemy will issue a single query

SELECT employee.id AS employee_id, employee.name AS employee_name, employee.hired_on AS employee_hired_on, employee.department_id AS employee_department_id, country_1.id AS country_1_id, country_1.name AS country_1_name, currency_1.id AS currency_1_id, currency_1.name AS currency_1_name, office_1.id AS office_1_id, office_1.name AS office_1_name, office_1.country_id AS office_1_country_id, office_1.currency_id AS office_1_currency_id, department_1.id AS department_1_id, department_1.name AS department_1_name, department_1.office_id AS department_1_office_id 
FROM employee 
LEFT OUTER JOIN department AS department_1 ON department_1.id = employee.department_id 
LEFT OUTER JOIN office AS office_1 ON office_1.id = department_1.office_id 
LEFT OUTER JOIN country AS country_1 ON country_1.id = office_1.country_id 
LEFT OUTER JOIN currency AS currency_1 ON currency_1.id = office_1.currency_id

recursivity elegance

while searching for an elegant solution, I stumbled upon this by Andrew Clark

incoming = {'E': {'D': {'A': {},
                        'O': {'Co': {},
                              'Cu': {}
                             }
                        }
                 }
            }

expected = [
   ('E', 'D', 'A'),
   ('E', 'D', 'O', 'Co'),
   ('E', 'D', 'O', 'Cu'),
]


def paths(tree, cur=()):
    if not tree:  # previous was a leaf, path to us is a solution
        yield cur
    else:  # we're a node
        for node, subtree in tree.iteritems():
            for path in paths(subtree, cur+(node,)):
                yield path

assert list(paths(incoming)) == expected

Customize django_tinymce widget in django admin

The lazy path using an HTMLField can be customized easily:

in settings.py

TINYMCE_DEFAULT_CONFIG = {
    'plugins': "xhtmlxtras",
    'theme': "advanced",
    "theme_advanced_buttons1" : "bold,italic,underline,separator,bullist,numlist,separator,outdent,indent,separator,undo,redo",
    "theme_advanced_buttons2" : "link,unlink,separator,removeformat,separator,sub,sup,separator,abbr",
    "theme_advanced_buttons3" : "",
    'cleanup_on_startup': True,
    'custom_undo_redo_levels': 10,
}
 

all options :

• http://www.tinymce.com/wiki.php/Configuration:Advanced_theme
• http://www.tinymce.com/wiki.php/Buttons/controls

default value for text function using lxml

Say we need to parse this XML

<pack xmlns="http://ns.qubic.tv/2010/item">
        <packitem>
            <duration>520</duration>
            <max_count>14</max_count>
        </packitem>
        <packitem>
            <duration></duration>
            <max_count>23</max_count>
        </packitem>
</pack>

if you want to parse it and retrieve the values in tuples

root = etree.fromstring(xml)
namespaces = {'i':"http://ns.qubic.tv/2010/item"}
packitems_duration = root.xpath('//i:pack/i:packitem/i:duration/text()', 
    namespaces=namespaces)
packitems_max_count = root.xpath('//b:pack/i:packitem/i:max_count/text()',
    namespaces=namespaces)
packitems = zip(packitems_duration, packitems_max_count)

>>> packitems
[('520','14')]

The problem is the zip result miss a value. That's because lxml returns nothing instead of None or empty string. Let's change that.

def lxml_empty_str(context, nodes):
    for node in nodes:
        node.text = node.text or ""
    return nodes

ns = etree.FunctionNamespace('http://ns.qubic.tv/lxmlfunctions')
ns['lxml_empty_str'] = lxml_empty_str

namespaces = {'i':"http://ns.qubic.tv/2010/item",
              'f': "http://ns.qubic.tv/lxmlfunctions"}
packitems_duration = root.xpath('f:lxml_empty_str('//b:pack/i:packitem/i:duration)/text()',
    namespaces={'b':billing_ns, 'f' : 'http://ns.qubic.tv/lxmlfunctions'})
packitems_max_count = root.xpath('f:lxml_empty_str('//b:pack/i:packitem/i:max_count)/text()',
    namespaces={'b':billing_ns, 'f' : 'http://ns.qubic.tv/lxmlfunctions'})
packitems = zip(packitems_duration, packitems_max_count)

>>> packitems
[('520','14'), ('','23')]

more info on extending lxml http://lxml.de/extensions.html#xpath-extension-functions