#!/usr/bin/env python """ A recursive descent parser example. Usage: python rparser.py [options] <inputfile> Options: -h, --help Display this help message. Example: python rparser.py myfile.txt The grammar: Prog ::= Command | Command Prog Command ::= Func_call Func_call ::= Term '(' Func_call_list ')' Func_call_list ::= Func_call | Func_call ',' Func_call_list Term = <word> """ import sys import string import types import getopt # # To use the IPython interactive shell to inspect your running # application, uncomment the following lines: # ## from IPython.Shell import IPShellEmbed ## ipshell = IPShellEmbed((), ## banner = '>>>>>>>> Into IPython >>>>>>>>', ## exit_msg = '<<<<<<<< Out of IPython <<<<<<<<') # # Then add the following line at the point in your code where # you want to inspect run-time values: # # ipshell('some message to identify where we are') # # For more information see: http://ipython.scipy.org/moin/ # # # Constants # # AST node types NoneNodeType = 0 ProgNodeType = 1 CommandNodeType = 2 FuncCallNodeType = 3 FuncCallListNodeType = 4 TermNodeType = 5 # Token types NoneTokType = 0 LParTokType = 1 RParTokType = 2 WordTokType = 3 CommaTokType = 4 EOFTokType = 5 # Dictionary to map node type values to node type names NodeTypeDict = { NoneNodeType: 'NoneNodeType', ProgNodeType: 'ProgNodeType', CommandNodeType: 'CommandNodeType', FuncCallNodeType: 'FuncCallNodeType', FuncCallListNodeType: 'FuncCallListNodeType', TermNodeType: 'TermNodeType', } # # Representation of a node in the AST (abstract syntax tree). # class ASTNode: def __init__(self, nodeType, *args): self.nodeType = nodeType self.children = [] for item in args: self.children.append(item) def show(self, level): self.showLevel(level) print 'Node -- Type %s' % NodeTypeDict[self.nodeType] level += 1 for child in self.children: if isinstance(child, ASTNode): child.show(level) elif type(child) == types.ListType: for item in child: item.show(level) else: self.showLevel(level) print 'Child:', child def showLevel(self, level): for idx in range(level): print ' ', # # The recursive descent parser class. # Contains the "recognizer" methods, which implement the grammar # rules (above), one recognizer method for each production rule. # class ProgParser: def __init__(self): pass def parseFile(self, infileName): self.infileName = infileName self.tokens = None self.tokenType = NoneTokType self.token = '' self.lineNo = -1 self.infile = file(self.infileName, 'r') self.tokens = genTokens(self.infile) try: self.tokenType, self.token, self.lineNo = self.tokens.next() except StopIteration: raise RuntimeError, 'Empty file' result = self.prog_reco() self.infile.close() self.infile = None return result def parseStream(self, instream): self.tokens = genTokens(instream, '<instream>') try: self.tokenType, self.token, self.lineNo = self.tokens.next() except StopIteration: raise RuntimeError, 'Empty file' result = self.prog_reco() return result def prog_reco(self): commandList = [] while 1: result = self.command_reco() if not result: break commandList.append(result) return ASTNode(ProgNodeType, commandList) def command_reco(self): if self.tokenType == EOFTokType: return None result = self.func_call_reco() return ASTNode(CommandNodeType, result) def func_call_reco(self): if self.tokenType == WordTokType: term = ASTNode(TermNodeType, self.token) self.tokenType, self.token, self.lineNo = self.tokens.next() if self.tokenType == LParTokType: self.tokenType, self.token, self.lineNo = self.tokens.next() result = self.func_call_list_reco() if result: if self.tokenType == RParTokType: self.tokenType, self.token, self.lineNo = \ self.tokens.next() return ASTNode(FuncCallNodeType, term, result) else: raise ParseError(self.lineNo, 'missing right paren') else: raise ParseError(self.lineNo, 'bad func call list') else: raise ParseError(self.lineNo, 'missing left paren') else: return None def func_call_list_reco(self): terms = [] while 1: result = self.func_call_reco() if not result: break terms.append(result) if self.tokenType != CommaTokType: break self.tokenType, self.token, self.lineNo = self.tokens.next() return ASTNode(FuncCallListNodeType, terms) # # The parse error exception class. # class ParseError(Exception): def __init__(self, lineNo, msg): RuntimeError.__init__(self, msg) self.lineNo = lineNo self.msg = msg def getLineNo(self): return self.lineNo def getMsg(self): return self.msg def is_word(token): for letter in token: if letter not in string.ascii_letters: return None return 1 # # Generate the tokens. # Usage: # gen = genTokens(infile) # tokType, tok, lineNo = gen.next() # ... def genTokens(infile): lineNo = 0 while 1: lineNo += 1 try: line = infile.next() except: yield (EOFTokType, None, lineNo) toks = line.split() for tok in toks: if is_word(tok): tokType = WordTokType elif tok == '(': tokType = LParTokType elif tok == ')': tokType = RParTokType elif tok == ',': tokType = CommaTokType yield (tokType, tok, lineNo) def test(infileName): parser = ProgParser() #ipshell('(test) #1\nCtrl-D to exit') result = None try: result = parser.parseFile(infileName) except ParseError, exp: sys.stderr.write('ParseError: (%d) %s\n' % \ (exp.getLineNo(), exp.getMsg())) if result: result.show(0) def usage(): print __doc__ sys.exit(1) def main(): args = sys.argv[1:] try: opts, args = getopt.getopt(args, 'h', ['help']) except: usage() relink = 1 for opt, val in opts: if opt in ('-h', '--help'): usage() if len(args) != 1: usage() inputfile = args[0] test(inputfile) if __name__ == '__main__': #import pdb; pdb.set_trace() main()
Comments and explanation:
And, here is a sample of the data we can apply this parser to:
aaa ( ) bbb ( ccc ( ) ) ddd ( eee ( ) , fff ( ggg ( ) , hhh ( ) , iii ( ) ) )
And, if we run the parser on the this input data, we see:
$ python workbook045.py workbook045.data Node -- Type ProgNodeType Node -- Type CommandNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: aaa Node -- Type FuncCallListNodeType Node -- Type CommandNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: bbb Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: ccc Node -- Type FuncCallListNodeType Node -- Type CommandNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: ddd Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: eee Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: fff Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: ggg Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: hhh Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: iii Node -- Type FuncCallListNodeType
Lexical analysis -- The tokenizer in our recursive descent parser example was (for demonstration purposes) overly simple. You can always write more complex tokenizers by hand. However, for more complex (and real) tokenizers, you may want to use a tool to build your tokenizer.
In this section we'll describe Plex and use it to produce a tokenizer for our recursive descent parser.
You can obtain Plex at http://www.cosc.canterbury.ac.nz/~greg/python/Plex/.
In order to use it, you may want to add Plex-1.1.4/Plex to your PYTHONPATH.
Here is a simple example from the Plex tutorial:
#!/usr/bin/env python """ Sample Plex lexer Usage: python plex_example.py inputfile """ import sys import Plex def count_lines(scanner, text): scanner.line_count += 1 print '-' * 60 def test(infileName): letter = Plex.Range("AZaz") digit = Plex.Range("09") name = letter + Plex.Rep(letter | digit) number = Plex.Rep1(digit) space = Plex.Any(" \t") endline = Plex.Str('\n') #comment = Plex.Str('"') + Plex.Rep( Plex.AnyBut('"')) + Plex.Str('"') resword = Plex.Str("if", "then", "else", "end") lexicon = Plex.Lexicon([ (endline, count_lines), (resword, 'keyword'), (name, 'ident'), (number, 'int'), ( Plex.Any("+-*/=<>"), 'operator'), (space, Plex.IGNORE), #(comment, 'comment'), (Plex.Str('('), 'lpar'), (Plex.Str(')'), 'rpar'), # comments surrounded by (* and *) (Plex.Str("(*"), Plex.Begin('comment')), Plex.State('comment', [ (Plex.Str("*)"), Plex.Begin('')), (Plex.AnyChar, Plex.IGNORE), ]), ]) infile = open(infileName, "r") scanner = Plex.Scanner(lexicon, infile, infileName) scanner.line_count = 0 while True: token = scanner.read() if token[0] is None: break position = scanner.position() posstr = ('(%d, %d)' % (position[1], position[2], )).ljust(10) tokstr = '"%s"' % token[1] tokstr = tokstr.ljust(20) print '%s tok: %s tokType: %s' % (posstr, tokstr, token[0],) print 'line_count: %d' % scanner.line_count def usage(): print __doc__ sys.exit(1) def main(): args = sys.argv[1:] if len(args) != 1: usage() infileName = args[0] test(infileName) if __name__ == '__main__': #import pdb; pdb.set_trace() main()
Here is a bit of data on which we can use the above lexer:
mass = (height * (* some comment *) width * depth) / density totalmass = totalmass + mass
And, when we apply the above test program to this data, here is what we see:
$ python plex_example.py plex_example.data (1, 0) tok: "mass" tokType: ident (1, 5) tok: "=" tokType: operator (1, 7) tok: "(" tokType: lpar (1, 8) tok: "height" tokType: ident (1, 15) tok: "*" tokType: operator (1, 36) tok: "width" tokType: ident (1, 42) tok: "*" tokType: operator (1, 44) tok: "depth" tokType: ident (1, 49) tok: ")" tokType: rpar (1, 51) tok: "/" tokType: operator (1, 53) tok: "density" tokType: ident ------------------------------------------------------------ (2, 0) tok: "totalmass" tokType: ident (2, 10) tok: "=" tokType: operator (2, 12) tok: "totalmass" tokType: ident (2, 22) tok: "+" tokType: operator (2, 24) tok: "mass" tokType: ident ------------------------------------------------------------ line_count: 2
Comments and explanation:
And, here are some comments on constructing the patterns used in a lexicon:
Now let's revisit our recursive descent parser, this time with a tokenizer built with Plex. The tokenizer is trivial, but will serve as an example of how to hook it into a parser:
#!/usr/bin/env python """ A recursive descent parser example using Plex. This example uses Plex to implement a tokenizer. Usage: python python_201_rparser_plex.py [options] <inputfile> Options: -h, --help Display this help message. Example: python python_201_rparser_plex.py myfile.txt The grammar: Prog ::= Command | Command Prog Command ::= Func_call Func_call ::= Term '(' Func_call_list ')' Func_call_list ::= Func_call | Func_call ',' Func_call_list Term = <word> """ import sys, string, types import getopt import Plex ## from IPython.Shell import IPShellEmbed ## ipshell = IPShellEmbed((), ## banner = '>>>>>>>> Into IPython >>>>>>>>', ## exit_msg = '<<<<<<<< Out of IPython <<<<<<<<') # # Constants # # AST node types NoneNodeType = 0 ProgNodeType = 1 CommandNodeType = 2 FuncCallNodeType = 3 FuncCallListNodeType = 4 TermNodeType = 5 # Token types NoneTokType = 0 LParTokType = 1 RParTokType = 2 WordTokType = 3 CommaTokType = 4 EOFTokType = 5 # Dictionary to map node type values to node type names NodeTypeDict = { NoneNodeType: 'NoneNodeType', ProgNodeType: 'ProgNodeType', CommandNodeType: 'CommandNodeType', FuncCallNodeType: 'FuncCallNodeType', FuncCallListNodeType: 'FuncCallListNodeType', TermNodeType: 'TermNodeType', } # # Representation of a node in the AST (abstract syntax tree). # class ASTNode: def __init__(self, nodeType, *args): self.nodeType = nodeType self.children = [] for item in args: self.children.append(item) def show(self, level): self.showLevel(level) print 'Node -- Type %s' % NodeTypeDict[self.nodeType] level += 1 for child in self.children: if isinstance(child, ASTNode): child.show(level) elif type(child) == types.ListType: for item in child: item.show(level) else: self.showLevel(level) print 'Child:', child def showLevel(self, level): for idx in range(level): print ' ', # # The recursive descent parser class. # Contains the "recognizer" methods, which implement the grammar # rules (above), one recognizer method for each production rule. # class ProgParser: def __init__(self): self.tokens = None self.tokenType = NoneTokType self.token = '' self.lineNo = -1 self.infile = None self.tokens = None def parseFile(self, infileName): self.tokens = None self.tokenType = NoneTokType self.token = '' self.lineNo = -1 self.infile = file(infileName, 'r') self.tokens = genTokens(self.infile, infileName) try: self.tokenType, self.token, self.lineNo = self.tokens.next() except StopIteration: raise RuntimeError, 'Empty file' result = self.prog_reco() self.infile.close() self.infile = None return result def parseStream(self, instream): self.tokens = None self.tokenType = NoneTokType self.token = '' self.lineNo = -1 self.tokens = genTokens(self.instream, '<stream>') try: self.tokenType, self.token, self.lineNo = self.tokens.next() except StopIteration: raise RuntimeError, 'Empty stream' result = self.prog_reco() self.infile.close() self.infile = None return result def prog_reco(self): commandList = [] while 1: result = self.command_reco() if not result: break commandList.append(result) return ASTNode(ProgNodeType, commandList) def command_reco(self): if self.tokenType == EOFTokType: return None result = self.func_call_reco() return ASTNode(CommandNodeType, result) def func_call_reco(self): if self.tokenType == WordTokType: term = ASTNode(TermNodeType, self.token) self.tokenType, self.token, self.lineNo = self.tokens.next() if self.tokenType == LParTokType: self.tokenType, self.token, self.lineNo = self.tokens.next() result = self.func_call_list_reco() if result: if self.tokenType == RParTokType: self.tokenType, self.token, self.lineNo = \ self.tokens.next() return ASTNode(FuncCallNodeType, term, result) else: raise ParseError(self.lineNo, 'missing right paren') else: raise ParseError(self.lineNo, 'bad func call list') else: raise ParseError(self.lineNo, 'missing left paren') else: return None def func_call_list_reco(self): terms = [] while 1: result = self.func_call_reco() if not result: break terms.append(result) if self.tokenType != CommaTokType: break self.tokenType, self.token, self.lineNo = self.tokens.next() return ASTNode(FuncCallListNodeType, terms) # # The parse error exception class. # class ParseError(Exception): def __init__(self, lineNo, msg): RuntimeError.__init__(self, msg) self.lineNo = lineNo self.msg = msg def getLineNo(self): return self.lineNo def getMsg(self): return self.msg # # Generate the tokens. # Usage - example # gen = genTokens(infile) # tokType, tok, lineNo = gen.next() # ... def genTokens(infile, infileName): letter = Plex.Range("AZaz") digit = Plex.Range("09") name = letter + Plex.Rep(letter | digit) lpar = Plex.Str('(') rpar = Plex.Str(')') comma = Plex.Str(',') comment = Plex.Str("#") + Plex.Rep(Plex.AnyBut("\n")) space = Plex.Any(" \t\n") lexicon = Plex.Lexicon([ (name, 'word'), (lpar, 'lpar'), (rpar, 'rpar'), (comma, 'comma'), (comment, Plex.IGNORE), (space, Plex.IGNORE), ]) scanner = Plex.Scanner(lexicon, infile, infileName) while 1: tokenType, token = scanner.read() name, lineNo, columnNo = scanner.position() if tokenType == None: tokType = EOFTokType token = None elif tokenType == 'word': tokType = WordTokType elif tokenType == 'lpar': tokType = LParTokType elif tokenType == 'rpar': tokType = RParTokType elif tokenType == 'comma': tokType = CommaTokType else: tokType = NoneTokType tok = token yield (tokType, tok, lineNo) def test(infileName): parser = ProgParser() #ipshell('(test) #1\nCtrl-D to exit') result = None try: result = parser.parseFile(infileName) except ParseError, exp: sys.stderr.write('ParseError: (%d) %s\n' % \ (exp.getLineNo(), exp.getMsg())) if result: result.show(0) def usage(): print __doc__ sys.exit(-1) def main(): args = sys.argv[1:] try: opts, args = getopt.getopt(args, 'h', ['help']) except: usage() for opt, val in opts: if opt in ('-h', '--help'): usage() if len(args) != 1: usage() infileName = args[0] test(infileName) if __name__ == '__main__': #import pdb; pdb.set_trace() main()
And, here is a sample of the data we can apply this parser to:
# Test for recursive descent parser and Plex. # Command #1 aaa() # Command #2 bbb (ccc()) # An end of line comment. # Command #3 ddd(eee(), fff(ggg(), hhh(), iii())) # End of test
And, when we run our parser, it produces the following:
$ python plex_recusive.py plex_recusive.data Node -- Type ProgNodeType Node -- Type CommandNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: aaa Node -- Type FuncCallListNodeType Node -- Type CommandNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: bbb Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: ccc Node -- Type FuncCallListNodeType Node -- Type CommandNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: ddd Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: eee Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: fff Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: ggg Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: hhh Node -- Type FuncCallListNodeType Node -- Type FuncCallNodeType Node -- Type TermNodeType Child: iii Node -- Type FuncCallListNodeType
Comments:
For complex parsing tasks, you may want to consider the following tools:
And, for lexical analysis, you may also want to look here:
In the sections below, we give examples and notes about the use of PLY and pyparsing.
In this section we will show how to implement our parser example with PLY.
First down-load PLY. It is available here: PLY (Python Lex-Yacc) -- http://www.dabeaz.com/ply/
Then add the PLY directory to your PYTHONPATH.
Learn how to construct lexers and parsers with PLY by reading doc/ply.html in the distribution of PLY and by looking at the examples in the distribution.
For those of you who want a more complex example, see A Python Parser for the RELAX NG Compact Syntax, which is implemented with PLY.
Now, here is our example parser. Comments and explanations are below:
#!/usr/bin/env python """ A parser example. This example uses PLY to implement a lexer and parser. The grammar: Prog ::= Command* Command ::= Func_call Func_call ::= Term '(' Func_call_list ')' Func_call_list ::= Func_call* Term = <word> Here is a sample "program" to use as input: # Test for recursive descent parser and Plex. # Command #1 aaa() # Command #2 bbb (ccc()) # An end of line comment. # Command #3 ddd(eee(), fff(ggg(), hhh(), iii())) # End of test """ import sys import types import getopt import ply.lex as lex import ply.yacc as yacc # # Globals # startlinepos = 0 # # Constants # # AST node types NoneNodeType = 0 ProgNodeType = 1 CommandNodeType = 2 CommandListNodeType = 3 FuncCallNodeType = 4 FuncCallListNodeType = 5 TermNodeType = 6 # Dictionary to map node type values to node type names NodeTypeDict = { NoneNodeType: 'NoneNodeType', ProgNodeType: 'ProgNodeType', CommandNodeType: 'CommandNodeType', CommandListNodeType: 'CommandListNodeType', FuncCallNodeType: 'FuncCallNodeType', FuncCallListNodeType: 'FuncCallListNodeType', TermNodeType: 'TermNodeType', } # # Representation of a node in the AST (abstract syntax tree). # class ASTNode: def __init__(self, nodeType, *args): self.nodeType = nodeType self.children = [] for item in args: self.children.append(item) def append(self, item): self.children.append(item) def show(self, level): self.showLevel(level) print 'Node -- Type: %s' % NodeTypeDict[self.nodeType] level += 1 for child in self.children: if isinstance(child, ASTNode): child.show(level) elif type(child) == types.ListType: for item in child: item.show(level) else: self.showLevel(level) print 'Value:', child def showLevel(self, level): for idx in range(level): print ' ', # # Exception classes # class LexerError(Exception): def __init__(self, msg, lineno, columnno): self.msg = msg self.lineno = lineno self.columnno = columnno def show(self): sys.stderr.write('Lexer error (%d, %d) %s\n' % \ (self.lineno, self.columnno, self.msg)) class ParserError(Exception): def __init__(self, msg, lineno, columnno): self.msg = msg self.lineno = lineno self.columnno = columnno def show(self): sys.stderr.write('Parser error (%d, %d) %s\n' % \ (self.lineno, self.columnno, self.msg)) # # Lexer specification # tokens = ( 'NAME', 'LPAR','RPAR', 'COMMA', ) # Tokens t_LPAR = r'\(' t_RPAR = r'\)' t_COMMA = r'\,' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' # Ignore whitespace t_ignore = ' \t' # Ignore comments ('#' to end of line) def t_COMMENT(t): r'\#[^\n]*' pass def t_newline(t): r'\n+' global startlinepos startlinepos = t.lexer.lexpos - 1 t.lineno += t.value.count("\n") def t_error(t): global startlinepos msg = "Illegal character '%s'" % (t.value[0]) columnno = t.lexer.lexpos - startlinepos raise LexerError(msg, t.lineno, columnno) # # Parser specification # def p_prog(t): 'prog : command_list' t[0] = ASTNode(ProgNodeType, t[1]) def p_command_list_1(t): 'command_list : command' t[0] = ASTNode(CommandListNodeType, t[1]) def p_command_list_2(t): 'command_list : command_list command' t[1].append(t[2]) t[0] = t[1] def p_command(t): 'command : func_call' t[0] = ASTNode(CommandNodeType, t[1]) def p_func_call_1(t): 'func_call : term LPAR RPAR' t[0] = ASTNode(FuncCallNodeType, t[1]) def p_func_call_2(t): 'func_call : term LPAR func_call_list RPAR' t[0] = ASTNode(FuncCallNodeType, t[1], t[3]) def p_func_call_list_1(t): 'func_call_list : func_call' t[0] = ASTNode(FuncCallListNodeType, t[1]) def p_func_call_list_2(t): 'func_call_list : func_call_list COMMA func_call' t[1].append(t[3]) t[0] = t[1] def p_term(t): 'term : NAME' t[0] = ASTNode(TermNodeType, t[1]) def p_error(t): global startlinepos msg = "Syntax error at '%s'" % t.value columnno = t.lexer.lexpos - startlinepos raise ParserError(msg, t.lineno, columnno) # # Parse the input and display the AST (abstract syntax tree) # def parse(infileName): startlinepos = 0 # Build the lexer lex.lex(debug=1) # Build the parser yacc.yacc() # Read the input infile = file(infileName, 'r') content = infile.read() infile.close() try: # Do the parse result = yacc.parse(content) # Display the AST result.show(0) except LexerError, exp: exp.show() except ParserError, exp: exp.show() USAGE_TEXT = __doc__ def usage(): print USAGE_TEXT sys.exit(-1) def main(): args = sys.argv[1:] try: opts, args = getopt.getopt(args, 'h', ['help']) except: usage() relink = 1 for opt, val in opts: if opt in ('-h', '--help'): usage() if len(args) != 1: usage() infileName = args[0] parse(infileName) if __name__ == '__main__': #import pdb; pdb.set_trace() main()
Applying this parser to the following input:
# Test for recursive descent parser and Plex. # Command #1 aaa() # Command #2 bbb (ccc()) # An end of line comment. # Command #3 ddd(eee(), fff(ggg(), hhh(), iii())) # End of test
produces the following output:
Node -- Type: ProgNodeType Node -- Type: CommandListNodeType Node -- Type: CommandNodeType Node -- Type: FuncCallNodeType Node -- Type: TermNodeType Value: aaa Node -- Type: CommandNodeType Node -- Type: FuncCallNodeType Node -- Type: TermNodeType Value: bbb Node -- Type: FuncCallListNodeType Node -- Type: FuncCallNodeType Node -- Type: TermNodeType Value: ccc Node -- Type: CommandNodeType Node -- Type: FuncCallNodeType Node -- Type: TermNodeType Value: ddd Node -- Type: FuncCallListNodeType Node -- Type: FuncCallNodeType Node -- Type: TermNodeType Value: eee Node -- Type: FuncCallNodeType Node -- Type: TermNodeType Value: fff Node -- Type: FuncCallListNodeType Node -- Type: FuncCallNodeType Node -- Type: TermNodeType Value: ggg Node -- Type: FuncCallNodeType Node -- Type: TermNodeType Value: hhh Node -- Type: FuncCallNodeType Node -- Type: TermNodeType Value: iii
Comments and explanation:
pyparsing is a relatively new parsing package for Python. It was implemented and is supported by Paul McGuire and it shows promise. It appears especially easy to use and seems especially appropriate in particular for quick parsing tasks, although it has features that make some complex parsing tasks easy. It follows a very natural Python style for constructing parsers.
Good documentation comes with the pyparsing distribution. See file HowToUseParsing.html. So, I won't try to repeat that here. What follows is an attempt to provide several quick examples to help you solve simple parsing tasks as quickly as possible.
You will also want to look at the samples in the examples directory, which are very helpful. My examples below are fairly simple. You can see more of the ability of pyparsing to handle complex tasks in the examples.
Where to get it - You can find pyparsing at: Pyparsing Wiki Home -- http://pyparsing.wikispaces.com/
How to install it - Put the pyparsing module somewhere on your PYTHONPATH.

Note: This example is for demonstration purposes only. If you really to need to parse comma delimited fields, you can probably do so much more easily with the CSV (comma separated values) module in the Python standard library.
Here is a simple grammar for lines containing fields separated by commas:
import sys from pyparsing import alphanums, ZeroOrMore, Word fieldDef = Word(alphanums) lineDef = fieldDef + ZeroOrMore("," + fieldDef) def test(): args = sys.argv[1:] if len(args) != 1: print 'usage: python pyparsing_test1.py <datafile.txt>' sys.exit(-1) infilename = sys.argv[1] infile = file(infilename, 'r') for line in infile: fields = lineDef.parseString(line) print fields test()
Here is some sample data:
abcd,defg 11111,22222,33333
And, when we run our parser on this data file, here is what we see:
$ python comma_parser.py sample1.data ['abcd', ',', 'defg'] ['11111', ',', '22222', ',', '33333']
Notes and explanation:
Note how the grammar is constructed from normal Python calls to function and object/class constructors. I've constructed the parser in-line because my example is simple, but constructing the parser in a function or even a module might make sense for more complex grammars. pyparsing makes it easy to use these these different styles.
Use "+" to specify a sequence. In our example, a lineDef is a fieldDef followed by ....
Use ZeroOrMore to specify repetition. In our example, a lineDef is a fieldDef followed by zero or more occurances of comma and fieldDef. There is also OneOrMore when you want to require at least one occurance.
Parsing comma delimited text happens so frequently that pyparsing provides a shortcut. Replace:
lineDef = fieldDef + ZeroOrMore("," + fieldDef)
with:
lineDef = delimitedList(fieldDef)
And note that delimitedList takes an optional argument delim used to specify the delimiter. The default is a comma.
This example parses expressions of the form func(arg1, arg2, arg3):
from pyparsing import Word, alphas, alphanums, nums, ZeroOrMore, Literal lparen = Literal("(") rparen = Literal(")") identifier = Word(alphas, alphanums + "_") integer = Word( nums ) functor = identifier arg = identifier | integer args = arg + ZeroOrMore("," + arg) expression = functor + lparen + args + rparen def test(): content = raw_input("Enter an expression: ") parsedContent = expression.parseString(content) print parsedContent test()
Explanation:
This example parses expressions having the following form:
Input format: [name] [phone] [city, state zip] Last, first<PHONE_NUMBER> city, ca 99999
Here is the parser:
import sys from pyparsing import alphas, nums, ZeroOrMore, Word, Group, Suppress, Combine lastname = Word(alphas) firstname = Word(alphas) city = Group(Word(alphas) + ZeroOrMore(Word(alphas))) state = Word(alphas, exact=2) zip = Word(nums, exact=5) name = Group(lastname + Suppress(",") + firstname) phone = Combine(Word(nums, exact=3) + "-" + Word(nums, exact=3) + "-" + Word(nums, exact=4)) location = Group(city + Suppress(",") + state + zip) record = name + phone + location def test(): args = sys.argv[1:] if len(args) != 1: print 'usage: python pyparsing_test3.py <datafile.txt>' sys.exit(-1) infilename = sys.argv[1] infile = file(infilename, 'r') for line in infile: line = line.strip() if line and line[0] != "#": fields = record.parseString(line) print fields test()
And, here is some sample input:
Jabberer, Jerry<PHONE_NUMBER> Bakersfield, CA 95111 Kackler, Kerry<PHONE_NUMBER> Fresno, CA 95112 Louderdale, Larry<PHONE_NUMBER> Los Angeles, CA 94001
Here is output from parsing the above input:
[['Jabberer', 'Jerry'], '111-222-3333', [['Bakersfield'], 'CA', '95111']] [['Kackler', 'Kerry'], '111-222-3334', [['Fresno'], 'CA', '95112']] [['Louderdale', 'Larry'], '111-222-3335', [['Los', 'Angeles'], 'CA', '94001']]
Comments:
This example (thanks to Paul McGuire) parses a more complex structure and produces a dictionary.
Here is the code:
from pyparsing import Literal, Word, Group, Dict, ZeroOrMore, alphas, nums,\ delimitedList import pprint testData = """ +-------+------+------+------+------+------+------+------+------+ | | A1 | B1 | C1 | D1 | A2 | B2 | C2 | D2 | +=======+======+======+======+======+======+======+======+======+ | min | 7 | 43 | 7 | 15 | 82 | 98 | 1 | 37 | | max | 11 | 52 | 10 | 17 | 85 | 112 | 4 | 39 | | ave | 9 | 47 | 8 | 16 | 84 | 106 | 3 | 38 | | sdev | 1 | 3 | 1 | 1 | 1 | 3 | 1 | 1 | +-------+------+------+------+------+------+------+------+------+ """ # Define grammar for datatable heading = (Literal( "+-------+------+------+------+------+------+------+------+------+") + "| | A1 | B1 | C1 | D1 | A2 | B2 | C2 | D2 |" + "+=======+======+======+======+======+======+======+======+======+").suppress() vert = Literal("|").suppress() number = Word(nums) rowData = Group( vert + Word(alphas) + vert + delimitedList(number,"|") + vert ) trailing = Literal( "+-------+------+------+------+------+------+------+------+------+").suppress() datatable = heading + Dict( ZeroOrMore(rowData) ) + trailing def main(): # Now parse data and print results data = datatable.parseString(testData) print "data:", data print "data.asList():", pprint.pprint(data.asList()) print "data keys:", data.keys() print "data['min']:", data['min'] print "data.max:", data.max if __name__ == '__main__': main()
When we run this, it produces the following:
data: [['min', '7', '43', '7', '15', '82', '98', '1', '37'], ['max', '11', '52', '10', '17', '85', '112', '4', '39'], ['ave', '9', '47', '8', '16', '84', '106', '3', '38'], ['sdev', '1', '3', '1', '1', '1', '3', '1', '1']] data.asList():[['min', '7', '43', '7', '15', '82', '98', '1', '37'], ['max', '11', '52', '10', '17', '85', '112', '4', '39'], ['ave', '9', '47', '8', '16', '84', '106', '3', '38'], ['sdev', '1', '3', '1', '1', '1', '3', '1', '1']] data keys: ['ave', 'min', 'sdev', 'max'] data['min']: ['7', '43', '7', '15', '82', '98', '1', '37'] data.max: ['11', '52', '10', '17', '85', '112', '4', '39']
Notes:
This section will help you to put a GUI (graphical user interface) in your Python program.
We will use a particular GUI library: PyGTK. We've chosen this because it is reasonably light-weight and our goal is to embed light-weight GUI interfaces in an (possibly) existing application.
For simpler GUI needs, consider EasyGUI, which is also described below.
For more heavy-weight GUI needs (for example, complete GUI applications), you may want to explore WxPython. See the WxPython home page at: http://www.wxpython.org/
Information about PyGTK is here: The PyGTK home page -- http://www.pygtk.org//.
In this section we explain how to pop up a simple dialog box from your Python application.
To do this, do the following:
Here is a sample that displays a message box:
#!/usr/bin/env python import sys import getopt import gtk class MessageBox(gtk.Dialog): def __init__(self, message="", buttons=(), pixmap=None, modal= True): gtk.Dialog.__init__(self) self.connect("destroy", self.quit) self.connect("delete_event", self.quit) if modal: self.set_modal(True) hbox = gtk.HBox(spacing=5) hbox.set_border_width(5) self.vbox.pack_start(hbox) hbox.show() if pixmap: self.realize() pixmap = Pixmap(self, pixmap) hbox.pack_start(pixmap, expand=False) pixmap.show() label = gtk.Label(message) hbox.pack_start(label) label.show() for text in buttons: b = gtk.Button(text) b.set_flags(gtk.CAN_DEFAULT) b.set_data("user_data", text) b.connect("clicked", self.click) self.action_area.pack_start(b) b.show() self.ret = None def quit(self, *args): self.hide() self.destroy() gtk.main_quit() def click(self, button): self.ret = button.get_data("user_data") self.quit() # create a message box, and return which button was pressed def message_box(title="Message Box", message="", buttons=(), pixmap=None, modal= True): win = MessageBox(message, buttons, pixmap=pixmap, modal=modal) win.set_title(title) win.show() gtk.main() return win.ret def test(): result = message_box(title='Test #1', message='Here is your message', buttons=('Ok', 'Cancel')) print 'result:', result USAGE_TEXT = """ Usage: python simple_dialog.py [options] Options: -h, --help Display this help message. Example: python simple_dialog.py """ def usage(): print USAGE_TEXT sys.exit(-1) def main(): args = sys.argv[1:] try: opts, args = getopt.getopt(args, 'h', ['help']) except: usage() relink = 1 for opt, val in opts: if opt in ('-h', '--help'): usage() if len(args) != 0: usage() test() if __name__ == '__main__': #import pdb; pdb.set_trace() main()
Some explanation:
And, here is an example that displays an text input dialog:
#!/usr/bin/env python import sys import getopt import gtk class EntryDialog( gtk.Dialog): def __init__(self, message="", default_text='', modal=True): gtk.Dialog.__init__(self) self.connect("destroy", self.quit) self.connect("delete_event", self.quit) if modal: self.set_modal(True) box = gtk.VBox(spacing=10) box.set_border_width(10) self.vbox.pack_start(box) box.show() if message: label = gtk.Label(message) box.pack_start(label) label.show() self.entry = gtk.Entry() self.entry.set_text(default_text) box.pack_start(self.entry) self.entry.show() self.entry.grab_focus() button = gtk.Button("OK") button.connect("clicked", self.click) button.set_flags(gtk.CAN_DEFAULT) self.action_area.pack_start(button) button.show() button.grab_default() button = gtk.Button("Cancel") button.connect("clicked", self.quit) button.set_flags(gtk.CAN_DEFAULT) self.action_area.pack_start(button) button.show() self.ret = None def quit(self, w=None, event=None): self.hide() self.destroy() gtk.main_quit() def click(self, button): self.ret = self.entry.get_text() self.quit() def input_box(title="Input Box", message="", default_text='', modal=True): win = EntryDialog(message, default_text, modal=modal) win.set_title(title) win.show() gtk.main() return win.ret def test(): result = input_box(title='Test #2', message='Enter a valuexxx:', default_text='a default value') if result is None: print 'Canceled' else: print 'result: "%s"' % result USAGE_TEXT = """ Usage: python simple_dialog.py [options] Options: -h, --help Display this help message. Example: python simple_dialog.py """ def usage(): print USAGE_TEXT sys.exit(-1) def main(): args = sys.argv[1:] try: opts, args = getopt.getopt(args, 'h', ['help']) except: usage() relink = 1 for opt, val in opts: if opt in ('-h', '--help'): usage() if len(args) != 0: usage() test() if __name__ == '__main__': #import pdb; pdb.set_trace() main()
Most of the explanation for the message box example is relevant to this example, too. Here are some differences:
This example shows a file selection dialog box:
#!/usr/bin/env python import sys import getopt import gtk class FileChooser(gtk.FileSelection): def __init__(self, modal=True, multiple=True): gtk.FileSelection.__init__(self) self.multiple = multiple self.connect("destroy", self.quit) self.connect("delete_event", self.quit) if modal: self.set_modal(True) self.cancel_button.connect('clicked', self.quit) self.ok_button.connect('clicked', self.ok_cb) if multiple: self.set_select_multiple(True) self.ret = None def quit(self, *args): self.hide() self.destroy() gtk.main_quit() def ok_cb(self, b): if self.multiple: self.ret = self.get_selections() else: self.ret = self.get_filename() self.quit() def file_sel_box(title="Browse", modal=False, multiple=True): win = FileChooser(modal=modal, multiple=multiple) win.set_title(title) win.show() gtk.main() return win.ret def file_open_box(modal=True): return file_sel_box("Open", modal=modal, multiple=True) def file_save_box(modal=True): return file_sel_box("Save As", modal=modal, multiple=False) def test(): result = file_open_box() print 'open result:', result result = file_save_box() print 'save result:', result USAGE_TEXT = """ Usage: python simple_dialog.py [options] Options: -h, --help Display this help message. Example: python simple_dialog.py """ def usage(): print USAGE_TEXT sys.exit(-1) def main(): args = sys.argv[1:] try: opts, args = getopt.getopt(args, 'h', ['help']) except: usage() relink = 1 for opt, val in opts: if opt in ('-h', '--help'): usage() if len(args) != 0: usage() test() if __name__ == '__main__': main() #import pdb #pdb.run('main()')
A little guidance:
Note that there are also predefined dialogs for font selection (FontSelectionDialog) and color selection (ColorSelectionDialog)
If your GUI needs are minimalist (maybe a pop-up dialog or two) and your application is imperative rather than event driven, then you may want to consider EasyGUI. As the name suggests, it is extremely easy to use.
How to know when you might be able to use EasyGUI:
EasyGUI plus documentation and examples are available at EasyGUI home page at SourceForge -- http://easygui.sourceforge.net/
EasyGUI provides functions for a variety of commonly needed dialog boxes, including:
See the documentation at the EasyGUI Web site for more features.
For a demonstration of EasyGUI's capabilities, run the easygui.py as a Python script:
$ python easygui.py
Here is a simple example that prompts the user for an entry, then shows the response in a message box:
import easygui def testeasygui(): response = easygui.enterbox(msg='Enter your name:', title='Name Entry') easygui.msgbox(msg=response, title='Your Response') testeasygui()
This example presents a dialog to allow the user to select a file:
import easygui def test(): response = easygui.fileopenbox(msg='Select a file') print 'file name: %s' % response test()
Python has an excellent range of implementation organization structures. These range from statements and control structures (at a low level) through functions, methods, and classes (at an intermediate level) and modules and packages at an upper level.
This section provides some guidance with the use of packages. In particular:
A Python package is a collection of Python modules in a disk directory.
In order to be able to import individual modules from a directory, the directory must contain a file named __init__.py. (Note that requirement does not apply to directories that are listed in PYTHONPATH.) The __init__.py serves several purposes:
One simple way to enable the user to import and use a package is to instruct the use to import individual modules from the package.
A second, slightly more advanced way to enable the user to import the package is to expose those features of the package in the __init__ module. Suppose that module mod1 contains functions fun1a and fun1b and suppose that module mod2 contains functions fun2a and fun2b. Then file __init__.py might contain the following:
from mod1 import fun1a, fun1b from mod2 import fun2a, fun2b
Then, if the following is evaluated in the user's code:
import testpackages
Then testpackages will contain fun1a, fun1b, fun2a, and fun2b.
For example, here is an interactive session that demostrates importing the package:
>>> import testpackages >>> print dir(testpackages) [`__builtins__', `__doc__', `__file__', `__name__', `__path__', `fun1a', `fun1b', `fun2a', `fun2b', `mod1', `mod2']
Distutils (Python Distribution Utilities) has special support for distrubuting and installing packages. Learn more here: Distributing Python Modules -- http://docs.python.org/distutils/index.html.
As our example, imagine that we have a directory containing the following:
Testpackages Testpackages/README Testpackages/MANIFEST.in Testpackages/setup.py Testpackages/testpackages/__init__.py Testpackages/testpackages/mod1.py Testpackages/testpackages/mod2.py
Notice the sub-directory Testpackages/testpackages containing the file __init__.py. This is the Python package that we will install.
We'll describe how to configure the above files so that they can be packaged as a single distribution file and so that the Python package they contain can be installed as a package by Distutils.
The MANIFEST.in file lists the files that we want included in our distribution. Here is the contents of our MANIFEST.in file:
include README MANIFEST MANIFEST.in include setup.py include testpackages/*.py
The setup.py file describes to Distutils (1) how to package the distribution file and (2) how to install the distribution. Here is the contents of our sample setup.py:
#!/usr/bin/env python from distutils.core import setup # [1] long_description = 'Tests for installing and distributing Python packages' setup(name = 'testpackages', # [2] version = '1.0a', description = 'Tests for Python packages', maintainer = 'Dave Kuhlman', maintainer_email =<EMAIL_ADDRESS>url = 'http://www.rexx.com/~dkuhlman', long_description = long_description, packages = ['testpackages'] # [3] )
Explanation:
Now, to create a distribution file, we run the following:
python setup.py sdist --formats=gztar
which will create a file testpackages-1.0a.tar.gz under the directory dist.
Then, you can give this distribution file to a potential user, who can install it by doing the following:
$ tar xvzf testpackages-1.0a.tar.gz $ cd testpackages-1.0a $ python setup.py build $ python setup.py install # as root
This document takes a workbook and exercise-with-solutions approach to Python training. It is hoped that those who feel a need for less explanation and more practical exercises will find this useful.
A few notes about the exercises:
The latest version of this document is at my Web site (URL above).
If you have comments or suggestions, please send them my way.
A name is any combination of letters, digits, and the underscore, but the first character must be a letter or an underscore. Names may be of any length.
Case is significant.
Exercises:
Solutions:
The following are keywords in Python and should not be used as variable names:
and del from not while as elif global or with assert else if pass yield break except import print class exec in raise continue finally is return def for lambda try
Exercises:
Solutions:
The following are operators in Python and will separate names:
+ - * ** / // % << >> & | ^ ~ < > <= >= == != <> and or is not in Also: () [] . (dot)
But, note that the Python style guide suggests that you place blanks around binary operators. One exception to this rule is function arguments and parameters for functions: it is suggested that you not put blanks around the equal sign (=) used to specify keyword arguments and default parameters.
Exercises:
Solutions:
In Python, normally we write one statement per line. In fact, Python assumes this. Therefore:
Extending a Python statement to a subsequent line -- Follow these two rules:
Exercises:
Extend the following statement to a second line using parentheses:
total_count = tree_count + vegetable_count + fruit_count
Extend the following statement to a second line using the backslash line continuation character:
total_count = tree_count + vegetable_count + fruit_count
Solutions:
Parentheses create an open context that tells Python that a statement extends to the next line:
total_count = (tree_count + vegetable_count + fruit_count)
A backslash as the last character on line tells Python that the current statement extends to the next line:
total_count = tree_count + \ vegetable_count + fruit_count
For extending a line on a subsequent line, which is better, parentheses or a backslash? Here is a quote:
"The preferred way of wrapping long lines is by using Python's implied line continuation inside parentheses, brackets and braces. If necessary, you can add an extra pair of parentheses around an expression, but sometimes using a backslash looks better."
-- PEP 8: Style Guide for Python Code -- http://www.python.org/dev/peps/pep-0008/
Python uses indentation to indicate program structure. That is to say, in order to nest a block of code inside a compound statement, you indent that nested code. This is different from many programming languages which use some sort of begin and end markers, for example curly brackets.
The standard coding practice for Python is to use four spaces per indentation level and to not use hard tabs. (See the Style Guide for Python Code.) Because of this, you will want to use a text editor that you can configure so that it will use four spaces for indentation. See here for a list of Python-friendly text editors: PythonEditors.
Exercises:
Given the following, nest the print statement inside the if statement:
if x > 0: print x
Nest these two lines:
z = x + y print z
inside the following function definition statement:
def show_sum(x, y):
Solutions:
Indentation indicates that one statement is nested inside another statement:
if x > 0: print x
Indentation indicates that a block of statements is nested inside another statement:
def show_sum(x, y): z = x + y print z
Here are a few rules:
Exercises:
Will the following code produce an error?
show_version() def show_version(): print 'Version 1.0a'
Will the following code produce an error?
def test(): show_version() def show_version(): print 'Version 1.0a' test()
Will the following code produce an error? Assume that show_config is not defined:
x = 3 if x > 5: show_config()
Solutions:
Answer: Yes, it generates an error. The name show_version would not be created and bound to a value until the def function definition statement binds a function object to it. That is done after the attempt to use (call) that object.
Answer: No. The function test() does call the function show_version(), but since test() is not called until after show_version() is defined, that is OK.
Answer: No. It's bad code, but in this case will not generate an error. Since x is less than 5, the body of the if statement is not evaluated.
N.B. This example shows why it is important during testing that every line of code in your Python program be evaluated. Here is good Pythonic advice: "If it's not tested, it's broken."
Each of the subsections in this section on built-in data types will have a similar structure:
The numbers you will use most commonly are likely to be integers and floats. Python also has long integers and complex numbers.
A few facts about numbers (in Python):
Python will convert to using a long integer automatically when needed. You do not need to worry about exceeding the size of a (standard) integer.
The size of the largest integer in your version of Python is in sys.maxint. To learn what it is, do:
>>> import sys >>> print sys.maxint<PHONE_NUMBER>854775807
The above show the maximum size of an integer on a 64-bit version of Python.
You can convert from integer to float by using the float constructor. Example:
>>> x = 25 >>> y = float(x) >>> print y 25.0
Python does "mixed arithmetic". You can add, multiply, and divide integers and floats. When you do, Python "promotes" the result to a float.
An integer is constructed with a series of digits or the integer constructor (int(x)). Be aware that a sequence of digits beginning with zero represents an octal value. Examples:
>>> x1 = 1234 >>> x2 = int('1234') >>> x3 = -25 >>> x1 1234 >>> x2 1234 >>> x3 -25
A float is constructed either with digits and a dot (example, 12.345) or with engineering/scientific notation or with the float constructor (float(x)). Examples:
>>> x1 = 2.0e3 >>> x1 = 1.234 >>> x2 = -1.234 >>> x3 = float('1.234') >>> x4 = 2.0e3 >>> x5 = 2.0e-3 >>> print x1, x2, x3, x4, x5 1.234 -1.234 1.234 2000.0 0.002
Exercises:
Construct these numeric values:
Solutions:
0
0.0, 0., or .0
101
1000.0
1e3 or 1.0e3
Asigning integer values to variables:
In [7]: value1 = 23 In [8]: value2 = -14 In [9]: value3 = 0 In [10]: value1 Out[10]: 23 In [11]: value2 Out[11]: -14 In [12]: value3 Out[12]: 0
Assigning expression values to variables:
value1 = 4 * (3 + 5) value2 = (value1 / 3.0) - 2
Assigning floats to variables:
value1 = 0.01 value2 = -3.0 value3 = 3e-4
Assigning expressions containing varialbes:
value4 = value1 * (value2 - value3) value4 = value1 + value2 + value3 - value4
Mixed arithmetic:
x = 5 y = 8 z = float(x) / y
You can also construct integers and floats using the class. Calling a class (using parentheses after a class name, for example) produces an instance of the class.
Exercises:
Solutions:
Use the int data type to construct an integer instance from a string:
int("123")
Use the float data type to construct a float instance from an integer:
float(123)
Use the int data type to construct an integer instance from a float:
int(12.345) # --> 12
Notice that the result is truncated to the integer part.
You can use most of the familiar operators with numbers, for example:
+ - * ** / // % << >> & | ^ ~ < > <= >= == != <>
Look here for an explanation of these operators when applied to numbers: Numeric Types -- int, float, long, complex -- http://docs.python.org/lib/typesnumeric.html.
Some operators take precedence over others. The table in the Web page just referenced above also shows that order of priority.
Here is a bit of that table:
All numeric types (except complex) support the following operations, sorted by ascending priority (operations in the same box have the same priority; all numeric operations have a higher priority than comparison operations): Operation Result --------- ------ x + y sum of x and y x - y difference of x and y x * y product of x and y x / y quotient of x and y x // y (floored) quotient of x and y x % y remainder of x / y -x x negated +x x unchanged abs(x) absolute value or magnitude of x int(x) x converted to integer long(x) x converted to long integer float(x) x converted to floating point complex(re,im) a complex number with real part re, imaginary part im. im defaults to zero. c.conjugate() conjugate of the complex number c divmod(x, y) the pair (x // y, x % y) pow(x, y) x to the power y x ** y x to the power y
Notice also that the same operator may perform a different function depending on the data type of the value to which it is applied.
Exercises:
Solutions:
Arithmetic expressions are follow standard infix algebraic syntax:
3 + 4 + 5
Use another infix expression:
2 + 3 * 4
Or:
2 + (3 * 4)
But, in this case the parentheses are not necessary because the * operator binds more tightly than the + operator.
Use parentheses to control order of evaluation:
(2 + 3) * 4
Note that the * operator has precedence over (binds tighter than) the + operator, so the parentheses are needed.
Python does mixed arithemetic. When you apply an operation to an integer and a float, it promotes the result to the "higher" data type, a float.
If you need to perform an operation on several integers, but want use a floating point operation, first convert one of the integers to a float using float(x), which effectively creates an instance of class float.
Try the following at your Python interactive prompt:
Exercises:
Given the following assignments:
x = 20 y = 50
Divide x by y giving a float result.
Solutions:
Promote one of the integers to float before performing the division:
z = float(x) / y
Most of the methods implemented by the data types (classes) int and float are special methods that are called through the use of operators. Special methods often have names that begin and end with a double underscore. To see a list of the special names and a bit of an indication of when each is called, do any of the following at the Python interactive prompt:
>>> help(int) >>> help(32) >>> help(float) >>> help(1.23) >>> dir(1) >>> dir(1.2)
Lists are a container data type that acts as a dynamic array. That is to say, a list is a sequence that can be indexed into and that can grow and shrink.
A tuple is an index-able container, like a list, except that a tuple is immutable.
A few characteristics of lists and tuples:
The literal representation of a list is square brackets containing zero or more items separated by commas.
Examples:
Try these at the Python interactive prompt:
>>> [11, 22, 33] >>> ['aa', 'bb', 'cc', ] >>> [100, 'apple', 200, 'banana', ] # The last comma is >>> optional.
A list can contain lists. In fact a list can contain any kind of object:
>>> [1, [2, 3], 4, [5, 6, 7, ], 8]
Lists are heterogenous, that is, different kinds of objects can be in the same list. Here is a list that contains a number, a string, and another list:
>>> [123, 'abc', [456, 789]]
Exercises:
Solutions:
We can define list literals at the Python or IPython interactive prompt:
Create a tuple using commas, optionally with parentheses:
In [1]: a1 = (11, 22, 33, ) In [2]: a1 Out[2]: (11, 22, 33)
Quoted characters separated by commas create a tuple of strings:
In [3]: a2 = ('aaa', 'bbb', 'ccc') In [4]: a2 Out[4]: ('aaa', 'bbb', 'ccc')
Items separated by commas inside square brackets create a list:
In [26]: a3 = [100, 200, 300, ] In [27]: a3 Out[27]: [100, 200, 300]
Strings separated by commas inside square brackets create a list of strings:
In [5]: a3 = ['basil', 'parsley', 'coriander'] In [6]: a3 Out[6]: ['basil', 'parsley', 'coriander'] In [7]:
A tuple or a list can contain tuples and lists:
In [8]: a5 = [(11, 22), (33, 44), (55,)] In [9]: a5 Out[9]: [(11, 22), (33, 44), (55,)]
A list or tuple can contain items of different types:
In [10]: a6 = [101, 102, 'abc', "def", (201, 202), ('ghi', 'jkl')] In [11]: a6 Out[11]: [101, 102, 'abc', 'def', (201, 202), ('ghi', 'jkl')]
In order to create a tuple containing exactly one item, we must use a comma:
In [13]: a7 = (6,) In [14]: a7 Out[14]: (6,)
In order to create an empty tuple, use the tuple class/type to create an instance of a empty tuple:
In [21]: a = tuple() In [22]: a Out[22]: () In [23]: type(a) Out[23]: <type 'tuple'>
There are several operators that are applicable to lists. Here is how to find out about them:
Exercises:
Solutions:
The plus operator, applied to two lists produces a new list that is a concatenation of two lists:
>>> [11, 22] + ['aa', 'bb']
Multiplying a list by an integer n creates a new list that repeats the original list n times:
>>> [11, 'abc', 4.5] * 3
The comparison operators can be used to compare lists:
>>> [11, 22] == [11, 22] >>> [11, 22] < [11, 33]
Again, use dir() and help() to learn about the methods supported by lists.
Examples:
Solutions:
The extend method adds elements from another list, or other iterable:
>>> a = [11, 22, 33, 44, ] >>> b = [55, 66] >>> a.extend(b) >>> a [11, 22, 33, 44, 55, 66]
Use the append method on a list to add/append an item to the end of a list:
>>> a = ['aa', 11] >>> a.append('bb') >>> a.append(22) >>> a ['aa', 11, 'bb', 22]
The insert method on a list enables us to insert items at a given position in a list:
>>> a = [11, 22, 33, 44, ] >>> a.insert(0, 'aa') >>> a ['aa', 11, 22, 33, 44] >>> a.insert(2, 'bb') >>> a ['aa', 11, 'bb', 22, 33, 44]
But, note that we use append to add items at the end of a list.
The pop method on a list returns the "right-most" item from a list and removes that item from the list:
>>> a = [11, 22, 33, 44, ] >>> >>> b = a.pop() >>> a [11, 22, 33] >>> b 44 >>> b = a.pop() >>> a [11, 22] >>> b 33
Note that the append and pop methods taken together can be used to implement a stack, that is a LIFO (last in first out) data structure.
A list comprehension is a convenient way to produce a list from an iterable (a sequence or other object that can be iterated over).
In its simplest form, a list comprehension resembles the header line of a for statement inside square brackets. However, in a list comprehension, the for statement header is prefixed with an expression and surrounded by square brackets. Here is a template:
[expr(x) for x in iterable]
where:
Here is an example:
>>> a = [11, 22, 33, 44] >>> b = [x * 2 for x in a] >>> b [22, 44, 66, 88]
Exercises:
Given the following list of strings:
names = ['alice', 'bertrand', 'charlene']
produce the following lists: (1) a list of all upper case names; (2) a list of capitalized (first letter upper case);
Given the following function which calculates the factorial of a number:
def t(n): if n <= 1: return n else: return n * t(n - 1)
and the following list of numbers:
numbers = [2, 3, 4, 5]
create a list of the factorials of each of the numbers in the list.
Solutions:
For our expression in a list comprehension, use the upper and capitalize methods:
>>> names = ['alice', 'bertrand', 'charlene'] >>> [name.upper() for name in names] ['ALICE', 'BERTRAND', 'CHARLENE'] >>> [name.capitalize() for name in names] ['Alice', 'Bertrand', 'Charlene']
The expression in our list comprehension calls the factorial function:
def t(n): if n <= 1: return n else: return n * t(n - 1) def test(): numbers = [2, 3, 4, 5] factorials = [t(n) for n in numbers] print 'factorials:', factorials if __name__ == '__main__': test()
A list comprehension can also contain an if clause. Here is a template:
[expr(x) for x in iterable if pred(x)]
where:
Only values for which the if clause evaluates to true are included in creating the resulting list.
Examples:
>>> a = [11, 22, 33, 44] >>> b = [x * 3 for x in a if x % 2 == 0] >>> b [66, 132]
Exercises:
Given two lists, generate a list of all the strings in the first list that are not in the second list. Here are two sample lists:
names1 = ['alice', 'bertrand', 'charlene', 'daniel'] names2 = ['bertrand', 'charlene']
Solutions:
The if clause of our list comprehension checks for containment in the list names2:
def test(): names1 = ['alice', 'bertrand', 'charlene', 'daniel'] names2 = ['bertrand', 'charlene'] names3 = [name for name in names1 if name not in names2] print 'names3:', names3 if __name__ == '__main__': test()
When run, this script prints out the following:
names3: ['alice', 'daniel']
A string is an ordered sequence of characters. Here are a few characteristics of strings:
Create strings with single quotes or double quotes. You can put single quotes inside double quotes and you can put double quotes inside single quotes. You can also escape characters with a backslash.
Exercises:
Solutions:
Create a string with double quotes to include single quotes inside the string:
>>> str1 = "that is jerry's ball"
Create a string enclosed with single quotes in order to include double quotes inside the string:
>>> str1 = 'say "goodbye", bullwinkle'
Take your choice. Escape either the single quotes or the double quotes with a backslash:
>>> str1 = 'say "hello" to jerry\'s mom' >>> str2 = "say \"hello\" to jerry's mom" >>> str1 'say "hello" to jerry\'s mom' >>> str2 'say "hello" to jerry\'s mom'
Triple quotes enable you to create a string that spans multiple lines. Use three single quotes or three double quotes to create a single quoted string.
Examples:
Solutions:
Use triple single quotes or triple double quotes to create multi-line strings:
String1 = '''This string extends across several lines. And, so it has end-of-line characters in it. ''' String2 = """ This string begins and ends with an end-of-line character. It can have both 'single' quotes and "double" quotes in it. """ def test(): print String1 print String2 if __name__ == '__main__': test()
Python does not have a distinct character type. In Python, a character is a string of length 1. You can use the ord() and chr() built-in functions to convert from character to integer and back.
Exercises:
Solutions:
The character "a" is a plain string of length 1:
>>> x = 'a'
The integer equivalent of the letter "A":
>>> x = "A" >>> ord(x) 65
You can concatenate strings with the "+" operator.
You can create multiple concatenated copies of a string with the "*" operator.
And, augmented assignment (+= and *=) also work.
Examples:
>>> 'cat' + ' and ' + 'dog' 'cat and dog' >>> '#' * 40 '########################################' >>> >>> s1 = 'flower' >>> s1 += 's' >>> s1 'flowers'
Exercises:
Given these strings:
>>> s1 = 'abcd' >>> s2 = 'efgh'
create a new string composed of the first string followed by (concatenated with) the second.
Create a single string containing 5 copies of the string 'abc'.
Use the multiplication operator to create a "line" of 50 dashes.
Here are the components of a path to a file on the file system: "home", "myusername", "Workdir", "notes.txt". Concatenate these together separating them with the path separator to form a complete path to that file. (Note that if you use the backslash to separate components of the path, you will need to use a double backslash, because the backslash is the escape character in strings.
Solutions:
The plus (+) operator applied to a string can be used to concatenate strings:
>>> s3 = s1 + s2 >>> s3 'abcdefgh'
The multiplication operator (*) applied to a string creates a new string that concatenates a string with itself some number of times:
>>> s1 = 'abc' * 5 >>> s1 'abcabcabcabcabc'
The multiplication operator (*) applied to a string can be used to create a "horizontal divider line":
>>> s1 = '-' * 50 >>> print s1 --------------------------------------------------
The sep member of the os module gives us a platform independent way to construct paths:
>>> import os >>> >>> a = ["home", "myusername", "Workdir", "notes.txt"] >>> path = a[0] + os.sep + a[1] + os.sep + a[2] + os.sep + a[3] >>> path 'home/myusername/Workdir/notes.txt'
And, a more concise solution:
>>> import os >>> a = ["home", "myusername", "Workdir", "notes.txt"] >>> os.sep.join(a) 'home/myusername/Workdir/notes.txt'
Notes:
String support a variety of operations. You can obtain a list of these methods by using the dir() built-in function on any string:
>>> dir("") ['__add__', '__class__', '__contains__', '__delattr__', '__doc__', '__eq__', '__ge__', '__getattribute__', '__getitem__', '__getnewargs__', '__getslice__', '__gt__', '__hash__', '__init__', '__le__', '__len__', '__lt__', '__mod__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__rmod__', '__rmul__', '__setattr__', '__str__', 'capitalize', 'center', 'count', 'decode', 'encode', 'endswith', 'expandtabs', 'find', 'index', 'isalnum', 'isalpha', 'isdigit', 'islower', 'isspace', 'istitle', 'isupper', 'join', 'ljust', 'lower', 'lstrip', 'partition', 'replace', 'rfind', 'rindex', 'rjust', 'rpartition', 'rsplit', 'rstrip', 'split', 'splitlines', 'startswith', 'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill']
And, you can get help on any specific method by using the help() built-in function. Here is an example:
>>> help("".strip) Help on built-in function strip: strip(...) S.strip([chars]) -> string or unicode Return a copy of the string S with leading and trailing whitespace removed. If chars is given and not None, remove characters in chars instead. If chars is unicode, S will be converted to unicode before stripping
Exercises:
Solutions:
The rstrip() method strips whitespace off the right side of a string:
>>> s1 = 'some text \n' >>> s1 'some text \n' >>> s2 = s1.rstrip() >>> s2 'some text'
The center(n) method centers a string within a padded string of width n:
>>> s1 = 'Dave' >>> s2 = s1.center(20) >>> s2 ' Dave '
The upper() method produces a new string that converts all alpha characters in the original to upper case:
>>> s1 = 'Banana' >>> s1 'Banana' >>> s2 = s1.upper() >>> s2 'BANANA'
The split(sep) method produces a list of strings that are separated by sep in the original string.
 Raw strings look like plain literal strings, but are prefixed with an "r" or "R". See String literals http://docs.python.org/reference/lexical_analysis.html#string-literals
Excercises:
Solutions:
We use an "r" prefix to define a raw string:
>>> print 'abc \\ def' abc \ def >>> print r'abc \ def' abc \ def
Unicode strings give us a consistent way to process character data from a variety of character encodings.
Excercises:
Solutions:
We can represent unicode string with either the "u" prefix or with a call to the unicode type:
def exercise1(): a = u'abcd' print a b = unicode('efgh') print b
We convert a string from another character encoding into unicode with the decode() string method:
import sys def exercise2(): a = 'abcd'.decode('utf-8') print a b = 'abcd'.decode(sys.getdefaultencoding()) print b
We can convert a unicode string to another character encoding with the encode() string method:
import sys def exercise3(): a = u'abcd' print a.encode('utf-8') print a.encode(sys.getdefaultencoding())
Here are two ways to check the type of a string:
import types def exercise4(): a = u'abcd' print type(a) is types.UnicodeType print type(a) is type(u'')
We can encode unicode characters in a string in several ways, for example, (1) by defining a utf-8 string and converting it to unicode or (2) defining a string with an embedded unicode character or (3) concatenating a unicode characher into a string:
def exercise5(): utf8_string = 'Ivan Krsti\xc4\x87' unicode_string = utf8_string.decode('utf-8') print unicode_string.encode('utf-8') print len(utf8_string) print len(unicode_string) unicode_string = u'aa\u0107bb' print unicode_string.encode('utf-8') unicode_string = 'aa' + unichr(263) + 'bb' print unicode_string.encode('utf-8')
Guidance for use of encodings and unicode:
Convert/decode from an external encoding to unicode early:
my_source_string.decode(encoding)
Do your work (Python processing) in unicode.
Convert/encode to an external encoding late (for example, just before saving to an external file):
my_unicode_string.encode(encoding)
For more information, see:
A dictionary is an un-ordered collection of key-value pairs.
A dictionary has a length, specifically the number of key-value pairs.
A dictionary provides fast look up by key.
The keys must be immutable object types.
Curley brackets are used to represent a dictionary. Each pair in the dictionary is represented by a key and value separated by a colon. Multiple pairs are separated by comas. For example, here is an empty dictionary and several dictionaries containing key/value pairs:
In [4]: d1 = {} In [5]: d2 = {'width': 8.5, 'height': 11} In [6]: d3 = {1: 'RED', 2: 'GREEN', 3: 'BLUE', } In [7]: d1 Out[7]: {} In [8]: d2 Out[8]: {'height': 11, 'width': 8.5} In [9]: d3 Out[9]: {1: 'RED', 2: 'GREEN', 3: 'BLUE'}
Notes:
Exercises:
Define a dictionary that has the following key-value pairs:
Key Value Eggplant Purple Tomato Red Parsley Green Lemon Yellow Pepper Green, Red, Yellow
Define a dictionary to represent the "enum" days of the week: Sunday, Monday, Tuesday, ...
Solutions:
A dictionary whose keys and values are strings can be used to represent this table:
vegetables = { 'Eggplant': 'Purple', 'Tomato': 'Red', 'Parsley': 'Green', 'Lemon': 'Yellow', 'Pepper': 'Green', }
Note that the open curly bracket enables us to continue this statement across multiple lines without using a backslash.
We might use strings for the names of the days of the week as keys:
DAYS = { 'Sunday': 1, 'Monday': 2, 'Tuesday': 3, 'Wednesday': 4, 'Thrusday': 5, 'Friday': 6, 'Saturday': 7, }
Dictionaries support the following "operators":
Length -- len(d) returns the number of pairs in a dictionary.
Indexing -- You can both set and get the value associated with a key by using the indexing operator [ ]. Examples:
In [12]: d3[2] Out[12]: 'GREEN' In [13]: d3[0] = 'WHITE' In [14]: d3[0] Out[14]: 'WHITE'
Test for key -- The in operator tests for the existence of a key in a dictionary. Example:
In [6]: trees = {'poplar': 'deciduous', 'cedar': 'evergreen'} In [7]: if 'cedar' in trees: ...: print 'The cedar is %s' % (trees['cedar'], ) ...: The cedar is evergreen
Exercises:
Create an empty dictionary, then use the indexing operator [ ] to in sert the following name-value pairs:
"red" -- "255:0:0" "green" -- "0:255:0" "blue" -- "0:0:255"
Print out the number of items in your dictionary.
Solutions:
We can use "[ ]" to set the value of a key in a dictionary:
def test(): colors = {} colors["red"] = "255:0:0" colors["green"] = "0:255:0" colors["blue"] = "0:0:255" print 'The value of red is "%s"' % (colors['red'], ) print 'The colors dictionary contains %d items.' % (len(colors), ) test()
When we run this, we see:
The value of red is "255:0:0" The colors dictionary contains 3 items.
The len() built-in function gives us the number of items in a dictionary. See the previous solution for an example of this.
Here is a table that describes the methods applicable to dictionarys:
| Operation | Result |
|---|---|
| len(a) | the number of items in a |
| a[k] | the item of a with key k |
| a[k] = v | set a[k] to v |
| del a[k] | remove a[k] from a |
| a.clear() | remove all items from a |
| a.copy() | a (shallow) copy of a |
| k in a | True if a has a key k, else False |
| k not in a | equivalent to not k in a |
| a.has_key(k) | equivalent to k in a, use that form in new code |
| a.items() | a copy of a's list of (key, value) pair |
| a.keys() | a copy of a's list of keys |
| a.update([b]) | updates a with key/value pairs from b, overwriting existing keys, returns None |
| a.fromkeys(seq[, value]) | creates a new dictionary with keys from seq and values set to value |
| a.values() | a copy of a's list of values |
| a.get(k[, x]) | a[k] if k in a, else x) |
| a.setdefault(k[, x]) | a[k] if k in a, else x (also setting it) |
| a.pop(k[, x]) | a[k] if k in a, else x (and remove k) (8) |
| a.popitem() | remove and return an arbitrary (key, value) pair |
| a.iteritems() | return an iterator over (key, value) pairs |
| a.iterkeys() | return an iterator over the mapping's keys |
| a.itervalues() | return an iterator over the mapping's values |
You can also find this table at the standard documentation Web site in the "Python Library Reference": Mapping Types -- dict http://docs.python.org/lib/typesmapping.html
Exercises:
Solutions:
We can use the d.items() method to retrieve a list of tuples containing key-value pairs, then use unpacking to capture the key and value:
Vegetables = { 'Eggplant': 'Purple', 'Tomato': 'Red', 'Parsley': 'Green', 'Lemon': 'Yellow', 'Pepper': 'Green', } def test(): for key, value in Vegetables.items(): print 'key:', key, ' value:', value test()
We retrieve a list of keys with the keys() method, the sort it with the list sort() method:
Vegetables = { 'Eggplant': 'Purple', 'Tomato': 'Red', 'Parsley': 'Green', 'Lemon': 'Yellow', 'Pepper': 'Green', } def test(): keys = Vegetables.keys() keys.sort() for key in keys: print 'key:', key, ' value:', Vegetables[key] test()
To test for the existence of a key in a dictionary, we can use either the in operator (preferred) or the d.has_key() method (old style):
Vegetables = { 'Eggplant': 'Purple', 'Tomato': 'Red', 'Parsley': 'Green', 'Lemon': 'Yellow', 'Pepper': 'Green', } def test(): if 'Eggplant' in Vegetables: print 'we have %s egplants' % Vegetables['Eggplant'] if 'Banana' not in Vegetables: print 'yes we have no bananas' if Vegetables.has_key('Parsley'): print 'we have leafy, %s parsley' % Vegetables['Parsley'] test()
Which will print out:
we have Purple egplants yes we have no bananas we have leafy, Green parsley
A Python file object represents a file on a file system.
A file object open for reading a text file is iterable. When we iterate over it, it produces the lines in the file.
A file may be opened in these modes:
The open() built-in function is used to create a file object. For example, the following code (1) opens a file for writing, then (2) for reading, then (3) for appending, and finally (4) for reading again:
def test(infilename): # 1. Open the file in write mode, which creates the file. outfile = open(infilename, 'w') outfile.write('line 1\n') outfile.write('line 2\n') outfile.write('line 3\n') outfile.close() # 2. Open the file for reading. infile = open(infilename, 'r') for line in infile: print 'Line:', line.rstrip() infile.close() # 3. Open the file in append mode, and add a line to the end of # the file. outfile = open(infilename, 'a') outfile.write('line 4\n') outfile.close() print '-' * 40 # 4. Open the file in read mode once more. infile = open(infilename, 'r') for line in infile: print 'Line:', line.rstrip() infile.close() test('tmp.txt')
Exercises:
Solutions:
Use the open() built-in function to open the file and create a file object. Use the read() method on the file object to read the entire file. Use the split() or splitlines() methods to split the file into lines:
>>> infile = open('tmp.txt', 'r') >>> content = infile.read() >>> infile.close() >>> lines = content.splitlines() >>> print lines ['line 1', 'line 2', 'line 3', '']
The f.readlines() method returns a list of lines in a file:
>>> infile = open('tmp.txt', 'r') >>> lines = infile.readlines() >>> infile.close() >>> print lines ['line 1\n', 'line 2\n', 'line 3\n']
Since a file object (open for reading) is itself an iterator, we can iterate over it in a for statement:
""" Test iteration over a text file. Usage: python test.py in_file_name """ import sys def test(infilename): infile = open(infilename, 'r') for line in infile: # Strip off the new-line character and any whitespace on # the right. line = line.rstrip() # Print only non-blank lines. if line: print line infile.close() def main(): args = sys.argv[1:] if len(args) != 1: print __doc__ sys.exit(1) infilename = args[0] test(infilename) if __name__ == '__main__': main()
Notes:
None is a singleton. There is only one instance of None. Use this value to indicate the absence of any other "real" value.
Test for None with the identity operator is.
Exercises:
Solutions:
The identity operators is and is not can be used to test for None:
>>> a = [11, None, 'abc', None, {}] >>> a [11, None, 'abc', None, {}] >>> count = 0 >>> for item in a: ... if item is None: ... count += 1 ... >>> >>> print count 2
Python has the two boolean values True and False. Many comparison operators return True and False.
Examples:
What value is returned by 3 > 2?
Answer: The boolean value True.
Given these variable definitions:
x = 3 y = 4 z = 5
What does the following print out:
print y > x and z > y
Answer -- Prints out "True"
The assignment statement uses the assignment operator =.
The assignment statement is a binding statement: it binds a value to a name within a namespace.
Exercises:
Solutions:
1. The = operator is an assignment statement that binds a value to a variable:
>>> vegetable = "eggplant"
There is also augmented assignment using the operators +=, -=, *=, /=, etc.
Exercises:
Solutions:
The += operator increments the value of an integer:
>>> count = 0 >>> count += 1 >>> count 1 >>> count += 1 >>> count 2
The += operator appends characters to the end of a string:
>>> buffer = 'abcde' >>> buffer += 'fgh' >>> buffer 'abcdefgh'
The += operator appends items in one list to another:
In [20]: a = [11, 22, 33] In [21]: b = [44, 55] In [22]: a += b In [23]: a Out[23]: [11, 22, 33, 44, 55]
The -= operator decrements the value of an integer:
>>> count = 5 >>> count 5 >>> count -= 1 >>> count 4
You can also assign a value to (1) an element of a list, (2) an item in a dictionary, (3) an attribute of an object, etc.
Exercises:
Create a list of three items, then assign a new value to the 2nd element in the list.
Create a dictionary, then assign values to the keys "vegetable" and "fruit" in that dictionary.
Use the following code to create an instance of a class:
class A(object): pass a = A()
Then assign values to an attribue named category in that instance.
Solutions:
Assignment with the indexing operator [] assigns a value to an element in a list:
>>> trees = ['pine', 'oak', 'elm'] >>> trees ['pine', 'oak', 'elm'] >>> trees[1] = 'cedar' >>> trees ['pine', 'cedar', 'elm']
Assignment with the indexing operator [] assigns a value to an item (a key-value pair) in a dictionary:
>>> foods = {} >>> foods {} >>> foods['vegetable'] = 'green beans' >>> foods['fruit'] = 'nectarine' >>> foods {'vegetable': 'green beans', 'fruit': 'nectarine'}
Assignment along with the dereferencing operator . (dot) enables us to assign a value to an attribute of an object:
>>> class A(object): ... pass ... >>> a = A() >>> a.category = 25 >>> a.__dict__ {'category': 25} >>> a.category 25
Warning: Be aware that the print statement will go away in Python version 3.0. It will be replaced by the built-in print() function.
The print statement sends output to standard output. It provides a somewhat more convenient way of producing output than using sys.stdout.write().
The print statement takes a series of zero or more objects separated by commas. Zero objects produces a blank line.
The print statement normally adds a newline at the end of its output. To eliminate that, add a comma at the end.
Exercises:
Solutions:
We can print a literal string:
>>> print 'Hello, there' Hello, there
We can print literals and the value of variables:
>>> description = 'cute' >>> print 'I am a', description, 'kid.' I am a cute kid.
The string formatting operator gives more control over formatting output:
>>> name = 'Alice' >>> print 'My name is "%s".' % (name, ) My name is "Alice".
The if statement is a compound statement that enables us to conditionally execute blocks of code.
The if statement also has optional elif: and else: clauses.
The condition in an if: or elif: clause can be any Python expression, in other words, something that returns a value (even if that value is None).
In the condition in an if: or elif: clause, the following values count as "false":
All other values count as true.
Exercises:
Given the following list:
>>> bananas = ['banana1', 'banana2', 'banana3',]
Print one message if it is an empty list and another messge if it is not.
Here is one way of defining a Python equivalent of an "enum":
NO_COLOR, RED, GREEN, BLUE = range(4)
Write an if: statement which implements the effect of a "switch" statement in Python. Print out a unique message for each color.
Solutions:
We can test for an empty or non-empty list:
>>> bananas = ['banana1', 'banana2', 'banana3',] >>> if not bananas: ... print 'yes, we have no bananas' ... else: ... print 'yes, we have bananas' ... yes, we have bananas
We can simulate a "switch" statement using if:elif: ...:
NO_COLOR, RED, GREEN, BLUE = range(4) def test(color): if color == RED: print "It's red." elif color == GREEN: print "It's green." elif color == BLUE: print "It's blue." def main(): color = BLUE test(color) if __name__ == '__main__': main()
Which, when run prints out the following:
It's blue.
The for: statement is the Python way to iterate over and process the elements of a collection or other iterable.
The basic form of the for: statement is the following:
for X in Y: statement o o o
where:
Exercises:
Solutions:
The for: statement can iterate over the items in a list:
In [13]: a = [11, 22, 33, ] In [14]: for value in a: ....: print 'value: %d' % value ....: ....: value: 11 value: 22 value: 33
The for: statement can iterate over the characters in a string:
In [16]: b = 'chocolate' In [17]: for chr1 in b: ....: print 'character: %s' % chr1 ....: ....: character: c character: h character: o character: c character: o character: l character: a character: t character: e
Notes:
When we need a sequential index, we can use the range() built-in function to create a list of integers. And, the xrange() built-in function produces an interator that produces a sequence of integers without creating the entire list. To iterate over a large sequence of integers, use xrange() instead of range().
Exercises:
Print out the integers from 0 to 5 in sequence.
Compute the sum of all the integers from 0 to 99999.
Given the following generator function:
import urllib Urls = [ 'http://yahoo.com', 'http://python.org', 'http://gimp.org', # The GNU image manipulation program ] def walk(url_list): for url in url_list: f = urllib.urlopen(url) stuff = f.read() f.close() yield stuff
Write a for: statement that uses this iterator generator to print the lengths of the content at each of the Web pages in that list.
Solutions:
The range() built-in function gives us a sequence to iterate over:
In [5]: for idx in range(6): ...: print 'idx: %d' % idx ...: ...: idx: 0 idx: 1 idx: 2 idx: 3 idx: 4 idx: 5
Since that sequence is a bit large, we'll use xrange() instead of range():
In [8]: count = 0 In [9]: for n in xrange(100000): ...: count += n ...: ...: In [10]: count Out[10]:<PHONE_NUMBER>
The for: statement enables us to iterate over iterables as well as collections:
import urllib Urls = [ 'http://yahoo.com', 'http://python.org', 'http://gimp.org', # The GNU image manipulation program ] def walk(url_list): for url in url_list: f = urllib.urlopen(url) stuff = f.read() f.close() yield stuff def test(): for url in walk(Urls): print 'length: %d' % (len(url), ) if __name__ == '__main__': test()
When I ran this script, it prints the following:
length: 9562 length: 16341 length: 12343
If you need an index while iterating over a sequence, consider using the enumerate() built-in function.
Exercises:
Given the following two lists of integers of the same length:
a = [1, 2, 3, 4, 5] b = [100, 200, 300, 400, 500]
Add the values in the first list to the corresponding values in the second list.
Solutions:
The enumerate() built-in function gives us an index and values from a sequence. Since enumerate() gives us an interator that produces a sequence of two-tuples, we can unpack those tuples into index and value variables in the header line of the for statement:
In [13]: a = [1, 2, 3, 4, 5] In [14]: b = [100, 200, 300, 400, 500] In [15]: In [16]: for idx, value in enumerate(a): ....: b[idx] += value ....: ....: In [17]: b Out[17]: [101, 202, 303, 404, 505]
A while: statement executes a block of code repeatedly as long as a condition is true.
Here is a template for the while: statement:
while condition: statement o o o
Where:
Exercises:
Solutions:
A while: loop with an index variable can be used to modify each element of a list:
def test_while(): numbers = [11, 22, 33, 44, ] print 'before: %s' % (numbers, ) idx = 0 while idx < len(numbers): numbers[idx] *= 2 idx += 1 print 'after: %s' % (numbers, )
But, notice that this task is easier using the for: statement and the built-in enumerate() function:
def test_for(): numbers = [11, 22, 33, 44, ] print 'before: %s' % (numbers, ) for idx, item in enumerate(numbers): numbers[idx] *= 2 print 'after: %s' % (numbers, )
The continue statement skips the remainder of the statements in the body of a loop and starts immediately at the top of the loop again.
A break statement in the body of a loop terminates the loop. It exits from the immediately containing loop.
break and continue can be used in both for: and while: statements.
Exercises:
Solutions:
The continue statement enables us to "skip" items that satisfy a condition or test:
def test(): numbers = [11, 22, 33, 44, 55, 66, ] print 'before: %s' % (numbers, ) for idx, item in enumerate(numbers): if item % 2 != 0: continue numbers[idx] *= 3 print 'after: %s' % (numbers, ) test()
The break statement enables us to exit from a loop when we find a zero:
def test(): numbers = [11, 22, 33, 0, 44, 55, 66, ] print 'numbers: %s' % (numbers, ) sum = 0 for item in numbers: if item == 0: break sum += item print 'sum: %d' % (sum, ) test()
The try:except: statement enables us to catch an exception that is thrown from within a block of code, or from code called from any depth withing that block.
The raise statement enables us to throw an exception.
An exception is a class or an instance of an exception class. If an exception is not caught, it results in a traceback and termination of the program.
There is a set of standard exceptions. You can learn about them here: Built-in Exceptions -- http://docs.python.org/lib/module-exceptions.html.
You can define your own exception classes. To do so, create an empty subclass of the class Exception.
Exercises:
Write a try:except: statement that attempts to open a file for reading and catches the exception thrown when the file does not exist.
Question: How do you find out the name of the exception that is thrown for an input/output error such as the failure to open a file?
Define an exception class. Then write a try:except: statement in which you throw and catch that specific exception.
Define an exception class and use it to implement a multi-level break from an inner loop, by-passing an outer loop.
Solutions:
Use the Python interactive interpreter to learn the exception type thrown when a I/O error occurs. Example:
>>> infile = open('xx_nothing__yy.txt', 'r') Traceback (most recent call last): File "<stdin>", line 1, in <module> IOError: [Errno 2] No such file or directory: 'xx_nothing__yy.txt' >>>
In this case, the exception type is IOError.
Now, write a try:except: block which catches that exception:
def test(): infilename = 'nothing_noplace.txt' try: infile = open(infilename, 'r') for line in infile: print line except IOError, exp: print 'cannot open file "%s"' % infilename test()
We define a exception class as a sub-class of class Exception, then throw it (with the raise statement) and catch it (with a try:except: statement):
class SizeError(Exception): pass def test_exception(size): try: if size <= 0: raise SizeError, 'size must be greater than zero' # Produce a different error to show that it will not be caught. x = y except SizeError, exp: print '%s' % (exp, ) print 'goodbye' def test(): test_exception(-1) print '-' * 40 test_exception(1) test()
When we run this script, it produces the following output:
$ python workbook027.py size must be greater than zero goodbye ---------------------------------------- Traceback (most recent call last): File "workbook027.py", line 20, in <module> test() File "workbook027.py", line 18, in test test_exception(1) File "workbook027.py", line 10, in test_exception x = y NameError: global name 'y' is not defined
Notes:
We define a sub-class of of class Exception, then raise it in an inner loop and catch it outside of an outer loop:
class BreakException1(Exception): pass def test(): a = [11, 22, 33, 44, 55, 66, ] b = [111, 222, 333, 444, 555, 666, ] try: for x in a: print 'outer -- x: %d' % x for y in b: if x > 22 and y > 444: raise BreakException1('leaving inner loop') print 'inner -- y: %d' % y print 'outer -- after' print '-' * 40 except BreakException1, exp: print 'out of loop -- exp: %s' % exp test()
Here is what this prints out when run:
outer -- x: 11 inner -- y: 111 inner -- y: 222 inner -- y: 333 inner -- y: 444 inner -- y: 555 inner -- y: 666 outer -- after ---------------------------------------- outer -- x: 22 inner -- y: 111 inner -- y: 222 inner -- y: 333 inner -- y: 444 inner -- y: 555 inner -- y: 666 outer -- after ---------------------------------------- outer -- x: 33 inner -- y: 111 inner -- y: 222 inner -- y: 333 inner -- y: 444 out of loop -- exp: leaving inner loop
A function has these characteristics:
A function is defined with the def: statement. Here is a simple example/template:
def function_name(arg1, arg2): local_var1 = arg1 + 1 local_var2 = arg2 * 2 return local_var1 + local_var2
And, here is an example of calling this function:
result = function_name(1, 2)
Here are a few notes of explanation:
Exercises:
Solutions:
The return statement enables us to return a value from a function:
def list_sum(values): sum = 0 for value in values: sum += value return sum def test(): a = [11, 22, 33, 44, ] print list_sum(a) if __name__ == '__main__': test()
You can provide a default value for an argument to a function.
If you do, that argument is optional (when the function is called).
Here are a few things to learn about optional arguments:
Provide a default value with an equal sign and a value. Example:
def sample_func(arg1, arg2, arg3='empty', arg4=0):
All parameters with default values must be after (to the right of) normal parameters.
Do not use a mutable object as a default value. Because the def: statement is evaluated only once and not each time the function is called, the mutable object might be shared across multiple calls to the function. Do not do this:
def sample_func(arg1, arg2=[]):
Instead, do this:
def sample_func(arg1, arg2=None): if arg2 is None: arg2 = []
Here is an example that illustrates how this might go wrong:
def adder(a, b=[]): b.append(a) return b def test(): print adder('aaa') print adder('bbb') print adder('ccc') test()
Which, when executed, displays the following:
['aaa'] ['aaa', 'bbb'] ['aaa', 'bbb', 'ccc']
Exercises:
Solutions:
We can pass a file as we would any other object. And, we can use a newline character as a default parameter value:
import sys def writer(outfile, msg='\n'): outfile.write(msg) def test(): writer(sys.stdout, 'aaaaa\n') writer(sys.stdout) writer(sys.stdout, 'bbbbb\n') test()
When run from the command line, this prints out the following:
aaaaa bbbbb
In this solution we are careful not to use a mutable object as a default value:
def add_to_dict(name, value, dic=None): if dic is None: dic = {} dic[name] = value return dic def test(): dic1 = {'albert': 'cute', } print add_to_dict('barry', 'funny', dic1) print add_to_dict('charlene', 'smart', dic1) print add_to_dict('darryl', 'outrageous') print add_to_dict('eddie', 'friendly') test()
If we run this script, we see:
{'barry': 'funny', 'albert': 'cute'} {'barry': 'funny', 'albert': 'cute', 'charlene': 'smart'} {'darryl': 'outrageous'} {'eddie': 'friendly'}
Notes:
A function, like any other object, can be passed as an argument to a function. This is due the the fact that almost all (maybe all) objects in Python are "first class objects". A first class object is one which we can:
Exercises:
Write a function that takes three arguments: (1) an input file, (2) an output file, and (3) a filter function:
The above function should read each line in the input text file, pass that line through the filter function, then write that (possibly) transformed line to the output file.
Now, write one or more "filter functions" that can be passed to the function described above.
Solutions:
This script adds or removes comment characters to the lines of a file:
import sys def filter(infile, outfile, filterfunc): for line in infile: line = filterfunc(line) outfile.write(line) def add_comment(line): line = '## %s' % (line, ) return line def remove_comment(line): if line.startswith('## '): line = line[3:] return line def main(): filter(sys.stdin, sys.stdout, add_comment) if __name__ == '__main__': main()
Running this might produce something like the following (note for MS Windows users: use type instead of cat):
$ cat tmp.txt line 1 line 2 line 3 $ cat tmp.txt | python workbook005.py ## line 1 ## line 2 ## line 3
Additional positional arguments passed to a function that are not specified in the function definition (the def: statement``), are collected in an argument preceded by a single asterisk. Keyword arguments passed to a function that are not specified in the function definition can be collected in a dictionary and passed to an argument preceded by a double asterisk.
Examples:
Solutions:
We use *args and **kwargs to collect extra arguments and extra keyword arguments:
def show_args(x, y=-1, *args, **kwargs): print '-' * 40 print 'x:', x print 'y:', y print 'args:', args print 'kwargs:', kwargs def test(): show_args(1) show_args(x=2, y=3) show_args(y=5, x=4) show_args(4, 5, 6, 7, 8) show_args(11, y=44, a=55, b=66) test()
Running this script produces the following:
$ python workbook006.py ---------------------------------------- x: 1 y: -1 args: () kwargs: {} ---------------------------------------- x: 2 y: 3 args: () kwargs: {} ---------------------------------------- x: 4 y: 5 args: () kwargs: {} ---------------------------------------- x: 4 y: 5 args: (6, 7, 8) kwargs: {} ---------------------------------------- x: 11 y: 44 args: () kwargs: {'a': 55, 'b': 66}
Notes:
We use args and kwargs to catch and pass on all arguments:
def func1(*args, **kwargs): print 'args: %s' % (args, ) print 'kwargs: %s' % (kwargs, ) def func2(*args, **kwargs): print 'before' func1(*args, **kwargs) print 'after' def test(): func2('aaa', 'bbb', 'ccc', arg1='ddd', arg2='eee') test()
When we run this, it prints the following:
before args: ('aaa', 'bbb', 'ccc') kwargs: {'arg1': 'ddd', 'arg2': 'eee'} after
Notes:
In a function definition, arguments must appear in the following order, from left to right:
In a function call, arguments must appear in the following order, from left to right:
If the arguments and return value of a function satisfy some description, then we can say that the function is polymorphic with respect to that description.
If the some of the methods of an object satisfy some description, then we can say that the object is polymorphic with respect to that description.
Basically, what this does is to enable us to use a function or an object anywhere that function satisfies the requirements given by a description.
Exercises:
Solutions:
We can pass a function as an argument to a function:
def fancy(obj): print 'fancy fancy -- %s -- fancy fancy' % (obj, ) def plain(obj): print 'plain -- %s -- plain' % (obj, ) def show(func, obj): func(obj) def main(): a = {'aa': 11, 'bb': 22, } show(fancy, a) show(plain, a) if __name__ == '__main__': main()
We can also put functions (function objects) in a data structure (for example, a list), and then pass that data structure to a function:
def fancy(obj): print 'fancy fancy -- %s -- fancy fancy' % (obj, ) def plain(obj): print 'plain -- %s -- plain' % (obj, ) Func_list = [fancy, plain, ] def show(funcs, obj): for func in funcs: func(obj) def main(): a = {'aa': 11, 'bb': 22, } show(Func_list, a) if __name__ == '__main__': main()
Notice that Python supports polymorphism (with or) without inheritance. This type of polymorphism is enabled by what is called duck-typing. For more on this see: Duck typing -- http://en.wikipedia.org/wiki/Duck_typing at Wikipedia.
A recursive function is a function that calls itself.
A recursive function must have a limiting condition, or else it will loop endlessly.
Each recursive call consumes space on the function call stack. Therefore, the number of recursions must have some reasonable upper bound.
Exercises:
Write a recursive function that prints information about each node in the following tree-structure data structure:
Tree = { 'name': 'animals', 'left_branch': { 'name': 'birds', 'left_branch': { 'name': 'seed eaters', 'left_branch': { 'name': 'house finch', 'left_branch': None, 'right_branch': None, }, 'right_branch': { 'name': 'white crowned sparrow', 'left_branch': None, 'right_branch': None, }, }, 'right_branch': { 'name': 'insect eaters', 'left_branch': { 'name': 'hermit thrush', 'left_branch': None, 'right_branch': None, }, 'right_branch': { 'name': 'black headed phoebe', 'left_branch': None, 'right_branch': None, }, }, }, 'right_branch': None, }
Solutions:
We write a recursive function to walk the whole tree. The recursive function calls itself to process each child of a node in the tree:
Tree = { 'name': 'animals', 'left_branch': { 'name': 'birds', 'left_branch': { 'name': 'seed eaters', 'left_branch': { 'name': 'house finch', 'left_branch': None, 'right_branch': None, }, 'right_branch': { 'name': 'white crowned sparrow', 'left_branch': None, 'right_branch': None, }, }, 'right_branch': { 'name': 'insect eaters', 'left_branch': { 'name': 'hermit thrush', 'left_branch': None, 'right_branch': None, }, 'right_branch': { 'name': 'black headed phoebe', 'left_branch': None, 'right_branch': None, }, }, }, 'right_branch': None, } Indents = [' ' * idx for idx in range(10)] def walk_and_show(node, level=0): if node is None: return print '%sname: %s' % (Indents[level], node['name'], ) level += 1 walk_and_show(node['left_branch'], level) walk_and_show(node['right_branch'], level) def test(): walk_and_show(Tree) if __name__ == '__main__': test()
Notes:
The "iterator protocol" defines what an iterator object must do in order to be usable in an "iterator context" such as a for statement. The iterator protocol is described in the standard library reference: Iterator Types -- http://docs.python.org/lib/typeiter.html
An easy way to define an object that obeys the iterator protocol is to write a generator function. A generator function is a function that contains one or more yield statements. If a function contains at least one yield statement, then that function when called, returns generator iterator, which is an object that obeys the iterator protocol, i.e. it's an iterator object.
Note that in recent versions of Python, yield is an expression. This enables the consumer to communicate back with the producer (the generator iterator). For more on this, see PEP: 342 Coroutines via Enhanced Generators - http://www.python.org/dev/peps/pep-0342/.
Exercises:
Implement a generator function -- The generator produced should yield all values from a list/iterable that satisfy a predicate. It should apply the transforms before return each value. The function takes these arguments:
values -- A list of values. Actually, it could be any iterable.
predicate -- A function that takes a single argument, performs a test on that value, and returns True or False.
transforms -- (optional) A list of functions. Apply each function in this list and returns the resulting value. So, for example, if the function is called like this:
result = transforms([11, 22], p, [f, g])
then the resulting generator might return:
g(f(11))
Implement a generator function that takes a list of URLs as its argument and generates the contents of each Web page, one by one (that is, it produces a sequence of strings, the HTML page contents).
Solutions:
Here is the implementation of a function which contains yield, and, therefore, produces a generator:
#!/usr/bin/env python """ filter_and_transform filter_and_transform(content, test_func, transforms=None) Return a generator that returns items from content after applying the functions in transforms if the item satisfies test_func . Arguments: 1. ``values`` -- A list of values 2. ``predicate`` -- A function that takes a single argument, performs a test on that value, and returns True or False. 3. ``transforms`` -- (optional) A list of functions. Apply each function in this list and returns the resulting value. So, for example, if the function is called like this:: result = filter_and_transforms([11, 22], p, [f, g]) then the resulting generator might return:: g(f(11)) """ def filter_and_transform(content, test_func, transforms=None): for x in content: if test_func(x): if transforms is None: yield x elif isiterable(transforms): for func in transforms: x = func(x) yield x else: yield transforms(x) def isiterable(x): flag = True try: x = iter(x) except TypeError, exp: flag = False return flag def iseven(n): return n % 2 == 0 def f(n): return n * 2 def g(n): return n ** 2 def test(): data1 = [11, 22, 33, 44, 55, 66, 77, ] for val in filter_and_transform(data1, iseven, f): print 'val: %d' % (val, ) print '-' * 40 for val in filter_and_transform(data1, iseven, [f, g]): print 'val: %d' % (val, ) print '-' * 40 for val in filter_and_transform(data1, iseven): print 'val: %d' % (val, ) if __name__ == '__main__': test()
Notes:
The following function uses the urllib module and the yield function to generate the contents of a sequence of Web pages:
import urllib Urls = [ 'http://yahoo.com', 'http://python.org', 'http://gimp.org', # The GNU image manipulation program ] def walk(url_list): for url in url_list: f = urllib.urlopen(url) stuff = f.read() f.close() yield stuff def test(): for x in walk(Urls): print 'length: %d' % (len(x), ) if __name__ == '__main__': test()
When I run this, I see:
$ python generator_example.py length: 9554 length: 16748 length: 11487
Classes provide Python's way to define new data types and to do OOP (object-oriented programming).
If you have made it this far, you have already used lots of objects. You have been a "consumer" of objects and their services. Now, you will learn how to define and implement new kinds of objects. You will become a "producer" of objects. You will define new classes and you will implement the capabilities (methods) of each new class.
A class is defined with the class statement. The first line of a class statement is a header (it has a colon at the end), and it specifies the name of the class being defined and an (optional) superclass. And that header introduces a compound statement: specifically, the body of the class statement which contains indented, nested statements, importantly, def statements that define the methods that can be called on instances of the objects implemented by this class.
Exercises:
Solutions:
A simple instance method can have the self parameter and no others:
class Demo(object): def show(self): print 'hello' def test(): a = Demo() a.show() test()
Notes:
A class can define methods with special names. You have seem some of these before. These names begin and end with a double underscore.
One important special name is __init__. It's the constructor for a class. It is called each time an instance of the class is created. Implementing this method in a class gives us a chance to initialize each instance of our class.
Exercises:
Solutions:
The constructor for a class is a method with the special name __init__:
class Plant(object): def __init__(self, name, size): self.name = name self.size = size def show(self): print 'name: "%s" size: %d' % (self.name, self.size, ) def test(): p1 = Plant('Eggplant', 25) p2 = Plant('Tomato', 36) plants = [p1, p2, ] for plant in plants: plant.show() test()
Notes:
It is a good idea to initialize all instance variables in the constructor. That enables someone reading our code to learn about all the instance variables of a class by looking in a single location:
# simple_node.py Indents = [' ' * n for n in range(10)] class Node(object): def __init__(self, name=None, children=None): self.name = name if children is None: self.children = [] else: self.children = children def show_name(self, indent): print '%sname: "%s"' % (Indents[indent], self.name, ) def show(self, indent=0): self.show_name(indent) indent += 1 for child in self.children: child.show(indent) def test(): n1 = Node('N1') n2 = Node('N2') n3 = Node('N3') n4 = Node('N4') n5 = Node('N5', [n1, n2,]) n6 = Node('N6', [n3, n4,]) n7 = Node('N7', [n5, n6,]) n7.show() if __name__ == '__main__': test()
Notes:
A subclass extends or specializes a superclass by adding additional methods to the superclass and by overriding methods (with the same name) that already exist in the superclass.
Exercises:
Solutions:
We can import our previous Node script, then implement classes that have the Node class as a superclass:
from simple_node import Node, Indents class Plant(Node): def __init__(self, name, height=-1, children=None): Node.__init__(self, name, children) self.height = height def show(self, indent=0): self.show_name(indent) print '%sheight: %s' % (Indents[indent], self.height, ) indent += 1 for child in self.children: child.show(indent) class Animal(Node): def __init__(self, name, color='no color', children=None): Node.__init__(self, name, children) self.color = color def show(self, indent=0): self.show_name(indent) print '%scolor: "%s"' % (Indents[indent], self.color, ) indent += 1 for child in self.children: child.show(indent) def test(): n1 = Animal('scrubjay', 'gray blue') n2 = Animal('raven', 'black') n3 = Animal('american kestrel', 'brown') n4 = Animal('red-shouldered hawk', 'brown and gray') n5 = Animal('corvid', 'none', [n1, n2,]) n6 = Animal('raptor', children=[n3, n4,]) n7a = Animal('bird', children=[n5, n6,]) n1 = Plant('valley oak', 50) n2 = Plant('canyon live oak', 40) n3 = Plant('jeffery pine', 120) n4 = Plant('ponderosa pine', 140) n5 = Plant('oak', children=[n1, n2,]) n6 = Plant('conifer', children=[n3, n4,]) n7b = Plant('tree', children=[n5, n6,]) n8 = Node('birds and trees', [n7a, n7b,]) n8.show() if __name__ == '__main__': test()
Notes:
Python also supports class-based polymorphism, which was, by the way, demonstrated in the previous example.
Exercises:
Solution:
We implement three simple classes and then create a list of instances of these classes:
class A(object): def show(self, msg): print 'class A -- msg: "%s"' % (msg, ) class B(object): def show(self, msg): print 'class B -- msg: "%s"' % (msg, ) class C(object): def show(self, msg): print 'class C -- msg: "%s"' % (msg, ) def test(): objs = [A(), B(), C(), A(), ] for idx, obj in enumerate(objs): msg = 'message # %d' % (idx + 1, ) obj.show(msg) if __name__ == '__main__': test()
Notes:
A method in a class can recusively call itself. This is very similar to the way in which we implemented recursive functions -- see: Recursive functions.
Exercises:
Solutions:
We implement a class with three instance variables: (1) name, (2) left branch, and (3) right branch. Then, we implement a show() method that displays the name and calls itself to show the children in each sub-tree:
Indents = [' ' * idx for idx in range(10)] class AnimalNode(object): def __init__(self, name, left_branch=None, right_branch=None): self.name = name self.left_branch = left_branch self.right_branch = right_branch def show(self, level=0): print '%sname: %s' % (Indents[level], self.name, ) level += 1 if self.left_branch is not None: self.left_branch.show(level) if self.right_branch is not None: self.right_branch.show(level) Tree = AnimalNode('animals', AnimalNode('birds', AnimalNode('seed eaters', AnimalNode('house finch'), AnimalNode('white crowned sparrow'), ), AnimalNode('insect eaters', AnimalNode('hermit thrush'), AnimalNode('black headed phoebe'), ), ), None, ) def test(): Tree.show() if __name__ == '__main__': test()
Instead of using a left branch and a right branch, in this solution we use a list to represent the children of a node:
class AnimalNode(object): def __init__(self, data, children=None): self.data = data if children is None: self.children = [] else: self.children = children def show(self, level=''): print '%sdata: %s' % (level, self.data, ) level += ' ' for child in self.children: child.show(level) Tree = AnimalNode('animals', [ AnimalNode('birds', [ AnimalNode('seed eaters', [ AnimalNode('house finch'), AnimalNode('white crowned sparrow'), AnimalNode('lesser gold finch'), ]), AnimalNode('insect eaters', [ AnimalNode('hermit thrush'), AnimalNode('black headed phoebe'), ]), ]) ]) def test(): Tree.show() if __name__ == '__main__': test()
Notes:
A class variable is one whose single value is shared by all instances of the class and, in fact, is shared by all who have access to the class (object).
"Normal" methods are instance methods. An instance method receives the instance as its first argument. A instance method is defined by using the def statement in the body of a class statement.
A class method receives the class as its first argument. A class method is defined by defining a normal/instance method, then using the classmethod built-in function. For example:
class ASimpleClass(object): description = 'a simple class' def show_class(cls, msg): print '%s: %s' % (cls.description , msg, ) show_class = classmethod(show_class)
A static method does not receive anything special as its first argument. A static method is defined by defining a normal/instance method, then using the staticmethod built-in function. For example:
class ASimpleClass(object): description = 'a simple class' def show_class(msg): print '%s: %s' % (ASimpleClass.description , msg, ) show_class = staticmethod(show_class)
In effect, both class methods and static methods are defined by creating a normal (instance) method, then creating a wrapper object (a class method or static method) using the classmethod or staticmethod built-in function.
Exercises:
Solutions:
We use a class variable named instance_count, rather than an instance variable, to keep a running total of instances. Then, we increment that variable each time an instance is created:
class CountInstances(object): instance_count = 0 def __init__(self, name='-no name-'): self.name = name CountInstances.instance_count += 1 def show(self): print 'name: "%s"' % (self.name, ) def show_instance_count(cls): print 'instance count: %d' % (cls.instance_count, ) show_instance_count = classmethod(show_instance_count) def test(): instances = [] instances.append(CountInstances('apple')) instances.append(CountInstances('banana')) instances.append(CountInstances('cherry')) instances.append(CountInstances()) for instance in instances: instance.show() CountInstances.show_instance_count() if __name__ == '__main__': test()
Notes:
When we run this script, it prints out the following:
name: "apple" name: "banana" name: "cherry" name: "-no name-" instance count: 4
The call to the classmethod built-in function effectively wraps the show_instance_count method in a class method, that is, in a method that takes a class object as its first argument rather than an instance object. To read more about classmethod, go to Built-in Functions -- http://docs.python.org/lib/built-in-funcs.html and search for "classmethod".
A static method takes neither an instance (self) nor a class as its first paramenter. And, static method is created with the staticmethod() built-in function (rather than with the classmethod() built-in):
class CountInstances(object): instance_count = 0 def __init__(self, name='-no name-'): self.name = name CountInstances.instance_count += 1 def show(self): print 'name: "%s"' % (self.name, ) def show_instance_count(): print 'instance count: %d' % ( CountInstances.instance_count, ) show_instance_count = staticmethod(show_instance_count) def test(): instances = [] instances.append(CountInstances('apple')) instances.append(CountInstances('banana')) instances.append(CountInstances('cherry')) instances.append(CountInstances()) for instance in instances: instance.show() CountInstances.show_instance_count() if __name__ == '__main__': test()
A decorator enables us to do what we did in the previous example with a somewhat simpler syntax.
For simple cases, the decorator syntax enables us to do this:
@functionwrapper def method1(self): o o o
instead of this:
def method1(self): o o o method1 = functionwrapper(method1)
So, we can write this:
@classmethod def method1(self): o o o
instead of this:
def method1(self): o o o method1 = classmethod(method1)
Exercises:
Solutions:
A decorator is an easier and cleaner way to define a class method (or a static method):
class CountInstances(object): instance_count = 0 def __init__(self, name='-no name-'): self.name = name CountInstances.instance_count += 1 def show(self): print 'name: "%s"' % (self.name, ) @classmethod def show_instance_count(cls): print 'instance count: %d' % (cls.instance_count, ) # Note that the following line has been replaced by # the classmethod decorator, above. # show_instance_count = classmethod(show_instance_count) def test(): instances = [] instances.append(CountInstances('apple')) instances.append(CountInstances('banana')) instances.append(CountInstances('cherry')) instances.append(CountInstances()) for instance in instances: instance.show() CountInstances.show_instance_count() if __name__ == '__main__': test()
Decorators can be used to "wrap" a function with another function.
When implementing a decorator, it is helpful to remember that the following decorator application:
@dec def func(arg1, arg2): pass
is equivalent to:
def func(arg1, arg2): pass func = dec(func)
Therefore, to implement a decorator, we write a function that returns a function object, since we replace the value originally bound to the function with this new function object. It may be helpful to take the view that we are creating a function that is a wrapper for the original function.
Exercises:
Solutions:
A function that contains and returns an inner function can be used to wrap a function:
def trace(func): def inner(*args, **kwargs): print '>>' func(*args, **kwargs) print '<<' return inner @trace def func1(x, y): print 'x:', x, 'y:', y func2((x, y)) @trace def func2(content): print 'content:', content def test(): func1('aa', 'bb') test()
Notes:
Decorators can also take arguments.
The following decorator with arguments:
@dec(argA, argB) def func(arg1, arg2): pass
is equivalent to:
def func(arg1, arg2): pass func = dec(argA, argB)(func)
Because the decorator's arguments are passed to the result of calling the decorator on the decorated function, you may find it useful to implement a decorator with arguments using a function inside a function inside a function.
Exercises:
Solutions:
Implement this decorator that takes arguments with a function containing a nested function which in turn contains a nested function:
def trace(msg): def inner1(func): def inner2(*args, **kwargs): print '>> [%s]' % (msg, ) retval = func(*args, **kwargs) print '<< [%s]' % (msg, ) return retval return inner2 return inner1 @trace('tracing func1') def func1(x, y): print 'x:', x, 'y:', y result = func2((x, y)) return result @trace('tracing func2') def func2(content): print 'content:', content return content * 3 def test(): result = func1('aa', 'bb') print 'result:', result test()
Decorators can be "stacked".
The following stacked decorators:
@dec2 @dec1 def func(arg1, arg2, ...): pass
are equivalent to:
def func(arg1, arg2, ...): pass func = dec2(dec1(func))
Exercises:
Solutions:
Reuse your tracing function from the previous exercise, then write a simple decorator that prints a row of dashes:
def trace(msg): def inner1(func): def inner2(*args, **kwargs): print '>> [%s]' % (msg, ) retval = func(*args, **kwargs) print '<< [%s]' % (msg, ) return retval return inner2 return inner1 def horizontal_line(func): def inner(*args, **kwargs): print '-' * 50 retval = func(*args, **kwargs) print '-' * 50 return retval return inner @trace('tracing func1') def func1(x, y): print 'x:', x, 'y:', y result = func2((x, y)) return result @horizontal_line @trace('tracing func2') def func2(content): print 'content:', content return content * 3 def test(): result = func1('aa', 'bb') print 'result:', result test()
Once again, a decorator with arguments can be implemented with a function nested inside a function which is nested inside a function. This remains the same whether the decorator is used as a stacked decorator or not. Here is a solution:
def trace(msg): def inner1(func): def inner2(*args, **kwargs): print '>> [%s]' % (msg, ) retval = func(*args, **kwargs) print '<< [%s]' % (msg, ) return retval return inner2 return inner1 def horizontal_line(line_chr): def inner1(func): def inner2(*args, **kwargs): print line_chr * 15 retval = func(*args, **kwargs) print line_chr * 15 return retval return inner2 return inner1 @trace('tracing func1') def func1(x, y): print 'x:', x, 'y:', y result = func2((x, y)) return result @horizontal_line('<**>') @trace('tracing func2') def func2(content): print 'content:', content return content * 3 def test(): result = func1('aa', 'bb') print 'result:', result test()
There is more about decorators here:
Definition: iterable (adjective) -- that which can be iterated over.
A good test of whether something is iterable is whether it can be used in a for: statement. For example, if we can write for item in X:, then X is iterable. Here is another simple test:
def isiterable(x): try: y = iter(x) except TypeError, exp: return False return True
Some kinds of iterables:
Exercises:
Solutions:
We implement a class that has __iter__() and next() methods:
import urllib class WebPages(object): def __init__(self, urls): self.urls = urls self.current_index = 0 def __iter__(self): self.current_index = 0 return self def next(self): if self.current_index >= len(self.urls): raise StopIteration url = self.urls[self.current_index] self.current_index += 1 f = urllib.urlopen(url) content = f.read() f.close() return content def test(): urls = [ 'http://www.python.org', 'http://en.wikipedia.org/', 'http://en.wikipedia.org/wiki/Python_(programming_language)', ] pages = WebPages(urls) for page in pages: print 'length: %d' % (len(page), ) pages = WebPages(urls) print '-' * 50 page = pages.next() print 'length: %d' % (len(page), ) page = pages.next() print 'length: %d' % (len(page), ) page = pages.next() print 'length: %d' % (len(page), ) page = pages.next() print 'length: %d' % (len(page), ) test()
The itertools module in the Python standard library has helpers for iterators: http://docs.python.org/library/itertools.html#module-itertools
Exercises:
SAX -- Parse an XML document with SAX, then show some information (tag, attributes, character data) for each element.
Minidom -- Parse an XML document with minidom, then walk the DOM tree and show some information (tag, attributes, character data) for each element.
Here is a sample XML document that you can use for input:
<?xml version="1.0"?> <people> <person id="1" value="abcd" ratio="3.2"> <name>Alberta</name> <interest>gardening</interest> <interest>reading</interest> <category>5</category> </person> <person id="2"> <name>Bernardo</name> <interest>programming</interest> <category></category> <agent> <firstname>Darren</firstname> <lastname>Diddly</lastname> </agent> </person> <person id="3" value="efgh"> <name>Charlie</name> <interest>people</interest> <interest>cats</interest> <interest>dogs</interest> <category>8</category> <promoter> <firstname>David</firstname> <lastname>Donaldson</lastname> <client> <fullname>Arnold Applebee</fullname> <refid>10001</refid> </client> </promoter> <promoter> <firstname>Edward</firstname> <lastname>Eddleberry</lastname> <client> <fullname>Arnold Applebee</fullname> <refid>10001</refid> </client> </promoter> </person> </people>
ElementTree -- Parse an XML document with ElementTree, then walk the DOM tree and show some information (tag, attributes, character data) for each element.
lxml -- Parse an XML document with lxml, then walk the DOM tree and show some information (tag, attributes, character data) for each element.
Modify document with ElementTree -- Use ElementTree to read a document, then modify the tree. Show the contents of the tree, and then write out the modified document.
XPath -- lxml supports XPath. Use the XPath support in lxml to address each of the following in the above XML instance document:
Solutions:
We can use the SAX support in the Python standard library:
#!/usr/bin/env python """ Parse and XML with SAX. Display info about each element. Usage: python test_sax.py infilename Examples: python test_sax.py people.xml """ import sys from xml.sax import make_parser, handler class TestHandler(handler.ContentHandler): def __init__(self): self.level = 0 def show_with_level(self, value): print '%s%s' % (' ' * self.level, value, ) def startDocument(self): self.show_with_level('Document start') self.level += 1 def endDocument(self): self.level -= 1 self.show_with_level('Document end') def startElement(self, name, attrs): self.show_with_level('start element -- name: "%s"' % (name, )) self.level += 1 def endElement(self, name): self.level -= 1 self.show_with_level('end element -- name: "%s"' % (name, )) def characters(self, content): content = content.strip() if content: self.show_with_level('characters: "%s"' % (content, )) def test(infilename): parser = make_parser() handler = TestHandler() parser.setContentHandler(handler) parser.parse(infilename) def usage(): print __doc__ sys.exit(1) def main(): args = sys.argv[1:] if len(args) != 1: usage() infilename = args[0] test(infilename) if __name__ == '__main__': main()
The minidom module contains a parse() function that enables us to read an XML document and create a DOM tree:
#!/usr/bin/env python """Process an XML document with minidom. Show the document tree. Usage: python minidom_walk.py [options] infilename """ import sys from xml.dom import minidom def show_tree(doc): root = doc.documentElement show_node(root, 0) def show_node(node, level): count = 0 if node.nodeType == minidom.Node.ELEMENT_NODE: show_level(level) print 'tag: %s' % (node.nodeName, ) for key in node.attributes.keys(): attr = node.attributes.get(key) show_level(level + 1) print '- attribute name: %s value: "%s"' % (attr.name, attr.value, ) if (len(node.childNodes) == 1 and node.childNodes[0].nodeType == minidom.Node.TEXT_NODE): show_level(level + 1) print '- data: "%s"' % (node.childNodes[0].data, ) for child in node.childNodes: count += 1 show_node(child, level + 1) return count def show_level(level): for x in range(level): print ' ', def test(): args = sys.argv[1:] if len(args) != 1: print __doc__ sys.exit(1) docname = args[0] doc = minidom.parse(docname) show_tree(doc) if __name__ == '__main__': #import pdb; pdb.set_trace() test()
ElementTree enables us to parse an XML document and create a DOM tree:
#!/usr/bin/env python """Process an XML document with elementtree. Show the document tree. Usage: python elementtree_walk.py [options] infilename """ import sys from xml.etree import ElementTree as etree def show_tree(doc): root = doc.getroot() show_node(root, 0) def show_node(node, level): show_level(level) print 'tag: %s' % (node.tag, ) for key, value in node.attrib.iteritems(): show_level(level + 1) print '- attribute -- name: %s value: "%s"' % (key, value, ) if node.text: text = node.text.strip() show_level(level + 1) print '- text: "%s"' % (node.text, ) if node.tail: tail = node.tail.strip() show_level(level + 1) print '- tail: "%s"' % (tail, ) for child in node.getchildren(): show_node(child, level + 1) def show_level(level): for x in range(level): print ' ', def test(): args = sys.argv[1:] if len(args) != 1: print __doc__ sys.exit(1) docname = args[0] doc = etree.parse(docname) show_tree(doc) if __name__ == '__main__': #import pdb; pdb.set_trace() test()
lxml enables us to parse an XML document and create a DOM tree. In fact, since lxml attempts to mimic the ElementTree API, our code is very similar to that in the solution to the ElementTree exercise:
#!/usr/bin/env python """Process an XML document with elementtree. Show the document tree. Usage: python lxml_walk.py [options] infilename """ # # Imports: import sys from lxml import etree def show_tree(doc): root = doc.getroot() show_node(root, 0) def show_node(node, level): show_level(level) print 'tag: %s' % (node.tag, ) for key, value in node.attrib.iteritems(): show_level(level + 1) print '- attribute -- name: %s value: "%s"' % (key, value, ) if node.text: text = node.text.strip() show_level(level + 1) print '- text: "%s"' % (node.text, ) if node.tail: tail = node.tail.strip() show_level(level + 1) print '- tail: "%s"' % (tail, ) for child in node.getchildren(): show_node(child, level + 1) def show_level(level): for x in range(level): print ' ', def test(): args = sys.argv[1:] if len(args) != 1: print __doc__ sys.exit(1) docname = args[0] doc = etree.parse(docname) show_tree(doc) if __name__ == '__main__': #import pdb; pdb.set_trace() test()
We can modify the DOM tree and write it out to a new file:
#!/usr/bin/env python """Process an XML document with elementtree. Show the document tree. Modify the document tree and then show it again. Write the modified XML tree to a new file. Usage: python elementtree_walk.py [options] infilename outfilename Options: -h, --help Display this help message. Example: python elementtree_walk.py myxmldoc.xml myotherxmldoc.xml """ import sys import os import getopt import time # Use ElementTree. from xml.etree import ElementTree as etree # Or uncomment to use Lxml. #from lxml import etree def show_tree(doc): root = doc.getroot() show_node(root, 0) def show_node(node, level): show_level(level) print 'tag: %s' % (node.tag, ) for key, value in node.attrib.iteritems(): show_level(level + 1) print '- attribute -- name: %s value: "%s"' % (key, value, ) if node.text: text = node.text.strip() show_level(level + 1) print '- text: "%s"' % (node.text, ) if node.tail: tail = node.tail.strip() show_level(level + 1) print '- tail: "%s"' % (tail, ) for child in node.getchildren(): show_node(child, level + 1) def show_level(level): for x in range(level): print ' ', def modify_tree(doc, tag, attrname, attrvalue): root = doc.getroot() modify_node(root, tag, attrname, attrvalue) def modify_node(node, tag, attrname, attrvalue): if node.tag == tag: node.attrib[attrname] = attrvalue for child in node.getchildren(): modify_node(child, tag, attrname, attrvalue) def test(indocname, outdocname): doc = etree.parse(indocname) show_tree(doc) print '-' * 50 date = time.ctime() modify_tree(doc, 'person', 'date', date) show_tree(doc) write_output = False if os.path.exists(outdocname): response = raw_input('Output file (%s) exists. Over-write? (y/n): ' % outdocname) if response == 'y': write_output = True else: write_output = True if write_output: doc.write(outdocname) print 'Wrote modified XML tree to %s' % outdocname else: print 'Did not write output file.' def usage(): print __doc__ sys.exit(1) def main(): args = sys.argv[1:] try: opts, args = getopt.getopt(args, 'h', ['help', ]) except: usage() for opt, val in opts: if opt in ('-h', '--help'): usage() if len(args) != 2: usage() indocname = args[0] outdocname = args[1] test(indocname, outdocname) if __name__ == '__main__': #import pdb; pdb.set_trace() main()
Notes:
When we parse and XML document with lxml, each element (node) has an xpath() method.
# test_xpath.py from lxml import etree def test(): doc = etree.parse('people.xml') root = doc.getroot() print root.xpath("//name/text()") print root.xpath("//@id") test()
And, when we run the above code, here is what we see:
$ python test_xpath.py ['Alberta', 'Bernardo', 'Charlie'] ['1', '2', '3']
For more on XPath see: XML Path Language (XPath) -- http://www.w3.org/TR/xpath
You can find information about database programming in Python here: Database Programming -- http://wiki.python.org/moin/DatabaseProgramming/.
For database access we use the Python Database API. You can find information about it here: Python Database API Specification v2.0 -- http://www.python.org/dev/peps/pep-0249/.
To use the database API we do the following:
Our examples use the gadfly database, which is written in Python. If you want to use gadfly, you can find it here: http://gadfly.sourceforge.net/. gadfly is a reasonable choice if you want an easy to use database on your local machine.
Another reasonable choice for a local database is sqlite3, which is in the Python standard library. Here is a descriptive quote from the SQLite Web site:
"SQLite is a software library that implements a self-contained, serverless, zero-configuration, transactional SQL database engine. SQLite is the most widely deployed SQL database engine in the world. The source code for SQLite is in the public domain."
You can learn about it here:
If you want or need to use another, enterprise class database, for example PostgreSQL, MySQL, Oracle, etc., you will need an interface module for your specific database. You can find information about database interface modules here: Database interfaces -- http://wiki.python.org/moin/DatabaseInterfaces
Excercises:
Solutions:
We can execute a SQL query and then retrieve all the rows with fetchall():
import gadfly def test(): connection = gadfly.connect("dbtest1", "plantsdbdir") cur = connection.cursor() cur.execute('select * from plantsdb order by p_name') rows = cur.fetchall() for row in rows: print '2. row:', row connection.close() test()
The cursor itself is an iterator. It iterates over the rows returned by a query. So, we execute a SQL query and then we use the cursor in a for: statement:
import gadfly def test(): connection = gadfly.connect("dbtest1", "plantsdbdir") cur = connection.cursor() cur.execute('select * from plantsdb order by p_name') for row in cur: print row connection.close() test()
The description attribute in the cursor is a container that has an item describing each field: