Removing comments... tokenize error

[MrJean1]

Re: whitespace , comment stripper, and EOL converter

Attached is another version of the stripper.py file. It contains my
change which seem to handle docstring correctly (at least on itself).


/Jean Brouwers

<pre>

############### ############### ############### ############### ##########
# Python source stripper / cleaner ;)
############### ############### ############### ############### ##########

import os
import sys
import token
import keyword
import StringIO
import tokenize
import traceback
__credits__ = \
'''
J¸rgen Hermann
M.E.Farmer
Jean Brouwers
'''
__version__ = '.8'
__author__ = 'M.E.Farmer'
__date__ = 'Apr 16, 2005,' \
'Jan 15 2005,' \
'Oct 24 2004' \

'''this docstring should be removed
'''

############### ############### ############### ############### ##########

class Stripper:
"""Python source stripper / cleaner
"""
def __init__(self, raw):
self.raw = raw

def format(self, out=sys.stdout, comments=0, docstrings=0,
spaces=1, untabify=1, eol='unix'):
""" strip comments,
strip docstrings,
strip extra whitespace and lines,
convert tabs to spaces,
convert EOL's in Python code.
"""
# Store line offsets in self.lines
self.lines = [0, 0]
pos = 0
self.temp = StringIO.String IO()
# Strips the first blank line if 1
self.lasttoken = 1
self.spaces = spaces
# 0 = no change, 1 = strip 'em
self.comments = comments # yep even these
# 0 = no change, 1 = strip 'em, 8 or 'pep8'= strip all but
"""'s
self.docstrings = docstrings

if untabify:
self.raw = self.raw.expand tabs()
self.raw = self.raw.rstrip ()+' '
self.out = out

# Have you ever had a multiple line ending script?
# They can be nasty so lets get them all the same.
self.raw = self.raw.replac e('\r\n', '\n')
self.raw = self.raw.replac e('\r', '\n')
self.lineend = '\n'

# Gather lines
while 1:
pos = self.raw.find(s elf.lineend, pos) + 1
if not pos: break
self.lines.appe nd(pos)

self.lines.appe nd(len(self.raw ))
self.pos = 0
self.lastOP = ''

# Wrap text in a filelike object
text = StringIO.String IO(self.raw)

# Parse the source.
## Tokenize calls the __call__
## method for each token till done.
try:
tokenize.tokeni ze(text.readlin e, self)
except tokenize.TokenE rror, ex:
traceback.print _exc()

# Ok now we write it to a file
# but we also need to clean the whitespace
# between the lines and at the ends.
self.temp.seek( 0)

# All this should be written into the
# __call__ method just haven't yet...

# Mac CR
if eol == 'mac':
self.lineend = '\r'
# Windows CR LF
elif eol == 'win':
self.lineend = '\r\n'
# Unix LF
else:
self.lineend = '\n'

for line in self.temp.readl ines():
if spaces == -1:
self.out.write( line.rstrip()+s elf.lineend)
else:
if not line.isspace():
self.lasttoken= 0
self.out.write( line.rstrip()+s elf.lineend)
else:
self.lasttoken+ =1
if self.lasttoken< =self.spaces and self.spaces:
self.out.write( self.lineend)

def __call__(self, toktype, toktext, (srow,scol), (erow,ecol),
line):
""" Token handler.
"""
# calculate new positions
oldpos = self.pos
newpos = self.lines[srow] + scol
self.pos = newpos + len(toktext)

##print "*token: %s text: %r line: %r" % \
(token.tok_name[toktype], toktext, line)

# kill comments
if self.comments:
if toktype == tokenize.COMMEN T:
return

# kill doc strings
if self.docstrings :
# a STRING must be a docstring
# if the most recent OP was ':'
if toktype == tokenize.STRING and self.lastOP == ':':
# pep8 frowns on triple single quotes
if (self.docstring s == 'pep8' or
self.docstrings == 8):
if not toktext.endswit h('"""'):
return
else:
return
elif toktype == token.OP:
# remember most recent OP
self.lastOP = toktext
elif self.lastOP == ':':
# newline and indent are OK inside docstring
if toktype not in [token.NEWLINE, token.INDENT]:
# otherwise the docstring ends
self.lastOP = ''
elif toktype == token.NEWLINE:
# consider any string starting
# on a new line as a docstring
self.lastOP = ':'

# handle newlines
if toktype in [token.NEWLINE, tokenize.NL]:
self.temp.write (self.lineend)
return

# send the original whitespace
if newpos > oldpos:
self.temp.write (self.raw[oldpos:newpos])

# skip indenting tokens
if toktype in [token.INDENT, token.DEDENT]:
self.pos = newpos
return

# send text to the temp file
self.temp.write (toktext)
return
############### ############### ############### ############### ##########

def Main():
import sys
if sys.argv[1]:
filein = open(sys.argv[1]).read()
Stripper(filein ).format(out=sy s.stdout,
comments=1, docstrings=1, untabify=1, eol='win')
############### ############### ############### ############### ##########

if __name__ == '__main__':
Main()

</pre>



M.E.Farmer wrote:[color=blue]
> I found the bug and hope I have squashed it.
> Single and qouble quoted strings that were assignments and spanned
> multilines using \ , were chopped after the first line.
> example:
> __date__ = 'Apr 16, 2005,' \
> 'Jan 15 2005,' \
> 'Oct 24 2004'
> became:
> __date__ = 'Apr 16, 2005,' \
>
> Not good :(
>
> tokenizer sends this as:
> name
> operator
> string
> string
> string
> newline
>
> I added test for string assignments that end in \.
> A flag is set and then all strings till a newline are ignored.
> Also rearranged the script a little.
> Maybe that will do it ...
> Updates available at[color=green]
> > The script is located at:
> > http://bellsouthpwp.net/m/e/mefjr75/python/stripper.py
> >
> > M.E.Farmer[/color][/color]

[M.E.Farmer]

Re: whitespace , comment stripper, and EOL converter

Hello Jean,
Glad to see your still playing along.
I have tested your script and it is broken too :(
Good idea about checking for the ':' , it just doesn't cover every
case.
This is the very reason I had not included docstring support before!
The problem is more diffcult than it first appears,
I am sure you have noticed ;)
Python is fairly flexible in it's layout and very dynamic in it's
execution.
This can lead to some hard to spot and hard to remove docstrings.

After staring at the problem for a day or so ( for the second time ),
*I am still stumped*

############### ############### ############### ############### #########
# this is a test I have put together for docstrings
############### ############### ############### ############### #########
"""This is a module doc it should be removed""" \
"This is really nasty but legal" \
'''Dang this is even worse''' + \
'this should be removed'#This is legal too
############### ############### ############### ############### #########
assignment = \
"""
this should stay
so should this
"""
more_assignment = 'keep me,' \
'keep me too,' \
'keep me.'

############### ############### ############### ############### #########
def func():
'This should be removed' \
"""This should be removed"""
pass
############### ############### ############### ############### ##########
def funq(d = {'MyPass':
"""This belongs to a dict and should stay"""
,'MyOtherPass':
'Kepp this string %s'\
"Keep this too" % 42 + """dfgffdgfdgdf g"""}):
"""This docstring is ignored""" + ''' by Python introspection
why?'''
pass
############### ############### ############### ############### ##########
def Usage():
"""This should be removed but how, removal will break the function.
This should be removed %s """# what do we do here
return Usage.__doc__% '42'
############### ############### ############### ############### ##########
class Klass:
u"This should be removed" \
''' this too '''
def __init__(self, num):
""" This is should be removed but how ? %d """ % num
return None
'People do this sometime for a block comment type of thing' \
"This type of string should be removed also"
def func2(self):
r'erase/this\line\sdfds f\sdf\dfsdf'
def inner():
u'''should be removed'''
return 42
return inner
############### ############### ############### ############### ##########
u'People do this sometime for a block comment type of thing' \
r"This type of string should be removed also" \
""" and this one too! """
# did I forget anything obvious ?
############### ############### ############### ############### #########

When the docstring is removed it should also consume the blank line
that is left behind.
Got to go to work, I'll think about it over the weekend.
If anyone else wants to play you are welcome to join.
Can pyparsing do this easily?( Paul probably has a 'six-line' solution
tucked away somewhere ;)
M.E.Farmer