#!/Library/Frameworks/Python.framework/Versions/3.5/bin/python
#============================================================================
#
# NAME
#
#     updateweb.py
#
# DESCRIPTION
#
#     Python script which updates my web sites.
#
#     It does miscellaneous cleanup on my master copy of the web site on disk,
#     including updating copyright information, then synchronizes the master
#     copy to my remote server web sites using FTP.
#
# USAGE
#
#     It's best to use the associated makefile.
#     But you can call this Python utility from the command line,
#
#     $ python updateweb.py          Clean up my master copy, then use it
#                                    to update my remote web server site.
#                                    Log warnings and errors.
#     $ python updateweb.py -v       Same, but log debug messages also.
#     $ python updateweb.py -c       Clean up my master copy only.
#     $ python updateweb.py -t       Run unit tests only.
#
#     We get username and password information from the file PARAMETERS_FILE.
#
#     Logs are written to the files,
#
#         logMaster.txt       Master web site cleanup log.
#         logPrimary.txt      Primary web server update log.
#
# AUTHOR
#
#     Sean E. O'Connor        23 Aug 2007  Version 1.0 released.
#     Sean E. O'Connor        18 May 2013  Version 4.2 released.
#     Sean E. O'Connor        07 Nov 2015  Version 4.3 released.
#     Sean E. O'Connor        22 Nov 2015  Version 4.4 released.
#     Sean E. O'Connor        07 Feb 2017  Version 4.5 released.
#     Sean E. O'Connor        04 Jun 2017  Version 4.6 released.
#
# LEGAL
#
#     updateweb.py Version 4.6 - A Python utility program which maintains my web site.
#     Copyright (C) 2007-2017 by Sean Erik O'Connor.  All Rights Reserved.
#
#     This program is free software: you can redistribute it and/or modify
#     it under the terms of the GNU General Public License as published by
#     the Free Software Foundation, either version 3 of the License, or
#     (at your option) any later version.
#
#     This program is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU General Public License for more details.
#
#     You should have received a copy of the GNU General Public License
#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#     The author's address is artificer!AT!seanerikoconnor!DOT!freeservers!DOT!com
#     with !DOT! replaced by . and the !AT! replaced by @
#
# NOTES
#
#    DOCUMENTATION
#
#    Python interpreter:               http://www.python.org
#    Python tutorial and reference:    htttp://docs.python.org/lib/lib.html
#    Python debugger:                  https://docs.python.org/3/library/pdb.html
#    Python regular expression howto:  http://www.amk.ca/python/howto/regex/
#
# SAMPLE DEBUGGING OF THIS SCRIPT
#
#    Run Python
#
#        seanoconnor:~/Desktop/Sean/WebSite/WebDesign/MaintainWebPage$ python
#        Python 3.5.0 (v3.5.0:374f501f4567, Sep 12 2015, 11:00:19) 
#        [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] on darwin
#        Type "help", "copyright", "credits" or "license" for more information.
#
#    Get the debugger started
#
#        >>> import pdb
#        >>> import updateweb
#        >>> pdb.run('updateweb.main()')
#        > <string>(1)<module>()
#
#    Import the class you want to debug within
#
#        (Pdb) from updateweb import WebSite
#
#    Set breakpoints
#
#        (Pdb) b WebSite.clean
#        Breakpoint 1 at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1067
#
#        (Pdb) b WebSite.isTempFile
#        Breakpoint 2 at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1126
#
#    List them
#
#        (Pdb) b
#        Num Type         Disp Enb   Where
#        1   breakpoint   keep yes   at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1067
#        2   breakpoint   keep yes   at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1126
#
#    Start running the program
#
#        (Pdb) c
#        
#            updateweb Version 4.6 - A Python utility program which maintains my web site.
#            Copyright (C) 2007-2017 by Sean Erik O'Connor.  All Rights Reserved.
#        
#            It deletes temporary files, rewrites old copyright lines and email address
#            lines in source files, then synchronizes all changes to my web sites.
#        
#            updateweb comes with ABSOLUTELY NO WARRANTY; for details see the
#            GNU General Public License.  This is free software, and you are welcome
#            to redistribute it under certain conditions; see the GNU General Public
#            License for details.
#        Scanning and cleaning local web site...
#
#    We stopped at the breakpoint
#
#        > /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py(1129)isTempFile()
#        -> fileName = fileInfo[ self.userSettings.FILE_NAME ]
#
#    List the source lines
#
#        (Pdb) l
#        1124           return False
#        1125   
#        1126B      def isTempFile( self, fileInfo ):
#        1127           """Identify a file name as a temporary file"""
#        1128   
#        1129 ->                fileName = fileInfo[ self.userSettings.FILE_NAME ]
#        1130   
#        1131           # Suffixes and names for temporary files be deleted.
#        1132           [pat, match] = patternMatch( self.userSettings.TEMP_FILE_SUFFIXES, fileName )
#        1133   
#        1134           pos = fileName.find( self.userSettings.VIM_TEMP_FILE_EXT )
#        1135           if pos >= 0:
#        1136                print( "temp? file name = {0:s} {1:d}\n".format( fileName, pos), end='', flush=True)
#
#    Disable the breakpoints
#
#        (Pdb) disable 1
#        Disabled breakpoint 1 at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1067
#
#        (Pdb) b
#        Num Type         Disp Enb   Where
#        1   breakpoint   keep no    at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1067
#               breakpoint already hit 1 time
#        2   breakpoint   keep yes   at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1126
#               breakpoint already hit 1 time
#               breakpoint already hit 5 times
#
#        (Pdb) disable 2
#
#        Disabled breakpoint 2 at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1126
#
#        (Pdb) b
#        Num Type         Disp Enb   Where
#        1   breakpoint   keep no    at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1067
#               breakpoint already hit 1 time
#        2   breakpoint   keep no    at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1126
#               breakpoint already hit 5 times
#
#    What we really want is to set a break at this line if the condition matches
#
#        (Pdb) b 1136
#        Breakpoint 3 at /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py:1136
#
#        (Pdb) condition 3 pos >= 0
#        New condition set for breakpoint 3.
#
#    Continue
#
#        (Pdb) c
#        > /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py(1136)isTempFile()
#        -> print( "temp? file name = {0:s} {1:d}\n".format( fileName, pos), end='', flush=True)
#        (Pdb) l
#        1131           # Suffixes and names for temporary files be deleted.
#        1132           [pat, match] = patternMatch( self.userSettings.TEMP_FILE_SUFFIXES, fileName )
#        1133   
#        1134           pos = fileName.find( self.userSettings.VIM_TEMP_FILE_EXT )
#        1135           if pos >= 0:
#        1136B->                    print( "temp? file name = {0:s} {1:d}\n".format( fileName, pos), end='', flush=True)
#        1137   
#        1138           # Vim editor temp files contain twiddles.
#        1139           if match or fileName.find( self.userSettings.VIM_TEMP_FILE_EXT ) >= 0:
#        1140               return True
#        1141   
#
#    Print a few variables
#
#        (Pdb) p pos
#        42
#
#        (Pdb) p fileName
#        'WebDesign/MaintainWebPage/.updateweb.py.un~'
#
#    Backtrace
#
#        (Pdb) bt
#          /Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/bdb.py(431)run()
#        -> exec(cmd, globals, locals)
#          <string>(1)<module>()
#          /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py(757)main()
#        -> changed = master.clean()
#          /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py(1095)clean()
#        -> if self.isTempFile( f ):
#        > /Users/seanoconnor/Desktop/Sean/WebSite/WebDesign/MaintainWebPage/updateweb.py(1136)isTempFile()
#        -> print( "temp? file name = {0:s} {1:d}\n".format( fileName, pos), end='', flush=True)
#
#    Keep going until we finish
#
#        (Pdb) c
#        ...done!
#        Scanning remote web site......done!
#        Synchronizing remote and local web sites...Uploading changed file WebDesign/MaintainWebPage/updateweb.py...
#        Uploading changed file WebDesign/MaintainWebPage/updateweb.py.txt...Uploading new file WebDesign/MaintainWebPage/logMaster.txt......done!
#
#    Quit
#
#        >>> quit()
#
#============================================================================

#----------------------------------------------------------------------------
#  Load Python Packages
#----------------------------------------------------------------------------

# OS stuff
import sys
import os
import platform
import optparse
import shutil

# Regular expressions
import string
import re

# FTP stuff
import ftplib

# Date and time
import time
import stat
import datetime

# Logging
import logging

# Unit testing
import unittest

# Enumerated types (v3.4)
from enum import Enum


#----------------------------------------------------------------------------
#  User settings.
#----------------------------------------------------------------------------

# Enum types for how to walk the directory tree.
class TreeWalk( Enum ):
    BREADTH_FIRST_SEARCH = 1
    DEPTH_FIRST_SEARCH   = 2

# 'Enum' types for properties of directories and files.
class FileType( Enum ):
    DIRECTORY=0
    FILE=1
    ON_MASTER_ONLY=2
    ON_REMOTE_ONLY=3
    ON_BOTH_MASTER_AND_REMOTE=4

# Megatons of user selectable settings.
class UserSettings:
    # Logging control.
    LOGFILENAME = ""
    VERBOSE          = False  # Verbose mode.  Prints out everything.
    CLEANONLY        = False  # Clean the local master web site only.
    UNITTEST         = False  # Run a unit test of a function.

    # When diving into the MathJax directory, web walking the deep directories
    # may exceed Python's default recursion limit of 1000.
    RECURSION_DEPTH = 5000
    sys.setrecursionlimit( RECURSION_DEPTH )

    # Fields in the file information (fileInfo) structure.
    FILE_NAME      = 0
    FILE_TYPE      = 1
    FILE_DATE_TIME = 2
    FILE_SIZE      = 3

    # Parameter file which contains web server account login information for FTP.
    PARAMETERS_FILE = "/private/param.txt"

    # Line numbers in the PARAMETERS_FILE, starting from 0.  All other lines are comments, and are skipped.
    SERVER              = 19
    USER                = 20
    PASSWORD            = 21
    FTP_ROOT            = 22
    FILE_SIZE_LIMIT     = 23

    # Map month names onto numbers.
    monthToNumber = { 'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 'Aug':8, 'Sep':9, 'Oct':10, 'Nov':11, 'Dec':12 }

    # Which private directories to skip over when updating the web page.  They will be listed as WARNINGs in the log.
    # These won't be uploaded to the web host.  e.g. MathJax, Subversion and git local admin directories, Cocoa and Java NetBeans build directories, etc.
    # We generally skip MathJax since it has thousands of small files.
    DIR_TO_SKIP     = "private|\.git|\.svn|MathJax|build|Debug|Release"

    # Which private files to skip over when updating the web page.  They will be listed as WARNINGs in the log.
    # These won't be uploaded to the web host. e.g. PaintshopPro, Photoshop, GIMP native images, Mathematica notebook.
    # Also anything labeled private or rendered.
    # Special case:  hidden file .htaccess doesn't show up on the output of ftp LIST, so we must upload manually.
    # FILE_TO_SKIP    = "\.psp|\.xcf|\.psd|\.svnignore|private|rendered|\.htaccess"
    FILE_TO_SKIP    = "\.svnignore|\.htaccess"

    # File extension for text files.
    TEXT_FILE_EXT  = ".txt"

    # Suffixes for temporary files which will be deleted during the cleanup phase.
    TEMP_FILE_SUFFIXES = r"""        # Use Python raw strings.
        \.                           # Match the dot in the file name.
                                     # Now begin matching the file name suffix.
                                     # (?: non-capturing match for the regex inside the parentheses, i.e. matching string cannot be retrieved later.
                                     # Now match any of the following file extensions:
        (?: o   | obj | lib | exe |  #     Object files generated by C, C++, etc compilers
                              pyc |  #     Object file generated by the Python compiler
                  ilk | pdb | sup |  #     Temp files from VC++ compiler
            idb | ncb | opt | plg |  #     Temp files from VC++ compiler
            sbr | bsc | map | bce |  #     Temp files from VC++ compiler
            res | aps | dep | db  |  #     Temp files from VC++ compiler
                              jbf |  #     Paintshop Pro
                      class | jar |  #     Java compiler
                              log |  #     WS_FTP
                              fas |  #     CLISP compiler
                        swp | swo |  #     Vim editor
                              aux |  #     TeX auxilliary files.
          DS_Store  | _\.DS_Store |  #     Mac OS finder folder settings.
                       _\.Trashes |  #     Mac OS recycle bin
        gdb_history)                 #     GDB history
        $                            #     Now we should see only the end of line.
        """

    # Special case:  Vim temporary files contain a twiddle anywhere in the name.
    VIM_TEMP_FILE_EXT = "~"

    # Suffixes for temporary directories which should be deleted during the cleanup phase.
    TEMP_DIR_SUFFIX   = r"""
        (?: Debug | Release |        # C++ compiler
           ipch   | \.vs    |        # Temp dirs from VC++ compiler
        \.Trashes | \.Trash)         # Mac OS recycle bin
        $
        """

    # File extension for an internally created temporary file.
    TEMP_FILE_EXT     = ".new"

    # Copy these file types to a file with the additional extension .txt because my server needs a file extension.
    # Make sure the file extension is supported in the .htaccess file.
    SOURCE_FILE_SUFFIXES = r"""
        (?: makefile$                                      # Any file called makefile is a source file.
              |
              \.                                           # Match the file name suffix after the .
                                                           # Now match any of these suffixes:
              (?: bashrc | bash_profile | bash_logout  )   # bash files
           $)
           """

    # Suffixes for HTML hypertext and CSS style sheet files.
    HYPERTEXT_SUFFIX=r"""
         \.                  # Match the filename suffix after the .
                             # Now match any of these suffixes:
         (?: html | htm |    #     HTML hypertext
              css)           #     CSS style sheet
         $
         """

    # Update my email address.
    # This is tricky:  Prevent matching and updating the name within in this Python source file by using the character class brackets.
    OLD_EMAIL_ADDRESS= r"""
        artifex\!AT\!sean[e]rikoconnor\!DOT\!freeservers\!DOT\!com
        """
    NEW_EMAIL_ADDRESS="artificer!AT!seanerikoconnor!DOT!freeservers!DOT!com"

    # Rewrite a line by replacing an old substring with a new substring.
    # In this particular case, mailbox.png ---> mailbox.png
    OLD_SUBSTRING=r"""
               mailbox
               \.
               jpg
               """

    NEW_SUBSTRING="mailbox.png"

    # Change old software version to new software version for all lines in files of the form,
    #      Primpoly Version nnnn.nnnn
    OLD_SOFTWARE_VERSION= r"""
        Primpoly
        \s+
        Version
        \s+
        ([0-9]+)   # The two part version number NNN.nnn
        \.
        ([0-9]+)
        """
    NEW_SOFTWARE_VERSION="Primpoly Version 13.0"

    # Match a copyright line.  Then extract the copyright symbol which can be (C) or &copy; and extract the old year.
    TWO_DIGIT_YEAR_FORMAT="%02d"
    COPYRIGHT_LINE= r"""
        Copyright                       # Copyright.
        \D+                             # Any non-digits.
        (?P<symbol> \(C\) | &copy;)     # Match and extract the copyright symbol.
        \D+                             # Any non-digits.
        (?P<oldYear>[0-9]+)             # Match and extract the old copyright year, then place it into variable 'oldYear'
        -                               # to
        ([0-9]+)                        # New copyright year.
        """

    # Match another type of copyright line.  Extract the copyright symbol which can be (C) or &copy; and the old year.
    COPYRIGHT_LINE2= r"""
        Copyright                       # Copyright.
        \D+                             # Any non-digits.
        (?P<symbol> \(C\) | &copy;)     # Match and extract the copyright symbol.
        \D+                             # Any non-digits.
        (?P<oldYear>[0-9]+)             # Match and extract the old copyright year, then place it into variable 'oldYear'
        """

    # Match a line containing the words,
    #    last updated YY
    # and extract the two digit year YY.
    LAST_UPDATED_LINE=r"""
        last\s+         # Match the words "last updated"
        updated\s+
        \d+             # Day number
        \s+             # One or more blanks or tabs
        [A-Za-z]+       # Month
        \s+             # One or more blanks or tabs
        (?P<year>\d+)   # Two digit year.  Place it into the variable 'year'
        """

    # Web server root directory.
    DEFAULT_ROOT_DIR   = "/"

    # The ftp listing occasionally shows the wrong date:
    #     For example, file foo.txt has just been modified on the current date, 1 January 2010 and uploaded to the server.
    #     But the updated file on the server shows a date of 30 December which is 1 day incorrect with no year listed.
    #     Thus we assume the current year 2010, which is correct, but the date makes it seem 11 months newer, which is incorrect.
    # So when we detect such a major discrepancy, we upload to be safe:  it's never an error to update the file
    # from the local master copy.  At worst, it wastes time.
    # Unfortunately, some old files will be uploaded repeatedly.  To avoid this, touch them to change to the current date.
    DAYS_NEWER_FOR_REMOTE_BEFORE_WE_SUSPECT_ITS_ACTUALLY_VERY_OLD = 150

    # Upload only if we are newer by more than a few minutes.  Allows for a little slop in time stamps on server or host.
    MINUTES_NEWER_FOR_MASTER_BEFORE_UPLOAD = 5.0
    DAYS_NEWER_FOR_MASTER_BEFORE_UPLOAD = (1.0 / 24.0) * (1.0 / 60.0) * MINUTES_NEWER_FOR_MASTER_BEFORE_UPLOAD

    # An ftp list command line should be at least this many chars, or we'll suspect and error.
    MIN_FTP_LINE_LENGTH = 7

    # Parse an ftp listing, extracting <bytes> <mon> <day> <hour> <min> <year> <filename>
    # ftp listings are generally similar to UNIX ls -l listings.
    #
    # Some examples:
    #
    # (1) Freeservers ftp listing,
    #
    #          0        1   2                3           4    5   6   7      8
    #     drwxr-xr-x    3 1000             1000         4096 Nov 18  2006 Electronics
    #     -rw-r--r--    1 1000             1000        21984 Jun  4 03:46 StyleSheet.css
    #     -rw-r--r--    1 1000             1000         2901 Sep 26 17:12 allclasses-frame.html
    #
    # (2) atspace ftp listing,
    #
    #     drwxr-xr-x    3  seanerikoconnor vusers         49 Apr  7  2006 Electronics
    #     -rw-r--r--    1  seanerikoconnor vusers      21984 Jun  4 04:03 StyleSheet.css
    #
    FTP_LISTING= r"""
        [drwx-]+            # Unix type file mode.
        \s+                 # One or more blanks or tabs.
        \d+                 # Number of links.
        \s+
        \w+                 # Owner.
        \s+
        \w+                 # Group.
        \s+
        (?P<bytes> \d+)     # File size in bytes, placed into the variable 'bytes'.
        \s+
        (?P<mon> \w+)       # Month modified, placed into the variable 'mon'.
        \s+
        (?P<day> \d+)       # Day modified, placed into the variable 'day'.
        \s+
        (
            (?P<hour> \d+)  # Hour modified, placed into the variable 'hour'.
            :
            (?P<min> \d+)   # Minute modified, placed into the variable 'min'.
        |
            (?P<year> \d+)  # If hours and minutes are absent (happens when year is not the current year), extract the year instead.
        )
        \s+
        (?P<filename> [A-Za-z0-9"'.\-_,~()=+#]+)    # Path and file name containing letters, numbers, and funny characters.
        $                                           # We must escape some of these characters with a backslash, \.
        """

    def __init__( self ):
        """Set up the user settings."""

        # Import the user settings from the parameter file.
        self.getPlatform()
        self.getMasterRootDir()
        self.getPrivateSettings()

    def getPrivateSettings( self ):
        """
        Read web account private settings from a secret offline parameter file.  Return an array of strings.
        e.g. self.privateSettings[ 19 ] = "seanerikoconnor.freeservers.com", where the index 19 = UserSettings.SERVER
        """

        # Private file which contains my account settings.
        inFileName = self.masterRootDir + self.PARAMETERS_FILE

        try:
            fin = open( inFileName, "r" )
        except IOError as detail:
            logging.error( "Cannot open the private settings file {0:s}: {1:s}.  Aborting...".format( inFileName, str( detail ) ) )
            sys.exit()

        # Read each line of the file, aborting if there is a read error.
        try:
            self.privateSettings = []
            line = fin.readline()
            while line:
                self.privateSettings.append( line.strip() )  # Strip off leading and trailing whitespace.
                line = fin.readline()
            fin.close()
        except Exception as detail:
            logging.error( "File I/O error reading private settings file {0:s}: {1:s}.  Aborting...".format( inFileName, str( detail ) ) )
            sys.exit()

        return

    def getPlatform( self ):
        """Find out which type of computer platform we are running on. """
        self.platformName = ""

        # Look at the computer name and try to figure out which of my platforms I'm running on.
        if platform.node().endswith( 'Artificer' ):       # Mac OS on MacBook Pro
            self.platformName = "Mac OS"
        elif platform.node() == "SEANOCONNORBC36":        # Windows 7 64-bit running in Parallels running in Mac OS.
            self.platformName = "Win"
        elif platform.node() == "seanoconnor-MacBookPro": # Ubuntu Linux on my old MacBook Pro
            self.platformName = "Linux"
        else:                                             # Guessing Mac OS
            self.platformName = "Mac OS"
            logging.error( "Can't determine the computer platform from node name |{0:s}|:  guessing Mac OS".format( platform.node() ))

        return

    def getMasterRootDir( self ):
        """Get the master web site root directory on this platform."""

        # Each platform has a definite root directory:
        # Mac OS
        if self.platformName == "Mac OS":
            self.masterRootDir = "/Users/seanoconnor/Desktop/Sean/WebSite"
        # Ubuntu Linux
        elif self.platformName == "Linux":
            self.masterRootDir = "/home/seanoconnor/Desktop/Sean/WebSite"
        # Windows on Parallels on Mac OS, /cygdrive/c/cygwin/home/Sean/WebSite
        elif self.platformName == "Win":
            self.masterRootDir = "C:/cygwin/home/Sean/Sean/WebSite"
        return

#----------------------------------------------------------------------------
#  Helper functions
#----------------------------------------------------------------------------

# Pattern match a regular expression on a string, ignoring case.
def patternMatch( regularExpression, searchString ):
    pat = re.compile( regularExpression, re.VERBOSE | re.IGNORECASE )
    match = pat.search( searchString )
    return [pat, match]


#----------------------------------------------------------------------------
#  Unit test some of the individual functions.
#----------------------------------------------------------------------------

class UnitTest( unittest.TestCase ):
    def setUp( self ):
        self.userSettings = UserSettings()
        self.userSettings.getPlatform()
        self.userSettings.getMasterRootDir()
        self.privateSettings = self.userSettings.privateSettings

    def tearDown( self ):
        self.userSettings = None
        self.privateSettings = None

    def test_userSettings( self ):
        print( "file size limit             =  {0:d} K".format( int( self.privateSettings[self.userSettings.FILE_SIZE_LIMIT] )))
        print( "Computer platform node name = |{0:s}|".format( platform.node() ))
        print( "Root directory              = |{0:s}|".format( self.userSettings.masterRootDir ))
        self.assertTrue( True )

    def test_extractSoftwareVersion( self ):
        print( "---------- Unit test:  Pattern match;  extracting the software version." )
        line    = "|     Primpoly Version 13.0 - A Program for Computing Primitive Polynomials."
        newLine = "|     Primpoly Version 13.0 - A Program for Computing Primitive Polynomials."
        [pat, match] = patternMatch( self.userSettings.OLD_SOFTWARE_VERSION, line )
        if match:
            newVersion = self.userSettings.NEW_SOFTWARE_VERSION
            sub = pat.sub( newVersion, line )
            self.assertEqual( line, newLine, "old line = {0:s} new line = {1:s}".format( line, sub ))
        else:
            self.assertTrue( False, "OH OH, no match!  Cannot change Primpoly version." )

    def test_extractFileNameFromFTPListing( self ):
        line = "-rw-r--r--    1 1000             1000         2901 Sep 26 17:12 allclasses-frame.html"
        extractedFileName = "allclasses-frame.html"
        [pat, match] = patternMatch( self.userSettings.FTP_LISTING, line )
        if match:
            filename = match.group( 'filename' )
            self.assertEqual( filename, extractedFileName, "line = {0:s} extracted file name = {1:s}".format( line, extractedFileName ))
        else:
            self.assertTrue( False, "OH OH, no match!  Cannot get the ftp filename." )

    def test_checkReplaceSubstring( self ):
        print( "------------ Unit test:  Pattern match;  checking for IE version." )
        oldline = "         <div class=\"icon\">Some text"
        newline = "         <div class=\"illustratedparagraph\">Some text"
        print( "oldline = {0:s}".format( oldline ))
        [pat, match] = patternMatch( self.userSettings.OLD_SUBSTRING, oldline )

        # Replace the old address with my new email address.
        if match:
            rewrittenline = pat.sub( self.userSettings.NEW_SUBSTRING, oldline )
            self.assertEqual( newline, rewrittenline, "newline = {0:s} rewrittenline = {1:s}".format( newline, rewrittenline ))
        else:
            self.assertTrue( False, "OH OH, no match!  Cannot rewrite custom." )


    def test_FileTimeAndDate( self ):
        print( "---------- Unit test:  parameters." )
        print( "Root directory = |{0:s}|".format( self.userSettings.masterRootDir ))
        fileName = self.userSettings.masterRootDir + "/Electronics/WebPageImages/PowerSupply1Schematic.psd"
        fileEpochTime = os.path.getmtime( fileName )
        fileTimeUTC = time.gmtime( fileEpochTime )[ 0 : 6 ]
        d = datetime.datetime( fileTimeUTC[0], fileTimeUTC[1], fileTimeUTC[2], fileTimeUTC[3], fileTimeUTC[4], fileTimeUTC[5])
        print( "file {0:s} datetime {1:s}\n".format( fileName, d.ctime() ) ) ;
        self.assertTrue( True )

#----------------------------------------------------------------------------
#  Main function
#----------------------------------------------------------------------------

def main():
    """Main program.  Clean up and update my web site."""

    # Print the obligatory legal notice.
    print( """
    updateweb Version 4.6 - A Python utility program which maintains my web site.
    Copyright (C) 2007-2017 by Sean Erik O'Connor.  All Rights Reserved.

    It deletes temporary files, rewrites old copyright lines and email address
    lines in source files, then synchronizes all changes to my web sites.

    updateweb comes with ABSOLUTELY NO WARRANTY; for details see the
    GNU General Public License.  This is free software, and you are welcome
    to redistribute it under certain conditions; see the GNU General Public
    License for details.
    """ )

    #---------------------------------------------------------------------
    #  Load default settings and start logging.
    #---------------------------------------------------------------------

    # Default user settings.
    userSettings = UserSettings()

    # Get command line options.
    opt = Opt( userSettings )

    # Load all unit test functions named test_* from UnitTest class, run the tests and exit.
    if userSettings.UNITTEST:
        suite = unittest.TestLoader().loadTestsFromTestCase( UnitTest )
        unittest.TextTestRunner(verbosity=2).run( suite )
        sys.exit()

    # Start logging to file.  Verbose turns on logging for
    # DEBUG, INFO, WARNING, ERROR, and CRITICAL levels.
    # Otherwise we log only WARNING, ERROR, and CRITICAL levels.
    if userSettings.VERBOSE:
        loglevel = logging.DEBUG
    else:
        loglevel = logging.WARNING

    # Pick the log file name.
    if userSettings.CLEANONLY:
        userSettings.LOGFILENAME = "logMaster.txt"
    else:
        userSettings.LOGFILENAME = "logPrimary.txt"

    logging.basicConfig( level=loglevel,
                         format='%(asctime)s %(levelname)-8s %(message)s',
                         datefmt='%a, %d %b %Y %H:%M:%S',
                         filename=userSettings.LOGFILENAME,
                         filemode='w' )

    logging.debug( "*** Begin logging ******************************" )

    #---------------------------------------------------------------------
    #  Scan the master web site, finding out all files and directories.
    #---------------------------------------------------------------------
    try:
        logging.debug( "Scanning master (local on disk) web site" )
        master = MasterWebSite( userSettings )

        print( "Scanning and cleaning local web site...", end='', flush=True ) ; # Suppress newline and flush output buffer so we can see the message right away.

        master.scan()

        # Clean up the directory by rewriting source code and hypertext and
        # removing temporary files.
        logging.debug( "Cleaning up master (local on disk) web site" )
        changed = master.clean()

        # Rescan if any changes happened.
        if changed:
            logging.debug( "Detected changes due to to cleanup." )
            master.quit()
            logging.debug( "Disposing of the old scan." )
            del master ;

            master = MasterWebSite( userSettings )
            logging.debug( "*** Rescanning ****************************" )
            master.scan()
        else:
            logging.debug( "No changes detected.  Keeping the original scan." )

        print( "...done!", flush=True ) ;

        # Master web site directories.
        masterDirectoryList  = master.directories

        # Master web site filenames only.
        masterFilesList = [ fileInfo[ userSettings.FILE_NAME ] for fileInfo in master.files ]

        logging.debug( "*** Master Directories **********************" )
        for d in masterDirectoryList:  logging.debug( "\t {0:s} (directory)".format( d ))

        logging.debug( "*** Master Files **********************" )
        for f in masterFilesList: logging.debug( "\t {0:s} (file)".format( f ))

        master.quit()

        # Clean up master web site only.  Don't update remote web sites.
        if userSettings.CLEANONLY:
            logging.debug( "Cleanup finished.  Exiting..." )
            sys.exit()

        #---------------------------------------------------------------------
        #  Scan the remote hosted web site.
        #---------------------------------------------------------------------

        logging.debug( "Reading private settings." )
        privateSettings = userSettings.privateSettings

        print( "Scanning remote web site...", end='', flush=True ) ;

        # Pick which web site to update.
        logging.debug( "Connecting to primary remote site." )
        remote = RemoteWebSite( userSettings,
                                privateSettings[userSettings.SERVER], privateSettings[userSettings.USER],
                                privateSettings[userSettings.PASSWORD], privateSettings[userSettings.FTP_ROOT] )

        logging.debug( "Scanning remote web site" )
        remote.scan()
        remote.quit()

        print( "...done!", flush=True ) ;

        remoteDirectoryList  = remote.directories
        remoteFilesList = [ fileInfo[ userSettings.FILE_NAME ] for fileInfo in remote.files ]

        logging.debug( "*** Remote Directories **********************" )
        for d in remoteDirectoryList: logging.debug( "\t remote dir:  {0:s}".format( d ))

        logging.debug( "*** Remote Files **********************" )
        for f in remoteFilesList: logging.debug( "\t remote file: {0:s}".format( f ))

        #---------------------------------------------------------------------
        # Synchronize the master and remote web sites.
        #---------------------------------------------------------------------

        print( "Synchronizing remote and local web sites...", end='', flush=True ) ;

        # Primary web site.
        logging.debug( "Connecting to primary remote site for synchronization." )
        u = UpdateWeb( userSettings,
                       privateSettings[userSettings.SERVER], privateSettings[userSettings.USER],
                       privateSettings[userSettings.PASSWORD], privateSettings[userSettings.FTP_ROOT],
                       privateSettings[userSettings.FILE_SIZE_LIMIT],
                       master.directories, master.files,
                       remote.directories, remote.files )

        logging.debug( "Synchronizing remote web site" )
        u.update()
        u.quit()

        print( "...done!", flush=True ) ;

        del u
        del remote
        del master ;

    except RecursionError as detail:
        logging.error( "Walking the directory tree got too deep for Python's recursion {0:s}.  Aborting...".format( str( detail ) ))
        sys.exit()

    return

#----------------------------------------------------------------------------
#  Command line option class
#----------------------------------------------------------------------------

class Opt( object ):
    """Get the command line options."""

    def __init__( self, userSettings ):
        """Get command line options"""
        commandLineParser = optparse.OptionParser()

        # Log all changes, not just warnings and errors.
        commandLineParser.add_option( "-v", "--verbose", dest="verbose",
                                      help="Turn on verbose mode to log everything",
                                      action="store_true" )

        commandLineParser.add_option( "-c", "--cleanonly", dest="cleanonly",
                                      help="Do a cleanup on the master web site only.",
                                      action="store_true" )

        commandLineParser.add_option( "-t", "--test", dest="test",
                                      help="Run unit tests of functions.",
                                      action="store_true" )

        (options, args) = commandLineParser.parse_args()

        if len(args) >= 1:
            commandLineParser.error( "ERROR:  updateweb.py should not have any arguments:  do python updateweb.py --help" )

        if options.verbose:
            userSettings.VERBOSE = True

        if options.cleanonly:
            userSettings.CLEANONLY  = True

        if options.test:
            userSettings.UNITTEST  = True

#----------------------------------------------------------------------------
#  Base class for web site processing.
#----------------------------------------------------------------------------

class WebSite( object ):
    """
    Abstract class used for analyzing both master (local to disk) and remote (ftp server) web sites.
    Contains the common web-walking functions which traverse the directory structures and files.
    Subclasses fill in the lower level functions which actually access the directories and files.
    Subclasses may also define additional functions unique to local web sites.
    """

    def __init__( self, settings ):
        """Set up root directories"""

        # Import the user settings.
        self.userSettings = settings

        # Queue keeps track of directories not yet processed.
        self.queue       = []

        # List of all directories traversed.
        self.directories = []

        # List of files traversed, with file information.
        self.files       = []

        # Find out the root directory and go there.
        self.rootDir = self.getRootDir()
        self.gotoRootDir( self.rootDir )

    def getCurrentYear( self ):
        """Get the current year."""
        return (int)(time.gmtime()[0])

    def getCurrentTwoDigitYear( self ):
        """Get the last two digits of the current year."""
        return self.getCurrentYear() % 100

    def isFileInfoType( self, fileInfo ):
        "Check if we have a file information structure or merely a simple file name."
        try:
            if isinstance( fileInfo, list ):
                return True
            elif isinstance( fileInfo, str ):
                return False
            else:
                logging.error( "isFileInfoType found a bad type.  Aborting..." )
                sys.exit()
        except TypeError as detail:
            logging.error( "isFileInfoType found a bad type {0:s}.  Aborting...".format( str( detail ) ))
            sys.exit()

    def getRootDir( self ):
        """Subclass:  Put code here to get the root directory"""
        return ""

    def gotoRootDir( self, root="" ):
        """Subclass:  Put code here to go to the root directory"""
        pass # Pythons's do-nothing statement.

    def oneLevelDown( self, dir ):
        """Subclass:  Fill in with a method which returns a list of the
        directories and files immediately beneath dir"""
        pass

    def walk( self, dir, type=TreeWalk.BREADTH_FIRST_SEARCH ):
        """Walk a directory in either depth first or breadth first order.  BFS is the default."""

        # Get all subfiles and subdirectories off this node.
        subdirectories, subfiles = self.oneLevelDown( dir )

        # Add all the subfiles in order.
        for f in subfiles:

            name = self.stripRoot( f )
            logging.debug( "Webwalking:  Adding file {0:s} to list.".format( name[self.userSettings.FILE_NAME] ))

            # Some files are private so skip them from consideration.
            pat=re.compile( self.userSettings.FILE_TO_SKIP )

            if pat.search( name[self.userSettings.FILE_NAME] ):
                logging.warning( "Webwalking:  Skipping private file {0:s}".format( name[self.userSettings.FILE_NAME] ))
            # Don't upload the log file due to file locking problems.
            elif name[self.userSettings.FILE_NAME].find( self.userSettings.LOGFILENAME ) >= 0:
                logging.debug( "Webwalking:  Skipping log file {0:s}".format( name[self.userSettings.FILE_NAME] ))
            # File size limit on some servers.
            else:
                self.files.append( name )

        # Queue up the subdirectories.
        for d in subdirectories:

            # Some directories are private so skip them from consideration.
            pat=re.compile( self.userSettings.DIR_TO_SKIP )
            if pat.search( d ):
                logging.warning( "Webwalking:  Skipping private dir {0:s}".format( d ))
            else:
                logging.debug( "Webwalking:  Pushing dir {0:s} on the queue.".format( d ))
                self.queue.append( d )

        # Search through the directories.
        while len( self.queue ) > 0:
            # For breadth first search, remove from beginning of queue.
            if type == TreeWalk.BREADTH_FIRST_SEARCH:
                d = self.queue.pop(0)

            # For depth first search, remove from end of queue.
            elif type == TreeWalk.DEPTH_FIRST_SEARCH:
                d = self.queue.pop()
            else:
                d = self.queue.pop(0)

            name = self.stripRoot( d )
            logging.debug( "Webwalking:  Adding relative directory {0:s} to list, full path = {1:s}.".format( name, d ) )
            self.directories.append( name )

            self.walk( d )

    def stripRoot( self, fileInfo ):
        """Return a path, but strip off the root directory"""

        root = self.rootDir

        # Extract the file name.
        if self.isFileInfoType( fileInfo ):
            name = fileInfo[ self.userSettings.FILE_NAME ]
        else:
            name = fileInfo

        # e.g. root = / and name = /Art/foo.txt yields stripped_path = Art/foo.txt
        # but root = /Sean and name = /Sean/Art/foo.txt yields stripped_path = Art/foo.txt
        lenroot = len( root )
        if root == self.userSettings.DEFAULT_ROOT_DIR:
            pass
        else:
            lenroot = lenroot + 1

        stripped_path = name[ lenroot: ]

        if self.isFileInfoType( fileInfo ):
            # Update the file name only.
            return [stripped_path, fileInfo[ self.userSettings.FILE_TYPE ],
                    fileInfo[ self.userSettings.FILE_DATE_TIME ], fileInfo[ self.userSettings.FILE_SIZE ]]
        else:
            return stripped_path

    def appendRootDir( self, rootDir, name ):
        """Append the root directory to a path"""

        # e.g. root = /, and name = Art/foo.txt yields /Art/foo.txt
        # but root = /Sean, and name = Art/foo.txt yields /Sean/Art/foo.txt
        if rootDir == self.userSettings.DEFAULT_ROOT_DIR:
            return rootDir + name
        else:
            return rootDir + "/" + name

    def scan(self):
        """Scan the directory tree recursively from the root"""
        logging.debug( "Webwalking:  Beginning recursive directory scan from root directory {0:s}".format( self.rootDir ))
        self.walk( self.rootDir )

    def modtime( self, f ):
        """Subclass:  Get file modification time"""
        pass

    def quit( self ):
        """Quit web site walking"""
        logging.debug( "Finished webwalking the master." )
        pass

    def removeDirectory( self, dirName ):
        """Subclass:  Remove a directory"""
        pass

    def removeFile( self, fileName ):
        """Subclass:  Remove a file"""
        pass

    def clean( self ):
        """Scan through all directories and files in the master on disk web site and process them."""
        numChanges = 0

        logging.debug( "Cleaning up the master web page." )

        if self.directories == None or self.files == None:
            logging.error( "Web site has no directories or files.  Aborting..." )
            sys.exit()

        for d in self.directories:

            if self.isTempDir( d ):
                # Add the full path prefix from the root.
                name = self.appendRootDir( self.getRootDir(), d )
                try:
                    logging.debug( "Removing temp dir {0:s} recursively".format( name ))
                    shutil.rmtree( name )
                    numChanges += 1
                except OSError as detail:
                    logging.error( "Cannot remove temp dir {0:s}: {1:s}".format( name, str( detail ) ))

        for f in self.files:

            # Add the full path prefix from the root.
            name = self.appendRootDir( self.getRootDir(), f[ self.userSettings.FILE_NAME ] )

            # Remove all temporary files.
            if self.isTempFile( f ):
                try:
                    logging.debug( "Removing temp file {0:s}".format( name ))
                    os.remove( name )
                    numChanges += 1
                except OSError as detail:
                    logging.error( "Cannot remove temp dir {0:s}: {1:s}".format(  name, str( detail ) ))

            # Update hypertext files.
            if self.isHypertextFile( f ):
                changed = self.rewriteHypertextFile( name )
                if changed:
                    numChanges += 1
                    logging.debug( "Rewrote hypertext file {0:s}".format( name ))

            # Update source code files.
            if self.isSourceFile( f ):
                changed = self.rewriteSourceFile( name )
                if changed:
                    numChanges += 1
                    logging.debug( "Rewrote source file {0:s}".format( name ))
                # After updating, copy to a text file.
                self.copyToTextFile( name )
                logging.debug( "Created a copy of the source file {0:s}{1:s}".format( name, self.userSettings.TEXT_FILE_EXT))

        # Flag that at least one file was changed.
        if numChanges > 0:
            return True

        return False

    def isTempFile( self, fileInfo ):
        """Identify a file name as a temporary file"""

        fileName = fileInfo[ self.userSettings.FILE_NAME ]

        # Suffixes and names for temporary files be deleted.
        [pat, match] = patternMatch( self.userSettings.TEMP_FILE_SUFFIXES, fileName )
        # Remove any files containing twiddles anywhere in the name.
        if match or fileName.find( self.userSettings.VIM_TEMP_FILE_EXT ) >= 0:
            return True

        return False

    def isTempDir( self, dirName ):
        """Identify a name as a temporary directory."""

        p = re.compile( self.userSettings.TEMP_DIR_SUFFIX, re.VERBOSE )
        return p.search( dirName )

    def isSourceFile(  self, fileInfo ):
        """Identify a source file name."""

        fileName = fileInfo[ self.userSettings.FILE_NAME ]
        p = re.compile( self.userSettings.SOURCE_FILE_SUFFIXES, re.VERBOSE)
        return p.search( fileName )

    def isHypertextFile( self, fileInfo ):
        """ Check if the file name is a hypertext file."""

        fileName = fileInfo[ self.userSettings.FILE_NAME ]
        p = re.compile( self.userSettings.HYPERTEXT_SUFFIX, re.VERBOSE)
        return p.search( fileName )

    def copyToTextFile( self, fileName ):
        """Make a copy of a file with a .txt extension"""
        pass

    def rewriteSourceFile( self, fileName ):
        """Rewrite both copyright lines, etc."""
        pass

    def cleanUpTempFile( self, tempFileName, fileName, changed ):
        """Remove the original file, rename the temporary file name to the original name.
        If there are no changes, just remove the temporary file.
        """
        pass

    def processLinesOfFile( self, inFileName, outFileName, processLineFunctionList=None ):
        """Process each line of a file with a list of functions.  Create a new temporary file.
        The default list is None which means make an exact copy.
        """
        pass

    def rewriteSubstring( self, line ):
        """Rewrite a line containing a pattern of your choice"""

        # Search for the old email address.
        [pat, match] = patternMatch( self.userSettings.OLD_SUBSTRING, line )

        # Replace the old address with my new email address.
        if match:
            newSubstring = self.userSettings.NEW_SUBSTRING
            sub = pat.sub( newSubstring, line )
            line = sub

        return line

    def rewriteEmailAddressLine( self, line ):
        """Rewrite lines containing old email addresses."""

        # Search for the old email address.
        [pat, match] = patternMatch( self.userSettings.OLD_EMAIL_ADDRESS, line )

        # Replace the old address with my new email address.
        if match:
            newAddress = self.userSettings.NEW_EMAIL_ADDRESS
            sub = pat.sub( newAddress, line )
            line = sub

        return line

    def rewriteVersionLine( self, line ):
        """Rewrite lines containing old version of software."""

        # Search for the old version.
        [pat, match] = patternMatch( self.userSettings.OLD_SOFTWARE_VERSION, line )

        # Replace the old address with my new email address.
        if match:
            newVersion = self.userSettings.NEW_SOFTWARE_VERSION
            sub = pat.sub( newVersion, line )
            line = sub

        return line

    def rewriteCopyrightLine( self, line ):
        """Rewrite copyright lines if they are out of date."""

        # Match the lines,
        #     Copyright (C) nnnn-mmmm by Sean Erik O'Connor.
        #     Copyright &copy; nnnn-mmmm by Sean Erik O'Connor.
        # and pull out the old year and save it.
        [pat, match] = patternMatch( self.userSettings.COPYRIGHT_LINE, line )

        # Found a match.
        if match:
            oldYear = int( match.group( 'oldYear' ))

            # Replace the old year with the current year.  We matched and extracted the
            # old copyright symbol into the variable 'symbol'.  We now insert it back using
            # the replacement text syntax with \g<symbol>.
            if oldYear < self.getCurrentYear():
                newCopyright = 'Copyright \g<symbol> \g<oldYear>-' + str( self.getCurrentYear() )
                sub = pat.sub( newCopyright, line )
                line = sub
        # Look for the other type of copyright line.
        else:
            #     Copyright (C) nnnn by Sean Erik O'Connor.
            #     Copyright &copy; mmmm by Sean Erik O'Connor.
            [pat, match] = patternMatch( self.userSettings.COPYRIGHT_LINE2, line )

            # Found a match.
            if match:
                oldYear = int( match.group( 'oldYear' ))

                # Replace the old year with the current year.
                if oldYear < self.getCurrentYear():
                    newCopyright = 'Copyright \g<symbol> ' + str( self.getCurrentYear() )
                    sub = pat.sub( newCopyright, line )
                    line = sub

        return line

    def rewriteLastUpdateLine( self, line ):
        """Rewrite the Last Updated line if the year is out of date."""

        # Match the last updated line and pull out the year.
        #      last updated 01 Jan 17.
        p = re.compile( self.userSettings.LAST_UPDATED_LINE, re.VERBOSE | re.IGNORECASE )
        m = p.search( line )

        if m:
            lastUpdateYear = int( m.group( 'year' ))

            # Convert to four digit years.
            if lastUpdateYear > 90:
                lastUpdateYear += 1900
            else:
                lastUpdateYear += 2000

            # If the year is old, rewrite to "01 Jan <current year>".
            if lastUpdateYear < self.getCurrentYear():
                twoDigitYear = self.userSettings.TWO_DIGIT_YEAR_FORMAT % self.getCurrentTwoDigitYear()
                sub = p.sub( 'last updated 01 Jan ' + twoDigitYear, line )
                line = sub

        return line

    def rewriteHypertextFile( self, fileName ):
        """Rewrite copyright lines, last updated lines, etc."""
        changed = False

        # Create a new temporary file name for the rewritten file.
        tempFileName = fileName + self.userSettings.TEMP_FILE_EXT

        # Apply changes to all lines of the file.  Apply change functions in the sequence listed.
        if self.processLinesOfFile( fileName, tempFileName,
                                    [self.rewriteCopyrightLine,
                                     self.rewriteLastUpdateLine,
                                     self.rewriteEmailAddressLine,
                                     self.rewriteSubstring,
                                     self.rewriteVersionLine] ):
            changed = True

        # Rename the temp file to the original file name.  If no changes, just delete the temp file.
        self.cleanUpTempFile( tempFileName, fileName, changed )

        return changed



#----------------------------------------------------------------------------
#  Subclass for local web site processing.
#----------------------------------------------------------------------------

class MasterWebSite( WebSite ):
    """Walk the master web directory on local disk down from the root.  Clean up temporary files and do other cleanup work."""


    def __init__( self, settings ):
        """Go to web page root and list all files and directories."""

        # Initialize the parent class.
        WebSite.__init__( self, settings )

        self.rootDir = self.getRootDir()
        logging.debug( "MasterWebSite.__init__():  \tRoot directory: {0:s}".format( self.rootDir))

    def getRootDir( self ):
        """Get the name of the root directory"""
        return self.userSettings.masterRootDir

    def gotoRootDir( self, rootDir ):
        """Go to the root directory"""

        # Go to the root directory.
        logging.debug( "MasterWebSite.gotoRootDir():  \tchdir to root directory:  {0:s}".format( rootDir))
        os.chdir( rootDir )

        # Read it back.
        self.rootDir = os.getcwd()
        logging.debug( "MasterWebSite.gotoRootDir():  \tgetcwd root directory:  {0:s}".format( self.rootDir ))

    def oneLevelDown( self, dir ):
        """List all files and subdirectories in the current directory, dir.  For files, collect file info
        such as time, date and size."""
        # Change to current directory.
        os.chdir( dir )

        # List all subdirectories and files.
        dirList = os.listdir( dir )
        directories = []
        files = []

        if dirList:
            for line in dirList:
                logging.debug( "MasterWebSite.oneLevelDown():  \tlistdir( {0:s} ) =  {1:s}".format( dir, line ))

                # Add the full path prefix from the root.
                name = self.appendRootDir( dir,  line )
                logging.debug( "MasterWebSite.oneLevelDown():  \tmaster dir/file (full path): {0:s}".format( name))

                # Is it a directory or a file?
                if os.path.isdir( name ):
                    directories.append( name )
                elif os.path.isfile( name ):
                    # First assemble the file information of name, time/date and size into a list.  Can index it like an array.
                    # e.g. fileInfo = [ '/WebDesign/EquationImages/equation001.png', 1, datetime.datetime(2010, 2, 3, 17, 15), 4675]
                    #     fileInfo[ 0 ] = '/WebDesign/EquationImages/equation001.png'
                    #     fileInfo[ 3 ] = 4675
                    fileInfo = [name,
                                FileType.FILE,
                                self.getFileDateTime( name ),
                                self.getFileSize( name ) ]
                    files.append( fileInfo )

        # Sort the names into order.
        if directories:
            directories.sort()
        if files:
            files.sort()

        return directories, files

    def getFileDateTime( self, fileName ):
        """Get a local file time and date in UTC."""

        fileEpochTime = os.path.getmtime( fileName )
        fileTimeUTC = time.gmtime( fileEpochTime )[ 0 : 6 ]
        # year, month,   day, hour,   minute, seconds
        d = datetime.datetime( fileTimeUTC[0], fileTimeUTC[1],
                               fileTimeUTC[2], fileTimeUTC[3],
                               fileTimeUTC[4], fileTimeUTC[5])
        return d

    def getFileSize( self, fileName ):
        """Get file size in bytes."""
        return os.path.getsize( fileName )

    def copyToTextFile( self, fileName ):
        """Make a copy of a file with a .txt extension"""

        # Remove the old copy with the text file extension.
        copyFileName = fileName + self.userSettings.TEXT_FILE_EXT
        try:
            os.remove( copyFileName )
        except OSError as detail:
            logging.error( "Cannot remove old text file copy {0:s}: {1:s}".format(  copyFileName, str( detail ) ) )

        # Create the new copy, which is an exact duplicate.
        self.processLinesOfFile( fileName, copyFileName )

        # Make the new copy have the same modification and access time and date as the original
        # since it is just an exact copy.
        # That way we won't upload copies with newer times constantly, just because they look as
        # though they've been recently modified.
        fileStat = os.stat( fileName ) ;
        os.utime( copyFileName, (fileStat[stat.ST_ATIME], fileStat[stat.ST_MTIME]))
        logging.debug( "Reset file time to original time for copy {0:s}".format( copyFileName ) )

    def rewriteSourceFile( self, fileName ):
        """Rewrite both copyright lines, etc."""
        changed = False

        # Create a temporary file name for the rewritten file.
        tempFileName = fileName + self.userSettings.TEMP_FILE_EXT

        # Apply changes to all lines of the file.  Apply change functions in the sequence listed.
        if self.processLinesOfFile( fileName, tempFileName,
                                    [self.rewriteCopyrightLine,
                                     self.rewriteEmailAddressLine,
                                     self.rewriteVersionLine] ):
            changed = True

        # Rename the temp file to the original file name.  If no changes, just delete the temp file.
        self.cleanUpTempFile( tempFileName, fileName, changed )

        return changed

    def cleanUpTempFile( self, tempFileName, fileName, changed ):
        """Remove the original file, rename the temporary file name to the original name.
        If there are no changes, just remove the temporary file.
        """

        if changed:
            # Remove the old file now that we have the rewritten file.
            try:
                os.remove( fileName )
                logging.debug( "Changes were made.  Remove original file {0:s}".format( fileName ))
            except OSError as detail:
                logging.error( "Cannot remove old file {0:s}: {1:s}.  Need to remove it manually.".format( fileName, str( detail ) ) )

            # Rename the new file to the old file name.
            try:
                os.rename( tempFileName, fileName )
                logging.debug( "Rename temp file {0:s} to original file {1:s}".format( tempFileName, fileName ))
            except OSError as detail:
                logging.error( "Cannot rename temporary file {0:s} to old file name {1:s}: {2:s}.  Need to do it manually".format( tempFileName, fileName, str( detail ) ))
        else:
            # No changes?  Remove the temporary file.
            try:
                os.remove( tempFileName )
                logging.debug( "No changes were made.  Remove temporary file {0:s}".format( tempFileName ))
            except OSError as detail:
                logging.error( "Cannot remove temporary file {0:s}: {1:s}.  Need to remove it manually.".format( (tempFileName, str( detail ))) )
        return

    def processLinesOfFile( self, inFileName, outFileName, processLineFunctionList=None ):
        """Process each line of a file with a list of functions.  Create a new temporary file.
        The default list is None which means make an exact copy.
        """

        # Assume no changes.
        changed = False

        try:
            fin = open( inFileName, "r" )
        except IOError as detail:
            logging.error( "processLinesOfFile():  \tCannot open file {0:s} for reading:  {1:s}".format( inFileName, str( detail ) ) )

        try:
            fout = open( outFileName, "w" )
        except IOError as detail:
            logging.error( "processLinesOfFile():  \tCannot open file {0:s} for writing:  {1:s}".format( outFileName, str( detail ) ) )

        # Read each line of the file, aborting if there is a read error.
        try:
            line = fin.readline()

            while line:
                original_line = line
                if processLineFunctionList == None:
                    # For a simple copy, just duplicate the line unchanged.
                    pass
                else:
                    # Otherwise, apply changes in succession to the line.
                    for processLineFunction in processLineFunctionList:
                        line = processLineFunction( line )

                if original_line != line:
                    logging.debug( "Rewrote the line >>>{0:s}<<< to >>>{1:s}<<<".format( original_line, line ) )
                    changed = True

                fout.write( line )

                line = fin.readline()

            fin.close()
            fout.close()
        except IOError as detail:
            logging.error(  "File I/O error during reading/writing file {0:s} in processLinesOfFile: {1:s}  Aborting...".format( inFileName, str( detail ) ) )
            sys.exit()

        if changed:
            logging.debug( "processLinesOfFile():  \tRewrote original file {0:s}.  Changes are in temporary copy {1:s}".format( inFileName, outFileName ) )

        # Return True if any lines were changed.
        return changed

#----------------------------------------------------------------------------
#   Subclass for remote web site processing.
#----------------------------------------------------------------------------

class RemoteWebSite( WebSite ):
    """Walk the remote web directory on a web server down from the root."""


    def __init__( self, settings, server, user, password, ftproot ):
        """Connect to FTP server and list all files and directories."""

        # Root directory of FTP server.
        self.rootDir = ftproot
        logging.debug( "Requesting remote web site ftp root dir {0:s}".format( self.rootDir ))

        # Connect to FTP server and log in.
        try:
            #self.ftp.set_debuglevel( 2 )
            self.ftp = ftplib.FTP( server )
            self.ftp.login( user, password )
        # Catch all exceptions with the parent class Exception:  all built-in, non-system-exiting exceptions are derived from this class.
        except Exception as detail:
            # Extract the string message from the exception class with str().
            logging.error( "Remote web site cannot login to ftp server: {0:s}  Aborting...".format( str( detail ) ))
            sys.exit()
        else:
            logging.debug( "Remote web site ftp login succeeded." )

        logging.debug( "Remote web site ftp welcome message {0:s}".format( self.ftp.getwelcome() ))

        # Initialize the superclass.
        WebSite.__init__( self, settings )



    def gotoRootDir( self, root ):
        """Go to the root directory"""

        try:
            # Go to the root directory.
            self.ftp.cwd( root )
            logging.debug( "ftp root directory (requested) = {0:s}".format( self.rootDir ))

            # Read it back.
            self.rootDir = self.ftp.pwd()
            logging.debug( "ftp root directory (read back from server): {0:s}".format( self.rootDir ))

        except Exception as detail:
            logging.error( "gotoRootDir(): \tCannot ftp cwd or pwd root dir {0:s} Aborting...".format( (root, str( detail ) )))
            sys.exit()


    def getRootDir( self ):
        """Get the root directory name"""

        return self.rootDir



    def quit(self):
        """Quit web site walking"""

        logging.debug( "Quitting remote site." )
        try:
            self.ftp.quit()
        except Exception as detail:
            logging.error( "Cannot ftp quit: {0:s}".format( str( detail ) ))



    def oneLevelDown( self, dir ):
        """List files and directories in a subdirectory using ftp"""

        try:
            # ftp listing from current dir.
            logging.debug( "RemoteWebSite.oneLevelDown():  \tftp cwd: {0:s}".format( dir))
            self.ftp.cwd( dir )
            dirList = []

            self.ftp.retrlines( 'LIST', dirList.append )
        except Exception as detail:
            logging.error( "oneLevelDown(): \tCannot ftp cwd or ftp LIST dir {0:s}:  {1:s} Aborting...".format( dir, str( detail )  ))
            sys.exit()

        directories = []
        files = []
        for line in dirList:
            logging.debug( "RemoteWebSite.oneLevelDown():  \tftp LIST: {0:s}".format( line))

            # Line should at least have the minimum FTP information.
            if len(line) >= self.userSettings.MIN_FTP_LINE_LENGTH:
                fileInfo = self.getFTPFileInformation( line )

                if fileInfo[ self.userSettings.FILE_NAME ] == "":
                    logging.error( "RemoteWebSite.oneLevelDown():  \tFTP LIST file name is NULL:" )

                logging.debug( "RemoteWebSite.oneLevelDown():  \tftp parsed file info: {0:s}".format( fileInfo[ self.userSettings.FILE_NAME ] ))

                # Prefix the full path prefix from the root to the directory name and add to the directory list.
                if fileInfo[ self.userSettings.FILE_TYPE ] == FileType.DIRECTORY:
                    dirname = self.appendRootDir( dir , fileInfo[ self.userSettings.FILE_NAME ] )
                    logging.debug( "RemoteWebSite.oneLevelDown():  \tftp dir (full path): {0:s}".format( dirname ))
                    directories.append( dirname )
                # Add file information to the list of files.
                else:
                    # Update the file name only:  add the full path prefix from the root.
                    fileInfo[ self.userSettings.FILE_NAME ] = self.appendRootDir( dir,  fileInfo[ self.userSettings.FILE_NAME ] )
                    logging.debug( "RemoteWebSite.oneLevelDown():  \tftp file (full path): {0:s}".format( fileInfo[ self.userSettings.FILE_NAME ] ))
                    files.append( fileInfo )
            else:
                logging.error( "RemoteWebSite.oneLevelDown():  \tFTP LIST line is too short:  {0:s}".format( line ))
        directories.sort()
        files.sort()
        return directories, files



    def modtime( self, f ):
        """Get the modification time of a file via ftp.  Return 0 if ftp cannot get it."""

        try:
            response = self.ftp.sendcmd( 'MDTM ' + f )
            # MDTM returns the last modified time of the file in the format
            # "213 YYYYMMDDhhmmss \r\n <error-response>
            # MM is 01 to 12, DD is 01 to 31, hh is 00 to 23, mm is 00 to 59, ss is 0 to 59.
            # error-response is 550 for info not available, and 500 or 501 if command cannot
            # be parsed.
            if response[:3] == '213':
                time = response[4:]
        except ftplib.error_perm:
            time = 0

        return time



    def getFTPFileInformation( self, line ):
        """Parse the ftp file listing and return file name, datetime and file size"""

        # Find out if we've a directory or a file.
        if line[0] == 'd':
            type = FileType.DIRECTORY
        else:
            type = FileType.FILE

        pattern = re.compile( self.userSettings.FTP_LISTING, re.VERBOSE )

        # Sensible defaults.
        filesize = 0
        filename = ""
        hour    = 0
        minute  = 0
        seconds = 0
        month   = 1
        day     = 1
        year    = self.getCurrentYear()

        # Extract time and date from the ftp listing.
        match = pattern.search( line )

        if match:
            filesize = int( match.group( 'bytes' ) )
            month    = self.userSettings.monthToNumber[ match.group( 'mon' ) ]
            day      = int( match.group( 'day' ) )

            # Pull out the year if we have it.  If so, the FTP listing will omit the month and day, so
            # let them default to Jan 1 as above.
            if match.group( 'year' ):
                year = int( match.group( 'year' ) )

            # If the FTP listing has no year, get the hour and minute.  Assume the current year;  which was set already above.
            if match.group( 'hour' ) and match.group( 'min' ):
                hour   = int( match.group( 'hour' ) )
                minute = int( match.group( 'min' ) )

            filename = match.group( 'filename' )

        # Package up the time and date nicely.
        d = datetime.datetime( year, month, day, hour, minute, seconds )

        return [filename, type, d, filesize]



class UpdateWeb( object ):
    """Given previously scanned master and remote directories, update the remote web site."""


    def __init__( self, settings, server, user, password, ftproot, fileSizeLimit,
                  masterDirectoryList, masterFileInfo, remoteDirectoryList, remoteFileInfo ):
        """Connect to remote site.  Accept previously scanned master and remote files and directories."""

        self.userSettings = settings

        # Connect to FTP server and log in.
        try:
            self.ftp = ftplib.FTP( server )
            self.ftp.login( user, password )
        except Exception as detail:
            logging.error( "Cannot login to ftp server: {0:s} Aborting...".format( str( detail )  ))
            sys.exit()
        else:
            logging.debug( "ftp login succeeded." )

        logging.debug( "ftp server welcome message:  {0:s}".format( self.ftp.getwelcome() ))

        # Master root directory.
        self.masterRootDir = self.userSettings.masterRootDir
        logging.debug( "Master (local to disk) root directory: {0:s}".format( self.masterRootDir))

        # Root directory of FTP server.
        self.ftpRootDir = ftproot
        logging.debug( "ftp root directory (requested) = {0:s}".format( self.ftpRootDir ))

        # Transform KB string to integer bytes.  e.g. "200" => 2048000
        self.fileSizeLimit = int( fileSizeLimit ) * 1024

        try:
            # Go to the root directory.
            self.ftp.cwd( self.ftpRootDir )

            # Read it back.
            self.ftpRootDir = self.ftp.pwd()
            logging.debug( "ftp root directory (read back from server): {0:s}".format( self.ftpRootDir ))
        except Exception as detail:
            logging.error( "UpdateWeb(): \tCannot ftp cwd or ftp LIST dir {0:s} Aborting...".format( (self.ftpRootDir, str( detail ) )) )

        self.masterDirectoryList = masterDirectoryList
        self.remoteDirectoryList = remoteDirectoryList
        self.masterFileInfo = masterFileInfo
        self.remoteFileInfo = remoteFileInfo



    def appendRootDir( self, rootDir, name ):
        """Append the root directory to a path"""

        # e.g. root = /, and name = Art/foo.txt yields /Art/foo.txt
        # but root = /Sean, and name = Art/foo.txt yields /Sean/Art/foo.txt
        if rootDir == self.userSettings.DEFAULT_ROOT_DIR:
            return rootDir + name
        else:
            return rootDir + "/" + name



    def fileInfo( self ):
        """Extract file names from file information.  Map file names onto file dates and times."""

        # Extract file names.
        self.masterFilesList = [ fileInfo[ self.userSettings.FILE_NAME ] for fileInfo in self.masterFileInfo ]
        self.remoteFilesList = [ fileInfo[ self.userSettings.FILE_NAME ] for fileInfo in self.remoteFileInfo ]

        # Dictionary mapping file names onto datetimes.
        # e.g.   d = dict(   [ [ 'one', 1 ], ['two', 2] ]    )
        #        d =>  {'two': 2, 'one': 1}
        #        len(d) =>  2
        #        d.items() => dict_items([('two', 2), ('one', 1)])
        #        for k,v in d.items(): print( "k: {0:s} v: {1:d}".format( str( k ), str( v ) ) )
        #            => k: two v: 2
        #            => k: one v: 1
        self.masterFileToDateTime = dict( [ (fileInfo[ self.userSettings.FILE_NAME ], fileInfo[ self.userSettings.FILE_DATE_TIME ]) \
                                          for fileInfo in self.masterFileInfo ] )
        self.remoteFileToDateTime = dict( [ (fileInfo[ self.userSettings.FILE_NAME ], fileInfo[ self.userSettings.FILE_DATE_TIME ]) \
                                          for fileInfo in self.remoteFileInfo ] )

        # Dictionary mapping master file names onto sizes.
        self.masterFileToSize = dict( [ (fileInfo[ self.userSettings.FILE_NAME ], fileInfo[ self.userSettings.FILE_SIZE ]) \
                                        for fileInfo in self.masterFileInfo ] )



    def update( self ):
        """Scan through the master web site, cleaning it up.
        Go to remote web site on my servers and synchronize all files.
        """

        self.fileInfo()

        # Which files and directories are different.
        self.changes()

        # Synchronize with the master.
        self.synchronize()



    def changes( self ):
        """Find the set of different directories and files on master and remote."""

        # Enter all master directories into the dictionary.
        dir_to_type = {}
        dir_to_type = dict( [ [ dir, FileType.ON_MASTER_ONLY ] for dir in self.masterDirectoryList ] )

        # Scan through all remote directories.
        for dir in self.remoteDirectoryList:
            if dir in dir_to_type:
                dir_to_type[ dir ] = FileType.ON_BOTH_MASTER_AND_REMOTE
            else:
                dir_to_type[ dir ] = FileType.ON_REMOTE_ONLY

        # Enter all master files into the dictionary.
        file_to_type = {}
        file_to_type = dict( [ [ file, FileType.ON_MASTER_ONLY ] for file in self.masterFilesList ] )

        # Scan through all remote files.
        for file in self.remoteFilesList:
            if file in file_to_type:
                file_to_type[ file ] = FileType.ON_BOTH_MASTER_AND_REMOTE
            else:
                file_to_type[ file ] = FileType.ON_REMOTE_ONLY

        logging.debug( "Raw dictionary dump of directories" )
        for k, v in dir_to_type.items(): logging.debug( "\t dir:  {0:s}  type: {1:s}".format( str( k ), str( v ) ))
        logging.debug( "Raw dictionary dump of files" )
        for k, v in file_to_type.items(): logging.debug( "\t file: {0:s}  type: {1:s}".format( str( k ), str( v ) ))

        # Create a list of master only directories keeping the ordering.
        self.masterOnlyDirs = []
        for dir in self.masterDirectoryList:
            if dir_to_type[ dir ] == FileType.ON_MASTER_ONLY:
                self.masterOnlyDirs.append( dir )

        # Create a list of remote directories keeping the ordering.
        self.remoteOnlyDirs = []
        for dir in self.remoteDirectoryList:
            if dir_to_type[ dir ] == FileType.ON_REMOTE_ONLY:
                self.remoteOnlyDirs.append( dir )

        # We don't care about common directories.

        # Create a list of master only files for master only keeping the ordering.
        self.masterOnlyFiles = []
        for file in self.masterFilesList:
            if file_to_type[ file ] == FileType.ON_MASTER_ONLY:
                self.masterOnlyFiles.append( file )

        # Create a list of remote only files for remote only keeping the ordering.
        self.remoteOnlyFiles = []
        for file in self.remoteFilesList:
            if file_to_type[ file ] == FileType.ON_REMOTE_ONLY:
                self.remoteOnlyFiles.append( file )

        # Create a list of common files for common files keeping the ordering.
        self.commonFiles = []
        for file in self.masterFilesList:
            if file_to_type[ file ] == FileType.ON_BOTH_MASTER_AND_REMOTE:
                self.commonFiles.append( file )

        logging.debug( "*** Master only directories ******************************" )
        for dir in self.masterOnlyDirs: logging.debug( "\t {0:s}".format( dir ))

        logging.debug( "*** Remote only directories ******************************" )
        for dir in self.remoteOnlyDirs: logging.debug( "\t {0:s}".format( dir ))

        logging.debug( "*** Master only files ******************************" )
        for file in self.masterOnlyFiles: logging.debug( "\t {0:s}".format( file ))

        logging.debug( "*** Remote only files ******************************" )
        for file in self.remoteOnlyFiles: logging.debug( "\t {0:s}".format( file ))

        logging.debug( "*** Common files ******************************" )
        for file in self.commonFiles:
            logging.debug( "\tname {0:s} master time {1:s} remote time {2:s}".format( \
                    file, self.masterFileToDateTime[ file ].ctime(), self.remoteFileToDateTime[ file ].ctime()))

    def synchronize( self ):
        """
        Synchronize files in the remote directory with the
        master directory.
        """
        # Compare the common files for time and date.
        for f in self.commonFiles:
            masterFileTime = self.masterFileToDateTime[ f ]
            remoteFileTime = self.remoteFileToDateTime[ f ]

            # How many fractional days different are we?
            days_different = \
                    abs( (remoteFileTime - masterFileTime).days + (remoteFileTime - masterFileTime).seconds / (60.0 * 60.0 * 24.0) )
            upload = False

            logging.debug( "Common file:  {0:s}.".format( f ))

            # Remote file time is newer.
            if remoteFileTime > masterFileTime:
                # Remote file time is MUCH newer:  suspect time is out of joint on the server, so upload local master file to be safe.
                if (days_different >= self.userSettings.DAYS_NEWER_FOR_REMOTE_BEFORE_WE_SUSPECT_ITS_ACTUALLY_VERY_OLD):
                    logging.warning( "Remote file {0:s} is much newer by {1:f} days.  Time out of joint on the server?  Preparing for upload."
                                      .format(f, days_different))
                    logging.warning( "On the other hand, it might just be an old file on the master web site.  Suggest you touch the file to avoid repeated uploads." ) ;
                    logging.warning( "\tmaster time {0:s} remote time {1:s}".format( masterFileTime.ctime(), remoteFileTime.ctime() ) )
                    upload = True
                # Remote file time is only slightly newer;  probably OK, just a little time inaccuracy on the server.
                else:
                    logging.debug( "Remote file {0:s} is slightly newer by {1:f} days.  Probably a wee bit of time inaccuracy on the server.  Wait -- don't upload yet." \
                                    .format(f,days_different))
                    logging.debug( "\tmaster time {0:s} remote time {1:s}".format( masterFileTime.ctime(), remoteFileTime.ctime() ))
                    upload = False
            # Master file time is newer.
            elif masterFileTime > remoteFileTime:
                # Master file time is newer (by several minutes), that it's likely to be changed;  upload.
                if (days_different >= self.userSettings.DAYS_NEWER_FOR_MASTER_BEFORE_UPLOAD):
                    logging.warning( "Master file {0:s} is newer by {1:f} days.  Preparing for upload.".format(f, days_different ))
                    logging.warning( "\tmaster time {0:s} remote time {1:s}".format( masterFileTime.ctime(), remoteFileTime.ctime() ))
                    upload = True
                else:
                    logging.debug( "Master file {0:s} is slightly newer by {1:f} days.  Wait -- don't upload yet.".format( f, days_different ))
                    logging.debug( "\tmaster time {0:s} remote time {1:s}".format( masterFileTime.ctime(), remoteFileTime.ctime() ))
                    upload = False

            #  But override the upload if the file is too big for the server.
            size = self.masterFileToSize[ f ]
            if size >= self.fileSizeLimit:
                logging.error( "upload():  Skipping upload of file {0:s} of size {1:d};  too large for server, limit is {2:d} bytes" \
                               .format(f, size, self.fileSizeLimit ))
                upload = False

            if upload:
                print( "Uploading changed file {0:s}...".format( f ), end='', flush=True )
                self.upload( f )

        # Remote directory is not in master.  Delete it.
        for d in self.remoteOnlyDirs:
            logging.debug( "Remote only dir.  Attempting to delete it:  {0:s}".format( d ))
            print( "Deleting remote directory {0:s}...".format( d ), end='', flush=True )
            self.rmdir( d )

        # Master directory missing on remote.  Create it.
        # Due to breadth first order scan, we'll create parent directories before child directories.
        for d in self.masterOnlyDirs:
            logging.debug( "Master only dir.  Creating dir {0:s} on remote.".format( d ))
            print( "Creating new remote directory {0:s}...".format( d ), end='', flush=True )
            self.mkdir( d )

        # Master file file missing on remote.  Upload it.
        for f in self.masterOnlyFiles:
            logging.debug( "Master only file.  Uploading {0:s} to remote.".format( f ))

            #  But override the upload if the file is too big for the server.
            size = self.masterFileToSize[ f ]
            if size >= self.fileSizeLimit:
                logging.error( "upload():  Skipping upload of file {0:s} of size {1:d};  too large for server, limit is {2:d} bytes" \
                               .format(f, size, self.fileSizeLimit ))
            else:
                print( "Uploading new file {0:s}...".format( f ), end='', flush=True )
                self.upload( f )

        # Remote contains a file not present on the master.  Delete the file.
        for f in self.remoteOnlyFiles:
            logging.debug( "Remote only file.  Deleting remote file {0:s}.".format( f ))
            print( "Deleting remote file {0:s}...".format( f ), end='', flush=True )
            self.delRemote( f )

    def delRemote( self, relativeFilePath ):
        """Delete a file using ftp."""

        logging.debug( "delRemote():  \trelative file path name: {0:s}".format( relativeFilePath ))

        # Parse the relative file path into file name and relative directory.
        relativeDir, fileName = os.path.split( relativeFilePath )
        logging.debug( "delRemote():  \tfile name: {0:s}".format( fileName ))
        logging.debug( "delRemote():  \trelative dir: {0:s}".format( relativeDir ))
        logging.debug( "delRemote():  \tremote root dir: {0:s}".format( self.ftpRootDir ))

        try:
            # Add the remote root path and go to the remote directory.
            remoteDir = self.appendRootDir( self.ftpRootDir, relativeDir )
            logging.debug( "delRemote():  \tftp cd remote dir: {0:s}".format( remoteDir ))
            self.ftp.cwd( remoteDir )
        except Exception as detail:
            logging.error( "delRemote():  \tCannot ftp chdir: {0:s}  Skipping...".format( str( detail ) ))
        else:
            try:
                logging.debug( "delRemote():  \tftp rm: {0:s}".format( fileName ))

                # Don't remove zero length file names.
                if len( fileName ) > 0:
                    self.ftp.delete( fileName )
                else:
                    logging.warning( "delRemote():  skipping ftp delete;  file NAME {0:s} had zero length".format( fileName ))
            except Exception as detail:
                logging.error( "delRemote():  \tCannot ftp rm: {0:s}".format( str( detail ) ))



    def mkdir( self, relativeDir ):
        """Create new remote directory using ftp."""

        logging.debug( "mkdir():  \trelative dir path name: {0:s}".format( relativeDir ))
        logging.debug( "mkdir():  \tremote root dir: {0:s}".format( self.ftpRootDir ))

        # Parse the relative dir path into prefix dir and suffix dir.
        path, dir = os.path.split( relativeDir )
        logging.debug( "mkdir():  \tremote prefix dir: {0:s}".format( path ))
        logging.debug( "mkdir():  \tremote dir:  {0:s}".format( dir ))

        try:
            # Add the remote root path and go to the remote directory.
            remoteDir = self.appendRootDir( self.ftpRootDir, path )
            logging.debug( "mkdir():  \tftp cd remote dir: {0:s}".format( remoteDir ))
            self.ftp.cwd( remoteDir )
        except Exception as detail:
            logging.error( "mkdir():  \tCannot ftp chrdir: {0:s}  Skipping...".format( str( detail ) ))
        else:
            try:
                logging.debug( "mkdir():  \tftp mkd: {0:s}".format( dir ))
                self.ftp.mkd( dir )
            except Exception as detail:
                logging.error( "mkdir():  \tCannot ftp mkdir: {0:s}".format( str( detail ) ))



    def rmdir( self, relativeDir ):
        """Delete an empty directory using ftp."""

        logging.debug( "rmdir():  \tintermediate dir path name: {0:s}".format( relativeDir ))
        logging.debug( "rmdir():  \tremote root dir: {0:s}".format( self.ftpRootDir ))

        # Parse the relative dir path into prefix dir and suffix dir.
        path, dir = os.path.split( relativeDir )
        logging.debug( "rmdir():  \tremote prefix dir: {0:s}".format( path ))
        logging.debug( "rmdir():  \tremote dir:  {0:s}".format( dir ))

        try:
            # Add the remote root path and go to the remote directory.
            remoteDir = self.appendRootDir( self.ftpRootDir, path )
            logging.debug( "rmdir():  \tftp cd remote dir: {0:s}".format( remoteDir ))
            self.ftp.cwd( remoteDir )
        except Exception as detail:
            logging.error( "rmdir():  \tCannot ftp chdir: {0:s}  Skipping...".format( str( detail ) ))
        else:
            try:
                logging.debug( "rmdir():  \tftp rmd: {0:s}".format( dir ))
                self.ftp.rmd( dir )
            except Exception as detail:
                logging.error( "rmdir():  \tCannot ftp rmdir dir {0:s}: {1:s}  Directory is probably not empty.  Do a manual delete." .format( dir, str( detail ) ))



    def download( self, relativeFilePath ):
        """Download a binary file using ftp."""

        logging.debug( "download():  \tfile name: {0:s}".format( relativeFilePath ))

        # Parse the relative file path into file name and relative directory.
        relativeDir, fileName = os.path.split( relativeFilePath )
        logging.debug( "download():  \tfile name: {0:s}".format( fileName ))
        logging.debug( "download():  \trelative dir: {0:s}".format( relativeDir ))
        logging.debug( "download():  \troot dir: {0:s}".format( self.ftpRootDir ))

        # Add the remote root path and go to the remote directory.
        remoteDir = self.appendRootDir( self.ftpRootDir, relativeDir )
        logging.debug( "download():  \tftp cd remote dir: {0:s}".format( remoteDir ))

        try:
            self.ftp.cwd( remoteDir )
        except Exception as detail:
            logging.error( "download():  \tCannot ftp chdir: {0:s}  Skipping...".format( str( detail ) ))
        else:
            # Add the master root path to get the local file name.
            # Open local binary file to write into.
            localFileName = self.appendRootDir( self.masterRootDir, relativeFilePath )
            logging.debug( "download():  \topen local file name: {0:s}".format( localFileName ))
            try:
                f = open( localFileName, "wb" )
                try:
                    # Calls f.write() on each block of the binary file.
                    #ftp.retrbinary( "RETR " + fileName, f.write )
                    pass
                except Exception as detail:
                    logging.error( "download():  \tCannot cannot ftp retrbinary: {0:s}".format( str( detail ) ))
                f.close()
            except IOError as detail:
                logging.error( "download():  \tCannot open local file {0:s} for reading:  {1:s}".format( localFileName, str( detail ) ))



    def upload( self, relativeFilePath ):
        """Upload  a binary file using ftp."""

        logging.debug( "upload():  \trelative file path name: {0:s}".format( relativeFilePath ))

        # Parse the relative file path into file name and relative directory.
        relativeDir, fileName = os.path.split( relativeFilePath )
        logging.debug( "upload():  \tfile name: {0:s}".format( fileName ))
        logging.debug( "upload():  \trelative dir: {0:s}".format( relativeDir ))
        logging.debug( "upload():  \tremote root dir: {0:s}".format( self.ftpRootDir ))

        # Add the remote root path and go to the remote directory.
        remoteDir = self.appendRootDir( self.ftpRootDir, relativeDir )
        logging.debug( "upload():  \tftp cd remote dir: {0:s}".format( remoteDir ))

        try:
            self.ftp.cwd( remoteDir )
        except Exception as detail:
            logging.error( "upload():  \tCannot ftp chdir: {0:s}  Skipping...".format( str( detail ) ))
        else:
            # Add the master root path to get the local file name.
            # Open local binary file to read from.
            localFileName = self.appendRootDir( self.masterRootDir, relativeFilePath )
            logging.debug( "upload():  \topen local file name: {0:s}".format( localFileName ))

            try:
                f = open( localFileName, "rb" )
                try:
                    # f.read() is called on each block of the binary file until EOF.
                    logging.debug( "upload():  \tftp STOR file {0:s}".format( fileName ))
                    self.ftp.storbinary( "STOR " + fileName, f )
                except Exception as detail:
                    logging.error( "upload():  \tCannot ftp storbinary: {0:s}".format( str( detail ) ))
                f.close()
            except IOError as detail:
                logging.error( "upload():  \tCannot open local file {0:s} for reading:  {1:s}".format( localFileName, str( detail ) ) )


    def quit( self ):
        """Log out of an ftp session"""

        logging.debug( "UpdateWeb::quit()" )
        try:
            self.ftp.quit()
        except Exception as detail:
            logging.error( "Cannot ftp quit because {0:s}".format( str( detail ) ))


#----------------------------------------------------------------------------
#     Call the main program.
#----------------------------------------------------------------------------

if __name__ == '__main__':
    main()