Marginalia

A friend was looking for a way to list the space usage on a windows server that only had FTP access. I had written something similar for a project long ago, and polished up to do the job.

This python script will walk an FTP directory in a top-down, depth-first pattern. It uses the ftplib library, which I believe is built-in to most or all python distributions. Configure the FTP_* variables near the top to set the server, port, user, password, and the delay between each FTP operation (to avoid hammering the server). The script recursively processes directories, creating a dirStruct tuple that contains the following items:

(pwd, subdirList, fileList, sizeInFilesHere, sizeTotal)
    pwd is a string like "/debian/dists/experimental"
    subdirList is a list of tuples just like this one
    fileList is a list of (filename, sizeInBytes) tuples
    sizeInFilesHere is a sum of all the files in this directory
    sizeTotal is a sum of all the files in this directory and all subdirectories

It also writes data to two CSV files:

• dirStruct_only_folders.csv
  • Contains entries for just the directories.
  • Local size is the total size of files in that folder (does not count subdirs).
  • Total size is the sum of local size and total size of all subdirs.
• dirStruct_complete.csv
  • Contains entries for both files and folders.
  • Files do not have a total size, only a local size.

#!/usr/bin/env python
#
# A script to recursively walk an FTP server directory structure, recording information
# about the file and directory sizes as it traverses the folders.
#
# Stores output in two CSV files:
#  dirStruct_only_folders.csv
#     Contains entries for just the directories.
#     Local size is the total size of files in that folder (does not count subdirs).
#     Total size is the sum of local size and total size of all subdirs.
#  dirStruct_complete.csv
#     Contains entries for both files and folders.
#     Files do not have a total size, only a local size.
#
# Customize the FTP_* variables below.
#
# Basically does a depth-first search.
#
# Written by Matthew L Beckler, matthew at mbeckler dot org.
# Released into the public domain, do whatever you like with this.
# Email me if you like the script or have suggestions to improve it.

from ftplib import FTP
from time import sleep


FTP_SERVER = "ftp.debian.org"
FTP_PORT = "21" # 21 is the default
FTP_USER = "" # leave empty for anon FTP server
FTP_PASS = ""
FTP_DELAY = 1 # how long to wait between calls to the ftp server

def parseListLine(line):
   # Files look like          "-rw-r--r--    1 1176     1176       176158 Mar 30 01:52 README.mirrors.html"
   # Directories look like    "drwxr-sr-x   15 1176     1176         4096 Feb 15 09:22 dists"
   # Returns (name, isDir, sizeBytes)
   items = line.split()
   return (items[8], items[0][0] == "d", int(items[4]))

# Since the silly ftp library makes us use a callback to handle each line of text from the server,
# we have a global lines buffer. Clear the buffer variable before doing each call.
lines = []
def appendLine(line):
   global lines
   lines.append(line)
def getListingParsed(ftp):
   """ This is a sensible interface to the silly line getting system. Returns a copy of the directory listing, parsed. """
   global lines
   lines = []
   ftp.dir(appendLine)
   myLines = lines[:]
   parsedLines = map(parseListLine, myLines)
   return parsedLines
   
def descendDirectories(ftp):
   # Will return a tuple for the current ftp directory, like this:
   # (pwd, subdirList, fileList, sizeInFilesHere, sizeTotal)
   #     pwd is a string like "/debian/dists/experimental"
   #     subdirList is a list of tuples just like this one
   #     fileList is a list of (filename, sizeInBytes) tuples
   #     sizeInFilesHere is a sum of all the files in this directory
   #     sizeTotal is a sum of all the files in this directory and all subdirectories

   sleep(FTP_DELAY) # be a nice client

   # make our directory structure to return
   pwd = ftp.pwd()
   subdirList = []
   fileList = []
   sizeInFilesHere = 0
   sizeTotal = 0

   print pwd + "/"
   items = getListingParsed(ftp)
   for name, isDir, sizeBytes in items:
      if not isDir:
         fileList.append( (name, sizeBytes) )
         sizeInFilesHere += sizeBytes
      else:
         # is a directory, so recurse
         ftp.cwd(name)
         struct = descendDirectories(ftp)
         ftp.cwd("..")
         subdirList.append(struct)
         sizeTotal += struct[4]

   # add in the size of all files here to sizeTotal
   sizeTotal += sizeInFilesHere
   return (pwd, subdirList, fileList, sizeInFilesHere, sizeTotal)

def pprintBytes(b):
   """ Pretty prints a number of bytes with a proper suffix, like K, M, G, T. """
   suffixes = ["", "K", "M", "G", "T", "?"]
   ix = 0
   while (b > 1024):
      b /= 1024.0
      ix += 1
   s = suffixes[min(len(suffixes) - 1, ix)]
   if int(b) == b:
      return "%d%s" % (b, s)
   else:
      return "%.1f%s" % (b, s)

def pprintDirStruct(dirStruct):
   """ Pretty print the directory structure. RECURSIVE FUNCTION! """
   print "{}/ ({} in {} files here, {} total)".format(dirStruct[0], pprintBytes(dirStruct[3]), len(dirStruct[2]), pprintBytes(dirStruct[4]))
   for ds in dirStruct[1]:
      pprintDirStruct(ds)

def saveDirStructToCSV(dirStruct, fid, includeFiles):
   """ Save the directory structure to a CSV file. RECURSIVE FUNCTION! """
   # Info about this directory itself
   fid.write("\"{}/\",{},{}\n".format(dirStruct[0], dirStruct[3], dirStruct[4]))
   pwd = dirStruct[0]

   # Info about files here
   if includeFiles:
      for name, size in dirStruct[2]:
         fid.write("\"{}\",{},\n".format(pwd + "/" + name, size))

   # Info about dirs here, recurse
   for ds in dirStruct[1]:
      saveDirStructToCSV(ds, fid, includeFiles)

print "Connecting to FTP server '%s' port %s..." % (FTP_SERVER, FTP_PORT)
ftp = FTP()
ftp.connect(FTP_SERVER, FTP_PORT)
if FTP_USER == "":
   ftp.login()
else:
   ftp.login(FTP_USER, FTP_PASS)

print "Walking directory structure..."
dirStruct = descendDirectories(ftp)

print ""
print "Finished descending directories, here is the info:"
pprintDirStruct(dirStruct)
print ""

FILENAME = "dirStruct_complete.csv"
print "Saving complete directory info (files and folders) to a CSV file: '%s'" % FILENAME
with open(FILENAME, "w") as fid:
   fid.write("\"Path\",\"Local size\",\"Total size\"\n")
   saveDirStructToCSV(dirStruct, fid, includeFiles=True)

FILENAME = "dirStruct_only_folders.csv"
print "Saving directory info (only folders) to a CSV file: '%s'" % FILENAME
with open(FILENAME, "w") as fid:
   fid.write("\"Path\",\"Local size\",\"Total size\"\n")
   saveDirStructToCSV(dirStruct, fid, includeFiles=False)

Sample CSV output:

"Path","Local size","Total size"
"/plugins/",5426535,7594527
"/plugins/foo-1.1.jar",7774,
"/plugins/CHANGELOG.txt",45169,

Local size is just the size of the file itself, or the size of all files in a directory. Total size is the total size of the files in a directory plus the total sizes of all subdirectories. Files do not have a total size entry.