#!/usr/bin/env python
#
# Windows Azure Linux Agent
#
# Copyright 2014 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.4+ and Openssl 1.0+
#
# Implements parts of RFC 2131, 1541, 1497 and
# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx
# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx
#

import array
import base64
import httplib
import os
import os.path
import platform
import pwd
import re
import shutil
import socket
import SocketServer
import struct
import string
import subprocess
import sys
import tempfile
import textwrap
import threading
import time
import traceback
import xml.dom.minidom
import fcntl
import inspect
import zipfile
import json

if not hasattr(subprocess,'check_output'):
    def check_output(*popenargs, **kwargs):
        r"""Backport from subprocess module from python 2.7"""
        if 'stdout' in kwargs:
            raise ValueError('stdout argument not allowed, it will be overridden.')
        process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
        output, unused_err = process.communicate()
        retcode = process.poll()
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
            raise subprocess.CalledProcessError(retcode, cmd, output=output)
        return output

    # Exception classes used by this module.
    class CalledProcessError(Exception):
        def __init__(self, returncode, cmd, output=None):
            self.returncode = returncode
            self.cmd = cmd
            self.output = output
        def __str__(self):
            return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)

    subprocess.check_output=check_output
    subprocess.CalledProcessError=CalledProcessError
    
GuestAgentName = "WALinuxAgent"
GuestAgentLongName = "Windows Azure Linux Agent"
GuestAgentVersion = "WALinuxAgent-2.0.5-PRE"
ProtocolVersion = "2012-11-30" #WARNING this value is used to confirm the correct fabric protocol.

Config = None
WaAgent = None
DiskActivated = False
Openssl = "openssl"
Children = []
ExtensionChildren = []
VMM_STARTUP_SCRIPT_NAME='install'
VMM_CONFIG_FILE_NAME='linuxosconfiguration.xml'
global RulesFiles
RulesFiles = [ "/lib/udev/rules.d/75-persistent-net-generator.rules",
               "/etc/udev/rules.d/70-persistent-net.rules" ]
VarLibDhcpDirectories = ["/var/lib/dhclient", "/var/lib/dhcpcd", "/var/lib/dhcp"]
EtcDhcpClientConfFiles = ["/etc/dhcp/dhclient.conf", "/etc/dhcp3/dhclient.conf"]
global LibDir
LibDir = "/var/lib/waagent"
global provisioned
provisioned=False
global provisionError
provisionError=None

WaagentConf = """\
#
# Windows Azure Linux Agent Configuration
#

Role.StateConsumer=None                 # Specified program is invoked with the argument "Ready" when we report ready status
                                        # to the endpoint server.
Role.ConfigurationConsumer=None         # Specified program is invoked with XML file argument specifying role configuration.
Role.TopologyConsumer=None              # Specified program is invoked with XML file argument specifying role topology.

Provisioning.Enabled=y                  #
Provisioning.DeleteRootPassword=y       # Password authentication for root account will be unavailable.
Provisioning.RegenerateSshHostKeyPair=y # Generate fresh host key pair.
Provisioning.SshHostKeyPairType=rsa     # Supported values are "rsa", "dsa" and "ecdsa".
Provisioning.MonitorHostName=y          # Monitor host name changes and publish changes via DHCP requests.

ResourceDisk.Format=y                   # Format if unformatted. If 'n', resource disk will not be mounted.
ResourceDisk.Filesystem=ext4            # Typically ext3 or ext4. FreeBSD images should use 'ufs2' here.
ResourceDisk.MountPoint=/mnt/resource   #
ResourceDisk.EnableSwap=n               # Create and use swapfile on resource disk.
ResourceDisk.SwapSizeMB=0               # Size of the swapfile.

LBProbeResponder=y                      # Respond to load balancer probes if requested by Windows Azure.

Logs.Verbose=n                          # Enable verbose logs

OS.RootDeviceScsiTimeout=300            # Root device timeout in seconds.
OS.OpensslPath=None                     # If "None", the system default version is used.
"""

############################################################
# BEGIN DISTRO CLASS DEFS
############################################################
############################################################    
#	AbstractDistro
############################################################    
class AbstractDistro(object):
    """
    AbstractDistro defines a skeleton neccesary for a concrete Distro class.

    Generic methods and attributes are kept here, distribution specific attributes
    and behavior are to be placed in the concrete child named distroDistro, where
    distro is the string returned by calling python platform.linux_distribution()[0].
    So for CentOS the derived class is called 'centosDistro'.
    """

    def __init__(self):
        """
        Generic Attributes go here.  These are based on 'majority rules'.
        This __init__() may be called or overriden by the child.
        """
        self.agent_service_name = os.path.basename(sys.argv[0]) 
        self.selinux=None
        self.service_cmd='/usr/sbin/service'
        self.ssh_service_restart_option='restart'
        self.ssh_service_name='ssh'
        self.ssh_config_file='/etc/ssh/sshd_config'
        self.hostname_file_path='/etc/hostname'
        self.dhcp_client_name='dhclient'
        self.requiredDeps = [ 'route', 'shutdown', 'ssh-keygen', 'useradd'
                              , 'openssl', 'sfdisk', 'fdisk', 'mkfs', 'chpasswd', 'sed', 'grep', 'sudo' ]
        self.init_script_file='/etc/init.d/waagent'
        self.agent_package_name='WALinuxAgent'
        self.fileBlackList = [ "/root/.bash_history", "/var/log/waagent.log",'/etc/resolv.conf' ]        
        self.agent_files_to_uninstall = ["/etc/waagent.conf", "/etc/logrotate.d/waagent", "/etc/sudoers.d/waagent"]
        self.grubKernelBootOptionsFile = '/etc/default/grub'
        self.grubKernelBootOptionsLine = 'GRUB_CMDLINE_LINUX_DEFAULT='
        self.getpidcmd = 'pidof'
        self.mount_dvd_cmd = 'mount'
        self.sudoers_dir_base = '/etc'
        self.waagent_conf_file = WaagentConf
        self.shadow_file_mode=0600
        
    def isSelinuxSystem(self):
        """
        Checks and sets self.selinux = True if SELinux is available on system.
        """
        if self.selinux == None:
            if Run("which getenforce",chk_err=False):
                self.selinux = False
            else:
                self.selinux = True
        return self.selinux
    
    def isSelinuxRunning(self):
        """
        Calls shell command 'getenforce' and returns True if 'Enforcing'.
        """
        if self.isSelinuxSystem():
            return RunGetOutput("getenforce")[1].startswith("Enforcing")
        else:
            return False
        
    def setSelinuxEnforce(self,state):
        """
        Calls shell command 'setenforce' with 'state' and returns resulting exit code.
        """
        if self.isSelinuxSystem():
            if state: s = '1'
            else: s='0'
            return Run("setenforce "+s)

    def setSelinuxContext(self,path,cn):
        """
        Calls shell 'chcon' with 'path' and 'cn' context.
        Returns exit result.
        """
        if self.isSelinuxSystem():
            return Run('chcon ' + cn + ' ' + path)
        
    def setHostname(self,name):
        """
        Shell call to hostname.
        Returns resulting exit code.
        """
        return Run('hostname ' + name)
        
    def publishHostname(self,name):
        """
        Set the contents of the hostname file to 'name'.
        Return 1 on failure.
        """
        try:
            r=SetFileContents(self.hostname_file_path, name)
            for f in EtcDhcpClientConfFiles:
                if os.path.exists(f) and FindStringInFile(f,r'^[^#]*?send\s*host-name.*?(<hostname>|gethostname[(,)])') == None :
                    r=ReplaceFileContentsAtomic('/etc/dhcp/dhclient.conf', "send host-name \"" + name + "\";\n"
                                                + "\n".join(filter(lambda a: not a.startswith("send host-name"), GetFileContents('/etc/dhcp/dhclient.conf').split('\n'))))
        except:
            return 1
        return r
        
    def installAgentServiceScriptFiles(self):
        """
        Create the waagent support files for service installation.
        Called by registerAgentService()
        Abstract Virtual Function.  Over-ridden in concrete Distro classes.
        """
        pass

    def registerAgentService(self):
        """
        Calls installAgentService to create service files.
        Shell exec service registration commands. (e.g. chkconfig --add waagent)
        Abstract Virtual Function.  Over-ridden in concrete Distro classes.
        """
        pass
    
    def uninstallAgentService(self):
        """
        Call service subsystem to remove waagent script.
        Abstract Virtual Function.  Over-ridden in concrete Distro classes.
        """
        pass

    def unregisterAgentService(self):
        """
        Calls self.stopAgentService and call self.uninstallAgentService()
        """
        self.stopAgentService()
        self.uninstallAgentService()
    
    def startAgentService(self):
        """
        Service call to start the Agent service
        """
        return Run(self.service_cmd + ' ' + self.agent_service_name + ' start')
        
    def stopAgentService(self):
        """
        Service call to stop the Agent service
        """
        return Run(self.service_cmd + ' '  + self.agent_service_name + ' stop',False)
    
    def restartSshService(self):
        """
        Service call to re(start) the SSH service
        """
        sshRestartCmd = self.service_cmd + " " + self.ssh_service_name + " " + self.ssh_service_restart_option
        retcode = Run(sshRestartCmd)
        if retcode > 0:
            Error("Failed to restart SSH service with return code:" + retcode)
        return retcode

    def sshDeployPublicKey(self,fprint,path):
        """
        Generic sshDeployPublicKey - over-ridden in some concrete Distro classes due to minor differences in openssl packages deployed
        """
        error=0
        SshPubKey = OvfEnv().OpensslToSsh(fprint)
        if SshPubKey != None:
            AppendFileContents(path, SshPubKey)
        else:
            Error("Failed: " + fprint + ".crt -> " + path)
            error = 1
        return error
    
    def checkPackageInstalled(self,p):
        """
        Query package database for prescence of an installed package.
        Abstract Virtual Function.  Over-ridden in concrete Distro classes.
        """
        pass

    def checkPackageUpdateable(self,p):
        """
        Online check if updated package of walinuxagent is available.
        Abstract Virtual Function.  Over-ridden in concrete Distro classes.
        """
        pass

    def deleteRootPassword(self):
        """
        Generic root password removal.
        """
        filepath="/etc/shadow"
        ReplaceFileContentsAtomic(filepath,"root:*LOCK*:14600::::::\n"
                                  + "\n".join(filter(lambda a: not a.startswith("root:"),GetFileContents(filepath).split('\n'))))
        os.chmod(filepath,self.shadow_file_mode)
        if self.isSelinuxSystem():
            self.setSelinuxContext(filepath,'system_u:object_r:shadow_t:s0')
        Log("Root password deleted.")
        return 0
    
    def changePass(self,user,password):
        return RunSendStdin("chpasswd",(user + ":" + password + "\n"))
    
    def load_ata_piix(self):
        return WaAgent.TryLoadAtapiix()

    def unload_ata_piix(self):
        """
        Generic function to remove ata_piix.ko.
        """
        return WaAgent.TryUnloadAtapiix()
        
    def deprovisionWarnUser(self):
        """
        Generic user warnings used at deprovision.
        """
        print("WARNING! Nameserver configuration in /etc/resolv.conf will be deleted.")

    def deprovisionDeleteFiles(self):
        """
        Files to delete when VM is deprovisioned
        """
        for a in VarLibDhcpDirectories:
            Run("rm -f " + a + "/*")

        # Clear LibDir, remove nameserver and root bash history
        
        for f in os.listdir(LibDir) + self.fileBlackList:
            try:
                os.remove(f)
            except:
                pass
        return 0
    
    def uninstallDeleteFiles(self):
        """
        Files to delete when agent is uninstalled.
        """
        for f in self.agent_files_to_uninstall:
            try:
                os.remove(f)
            except:
                pass
        return 0
    
    def checkDependencies(self):
        """
        Generic dependency check.
        Return 1 unless all dependencies are satisfied.
        """
        if self.checkPackageInstalled('NetworkManager'):
            Error(GuestAgentLongName + " is not compatible with network-manager.")
            return 1
        try:
            m= __import__('pyasn1')
        except ImportError:
            Error(GuestAgentLongName + " requires python-pyasn1 for your Linux distribution.")
            return 1
        for a in self.requiredDeps:
            if Run("which " + a + " > /dev/null 2>&1",chk_err=False):
                Error("Missing required dependency: " + a)
                return 1
        return 0

    def packagedInstall(self,buildroot):
        """
        Called from setup.py for use by RPM.
        Copies generated files waagent.conf, under the buildroot.
        """
        if not os.path.exists(buildroot+'/etc'):
            os.mkdir(buildroot+'/etc')
        SetFileContents(buildroot+'/etc/waagent.conf', MyDistro.waagent_conf_file)
        
        if not os.path.exists(buildroot+'/etc/logrotate.d'):
            os.mkdir(buildroot+'/etc/logrotate.d')
        SetFileContents(buildroot+'/etc/logrotate.d/waagent', WaagentLogrotate)
    
        self.init_script_file=buildroot+self.init_script_file
        # this allows us to call installAgentServiceScriptFiles()
        if not os.path.exists(os.path.dirname(self.init_script_file)):
            os.mkdir(os.path.dirname(self.init_script_file))
        self.installAgentServiceScriptFiles()

    def GetIpv4Address(self):
        """
        Return the ip of the 
        first active non-loopback interface.
        """
        addr=''
        iface,addr=GetFirstActiveNetworkInterfaceNonLoopback()
        return addr

    def GetMacAddress(self):
        return GetMacAddress()

    def GetInterfaceName(self):
        return GetFirstActiveNetworkInterfaceNonLoopback()[0]

    def CreateAccount(self,user, password, expiration, thumbprint):
        return CreateAccount(user, password, expiration, thumbprint)
    
    def DeleteAccount(self,user):
        return DeleteAccount(user)

    def ActivateResourceDisk(self):
        """
        Format, mount, and if specified in the configuration
        set resource disk as swap.
        """
        global DiskActivated
        format = Config.get("ResourceDisk.Format")
        if format == None or format.lower().startswith("n"):
            DiskActivated = True
            return
        device = DeviceForIdePort(1)
        if device == None:
            Error("ActivateResourceDisk: Unable to detect disk topology.")
            return
        device = "/dev/" + device
        for entry in RunGetOutput("mount")[1].split():
            if entry.startswith(device + "1"):
                Log("ActivateResourceDisk: " + device + "1 is already mounted.")
                DiskActivated = True
                return
        mountpoint = Config.get("ResourceDisk.MountPoint")
        if mountpoint == None:
            mountpoint = "/mnt/resource"
        CreateDir(mountpoint, "root", 0755)
        fs = Config.get("ResourceDisk.Filesystem")
        if fs == None:
            fs = "ext3"
        if RunGetOutput("sfdisk -q -c " + device + " 1")[1].rstrip() == "7" and fs != "ntfs":
            Run("sfdisk -c " + device + " 1 83")
            Run("mkfs." + fs + " " + device + "1")
        if Run("mount " + device + "1 " + mountpoint):
            Error("ActivateResourceDisk: Failed to mount resource disk (" + device + "1).")
            return
        Log("Resource disk (" + device + "1) is mounted at " + mountpoint + " with fstype " + fs)
        DiskActivated = True
        swap = Config.get("ResourceDisk.EnableSwap")
        if swap == None or swap.lower().startswith("n"):
            return
        sizeKB = int(Config.get("ResourceDisk.SwapSizeMB")) * 1024
        if os.path.isfile(mountpoint + "/swapfile") and os.path.getsize(mountpoint + "/swapfile") != (sizeKB * 1024):
            os.remove(mountpoint + "/swapfile")
        if not os.path.isfile(mountpoint + "/swapfile"):
            Run("dd if=/dev/zero of=" + mountpoint + "/swapfile bs=1024 count=" + str(sizeKB))
            Run("mkswap " + mountpoint + "/swapfile")
        if not Run("swapon " + mountpoint + "/swapfile"):
            Log("Enabled " + str(sizeKB) + " KB of swap at " + mountpoint + "/swapfile")
        else:
            Error("ActivateResourceDisk: Failed to activate swap at " + mountpoint + "/swapfile")

    def Install(self):
        return Install()

    def mediaHasFilesystem(self,dsk):
        if len(dsk) == 0 :
            return False
        if Run("LC_ALL=C fdisk -l " + dsk + " | grep Disk"):
            return False
        return True
    
    def mountDVD(self,dvd,location):
        return RunGetOutput(self.mount_dvd_cmd + ' ' + dvd + ' ' + location)

    def GetHome(self):
        return GetHome()

    def getDhcpClientName(self):
        return self.dhcp_client_name

    def initScsiDiskTimeout(self):
        """
        Set the SCSI disk timeout when the agent starts running
        """
        self.setScsiDiskTimeout()

    def setScsiDiskTimeout(self):
        """
        Iterate all SCSI disks(include hot-add) and set their timeout if their value are different from the OS.RootDeviceScsiTimeout
        """
        try:
            scsiTimeout = Config.get("OS.RootDeviceScsiTimeout")
            for diskName in [disk for disk in os.listdir("/sys/block") if disk.startswith("sd")]:
                self.setBlockDeviceTimeout(diskName, scsiTimeout)
        except:
            pass

    def setBlockDeviceTimeout(self, device, timeout):
        """
        Set SCSI disk timeout by set /sys/block/sd*/device/timeout
        """
        if timeout != None and device:
            filePath = "/sys/block/" + device + "/device/timeout"
            if(GetFileContents(filePath).splitlines()[0].rstrip() != timeout):
                SetFileContents(filePath,timeout)
                Log("SetBlockDeviceTimeout: Update the device " + device + " with timeout " + timeout)

############################################################    
#	SuSEDistro
############################################################    
suse_init_file = """\
#! /bin/sh
#
# Windows Azure Linux Agent sysV init script
#
# Copyright 2013 Microsoft Corporation
# Copyright SUSE LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# /etc/init.d/waagent
#
#  and symbolic link
#
# /usr/sbin/rcwaagent
#
# System startup script for the waagent
#
### BEGIN INIT INFO
# Provides: WindowsAzureLinuxAgent
# Required-Start: $network sshd
# Required-Stop: $network sshd
# Default-Start: 3 5
# Default-Stop: 0 1 2 6
# Description: Start the WindowsAzureLinuxAgent
### END INIT INFO

PYTHON=/usr/bin/python
WAZD_BIN=/usr/sbin/waagent
WAZD_CONF=/etc/waagent.conf
WAZD_PIDFILE=/var/run/waagent.pid

test -x "$WAZD_BIN" || { echo "$WAZD_BIN not installed"; exit 5; }
test -e "$WAZD_CONF" || { echo "$WAZD_CONF not found"; exit 6; }

. /etc/rc.status

# First reset status of this service
rc_reset

# Return values acc. to LSB for all commands but status:
# 0 - success
# 1 - misc error
# 2 - invalid or excess args
# 3 - unimplemented feature (e.g. reload)
# 4 - insufficient privilege
# 5 - program not installed
# 6 - program not configured
#
# Note that starting an already running service, stopping
# or restarting a not-running service as well as the restart
# with force-reload (in case signalling is not supported) are
# considered a success.


case "$1" in
    start)
        echo -n "Starting WindowsAzureLinuxAgent"
        ## Start daemon with startproc(8). If this fails
        ## the echo return value is set appropriate.
        startproc -f ${PYTHON} ${WAZD_BIN} -daemon
        rc_status -v
        ;;
    stop)
        echo -n "Shutting down WindowsAzureLinuxAgent"
        ## Stop daemon with killproc(8) and if this fails
        ## set echo the echo return value.
        killproc -p ${WAZD_PIDFILE} ${PYTHON} ${WAZD_BIN}
        rc_status -v
        ;;
    try-restart)
        ## Stop the service and if this succeeds (i.e. the
        ## service was running before), start it again.
        $0 status >/dev/null && $0 restart
        rc_status
        ;;
    restart)
        ## Stop the service and regardless of whether it was
        ## running or not, start it again.
        $0 stop
        sleep 1
        $0 start
        rc_status
        ;;
    force-reload|reload)
        rc_status
        ;;
    status)
        echo -n "Checking for service WindowsAzureLinuxAgent "
        ## Check status with checkproc(8), if process is running
        ## checkproc will return with exit status 0.

        checkproc -p ${WAZD_PIDFILE} ${PYTHON} ${WAZD_BIN}
        rc_status -v
        ;;
    probe)
        ;;
    *)
        echo "Usage: $0 {start|stop|status|try-restart|restart|force-reload|reload}"
        exit 1
        ;;
esac
rc_exit
"""
class SuSEDistro(AbstractDistro):
    """
    SuSE Distro concrete class
    Put SuSE specific behavior here...
    """
    def __init__(self):
        super(SuSEDistro,self).__init__()
        self.service_cmd='/sbin/service'
        self.ssh_service_name='sshd'
        self.kernel_boot_options_file='/boot/grub/menu.lst'
        self.hostname_file_path='/etc/HOSTNAME'
        self.requiredDeps += [ "/sbin/insserv" ]
        self.init_file=suse_init_file
        self.dhcp_client_name='dhcpcd'
        if (DistInfo(fullname=1)[0] == 'SUSE Linux Enterprise Server') and (DistInfo()[1] >= '12'):
            self.dhcp_client_name='wickedd-dhcp4'
        self.grubKernelBootOptionsFile = '/boot/grub/menu.lst'
        self.grubKernelBootOptionsLine = 'kernel'
        self.getpidcmd='pidof '
        
    def checkPackageInstalled(self,p):
        if Run("rpm -q " + p,chk_err=False):
            return 0
        else:
            return 1

    def checkPackageUpdateable(self,p):
        if Run("zypper list-updates | grep " + p,chk_err=False):
            return 1
        else:
            return 0
        

    def installAgentServiceScriptFiles(self):
        try:
            SetFileContents(self.init_script_file, self.init_file)
            os.chmod(self.init_script_file, 0744)
        except:
            pass
        
    def registerAgentService(self):
        self.installAgentServiceScriptFiles()
        return Run('insserv ' + self.agent_service_name)

    def uninstallAgentService(self):
        return Run('insserv -r ' + self.agent_service_name)

    def unregisterAgentService(self):
        self.stopAgentService()
        return self.uninstallAgentService()
    
############################################################    
#	redhatDistro
############################################################    

redhat_init_file= """\
#!/bin/bash
#
# Init file for WindowsAzureLinuxAgent.
#
# chkconfig: 2345 60 80
# description: WindowsAzureLinuxAgent
#

# source function library
. /etc/rc.d/init.d/functions

RETVAL=0
FriendlyName="WindowsAzureLinuxAgent"
WAZD_BIN=/usr/sbin/waagent

start()
{
    echo -n $"Starting $FriendlyName: "
    $WAZD_BIN -daemon &
}

stop()
{
    echo -n $"Stopping $FriendlyName: "
    killproc -p /var/run/waagent.pid $WAZD_BIN
    RETVAL=$?
    echo
    return $RETVAL
}

case "$1" in
    start)
        start
        ;;
    stop)
        stop
        ;;
    restart)
        stop
        start
        ;;
    reload)
        ;;
    report)
        ;;
    status)
        status $WAZD_BIN
        RETVAL=$?
        ;;
    *)
        echo $"Usage: $0 {start|stop|restart|status}"
        RETVAL=1
esac
exit $RETVAL
"""

class redhatDistro(AbstractDistro):
    """
    Redhat Distro concrete class
    Put Redhat specific behavior here...
    """
    def __init__(self):
        super(redhatDistro,self).__init__()
        self.service_cmd='/sbin/service'
        self.ssh_service_restart_option='condrestart'
        self.ssh_service_name='sshd'
        self.hostname_file_path= None if DistInfo()[1] < '7.0' else '/etc/hostname'
        self.init_file=redhat_init_file
        self.grubKernelBootOptionsFile = '/boot/grub/menu.lst'
        self.grubKernelBootOptionsLine = 'kernel'

    def publishHostname(self,name):
        super(redhatDistro,self).publishHostname(name)
        if DistInfo()[1] < '7.0' :
            filepath = "/etc/sysconfig/network"
            if os.path.isfile(filepath):
                ReplaceFileContentsAtomic(filepath, "HOSTNAME=" + name + "\n"
                    + "\n".join(filter(lambda a: not a.startswith("HOSTNAME"), GetFileContents(filepath).split('\n'))))

        ethernetInterface = MyDistro.GetInterfaceName()
        filepath = "/etc/sysconfig/network-scripts/ifcfg-" + ethernetInterface
        if os.path.isfile(filepath):
            ReplaceFileContentsAtomic(filepath, "DHCP_HOSTNAME=" + name + "\n"
                    + "\n".join(filter(lambda a: not a.startswith("DHCP_HOSTNAME"), GetFileContents(filepath).split('\n'))))
        return 0

    def installAgentServiceScriptFiles(self):
        SetFileContents(self.init_script_file, self.init_file)
        os.chmod(self.init_script_file, 0744)
        return 0

    def registerAgentService(self):
        self.installAgentServiceScriptFiles()
        return Run('chkconfig --add waagent')
   
    def uninstallAgentService(self):
        return Run('chkconfig --del ' + self.agent_service_name)

    def unregisterAgentService(self):
        self.stopAgentService()
        return self.uninstallAgentService()
    
    def checkPackageInstalled(self,p):
        if Run("yum list installed " + p,chk_err=False):
            return 0
        else:
            return 1

    def checkPackageUpdateable(self,p):
        if Run("yum check-update | grep "+ p,chk_err=False):
            return 1
        else:
            return 0



############################################################    
#	centosDistro
############################################################    

class centosDistro(redhatDistro):
    """
    CentOS Distro concrete class
    Put CentOS specific behavior here...
    """
    def __init__(self):
        super(centosDistro,self).__init__()

############################################################    
#	debianDistro
############################################################    
debian_init_file = """\
#!/bin/sh
### BEGIN INIT INFO
# Provides:          WindowsAzureLinuxAgent
# Required-Start:    $network $syslog
# Required-Stop:     $network $syslog
# Should-Start:      $network $syslog
# Should-Stop:       $network $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: WindowsAzureLinuxAgent
# Description:       WindowsAzureLinuxAgent
### END INIT INFO

. /lib/lsb/init-functions

OPTIONS="-daemon"
WAZD_BIN=/usr/sbin/waagent
WAZD_PID=/var/run/waagent.pid

case "$1" in
    start)
        log_begin_msg "Starting WindowsAzureLinuxAgent..."
        pid=$( pidofproc $WAZD_BIN )
        if [ -n "$pid" ] ; then
              log_begin_msg "Already running."
              log_end_msg 0
              exit 0
        fi
        start-stop-daemon --start --quiet --oknodo --background --exec $WAZD_BIN -- $OPTIONS
        log_end_msg $?
        ;;

    stop)
        log_begin_msg "Stopping WindowsAzureLinuxAgent..."
        start-stop-daemon --stop --quiet --oknodo --pidfile $WAZD_PID
        ret=$?
        rm -f $WAZD_PID
        log_end_msg $ret
        ;;
    force-reload)
        $0 restart
        ;;
    restart)
        $0 stop
        $0 start
        ;;
    status)
        status_of_proc $WAZD_BIN && exit 0 || exit $?
        ;;
    *)
        log_success_msg "Usage: /etc/init.d/waagent {start|stop|force-reload|restart|status}"
        exit 1
        ;;
esac

exit 0
"""

class debianDistro(AbstractDistro):
    """
    debian Distro concrete class
    Put debian specific behavior here...
    """
    def __init__(self):
        super(debianDistro,self).__init__()
        self.requiredDeps += [ "/usr/sbin/update-rc.d" ]
        self.init_file=debian_init_file
        self.agent_package_name='walinuxagent'
        self.dhcp_client_name='dhclient'
        self.getpidcmd='pidof '
        self.shadow_file_mode=0640

    def checkPackageInstalled(self,p):
        """
        Check that the package is installed.
        Return 1 if installed, 0 if not installed.
        This method of using dpkg-query
        allows wildcards to be present in the
        package name.
        """
        if not Run("dpkg-query -W -f='${Status}\n' '" + p + "' | grep ' installed' 2>&1",chk_err=False):
            return 1
        else:
            return 0
        
    def checkDependencies(self):
        """
        Debian dependency check.  python-pyasn1 is NOT needed.
        Return 1 unless all dependencies are satisfied.
        NOTE: using network*manager will catch either package name in Ubuntu or debian.
        """
        if self.checkPackageInstalled('network*manager'):
            Error(GuestAgentLongName + " is not compatible with network-manager.")
            return 1
        for a in self.requiredDeps:
            if Run("which " + a + " > /dev/null 2>&1",chk_err=False):
                Error("Missing required dependency: " + a)
                return 1
        return 0

    def checkPackageUpdateable(self,p):
        if Run("apt-get update ; apt-get upgrade -us | grep " + p,chk_err=False):
            return 1
        else:
            return 0
            
    def installAgentServiceScriptFiles(self):
        """
        If we are packaged - the service name is walinuxagent, do nothing.
        """
        if self.agent_service_name == 'walinuxagent':
            return 0
        try:
            SetFileContents(self.init_script_file, self.init_file)
            os.chmod(self.init_script_file, 0744)
        except OSError, e:
            ErrorWithPrefix('installAgentServiceScriptFiles','Exception: '+str(e)+' occured creating ' + self.init_script_file)
            return 1
        return 0
    
    def registerAgentService(self):
        if self.installAgentServiceScriptFiles() == 0:
            return Run('update-rc.d waagent defaults')
        else :
            return 1
    
    def uninstallAgentService(self):
        return Run('update-rc.d -f ' + self.agent_service_name + ' remove')

    def unregisterAgentService(self):
        self.stopAgentService()
        return self.uninstallAgentService()
    
    def sshDeployPublicKey(self,fprint,path):
        """
        We support PKCS8.
        """
        if Run("ssh-keygen -i -m PKCS8 -f " + fprint + " >> " + path):
            return 1
        else :
            return 0
        
############################################################    
#	UbuntuDistro
############################################################    
ubuntu_upstart_file = """\
#walinuxagent - start Windows Azure agent

description "walinuxagent"
author "Ben Howard <ben.howard@canonical.com>"

start on (filesystem and started rsyslog)

pre-start script

	WALINUXAGENT_ENABLED=1
    [ -r /etc/default/walinuxagent ] && . /etc/default/walinuxagent

    if [ "$WALINUXAGENT_ENABLED" != "1" ]; then
        exit 1
    fi

    if [ ! -x /usr/sbin/waagent ]; then
        exit 1
    fi

    #Load the udf module
    modprobe -b udf
end script

exec /usr/sbin/waagent -daemon
"""

class UbuntuDistro(debianDistro):
    """
    Ubuntu Distro concrete class
    Put Ubuntu specific behavior here...
    """
    def __init__(self):
        super(UbuntuDistro,self).__init__()
        self.init_script_file='/etc/init/waagent.conf'
        self.init_file=ubuntu_upstart_file
        self.fileBlackList = [ "/root/.bash_history", "/var/log/waagent.log"]
        self.dhcp_client_name=None
        self.getpidcmd='pidof '

    def registerAgentService(self):
        return self.installAgentServiceScriptFiles()

    def startAgentService(self):
        """
        Use upstart syntax.
        """
        return Run('start ' + self.agent_service_name)
        
    def stopAgentService(self):
        """
        Use upstart syntax.
        """
        return Run('stop ' + self.agent_service_name)
    
    def uninstallAgentService(self):
        """
        If we are packaged - the service name is walinuxagent, do nothing.
        """
        if self.agent_service_name == 'walinuxagent':
            return 0
        os.remove('/etc/init/' + self.agent_service_name + '.conf')

    def unregisterAgentService(self):
        """
        If we are packaged - the service name is walinuxagent, do nothing.
        """
        if self.agent_service_name == 'walinuxagent':
            return
        self.stopAgentService()
        return self.uninstallAgentService()
    
    def deprovisionWarnUser(self):
        """
        Ubuntu specific warning string from Deprovision.
        """
        print("WARNING! Nameserver configuration in /etc/resolvconf/resolv.conf.d/{tail,originial} will be deleted.")

    def deprovisionDeleteFiles(self):
        """
        Ubuntu uses resolv.conf by default, so removing /etc/resolv.conf will
        break resolvconf. Therefore, we check to see if resolvconf is in use,
        and if so, we remove the resolvconf artifacts.
        """
        if os.path.realpath('/etc/resolv.conf') != '/run/resolvconf/resolv.conf':
            Log("resolvconf is not configured. Removing /etc/resolv.conf")
            self.fileBlackList.append('/etc/resolv.conf')
        else:
            Log("resolvconf is enabled; leaving /etc/resolv.conf intact")
            resolvConfD = '/etc/resolvconf/resolv.conf.d/'
            self.fileBlackList.extend([resolvConfD + 'tail', resolvConfD + 'originial'])
        for f in os.listdir(LibDir)+self.fileBlackList:
            try:
                os.remove(f)
            except:
                pass
        return 0

    def getDhcpClientName(self):
        if self.dhcp_client_name != None :
            return self.dhcp_client_name
        if DistInfo()[1] == '12.04' :
            self.dhcp_client_name='dhclient3'
        else :
            self.dhcp_client_name='dhclient'
        return self.dhcp_client_name

############################################################    
#	LinuxMintDistro
############################################################    

class LinuxMintDistro(UbuntuDistro):
    """
    LinuxMint Distro concrete class
    Put LinuxMint specific behavior here...
    """
    def __init__(self):
        super(LinuxMintDistro,self).__init__()

############################################################    
#	fedoraDistro
############################################################    

class fedoraDistro(redhatDistro):
    """
    FedoraDistro concrete class
    Put Fedora specific behavior here...
    """
    def __init__(self):
        super(fedoraDistro,self).__init__()

############################################################    
#	FreeBSD
############################################################    
FreeBSDWaagentConf = """\
#
# Windows Azure Linux Agent Configuration
#

Role.StateConsumer=None                 # Specified program is invoked with the argument "Ready" when we report ready status
                                        # to the endpoint server.
Role.ConfigurationConsumer=None         # Specified program is invoked with XML file argument specifying role configuration.
Role.TopologyConsumer=None              # Specified program is invoked with XML file argument specifying role topology.

Provisioning.Enabled=y                  #
Provisioning.DeleteRootPassword=y       # Password authentication for root account will be unavailable.
Provisioning.RegenerateSshHostKeyPair=y # Generate fresh host key pair.
Provisioning.SshHostKeyPairType=rsa     # Supported values are "rsa", "dsa" and "ecdsa".
Provisioning.MonitorHostName=y          # Monitor host name changes and publish changes via DHCP requests.

ResourceDisk.Format=y                   # Format if unformatted. If 'n', resource disk will not be mounted.
ResourceDisk.Filesystem=ufs2            #
ResourceDisk.MountPoint=/mnt/resource   #
ResourceDisk.EnableSwap=n               # Create and use swapfile on resource disk.
ResourceDisk.SwapSizeMB=0               # Size of the swapfile.

LBProbeResponder=y                      # Respond to load balancer probes if requested by Windows Azure.

Logs.Verbose=n                          # Enable verbose logs

OS.RootDeviceScsiTimeout=300            # Root device timeout in seconds.
OS.OpensslPath=None                     # If "None", the system default version is used.
"""

bsd_init_file="""\
#! /bin/sh

# PROVIDE: waagent
# REQUIRE: DAEMON cleanvar sshd
# BEFORE: LOGIN
# KEYWORD: nojail

. /etc/rc.subr
export PATH=$PATH:/usr/local/bin
name="waagent"
rcvar="waagent_enable"
command="/usr/sbin/${name}"
command_interpreter="/usr/local/bin/python"
waagent_flags=" daemon &"

pidfile="/var/run/waagent.pid"

load_rc_config $name
run_rc_command "$1"

"""
bsd_activate_resource_disk_txt="""\
#!/usr/bin/env python

import os
import sys
import imp

# waagent has no '.py' therefore create waagent module import manually.
__name__='setupmain' #prevent waagent.__main__ from executing
waagent=imp.load_source('waagent','/tmp/waagent') 
waagent.LoggerInit('/var/log/waagent.log','/dev/console')
from waagent import RunGetOutput,Run
Config=waagent.ConfigurationProvider()
format = Config.get("ResourceDisk.Format")
if format == None or format.lower().startswith("n"):
    sys.exit(0)
device_base = 'da1'
device = "/dev/" + device_base
for entry in RunGetOutput("mount")[1].split():
    if entry.startswith(device + "s1"):
        waagent.Log("ActivateResourceDisk: " + device + "s1 is already mounted.")
        sys.exit(0)
mountpoint = Config.get("ResourceDisk.MountPoint")
if mountpoint == None:
    mountpoint = "/mnt/resource"
waagent.CreateDir(mountpoint, "root", 0755)
fs = Config.get("ResourceDisk.Filesystem")
if waagent.FreeBSDDistro().mediaHasFilesystem(device) == False :
    Run("newfs " + device + "s1")
if Run("mount " + device + "s1 " + mountpoint):
    waagent.Error("ActivateResourceDisk: Failed to mount resource disk (" + device + "s1).")
    sys.exit(0)
waagent.Log("Resource disk (" + device + "s1) is mounted at " + mountpoint + " with fstype " + fs)
swap = Config.get("ResourceDisk.EnableSwap")
if swap == None or swap.lower().startswith("n"):
    sys.exit(0)
sizeKB = int(Config.get("ResourceDisk.SwapSizeMB")) * 1024
if os.path.isfile(mountpoint + "/swapfile") and os.path.getsize(mountpoint + "/swapfile") != (sizeKB * 1024):
    os.remove(mountpoint + "/swapfile")
if not os.path.isfile(mountpoint + "/swapfile"):
    Run("dd if=/dev/zero of=" + mountpoint + "/swapfile bs=1024 count=" + str(sizeKB))
if Run("mdconfig -a -t vnode -f " + mountpoint + "/swapfile -u 0"):
    waagent.Error("ActivateResourceDisk: Configuring swap - Failed to create md0")
if not Run("swapon /dev/md0"):
    waagent.Log("Enabled " + str(sizeKB) + " KB of swap at " + mountpoint + "/swapfile")
else:
    waagent.Error("ActivateResourceDisk: Failed to activate swap at " + mountpoint + "/swapfile")
"""

class FreeBSDDistro(AbstractDistro):
    """
    """
    def __init__(self):
        """
        Generic Attributes go here.  These are based on 'majority rules'.
        This __init__() may be called or overriden by the child.
        """
        super(FreeBSDDistro,self).__init__()
        self.agent_service_name = os.path.basename(sys.argv[0]) 
        self.selinux=False
        self.ssh_service_name='sshd'
        self.ssh_config_file='/etc/ssh/sshd_config'
        self.hostname_file_path='/etc/hostname'
        self.dhcp_client_name='dhclient'
        self.requiredDeps = [ 'route', 'shutdown', 'ssh-keygen', 'pw'
                              , 'openssl', 'fdisk', 'sed', 'grep' , 'sudo']
        self.init_script_file='/etc/rc.d/waagent'
        self.init_file=bsd_init_file
        self.agent_package_name='WALinuxAgent'
        self.fileBlackList = [ "/root/.bash_history", "/var/log/waagent.log",'/etc/resolv.conf' ]        
        self.agent_files_to_uninstall = ["/etc/waagent.conf", "/usr/local/etc/sudoers.d/waagent"]
        self.grubKernelBootOptionsFile = '/boot/loader.conf'
        self.grubKernelBootOptionsLine = ''
        self.getpidcmd = 'pgrep -n'
        self.mount_dvd_cmd = 'dd bs=2048 count=33 skip=295 if=' # custom data max len is 64k 
        self.sudoers_dir_base = '/usr/local/etc'
        self.waagent_conf_file = FreeBSDWaagentConf
        
    def installAgentServiceScriptFiles(self):
        SetFileContents(self.init_script_file, self.init_file)
        os.chmod(self.init_script_file, 0777)
        AppendFileContents("/etc/rc.conf","waagent_enable='YES'\n")
        return 0

    def registerAgentService(self):
        self.installAgentServiceScriptFiles()
        return Run("services_mkdb " + self.init_script_file)

        
    def sshDeployPublicKey(self,fprint,path):
        """
        We support PKCS8.
        """
        if Run("ssh-keygen -i -m PKCS8 -f " + fprint + " >> " + path):
            return 1
        else :
            return 0

    def deleteRootPassword(self):
        """
        BSD root password removal.
        """
        filepath="/etc/master.passwd"
        ReplaceStringInFile(filepath,r'root:.*?:','root::')
        #ReplaceFileContentsAtomic(filepath,"root:*LOCK*:14600::::::\n"
        #                          + "\n".join(filter(lambda a: not a.startswith("root:"),GetFileContents(filepath).split('\n'))))
        os.chmod(filepath,self.shadow_file_mode)
        if self.isSelinuxSystem():
            self.setSelinuxContext(filepath,'system_u:object_r:shadow_t:s0')
        RunGetOutput("pwd_mkdb -u root /etc/master.passwd")
        Log("Root password deleted.")
        return 0

    def changePass(self,user,password):
        return RunSendStdin("pw usermod " + user + " -h 0 ",password)
    
    def load_ata_piix(self):
        return 0

    def unload_ata_piix(self):
        return 0

    def checkDependencies(self):
        """
        FreeBSD dependency check.
        Return 1 unless all dependencies are satisfied.
        """
        for a in self.requiredDeps:
            if Run("which " + a + " > /dev/null 2>&1",chk_err=False):
                Error("Missing required dependency: " + a)
                return 1
        return 0

    def packagedInstall(self,buildroot):
        pass

    def GetInterfaceName(self):
        """
        Return the ip of the 
        active ethernet interface.
        """
        iface,inet,mac=self.GetFreeBSDEthernetInfo()
        return iface

    def GetIpv4Address(self):
        """
        Return the ip of the 
        active ethernet interface.
        """
        iface,inet,mac=self.GetFreeBSDEthernetInfo()
        return inet

    def GetMacAddress(self):
        """
        Return the ip of the 
        active ethernet interface.
        """
        iface,inet,mac=self.GetFreeBSDEthernetInfo()
        l=mac.split(':')
        r=[]
        for i in l:
            r.append(string.atoi(i,16))
        return r

    def GetFreeBSDEthernetInfo(self):
        """
        There is no SIOCGIFCONF
        on freeBSD - just parse ifconfig.
        Returns strings: iface, inet4_addr, and mac
        or 'None,None,None' if unable to parse.
        We will sleep and retry as the network must be up.
        """
        code,output=RunGetOutput("ifconfig",chk_err=False)
        Log(output)
        retries=10
        cmd='ifconfig | grep -A1 -B2 ether | grep -B3 inet | grep -A3 UP '
        code=1

        while code > 0 :
            if code > 0 and retries == 0:
                Error("GetFreeBSDEthernetInfo - Failed to detect ethernet interface")
                return None, None, None
            code,output=RunGetOutput(cmd,chk_err=False)
            retries-=1
            if code > 0 and retries > 0 :
                Log("GetFreeBSDEthernetInfo - Error: retry ethernet detection " + str(retries))
                if retries == 9 :
                    c,o=RunGetOutput("ifconfig | grep -A1 -B2 ether",chk_err=False)
                    if c == 0:
                        t=o.replace('\n',' ')
                        t=t.split()
                        i=t[0][:-1]
                        Log(RunGetOutput('id')[1])
                        Run('dhclient '+i)
                time.sleep(10)

        j=output.replace('\n',' ')
        j=j.split()
        iface=j[0][:-1]

        for i in range(len(j)):
            if j[i] == 'inet' :
                inet=j[i+1]
            elif j[i] == 'ether' :
                mac=j[i+1]

        return iface, inet, mac

    def CreateAccount(self,user, password, expiration, thumbprint):
        """
        Create a user account, with 'user', 'password', 'expiration', ssh keys
        and sudo permissions.
        Returns None if successful, error string on failure.
        """
        userentry = None
        try:
            userentry = pwd.getpwnam(user)
        except:
            pass
        uidmin = None
        try:
            uidmin = int(GetLineStartingWith("UID_MIN", "/etc/login.defs").split()[1])
        except:
            pass
        if uidmin == None:
            uidmin = 100
        if userentry != None and userentry[2] < uidmin:
            Error("CreateAccount: " + user + " is a system user. Will not set password.")
            return "Failed to set password for system user: " + user + " (0x06)."
        if userentry == None:
            command = "pw useradd " + user + " -m"
            if expiration != None:
                command += " -e " + expiration.split('.')[0]
            if Run(command):
                Error("Failed to create user account: " + user)
                return "Failed to create user account: " + user + " (0x07)."
            else:
                Log("CreateAccount: " + user + " already exists. Will update password.")
        
        if password != None:
            self.changePass(user,password)
        try:
            # for older distros create sudoers.d
            if not os.path.isdir(MyDistro.sudoers_dir_base+'/sudoers.d/'):
                # create the /etc/sudoers.d/ directory
                os.mkdir(MyDistro.sudoers_dir_base+'/sudoers.d')
                # add the include of sudoers.d to the /etc/sudoers
                SetFileContents(MyDistro.sudoers_dir_base+'/sudoers',GetFileContents(MyDistro.sudoers_dir_base+'/sudoers')+'\n#includedir ' + MyDistro.sudoers_dir_base + '/sudoers.d\n')
            if password == None:
                SetFileContents(MyDistro.sudoers_dir_base+"/sudoers.d/waagent", user + " ALL = (ALL) NOPASSWD: ALL\n")
            else:
                SetFileContents(MyDistro.sudoers_dir_base+"/sudoers.d/waagent", user + " ALL = (ALL) ALL\n")
            os.chmod(MyDistro.sudoers_dir_base+"/sudoers.d/waagent", 0440)
        except:
            Error("CreateAccount: Failed to configure sudo access for user.")
            return "Failed to configure sudo privileges (0x08)."
        home = MyDistro.GetHome()
        if thumbprint != None:
            dir = home + "/" + user + "/.ssh"
            CreateDir(dir, user, 0700)
            pub = dir + "/id_rsa.pub"
            prv = dir + "/id_rsa"
            Run("ssh-keygen -y -f " + thumbprint + ".prv > " + pub)
            SetFileContents(prv, GetFileContents(thumbprint + ".prv"))
            for f in [pub, prv]:
                os.chmod(f, 0600)
                ChangeOwner(f, user)
            SetFileContents(dir + "/authorized_keys", GetFileContents(pub))
            ChangeOwner(dir + "/authorized_keys", user)
        Log("Created user account: " + user)
        return None
    
    def DeleteAccount(self,user):
        """
        Delete the 'user'.
        Clear utmp first, to avoid error.
        Removes the /etc/sudoers.d/waagent file.
        """
        userentry = None
        try:
            userentry = pwd.getpwnam(user)
        except:
            pass
        if userentry == None:
            Error("DeleteAccount: " + user + " not found.")
            return
        uidmin = None
        try:
            uidmin = int(GetLineStartingWith("UID_MIN", "/etc/login.defs").split()[1])
        except:
            pass
        if uidmin == None:
            uidmin = 100
        if userentry[2] < uidmin:
            Error("DeleteAccount: " + user + " is a system user. Will not delete account.")
            return
        Run("> /var/run/utmp") #Delete utmp to prevent error if we are the 'user' deleted
        Run("rmuser -y " + user)
        try:
            os.remove(MyDistro.sudoers_dir_base+"/sudoers.d/waagent")
        except:
            pass
        return

    def ActivateResourceDiskNoThread(self):
        """
        Format, mount, and if specified in the configuration
        set resource disk as swap.
        """
        global DiskActivated
        Run('cp /usr/sbin/waagent /tmp/')
        SetFileContents('/tmp/bsd_activate_resource_disk.py',bsd_activate_resource_disk_txt)
        Run('chmod +x /tmp/bsd_activate_resource_disk.py')
        pid = subprocess.Popen(["/tmp/bsd_activate_resource_disk.py", ""]).pid
        Log("Spawning bsd_activate_resource_disk.py")
        DiskActivated = True
        return

    def Install(self):
        """
        Install the agent service.
        Check dependencies.
        Create /etc/waagent.conf and move old version to
        /etc/waagent.conf.old
        Copy RulesFiles to /var/lib/waagent
        Create /etc/logrotate.d/waagent
        Set /etc/ssh/sshd_config ClientAliveInterval to 180
        Call ApplyVNUMAWorkaround()
        """
        if MyDistro.checkDependencies():
            return 1
        os.chmod(sys.argv[0], 0755)
        SwitchCwd()
        for a in RulesFiles:
            if os.path.isfile(a):
                if os.path.isfile(GetLastPathElement(a)):
                    os.remove(GetLastPathElement(a))
                shutil.move(a, ".")
                Warn("Moved " + a + " -> " + LibDir + "/" + GetLastPathElement(a) )
        MyDistro.registerAgentService()
        if os.path.isfile("/etc/waagent.conf"):
            try:
                os.remove("/etc/waagent.conf.old")
            except:
                pass
            try:
                os.rename("/etc/waagent.conf", "/etc/waagent.conf.old")
                Warn("Existing /etc/waagent.conf has been renamed to /etc/waagent.conf.old")
            except:
                pass
        SetFileContents("/etc/waagent.conf", self.waagent_conf_file)
        if os.path.exists('/usr/local/etc/logrotate.d/'):
            SetFileContents("/usr/local/etc/logrotate.d/waagent", WaagentLogrotate)
        filepath = "/etc/ssh/sshd_config"
        ReplaceFileContentsAtomic(filepath, "\n".join(filter(lambda a: not
            a.startswith("ClientAliveInterval"),
            GetFileContents(filepath).split('\n'))) + "\nClientAliveInterval 180\n")
        Log("Configured SSH client probing to keep connections alive.")
        #ApplyVNUMAWorkaround()
        return 0
    
    def mediaHasFilesystem(self,dsk):
        if Run('LC_ALL=C fdisk -p ' + dsk + ' | grep "invalid fdisk partition table found" ',False):
            return False
        return True
    
    def mountDVD(self,dvd,location):
        #At this point we cannot read a joliet option udf DVD in freebsd10 - so we 'dd' it into our location
        retcode,out = RunGetOutput(self.mount_dvd_cmd + dvd + ' of=' + location + '/ovf-env.xml')
        if retcode != 0:
            return retcode,out

        ovfxml = (GetFileContents(location+"/ovf-env.xml",asbin=False))
        ovfxml = re.sub('>.*\Z', '', ovfxml)
        ovfxml += '>'
        if ord(ovfxml[0]) > 128 and ord(ovfxml[1]) > 128 and ord(ovfxml[2]) > 128 :
            ovfxml = ovfxml[3:] # BOM is not stripped. First three bytes are > 128 and not unicode chars so we ignore them.
        ovfxml = ovfxml.strip(chr(0x00))
        ovfxml = "".join(filter(lambda x: ord(x)<128, ovfxml))
        SetFileContents(location+"/ovf-env.xml", ovfxml)
        return retcode,out

    def GetHome(self):
        return '/home'

    def initScsiDiskTimeout(self):
        """
        Set the SCSI disk timeout by updating the kernal config
        """
        timeout = Config.get("OS.RootDeviceScsiTimeout") 
        if timeout:
            Run("sysctl kern.cam.da.default_timeout=" + timeout)

    def setScsiDiskTimeout(self):
        return

    def setBlockDeviceTimeout(self, device, timeout):
        return

############################################################
# END DISTRO CLASS DEFS
############################################################  

# This lets us index into a string or an array of integers transparently.
def Ord(a):
    """
    Allows indexing into a string or an array of integers transparently.
    Generic utility function.
    """
    if type(a) == type("a"):
        a = ord(a)
    return a

def IsLinux():
    """
    Returns True if platform is Linux.
    Generic utility function.
    """
    return (platform.uname()[0] == "Linux")

def GetLastPathElement(path):
    """
    Similar to basename.
    Generic utility function.
    """
    return path.rsplit('/', 1)[1]

def GetFileContents(filepath,asbin=False):
    """
    Read and return contents of 'filepath'.
    """
    mode='r'
    if asbin:
        mode+='b'
    c=None
    try:
        with open(filepath, mode) as F :
            c=F.read()
    except IOError, e:
        ErrorWithPrefix('GetFileContents','Reading from file ' + filepath + ' Exception is ' + str(e))
        return None        
    return c

def SetFileContents(filepath, contents):
    """
    Write 'contents' to 'filepath'.
    """
    if type(contents) == str :
        contents=contents.encode('latin-1', 'ignore')
    try:
        with open(filepath, "wb+") as F :
            F.write(contents)
    except IOError, e:
        ErrorWithPrefix('SetFileContents','Writing to file ' + filepath + ' Exception is ' + str(e))
        return None
    return 0

def AppendFileContents(filepath, contents):
    """
    Append 'contents' to 'filepath'.
    """
    if type(contents) == str :
        contents=contents.encode('latin-1')
    try: 
        with open(filepath, "a+") as F :
            F.write(contents)
    except IOError, e:
        ErrorWithPrefix('AppendFileContents','Appending to file ' + filepath + ' Exception is ' + str(e))
        return None
    return 0

def ReplaceFileContentsAtomic(filepath, contents):
    """
    Write 'contents' to 'filepath' by creating a temp file, and replacing original.
    """
    handle, temp = tempfile.mkstemp(dir = os.path.dirname(filepath))
    if type(contents) == str :
        contents=contents.encode('latin-1')
    try:
        os.write(handle, contents)
    except IOError, e:
        ErrorWithPrefix('ReplaceFileContentsAtomic','Writing to file ' + filepath + ' Exception is ' + str(e))
        return None
    finally:
        os.close(handle)
    try:
        os.rename(temp, filepath)
        return None
    except IOError, e:
        ErrorWithPrefix('ReplaceFileContentsAtomic','Renaming ' + temp+ ' to ' + filepath + ' Exception is ' + str(e))
    try:
        os.remove(filepath)
    except IOError, e:
        ErrorWithPrefix('ReplaceFileContentsAtomic','Removing '+ filepath + ' Exception is ' + str(e))
    try:
        os.rename(temp,filepath)
    except IOError, e:
        ErrorWithPrefix('ReplaceFileContentsAtomic','Removing '+ filepath + ' Exception is ' + str(e))
        return 1
    return 0

def GetLineStartingWith(prefix, filepath):
    """
    Return line from 'filepath' if the line startswith 'prefix'
    """
    for line in GetFileContents(filepath).split('\n'):
        if line.startswith(prefix):
            return line
    return None

def Run(cmd,chk_err=True):
    """
    Calls RunGetOutput on 'cmd', returning only the return code.
    If chk_err=True then errors will be reported in the log.
    If chk_err=False then errors will be suppressed from the log.
    """
    retcode,out=RunGetOutput(cmd,chk_err)
    return retcode

def RunGetOutput(cmd,chk_err=True):
    """
    Wrapper for subprocess.check_output.
    Execute 'cmd'.  Returns return code and STDOUT, trapping expected exceptions.
    Reports exceptions to Error if chk_err parameter is True
    """
    LogIfVerbose(cmd)
    try:                                     
        output=subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
    except subprocess.CalledProcessError,e :
        if chk_err :
            Error('CalledProcessError.  Error Code is ' + str(e.returncode)  )
            Error('CalledProcessError.  Command string was ' + e.cmd  )
            Error('CalledProcessError.  Command result was ' + (e.output[:-1]).decode('latin-1'))
        return e.returncode,e.output.decode('latin-1')
    return 0,output.decode('latin-1')

def RunSendStdin(cmd,input,chk_err=True):
    """
    Wrapper for subprocess.Popen.
    Execute 'cmd', sending 'input' to STDIN of 'cmd'.
    Returns return code and STDOUT, trapping expected exceptions.
    Reports exceptions to Error if chk_err parameter is True
    """
    LogIfVerbose(cmd+input)
    try:                                     
        me=subprocess.Popen([cmd], shell=True, stdin=subprocess.PIPE,stderr=subprocess.STDOUT,stdout=subprocess.PIPE)
        output=me.communicate(input)
    except OSError , e :
        if chk_err :
            Error('CalledProcessError.  Error Code is ' + str(me.returncode)  )
            Error('CalledProcessError.  Command string was ' + cmd  )
            Error('CalledProcessError.  Command result was ' + output[0].decode('latin-1'))
            return 1,output[0].decode('latin-1')
    if me.returncode is not 0 and chk_err is True:
            Error('CalledProcessError.  Error Code is ' + str(me.returncode)  )
            Error('CalledProcessError.  Command string was ' + cmd  )
            Error('CalledProcessError.  Command result was ' + output[0].decode('latin-1'))
    return me.returncode,output[0].decode('latin-1')

def GetNodeTextData(a):
    """
    Filter non-text nodes from DOM tree
    """
    for b in a.childNodes:
        if b.nodeType == b.TEXT_NODE:
            return b.data

def GetHome():
    """
    Attempt to guess the $HOME location.
    Return the path string.
    """
    home = None
    try:
        home = GetLineStartingWith("HOME", "/etc/default/useradd").split('=')[1].strip()
    except:
        pass
    if (home == None) or (home.startswith("/") == False):
        home = "/home"
    return home

def ChangeOwner(filepath, user):
    """
    Lookup user.  Attempt chown 'filepath' to 'user'.
    """
    p = None
    try:
        p = pwd.getpwnam(user)
    except:
        pass
    if p != None:
        os.chown(filepath, p[2], p[3])

def CreateDir(dirpath, user, mode):
    """
    Attempt os.makedirs, catch all exceptions.
    Call ChangeOwner afterwards.
    """
    try:
        os.makedirs(dirpath, mode)
    except:
        pass
    ChangeOwner(dirpath, user)

def CreateAccount(user, password, expiration, thumbprint):
    """
    Create a user account, with 'user', 'password', 'expiration', ssh keys
    and sudo permissions.
    Returns None if successful, error string on failure.
    """
    userentry = None
    try:
        userentry = pwd.getpwnam(user)
    except:
        pass
    uidmin = None
    try:
        uidmin = int(GetLineStartingWith("UID_MIN", "/etc/login.defs").split()[1])
    except:
        pass
    if uidmin == None:
        uidmin = 100
    if userentry != None and userentry[2] < uidmin:
        Error("CreateAccount: " + user + " is a system user. Will not set password.")
        return "Failed to set password for system user: " + user + " (0x06)."
    if userentry == None:
        command = "useradd -m " + user
        if expiration != None:
            command += " -e " + expiration.split('.')[0]
        if Run(command):
            Error("Failed to create user account: " + user)
            return "Failed to create user account: " + user + " (0x07)."
    else:
        Log("CreateAccount: " + user + " already exists. Will update password.")
    if password != None:
        RunSendStdin("chpasswd",(user + ":" + password + "\n"))
    try:
        # for older distros create sudoers.d
        if not os.path.isdir('/etc/sudoers.d/'):
            # create the /etc/sudoers.d/ directory
            os.mkdir('/etc/sudoers.d/')
            # add the include of sudoers.d to the /etc/sudoers
            SetFileContents('/etc/sudoers',GetFileContents('/etc/sudoers')+'\n#includedir /etc/sudoers.d\n')
        if password == None:
            SetFileContents("/etc/sudoers.d/waagent", user + " ALL = (ALL) NOPASSWD: ALL\n")
        else:
            SetFileContents("/etc/sudoers.d/waagent", user + " ALL = (ALL) ALL\n")
        os.chmod("/etc/sudoers.d/waagent", 0440)
    except:
        Error("CreateAccount: Failed to configure sudo access for user.")
        return "Failed to configure sudo privileges (0x08)."
    home = MyDistro.GetHome()
    if thumbprint != None:
        dir = home + "/" + user + "/.ssh"
        CreateDir(dir, user, 0700)
        pub = dir + "/id_rsa.pub"
        prv = dir + "/id_rsa"
        Run("ssh-keygen -y -f " + thumbprint + ".prv > " + pub)
        SetFileContents(prv, GetFileContents(thumbprint + ".prv"))
        for f in [pub, prv]:
            os.chmod(f, 0600)
            ChangeOwner(f, user)
        SetFileContents(dir + "/authorized_keys", GetFileContents(pub))
        ChangeOwner(dir + "/authorized_keys", user)
    Log("Created user account: " + user)
    return None

def DeleteAccount(user):
    """
    Delete the 'user'.
    Clear utmp first, to avoid error.
    Removes the /etc/sudoers.d/waagent file.
    """
    userentry = None
    try:
        userentry = pwd.getpwnam(user)
    except:
        pass
    if userentry == None:
        Error("DeleteAccount: " + user + " not found.")
        return
    uidmin = None
    try:
        uidmin = int(GetLineStartingWith("UID_MIN", "/etc/login.defs").split()[1])
    except:
        pass
    if uidmin == None:
        uidmin = 100
    if userentry[2] < uidmin:
        Error("DeleteAccount: " + user + " is a system user. Will not delete account.")
        return
    Run("> /var/run/utmp") #Delete utmp to prevent error if we are the 'user' deleted
    Run("userdel -f -r " + user)
    try:
        os.remove("/etc/sudoers.d/waagent")
    except:
        pass
    return

def IsInRangeInclusive(a, low, high):
    """
    Return True if 'a' in 'low' <= a >= 'high'
    """
    return (a >= low and a <= high)

def IsPrintable(ch):
    """
    Return True if character is displayable.
    """
    return IsInRangeInclusive(ch, Ord('A'), Ord('Z')) or IsInRangeInclusive(ch, Ord('a'), Ord('z')) or IsInRangeInclusive(ch, Ord('0'), Ord('9'))

def HexDump(buffer, size):
    """
    Return Hex formated dump of a 'buffer' of 'size'.
    """
    if size < 0:
        size = len(buffer)
    result = ""
    for i in range(0, size):
        if (i % 16) == 0:
            result += "%06X: " % i
        byte = buffer[i]
        if type(byte) == str:
            byte = ord(byte.decode('latin1'))
        result += "%02X " % byte
        if (i & 15) == 7:
            result += " "
        if ((i + 1) % 16) == 0 or (i + 1) == size:
            j = i
            while ((j + 1) % 16) != 0:
                result += "   "
                if (j & 7) == 7:
                    result += " "
                j += 1
            result += " "
            for j in range(i - (i % 16), i + 1):
                byte=buffer[j]
                if type(byte) == str:
                    byte = ord(byte.decode('latin1'))
                k = '.'
                if IsPrintable(byte):
                    k = chr(byte)
                result += k
            if (i + 1) != size:
                result += "\n"
    return result

class Logger(object):
    """
    The Agent's logging assumptions are:
    For Log, and LogWithPrefix all messages are logged to the
    self.file_path and to the self.con_path.  Setting either path
    parameter to None skips that log.  If Verbose is enabled, messages
    calling the LogIfVerbose method will be logged to file_path yet
    not to con_path.  Error and Warn messages are normal log messages
    with the 'ERROR:' or 'WARNING:' prefix added.
    """

    def __init__(self,filepath,conpath,verbose=False):
        """
        Construct an instance of Logger.
        """
        self.file_path=filepath
        self.con_path=conpath
        self.verbose=verbose
        
    def ThrottleLog(self,counter):
        """
        Log everything up to 10, every 10 up to 100, then every 100.
        """
        return (counter < 10) or ((counter < 100) and ((counter % 10) == 0)) or ((counter % 100) == 0)
    
    def LogToFile(self,message):
        """
        Write 'message' to logfile.
        """
        if self.file_path:
            with open(self.file_path, "a") as F :
                F.write(message.encode('ascii','ignore') + "\n")
                F.close()
            
    def LogToCon(self,message):
        """
        Write 'message' to /dev/console.
        This supports serial port logging if the /dev/console
        is redirected to ttys0 in kernel boot options.
        """
        if self.con_path:
            with open(self.con_path, "w") as C :
                C.write(message.encode('ascii','ignore') + "\n")
                C.close()
                
    def Log(self,message):
        """
        Standard Log function.
        Logs to self.file_path, and con_path
        """
        self.LogWithPrefix("", message)
    
    def LogWithPrefix(self,prefix, message):
        """
        Prefix each line of 'message' with current time+'prefix'.
        """
        t = time.localtime()
        t = "%04u/%02u/%02u %02u:%02u:%02u " % (t.tm_year, t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec)
        t += prefix
        for line in message.split('\n'):
            line = t + line
            self.LogToFile(line)
            self.LogToCon(line)
            
    def NoLog(self,message):
        """
        Don't Log.
        """
        pass
    
    def LogIfVerbose(self,message):
        """
        Only log 'message' if global Verbose is True.
        """
        self.LogWithPrefixIfVerbose('',message)
    
    def LogWithPrefixIfVerbose(self,prefix, message):
        """
        Only log 'message' if global Verbose is True.
        Prefix each line of 'message' with current time+'prefix'.
        """
        if self.verbose == True:
            t = time.localtime()
            t = "%04u/%02u/%02u %02u:%02u:%02u " % (t.tm_year, t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec)
            t += prefix
            for line in message.split('\n'):
                line = t + line
                self.LogToFile(line)
                self.LogToCon(line)
                
    def Warn(self,message):
        """
        Prepend the text "WARNING:" to the prefix for each line in 'message'.
        """
        self.LogWithPrefix("WARNING:", message)
    
    def Error(self,message):
        """
        Call ErrorWithPrefix(message).
        """
        ErrorWithPrefix("", message)
    
    def ErrorWithPrefix(self,prefix, message):
        """
        Prepend the text "ERROR:" to the prefix for each line in 'message'.
        Errors written to logfile, and /dev/console
        """
        self.LogWithPrefix("ERROR:", message)
    
def LoggerInit(log_file_path,log_con_path,verbose=False):
    """
    Create log object and export its methods to global scope.
    """
    global Log,LogWithPrefix,LogIfVerbose,LogWithPrefixIfVerbose,Error,ErrorWithPrefix,Warn,NoLog,ThrottleLog,myLogger
    l=Logger(log_file_path,log_con_path,verbose)
    Log,LogWithPrefix,LogIfVerbose,LogWithPrefixIfVerbose,Error,ErrorWithPrefix,Warn,NoLog,ThrottleLog,myLogger = l.Log,l.LogWithPrefix,l.LogIfVerbose,l.LogWithPrefixIfVerbose,l.Error,l.ErrorWithPrefix,l.Warn,l.NoLog,l.ThrottleLog,l

def Linux_ioctl_GetInterfaceMac(ifname):
    """
    Return the mac-address bound to the socket.
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    info = fcntl.ioctl(s.fileno(), 0x8927,  struct.pack('256s', (ifname[:15]+('\0'*241)).encode('latin-1')))
    return ''.join(['%02X' % Ord(char) for char in info[18:24]])

def GetFirstActiveNetworkInterfaceNonLoopback():
    """
    Return the interface name, and ip addr of the
    first active non-loopback interface.
    """
    iface=''
    expected=16 # how many devices should I expect...
    struct_size=40 # for 64bit the size is 40 bytes
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    buff=array.array('B', b'\0' * (expected*struct_size))
    retsize=(struct.unpack('iL', fcntl.ioctl(s.fileno(), 0x8912, struct.pack('iL',expected*struct_size,buff.buffer_info()[0]))))[0]
    if retsize == (expected*struct_size) :
        Warn('SIOCGIFCONF returned more than ' + str(expected) + ' up network interfaces.')
    s=buff.tostring()
    for i in range(0,struct_size*expected,struct_size):
        iface=s[i:i+16].split(b'\0', 1)[0]
        if iface == b'lo':
            continue
        else :
            break
    return iface.decode('latin-1'), socket.inet_ntoa(s[i+20:i+24])

def GetIpv4Address():
    """
    Return the ip of the 
    first active non-loopback interface.
    """
    iface,addr=GetFirstActiveNetworkInterfaceNonLoopback()
    return addr

def HexStringToByteArray(a):
    """
    Return hex string packed into a binary struct.
    """
    b = b""
    for c in range(0, len(a) // 2):
        b += struct.pack("B", int(a[c * 2:c * 2 + 2], 16))
    return b

def GetMacAddress():
    """
    Convienience function, returns mac addr bound to
    first non-loobback interface.
    """
    ifname=''
    while len(ifname) < 2 :
        ifname=GetFirstActiveNetworkInterfaceNonLoopback()[0]
    a = Linux_ioctl_GetInterfaceMac(ifname)        
    return HexStringToByteArray(a)

def DeviceForIdePort(n):
    """
    Return device name attached to ide port 'n'.
    """
    if n > 3:
        return None
    g0 = "00000000"
    if n > 1:
        g0 = "00000001"
        n = n - 2
    device = None
    path = "/sys/bus/vmbus/devices/"
    for vmbus in os.listdir(path):
        guid = GetFileContents(path + vmbus + "/device_id").lstrip('{').split('-')
        if guid[0] == g0 and guid[1] == "000" + str(n):
            for root, dirs, files in os.walk(path + vmbus):
                if root.endswith("/block"):
                    device = dirs[0]
                    break
                else : #older distros
                    for d in dirs:
                        if ':' in d and "block" == d.split(':')[0]:
                            device = d.split(':')[1]
                            break
            break
    return device

class Util(object):
    """
    Http communication class.
    Base of GoalState, and Agent classes.
    """
    def _HttpGet(self, url, headers):
        """
        Do HTTP get on 'url' with 'headers'.
        On error, sleep 10 and maxRetry times.
        Return the output buffer or None.
        """
        LogIfVerbose("HttpGet(" + url + ")")
        maxRetry = 2
        if url.startswith("http://"):
            url = url[7:]
            url = url[url.index("/"):]
        for retry in range(0, maxRetry + 1):
            strRetry = str(retry)
            log = [NoLog, Error][retry > 0]
            log("retry HttpGet(" + url + "),retry=" + strRetry)
            response = None
            strStatus = "None"
            try:
                httpConnection = httplib.HTTPConnection(self.Endpoint)
                if headers == None:
                    request = httpConnection.request("GET", url)
                else:
                    request = httpConnection.request("GET", url, None, headers)
                response = httpConnection.getresponse()
                strStatus = str(response.status)
            except httplib.HTTPException, e:
                Error('HTTPException ' + e.message + ' args: ' + repr(e.args))
            except IOError, e:
                Error('socket IOError ' + e.message + ' args: ' + repr(e.args))
            log("response HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus)
            if response == None or response.status != httplib.OK:
                Error("HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                if retry == maxRetry:
                    Error("return HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                    return None
                else:
                    Error("sleep 10 seconds HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                    time.sleep(10)
            else:
                log("return HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                return response.read()

    def HttpGetWithoutHeaders(self, url):
        """
        Return data from an HTTP get on 'url'.
        """
        return self._HttpGet(url, None)

    def HttpGetWithHeaders(self, url):
        """
        Return data from an HTTP get on 'url' with
        x-ms-agent-name and x-ms-version
        headers.
        """
        return self._HttpGet(url, {"x-ms-agent-name": GuestAgentName, "x-ms-version": ProtocolVersion})

    def HttpSecureGetWithHeaders(self, url, transportCert):
        """
        Return output of get using ssl cert.
        """
        return self._HttpGet(url, {"x-ms-agent-name": GuestAgentName,
                                   "x-ms-version": ProtocolVersion,
                                   "x-ms-cipher-name": "DES_EDE3_CBC",
                                   "x-ms-guest-agent-public-x509-cert": transportCert})

    def HttpPost(self, url, data):
        """
        Send http POST to server, sleeping 10 retrying maxRetry times upon error.
        """
        LogIfVerbose("HttpPost(" + url + ")")
        maxRetry = 2
        for retry in range(0, maxRetry + 1):
            strRetry = str(retry)
            log = [NoLog, Error][retry > 0]
            log("retry HttpPost(" + url + "),retry=" + strRetry)
            response = None
            strStatus = "None"
            try:
                httpConnection = httplib.HTTPConnection(self.Endpoint)
                request = httpConnection.request("POST", url, data, {"x-ms-agent-name": GuestAgentName,
                                                                     "Content-Type": "text/xml; charset=utf-8",
                                                                     "x-ms-version": ProtocolVersion})
                response = httpConnection.getresponse()
                strStatus = str(response.status)
            except httplib.HTTPException, e:
                Error('HTTPException ' + e.message + ' args: ' + repr(e.args))
            except IOError, e:
                Error('socket IOError ' + e.message + ' args: ' + repr(e.args))
            log("response HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus)
            if response == None or (response.status != httplib.OK and response.status != httplib.ACCEPTED):
                Error("HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                if retry == maxRetry:
                    Error("return HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                    return None
                else:
                    Error("sleep 10 seconds HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                    time.sleep(10)
            else:
                log("return HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                return response

    def HttpPutBlockBlob(self, url, data):
        """
        Send http PUT to server, sleeping 10 retrying maxRetry times upon error.
        """
        LogIfVerbose("HttpPutBlockBlob(" + url + ")")
        maxRetry = 2
        for retry in range(0, maxRetry + 1):
            strRetry = str(retry)
            log = [NoLog, Error][retry > 0]
            log("retry HttpPutBlockBlob(" + url + "),retry=" + strRetry)
            response = None
            strStatus = "None"
            try:
                httpConnection = httplib.HTTPConnection(self.Endpoint)
                request = httpConnection.request("PUT", url, data, {"x-ms-blob-type" : "BlockBlob", "x-ms-date" : time.strftime("%Y-%M-%dT%H:%M:%SZ", time.gmtime()) ,"Content-Length": str(len(data))} )
                response = httpConnection.getresponse()
                strStatus = str(response.status)
            except httplib.HTTPException, e:
                Error('HTTPException ' + e.message + ' args: ' + repr(e.args))
            except IOError, e:
                Error('socket IOError ' + e.message + ' args: ' + repr(e.args))
            log("response HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus)
            if response == None or (response.status != httplib.OK and response.status != httplib.CREATED):
                Error("HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                if retry == maxRetry:
                    Error("return HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                    return None
                else:
                    Error("sleep 10 seconds HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                    time.sleep(10)
            else:
                log("return HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus)
                return response

class TCPHandler(SocketServer.BaseRequestHandler):
    """
    Callback object for LoadBalancerProbeServer.
    Recv and send LB probe messages.
    """
    def __init__(self,lb_probe):
        super(TCPHandler,self).__init__()
        self.lb_probe=lb_probe
        
    def GetHttpDateTimeNow(self):
        """
        Return formatted gmtime "Date: Fri, 25 Mar 2011 04:53:10 GMT"
        """
        return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())

    def handle(self):
        """
        Log LB probe messages, read the socket buffer,
        send LB probe response back to server.
        """
        self.lb_probe.ProbeCounter = (self.lb_probe.ProbeCounter + 1) % 1000000
        log = [NoLog, LogIfVerbose][ThrottleLog(self.lb_probe.ProbeCounter)]
        strCounter = str(self.lb_probe.ProbeCounter)
        if self.lb_probe.ProbeCounter == 1:
            Log("Receiving LB probes.")
        log("Received LB probe # " + strCounter)
        self.request.recv(1024)
        self.request.send("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nContent-Type: text/html\r\nDate: " + self.GetHttpDateTimeNow() + "\r\n\r\nOK")
            
class LoadBalancerProbeServer(object):
    """
    Threaded object to receive and send LB probe messages.
    Load Balancer messages but be recv'd by
    the load balancing server, or this node may be shut-down.
    """
    def __init__(self, port):
        self.ProbeCounter = 0
        self.server = SocketServer.TCPServer((self.get_ip(), port), TCPHandler)
        self.server_thread = threading.Thread(target = self.server.serve_forever)
        self.server_thread.setDaemon(True)
        self.server_thread.start()
        
    def shutdown(self):
        self.server.shutdown()

    def get_ip(self):
        for retry in range(1,6):
            ip = MyDistro.GetIpv4Address()
            if ip == None :
                Log("LoadBalancerProbeServer: GetIpv4Address() returned None, sleeping 10 before retry " + str(retry+1) )
                time.sleep(10)
            else:
                return ip

class ConfigurationProvider(object):
    """
    Parse amd store key:values in /etc/waagent.conf.
    """
    def __init__(self):
        self.values = dict()
        if os.path.isfile("/etc/waagent.conf") == False:
            raise Exception("Missing configuration in /etc/waagent.conf")
        try:
            for line in GetFileContents("/etc/waagent.conf").split('\n'):
                if not line.startswith("#") and "=" in line:
                    parts = line.split()[0].split('=')
                    value = parts[1].strip("\" ")
                    if value != "None":
                        self.values[parts[0]] = value
                    else:
                        self.values[parts[0]] = None
        except:
            Error("Unable to parse /etc/waagent.conf")
            raise
        return

    def get(self, key):
        return self.values.get(key)

class EnvMonitor(object):
    """
    Montor changes to dhcp and hostname.
    If dhcp clinet process re-start has occurred, reset routes, dhcp with fabric.
    """
    def __init__(self):
        self.shutdown = False
        self.HostName = socket.gethostname()
        self.server_thread = threading.Thread(target = self.monitor)
        self.server_thread.setDaemon(True)
        self.server_thread.start()
        self.published = False

    def monitor(self):
        """
        Monitor dhcp client pid and hostname.
        If dhcp clinet process re-start has occurred, reset routes, dhcp with fabric.
        """
        publish = ConfigurationProvider().get("Provisioning.MonitorHostName")
        dhcpcmd = MyDistro.getpidcmd+ ' ' + MyDistro.getDhcpClientName()
        dhcppid = RunGetOutput(dhcpcmd)[1]
        while not self.shutdown:
            for a in RulesFiles:
                if os.path.isfile(a):
                    if os.path.isfile(GetLastPathElement(a)):
                        os.remove(GetLastPathElement(a))
                    shutil.move(a, ".")
                    Log("EnvMonitor: Moved " + a + " -> " + LibDir)
            MyDistro.setScsiDiskTimeout()
            if publish != None and publish.lower().startswith("y"):
                try:
                    if socket.gethostname() != self.HostName:
                        Log("EnvMonitor: Detected host name change: " + self.HostName + " -> " + socket.gethostname())
                        self.HostName = socket.gethostname()
                        WaAgent.UpdateAndPublishHostName(self.HostName)
                        dhcppid = RunGetOutput(dhcpcmd)[1]
                        self.published = True
                except:
                    pass
            else:
                self.published = True
            pid = ""
            if not os.path.isdir("/proc/" + dhcppid.strip()):
                pid = RunGetOutput(dhcpcmd)[1]
            if pid != "" and pid != dhcppid:
                Log("EnvMonitor: Detected dhcp client restart. Restoring routing table.")
                WaAgent.RestoreRoutes()
                dhcppid = pid
            for child in Children:
                if child.poll() != None:
                    Children.remove(child)
            time.sleep(5)

    def SetHostName(self, name):
        """
        Generic call to MyDistro.setHostname(name).
        Complian to Log on error.
        """
        if socket.gethostname() == name:
            self.published = True
        elif MyDistro.setHostname(name):
            Error("Error: SetHostName: Cannot set hostname to " + name)
            return ("Error: SetHostName: Cannot set hostname to " + name)

    def IsHostnamePublished(self):
        """
        Return self.published  
        """
        return self.published

    def ShutdownService(self):
        """
        Stop server comminucation and join the thread to main thread.
        """
        self.shutdown = True
        self.server_thread.join()

class Certificates(object):
    """
    Object containing certificates of host and provisioned user.
    Parses and splits certificates into files.
    """
    #     <CertificateFile xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="certificates10.xsd">
    #     <Version>2010-12-15</Version>
    #     <Incarnation>2</Incarnation>
    #     <Format>Pkcs7BlobWithPfxContents</Format>
    #     <Data>MIILTAY...
    #     </Data>
    #     </CertificateFile>

    def __init__(self):
        self.reinitialize()

    def reinitialize(self):
        """
        Reset the Role, Incarnation
        """
        self.Incarnation = None
        self.Role = None

    def Parse(self, xmlText):
        """
        Parse multiple certificates into seperate files.
        """
        self.reinitialize()
        SetFileContents("Certificates.xml", xmlText)
        dom = xml.dom.minidom.parseString(xmlText)
        for a in [ "CertificateFile", "Version", "Incarnation",
                   "Format", "Data", ]:
            if not dom.getElementsByTagName(a):
                Error("Certificates.Parse: Missing " + a)
                return None
        node = dom.childNodes[0]
        if node.localName != "CertificateFile":
            Error("Certificates.Parse: root not CertificateFile")
            return None
        SetFileContents("Certificates.p7m",
            "MIME-Version: 1.0\n"
            + "Content-Disposition: attachment; filename=\"Certificates.p7m\"\n"
            + "Content-Type: application/x-pkcs7-mime; name=\"Certificates.p7m\"\n"
            + "Content-Transfer-Encoding: base64\n\n"
            + GetNodeTextData(dom.getElementsByTagName("Data")[0]))
        if Run(Openssl + " cms -decrypt -in Certificates.p7m -inkey TransportPrivate.pem -recip TransportCert.pem | " + Openssl + " pkcs12 -nodes -password pass: -out Certificates.pem"):
            Error("Certificates.Parse: Failed to extract certificates from CMS message.")
            return self
        # There may be multiple certificates in this package. Split them.
        file = open("Certificates.pem")
        pindex = 1
        cindex = 1
        output = open("temp.pem", "w")
        for line in file.readlines():
            output.write(line)
            if re.match(r'[-]+END .*?(KEY|CERTIFICATE)[-]+$',line):
                output.close()
                if re.match(r'[-]+END .*?KEY[-]+$',line):
                    os.rename("temp.pem", str(pindex) + ".prv")
                    pindex += 1
                else:
                    os.rename("temp.pem", str(cindex) + ".crt")
                    cindex += 1
                output = open("temp.pem", "w")
        output.close()
        os.remove("temp.pem")
        keys = dict()
        index = 1
        filename = str(index) + ".crt"
        while os.path.isfile(filename):
            thumbprint = (RunGetOutput(Openssl + " x509 -in " + filename + " -fingerprint -noout")[1]).rstrip().split('=')[1].replace(':', '').upper()
            pubkey=RunGetOutput(Openssl + " x509 -in " + filename + " -pubkey -noout")[1]
            keys[pubkey] = thumbprint
            os.rename(filename, thumbprint + ".crt")
            os.chmod(thumbprint + ".crt", 0600)
            MyDistro.setSelinuxContext(thumbprint + '.crt','unconfined_u:object_r:ssh_home_t:s0')
            index += 1
            filename = str(index) + ".crt"
        index = 1
        filename = str(index) + ".prv"
        while os.path.isfile(filename):
            pubkey = RunGetOutput(Openssl + " rsa -in " + filename + " -pubout 2> /dev/null ")[1]
            os.rename(filename, keys[pubkey] + ".prv")
            os.chmod(keys[pubkey] + ".prv", 0600)
            MyDistro.setSelinuxContext( keys[pubkey] + '.prv','unconfined_u:object_r:ssh_home_t:s0')
            index += 1
            filename = str(index) + ".prv"
        return self

class SharedConfig(object):
    """
    Parse role endpoint server and goal state config.
    """
    #
    # <SharedConfig version="1.0.0.0" goalStateIncarnation="1">
    #   <Deployment name="db00a7755a5e4e8a8fe4b19bc3b330c3" guid="{ce5a036f-5c93-40e7-8adf-2613631008ab}" incarnation="2">
    #     <Service name="MyVMRoleService" guid="{00000000-0000-0000-0000-000000000000}" />
    #     <ServiceInstance name="db00a7755a5e4e8a8fe4b19bc3b330c3.1" guid="{d113f4d7-9ead-4e73-b715-b724b5b7842c}" />
    #   </Deployment>
    #   <Incarnation number="1" instance="MachineRole_IN_0" guid="{a0faca35-52e5-4ec7-8fd1-63d2bc107d9b}" />
    #   <Role guid="{73d95f1c-6472-e58e-7a1a-523554e11d46}" name="MachineRole" settleTimeSeconds="10" />
    #   <LoadBalancerSettings timeoutSeconds="0" waitLoadBalancerProbeCount="8">
    #     <Probes>
    #       <Probe name="MachineRole" />
    #       <Probe name="55B17C5E41A1E1E8FA991CF80FAC8E55" />
    #       <Probe name="3EA4DBC19418F0A766A4C19D431FA45F" />
    #     </Probes>
    #   </LoadBalancerSettings>
    #   <OutputEndpoints>
    #     <Endpoint name="MachineRole:Microsoft.WindowsAzure.Plugins.RemoteAccess.Rdp" type="SFS">
    #       <Target instance="MachineRole_IN_0" endpoint="Microsoft.WindowsAzure.Plugins.RemoteAccess.Rdp" />
    #     </Endpoint>
    #   </OutputEndpoints>
    #   <Instances>
    #     <Instance id="MachineRole_IN_0" address="10.115.153.75">
    #       <FaultDomains randomId="0" updateId="0" updateCount="0" />
    #       <InputEndpoints>
    #         <Endpoint name="a" address="10.115.153.75:80" protocol="http" isPublic="true" loadBalancedPublicAddress="70.37.106.197:80" enableDirectServerReturn="false" isDirectAddress="false" disableStealthMode="false">
    #           <LocalPorts>
    #             <LocalPortRange from="80" to="80" />
    #           </LocalPorts>
    #         </Endpoint>
    #         <Endpoint name="Microsoft.WindowsAzure.Plugins.RemoteAccess.Rdp" address="10.115.153.75:3389" protocol="tcp" isPublic="false" enableDirectServerReturn="false" isDirectAddress="false" disableStealthMode="false">
    #           <LocalPorts>
    #             <LocalPortRange from="3389" to="3389" />
    #           </LocalPorts>
    #         </Endpoint>
    #         <Endpoint name="Microsoft.WindowsAzure.Plugins.RemoteForwarder.RdpInput" address="10.115.153.75:20000" protocol="tcp" isPublic="true" loadBalancedPublicAddress="70.37.106.197:3389" enableDirectServerReturn="false" isDirectAddress="false" disableStealthMode="false">
    #           <LocalPorts>
    #             <LocalPortRange from="20000" to="20000" />
    #           </LocalPorts>
    #         </Endpoint>
    #       </InputEndpoints>
    #     </Instance>
    #   </Instances>
    # </SharedConfig>
    #
    def __init__(self):
        self.reinitialize()

    def reinitialize(self):
        """
        Reset members.
        """
        self.Deployment = None
        self.Incarnation = None
        self.Role = None
        self.LoadBalancerSettings = None
        self.OutputEndpoints = None
        self.Instances = None

    def Parse(self, xmlText):
        """
        Parse and write configuration to file SharedConfig.xml.
        """
        self.reinitialize()
        SetFileContents("SharedConfig.xml", xmlText)
        dom = xml.dom.minidom.parseString(xmlText)
        for a in [ "SharedConfig", "Deployment", "Service",
                   "ServiceInstance", "Incarnation", "Role", ]:
            if not dom.getElementsByTagName(a):
                Error("SharedConfig.Parse: Missing " + a)
                return None
        node = dom.childNodes[0]
        if node.localName != "SharedConfig":
            Error("SharedConfig.Parse: root not SharedConfig")
            return None
        program = Config.get("Role.TopologyConsumer")
        if program != None:
            try:
                Children.append(subprocess.Popen([program, LibDir + "/SharedConfig.xml"]))
            except OSError, e :
                ErrorWithPrefix('Agent.Run','Exception: '+ str(e) +' occured launching ' + program )
        return self

class ExtensionsConfig(object):
    """
    Parse ExtensionsConfig, downloading and unpacking them to /var/lib/waagent.
    Install if <enabled>true</enabled>, remove if it is set to false.
    """
    #<?xml version="1.0" encoding="utf-8"?>
    #<Extensions version="1.0.0.0" goalStateIncarnation="6"><Plugins>
    #  <Plugin name="OSTCExtensions.ExampleHandlerLinux" version="1.5"
    #location="http://previewusnorthcache.blob.core.test-cint.azure-test.net/d84b216d00bf4d96982be531539e1513/OSTCExtensions_ExampleHandlerLinux_usnorth_manifest.xml"
    #config="" state="enabled" autoUpgrade="false" runAsStartupTask="false" isJson="true" />
    #</Plugins>
    #<PluginSettings>
    #  <Plugin name="OSTCExtensions.ExampleHandlerLinux" version="1.5">
    #    <RuntimeSettings seqNo="2">{"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"1BE9A13AA1321C7C515EF109746998BAB6D86FD1",
    #"protectedSettings":"MIIByAYJKoZIhvcNAQcDoIIBuTCCAbUCAQAxggFxMIIBbQIBADBVMEExPzA9BgoJkiaJk/IsZAEZFi9XaW5kb3dzIEF6dXJlIFNlcnZpY2UgTWFuYWdlbWVudCBmb3IgR
    #Xh0ZW5zaW9ucwIQZi7dw+nhc6VHQTQpCiiV2zANBgkqhkiG9w0BAQEFAASCAQCKr09QKMGhwYe+O4/a8td+vpB4eTR+BQso84cV5KCAnD6iUIMcSYTrn9aveY6v6ykRLEw8GRKfri2d6
    #tvVDggUrBqDwIgzejGTlCstcMJItWa8Je8gHZVSDfoN80AEOTws9Fp+wNXAbSuMJNb8EnpkpvigAWU2v6pGLEFvSKC0MCjDTkjpjqciGMcbe/r85RG3Zo21HLl0xNOpjDs/qqikc/ri43Y76E/X
    #v1vBSHEGMFprPy/Hwo3PqZCnulcbVzNnaXN3qi/kxV897xGMPPC3IrO7Nc++AT9qRLFI0841JLcLTlnoVG1okPzK9w6ttksDQmKBSHt3mfYV+skqs+EOMDsGCSqGSIb3DQEHATAUBggqh
    #kiG9w0DBwQITgu0Nu3iFPuAGD6/QzKdtrnCI5425fIUy7LtpXJGmpWDUA==","publicSettings":{"port":"3000"}}}]}</RuntimeSettings>
    #  </Plugin>
    #</PluginSettings>
    #<StatusUploadBlob>https://ostcextensions.blob.core.test-cint.azure-test.net/vhds/eg-plugin7-vm.eg-plugin7-vm.eg-plugin7-vm.status?sr=b&amp;sp=rw&amp;
    #se=9999-01-01&amp;sk=key1&amp;sv=2012-02-12&amp;sig=wRUIDN1x2GC06FWaetBP9sjjifOWvRzS2y2XBB4qoBU%3D</StatusUploadBlob></Extensions>

    def __init__(self):
        self.reinitialize()

    def reinitialize(self):
        """
        Reset members.
        """
        self.Extensions = None
        self.Plugins = None
        self.Util = None
        
    def Parse(self, xmlText):
        """
        Write configuration to file ExtensionsConfig.xml.
        If state is enabled:
            if the plugin is installed:
                if the new plugin's version is higher:
                    download the new archive
                    do the updateCommand.
                    disable the old plugin and remove
                    enable the new plugin
                if the new plugin's version is the same or lower:
                    create the new .settings file from the configuration received
                    do the enableCommand
            if the plugin is not installed:
                download/unpack archive and call the installCommand/Enable
        if state is disabled:
            call disableCommand
        """
        self.reinitialize()
        self.Util=Util()
        dom = xml.dom.minidom.parseString(xmlText)
        LogIfVerbose(xmlText)
        try:
            self.Extensions=dom.getElementsByTagName("Extensions")
            pg = dom.getElementsByTagName("Plugins")
            self.Plugins = pg[0].getElementsByTagName("Plugin")
            incarnation=self.Extensions[0].getAttribute("goalStateIncarnation")
            SetFileContents('ExtensionsConfig.'+incarnation+'.xml', xmlText)
        except:
            Error('Error parsing ExtensionsConfig.')
            return None
        for p in self.Plugins:
            if len(p.getAttribute("location"))<1:  # this plugin is inside the PluginSettings
                continue
            previous_version = None
            version=p.getAttribute("version")
            name=p.getAttribute("name")
            if p.getAttribute("isJson") != 'true':
                Error("Plugin " + name+" version: " +version+" is not a JSON Extension.  Skipping.")
                continue
            Log("Found Plugin: " + name + ' version: ' + version)
            if p.getAttribute("state") == 'disabled' or p.getAttribute("state") == 'uninstall': 
                #disable 
                zip_dir=LibDir+"/" + name + '-' + version
                mfile=None
                for root, dirs, files in os.walk(zip_dir):
                    for f in files:
                        if f in ('HandlerManifest.json'):
                            mfile=os.path.join(root,f)
                    if mfile != None:
                        break
                if mfile == None :
                    Error('HandlerManifest.json not found.')
                    continue
                manifest = GetFileContents(mfile)
                p.setAttribute('manifestdata',manifest)
                if self.launchCommand(name,version,'disableCommand') == None :
                    Error('Unable to disable '+name)
                else :
                    Log(name+' is disabled')
                # uninstall if needed
                if p.getAttribute("state") == 'uninstall':
                    if self.launchCommand(name,version,'uninstallCommand') == None :
                        Error('Unable to uninstall '+name)
                    else :
                        Log(name+' uninstallCommand completed .')
                    # remove the plugin
                    Run('rm -rf ' + LibDir + '/' + name +'-'+ version + '*')
                    Log(name +'-'+ version + ' extension files deleted.')
                continue    
            # state is enabled
            # if the same plugin exists and the version is newer or
            # does not exist then download and unzip the new plugin
            plg_dir=None
            for root, dirs, files in os.walk(LibDir):
                for d in dirs:
                    if name in d:
                        plg_dir=os.path.join(root,d)
                    if plg_dir != None:
                        break
            if plg_dir != None :
                previous_version=plg_dir.rsplit('-')[-1]
            if plg_dir == None or version > previous_version :
                location=p.getAttribute("location")
                Log("Downloading plugin manifest: " + name + " from " + location)
                self.Util.Endpoint=location.split('/')[2]
                Log("Plugin server is: " +  self.Util.Endpoint)
                manifest=self.Util.HttpGetWithoutHeaders(location)
                if manifest == None:
                    Error("Unable to download plugin manifest" + name + " from primary location.  Attempting with failover location.")
                    failoverlocation=p.getAttribute("failoverlocation")
                    self.Util.Endpoint=failoverlocation.split('/')[2]
                    Log("Plugin failover server is: " +  self.Util.Endpoint)
                    manifest=self.Util.HttpGetWithoutHeaders(failoverlocation)
                if manifest == None:
                    Error("Unable to download plugin manifest" + name + " from failover location")
                    continue
                Log("Plugin manifest" + name + "downloaded successfully length = " + str(len(manifest)))
                filepath=LibDir+"/" + name + '.' + incarnation + '.manifest'
                if os.path.splitext(location)[-1] == '.xml' : #if this is an xml file we may have a BOM
                    if ord(manifest[0]) > 128 and ord(manifest[1]) > 128 and ord(manifest[2]) > 128:
                        manifest=manifest[3:]
                SetFileContents(filepath,manifest)
                #Get the bundle url from the manifest
                p.setAttribute('manifestdata',manifest)
                man_dom = xml.dom.minidom.parseString(manifest)
                bundle_uri = ""
                for mp in man_dom.getElementsByTagName("Plugin"):
                    if GetNodeTextData(mp.getElementsByTagName("Version")[0]) == version:
                        bundle_uri = GetNodeTextData(mp.getElementsByTagName("Uri")[0])
                        break
                if len(bundle_uri) < 1 :
                    Error("Unable to fetch Bundle URI from manifest for " + name + " v " + version)
                    continue
                Log("Bundle URI = " + bundle_uri)
                # Download the zipfile archive and save as '.zip'
                bundle=self.Util.HttpGetWithoutHeaders(bundle_uri)
                if bundle == None:
                    Error("Unable to download plugin bundle" + bundle_uri )
                    continue
                b=bytearray(bundle)
                filepath=LibDir+"/" + os.path.basename(bundle_uri) + '.zip'
                SetFileContents(filepath,b)
                Log("Plugin bundle" + bundle_uri + "downloaded successfully length = " + str(len(bundle)))
                # unpack the archive
                z=zipfile.ZipFile(filepath)
                zip_dir=LibDir+"/" + name + '-' + version
                z.extractall(zip_dir)
                Log('Extracted ' + bundle_uri + ' to ' + zip_dir) 
                # zip no file perms in .zip so set all the scripts to +x
                Run( "find " + zip_dir +" -type f | xargs chmod  u+x ")
                #write out the base64 config data so the plugin can process it.
                mfile=None
                for root, dirs, files in os.walk(zip_dir):
                    for f in files:
                        if f in ('HandlerManifest.json'):
                            mfile=os.path.join(root,f)
                    if mfile != None:
                        break
                if mfile == None :
                    Error('HandlerManifest.json not found.')
                    continue
                manifest = GetFileContents(mfile)
                p.setAttribute('manifestdata',manifest)
                # create the status and config dirs
                Run('mkdir -p ' + root + '/status')
                Run('mkdir -p ' + root + '/config')
                # write out the configuration data to goalStateIncarnation.settings file in the config path.
                config=''
                seqNo='0'
                if len(dom.getElementsByTagName("PluginSettings")) != 0 :
                    pslist=dom.getElementsByTagName("PluginSettings")[0].getElementsByTagName("Plugin")
                    for ps in pslist:
                        if name == ps.getAttribute("name") and version == ps.getAttribute("version"):
                            Log("Found RuntimeSettings for " + name + " V " + version)
                            config=GetNodeTextData(ps.getElementsByTagName("RuntimeSettings")[0])
                            seqNo=ps.getElementsByTagName("RuntimeSettings")[0].getAttribute("seqNo") 
                            break
                if config == '':
                    Error("No RuntimeSettings for " + name + " V " + version)
                SetFileContents(root +"/config/" + incarnation +".settings",  config )
                #create HandlerEnvironment.json
                handler_env='[{  "name": "'+name+'", "seqNo": "'+seqNo+'", "version": 1.0,  "handlerEnvironment": {    "logFolder": "/var/log",    "configFolder": "' + root + '/config",    "statusFolder": "' + root + '/status",    "heartbeatFile": "'+ root + '/heartbeat.log"}}]'
                SetFileContents(root+'/HandlerEnvironment.json',handler_env)
                cmd = ''
                getcmd='installCommand'
                if plg_dir != None and version > plg_dir.rsplit('-')[-1]:
                    getcmd='updateCommand'   
                    # disable the old plugin if it exists
                    if previous_version != None:
                        if self.launchCommand(name,previous_version,'disableCommand') == None :
                            Error('Unable to disable old plugin '+name+' version ' + previous_version)
                        else :
                            Log(name+' version ' + previous_version + ' is disabled')
                
                if getcmd=='updateCommand':
                    if self.launchCommand(name,version,getcmd,previous_version) == None :
                        Error('Update failed for '+name+'-'+version)
                    else :
                        Log('Update complete'+name+'-'+version)
                    # if we updated - call unistall for the old plugin
                    if self.launchCommand(name,previous_version,'uninstallCommand') == None :
                        Error('Uninstall failed for '+name+'-'+previous_version)
                    else :
                        Log('Uninstall complete'+ name +'-' + previous_version)
                else :  # run install 
                    if self.launchCommand(name,version,getcmd) == None :
                        Error('Installation failed for '+name+'-'+version)
                    else :
                        Log('Installation completed for '+name+'-'+version)
            #end if plg_dir == none or version > = prev
            # change incarnation of settings file so it knows how to name status...
            zip_dir=LibDir+"/" + name + '-' + version
            mfile=None
            for root, dirs, files in os.walk(zip_dir):
                for f in files:
                    if f in ('HandlerManifest.json'):
                        mfile=os.path.join(root,f)
                if mfile != None:
                    break
            if mfile == None :
                Error('HandlerManifest.json not found.')
                continue
            manifest = GetFileContents(mfile)
            p.setAttribute('manifestdata',manifest)
            config=''
            seqNo='0'
            if len(dom.getElementsByTagName("PluginSettings")) != 0 :
                try:
                    pslist=dom.getElementsByTagName("PluginSettings")[0].getElementsByTagName("Plugin")
                except:
                    Error('Error parsing ExtensionsConfig.')
                    continue
                for ps in pslist:
                    if name == ps.getAttribute("name") and version == ps.getAttribute("version"):
                        Log("Found RuntimeSettings for " + name + " V " + version)
                        config=GetNodeTextData(ps.getElementsByTagName("RuntimeSettings")[0])
                        seqNo=ps.getElementsByTagName("RuntimeSettings")[0].getAttribute("seqNo") 
                        break
            if config == '':
                Error("No RuntimeSettings for " + name + " V " + version)
            SetFileContents(root +"/config/" + incarnation +".settings",  config )

            # state is still enable
            if self.launchCommand(name,version,'enableCommand') == None :
                Error('Enable failed for '+name+'-'+version)
            else :
                Log('Enable completed for '+name+'-'+version)
            # this plugin processing is complete
            Log('Processing completed for '+name+'-'+version)
        #end plugin processing loop
        Log('Finished processing ExtensionsConfig.xml')
        return self

    def launchCommand(self,name,version,command,prev_version=None):
        # get the manifest and read the command
        mfile=None
        zip_dir=LibDir+"/" + name + '-' + version
        for root, dirs, files in os.walk(zip_dir):
            for f in files:
                if f in ('HandlerManifest.json'):
                    mfile=os.path.join(root,f)
            if mfile != None:
                break
        if mfile == None :
            Error('HandlerManifest.json not found.')
            return None
        manifest = GetFileContents(mfile)
        try:
            jsn = json.loads(manifest)
        except:
            Error('Error parsing HandlerManifest.json.')
            return None
        if type(jsn)==list:
            jsn=jsn[0]
        if jsn.has_key('handlerManifest') :
            cmd = jsn['handlerManifest'][command]
        else :
            Error('Key handlerManifest not found.  Handler cannot be installed.')
        if len(cmd) == 0 :
            Error('Unable to read ' + command )
            return None

        # for update we send the path of the old installation
        arg=''
        if prev_version != None :
            arg=' ' + LibDir+'/' + name + '-' + prev_version
        dirpath=os.path.dirname(mfile)
        LogIfVerbose('Command is '+ dirpath+'/'+ cmd)
        # launch
        pid=None
        try:
            pid = subprocess.Popen(dirpath+'/'+cmd+arg,shell=True,cwd=dirpath).pid
        except Exception as e:
            Error('Exception launching ' + cmd + str(e))
        if pid == None or pid < 1 :
            ExtensionChildren.append((-1,root))
            Error('Error launching ' + cmd + '.')
        else :
            ExtensionChildren.append((pid,root))
            Log("Spawned "+ cmd + " PID " + str(pid))
        # wait until install/upgrade is finished
        retry = 5*60/10
        while pid != None and pid > 0:
            if retry==0:
                break
            if Run("ps --no-headers " + str(pid),chk_err=False) == 0:
                Log(cmd + ' still running with PID ' + str(pid))
                time.sleep(10)
            else :
                pid=0
            retry-=1
            
        if retry==0:
            Error('More than five minutes has passed.  Killing ' + str(pid))
            os.kill(pid,9)
            return None
        Log(command + ' completed.')
        return 0

    def ReportHandlerStatus(self):
        """
        Collect all status reports.
        """
        # { "version": "1.0", "timestampUTC": "2014-03-31T21:28:58Z", 
        # "aggregateStatus": { 
        # "guestAgentStatus": { "version": "2.0.4PRE", "status": "Ready", "formattedMessage": { "lang": "en-US", "message": "GuestAgent is running and accepting new configurations." } }, 
        # "handlerAggregateStatus": [{ 
        # "handlerName": "ExampleHandlerLinux", "handlerVersion": "1.0", "status": "Ready", "runtimeSettingsStatus": { 
        # "sequenceNumber": "2", "settingsStatus": { "timestampUTC": "2014-03-31T23:46:00Z", "status": { "name": "ExampleHandlerLinux", "operation": "Command Execution Finished", "configurationAppliedTime": "2014-03-31T23:46:00Z", "status": "success", "formattedMessage": { "lang": "en-US", "message": "Finished executing command" }, 
        # "substatus": [
        # { "name": "StdOut", "status": "success", "formattedMessage": { "lang": "en-US", "message": "Goodbye world!" }  }, 
        # { "name": "StdErr", "status": "success", "formattedMessage": { "lang": "en-US", "message": "" } }
        # ] 
        # } } } }
        # ]
        #  }}

        try:
            incarnation=self.Extensions[0].getAttribute("goalStateIncarnation")
        except:
            Error('Error parsing ExtensionsConfig.  Unable to send status reports')
            return None
        status=''
        statuses=''
        for p in self.Plugins:
            if p.getAttribute("state") == 'uninstall':
                continue
            version=p.getAttribute("version")
            name=p.getAttribute("name")
            if p.getAttribute("isJson") != 'true':
                Error("Plugin " + name+" version: " +version+" is not a JSON Extension.  Skipping.")
                continue
            status_file=LibDir+'/'+name+'-'+version+'/status/'+incarnation+'.status'
            if os.path.exists(status_file) !=True and os.path.exists(status_file+'_sent') != True:
                if p.getAttribute("state") == 'disabled' :
                    Log(name+'-'+version+' is disabled.  No status to report.')
                    continue
                Error("Unable to locate " + status_file)
                Error('Status report '+status_file+' Not sent!')
                continue
            elif os.path.exists(status_file+'_sent') == True:
                continue
            if len(statuses)>0:
                statuses+=','
            statuses+=GetFileContents(status_file)
            os.rename(status_file,status_file+'_sent')
            
        if len(statuses)<1:
            Log('No Handler status to report')
            return None
        tstamp=time.strftime("%Y-%M-%dT%H:%M:%SZ", time.gmtime())
        #header
        #agent state
        if provisioned == False:
            if provisionError == None :
                agent_state='Provisioning'
                agent_msg='Guest Agent is starting.'
            else:
                agent_state='Provisioning Error.'
                agent_msg=provisionError
        else:
            agent_state='Ready'
            agent_msg='GuestAgent is running and accepting new configurations.'
            
        status='{"version":"1.0","timestampUTC":"'+tstamp+'","aggregateStatus":{"guestAgentStatus":{"version":"'+GuestAgentVersion+'","status":"'+agent_state+'","formattedMessage":{"lang":"en-US","message":"'+agent_msg+'"}},"handlerAggregateStatus":['+statuses+']}}'
        try:
            uri=GetNodeTextData(self.Extensions[0].getElementsByTagName("StatusUploadBlob")[0]).replace('&amp;','&')
        except:
            Error('Error parsing ExtensionsConfig.  Unable to send status reports')
            return None
        self.Util.Endpoint=uri.split('/')[2]
        self.Util.HttpPutBlockBlob(uri, status)
        Log('Status report '+status+' sent to ' + uri)
        return True
    
    def CheckHeartbeat(self):
        try:
            incarnation=self.Extensions[0].getAttribute("goalStateIncarnation")
            uri=GetNodeTextData(self.Extensions[0].getElementsByTagName("StatusUploadBlob")[0])
        except:
            Error('Error parsing ExtensionsConfig.  Unable to check hearbeat.')
            return None
        for p in self.Plugins:
            if p.getAttribute("state") == 'disabled' or p.getAttribute("state") == 'uninstall':
                continue
            version=p.getAttribute("version")
            name=p.getAttribute("name")
            if p.getAttribute("isJson") != 'true':
                Error("Plugin " + name+" version: " +version+" is not a JSON Extension.  Skipping.")
                continue
            try:
                if len(p.getAttribute("manifestdata"))<1 or  json.loads(p.getAttribute("manifestdata"))[0]['handlerManifest']['reportHeartbeat']!=True :
                    Error("JSON error, unable to process manifestdata")
                    continue
            except:
                continue
            
            heartbeat_file=LibDir+'/'+name+'-'+version+'/heartbeat.log'
            status_file=LibDir+'/'+name+'-'+version+'/status/'+incarnation+'.status'
            if not os.path.exists(heartbeat_file):
                Error('Missing '+ heartbeat_file)
                continue
            else:
                heartbeat=GetFileContents(heartbeat_file)
                try:
                    hb=json.loads(heartbeat)
                except:
                    Error("JSON error, unable to process " + heartbeat_file)
                try:
                    d=int(time.time()-os.stat(heartbeat_file).st_mtime)
                except:
                    Error("Unable to stat " + heartbeat_file)
                    continue
                
                if d >= 700: # stop sending heartbeats 
                    return 'NotReady'
                if d < 120:   # within 2 mins considered active
                    return 'Ready'
                if d < 600:    # less than 10 mins unknown
                    state='Unknown'
                else: # more than 10 mins with no update considered notready
                    state='NotReady'
                try:
                    stat_rept='{"handlerName":"' + name + '","handlerVersion":"'+version+ '","status":"' +hb[0]['heartbeat']['status'] + '","code":' + hb[0]['heartbeat']['code'] + ',"formattedMessage":{"lang":"en-US","message":"' + hb[0]['heartbeat']['Message'] + '"}}'
                    cur_file=status_file+'_current'
                    with open(cur_file,'w+') as f:
                        f.write(stat_rept)
                    # if inc.status exists, rename the inc.status to inc.status_sent
                    if os.path.exists(status_file) == True:
                        os.rename(status_file,status_file+'_sent')
                    # rename inc.status_current to inc.status
                    os.rename(cur_file,status_file)
                    # remove  inc.status_sent
                    if os.path.exists(status_file+'_sent') == True:
                        os.unlink(status_file+'_sent')
                except:
                    Error("Unable to create " + status_file)
                    continue
   
class HostingEnvironmentConfig(object):
    """
    Parse Hosting enviromnet config and store in
    HostingEnvironmentConfig.xml
    """
    #
    # <HostingEnvironmentConfig version="1.0.0.0" goalStateIncarnation="1">
    #   <StoredCertificates>
    #     <StoredCertificate name="Stored0Microsoft.WindowsAzure.Plugins.RemoteAccess.PasswordEncryption" certificateId="sha1:C093FA5CD3AAE057CB7C4E04532B2E16E07C26CA" storeName="My" configurationLevel="System" />
    #   </StoredCertificates>
    #   <Deployment name="db00a7755a5e4e8a8fe4b19bc3b330c3" guid="{ce5a036f-5c93-40e7-8adf-2613631008ab}" incarnation="2">
    #     <Service name="MyVMRoleService" guid="{00000000-0000-0000-0000-000000000000}" />
    #     <ServiceInstance name="db00a7755a5e4e8a8fe4b19bc3b330c3.1" guid="{d113f4d7-9ead-4e73-b715-b724b5b7842c}" />
    #   </Deployment>
    #   <Incarnation number="1" instance="MachineRole_IN_0" guid="{a0faca35-52e5-4ec7-8fd1-63d2bc107d9b}" />
    #   <Role guid="{73d95f1c-6472-e58e-7a1a-523554e11d46}" name="MachineRole" hostingEnvironmentVersion="1" software="" softwareType="ApplicationPackage" entryPoint="" parameters="" settleTimeSeconds="10" />
    #   <HostingEnvironmentSettings name="full" Runtime="rd_fabric_stable.110217-1402.RuntimePackage_1.0.0.8.zip">
    #     <CAS mode="full" />
    #     <PrivilegeLevel mode="max" />
    #     <AdditionalProperties><CgiHandlers></CgiHandlers></AdditionalProperties>
    #   </HostingEnvironmentSettings>
    #   <ApplicationSettings>
    #     <Setting name="__ModelData" value="&lt;m role=&quot;MachineRole&quot; xmlns=&quot;urn:azure:m:v1&quot;>&lt;r name=&quot;MachineRole&quot;>&lt;e name=&quot;a&quot; />&lt;e name=&quot;b&quot; />&lt;e name=&quot;Microsoft.WindowsAzure.Plugins.RemoteAccess.Rdp&quot; />&lt;e name=&quot;Microsoft.WindowsAzure.Plugins.RemoteForwarder.RdpInput&quot; />&lt;/r>&lt;/m>" />
    #     <Setting name="Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString" value="DefaultEndpointsProtocol=http;AccountName=osimages;AccountKey=DNZQ..." />
    #     <Setting name="Microsoft.WindowsAzure.Plugins.RemoteForwarder.Enabled" value="true" />
    #   </ApplicationSettings>
    #   <ResourceReferences>
    #     <Resource name="DiagnosticStore" type="directory" request="Microsoft.Cis.Fabric.Controller.Descriptions.ServiceDescription.Data.Policy" sticky="true" size="1" path="db00a7755a5e4e8a8fe4b19bc3b330c3.MachineRole.DiagnosticStore\" disableQuota="false" />
    #   </ResourceReferences>
    # </HostingEnvironmentConfig>
    #
    def __init__(self):
        self.reinitialize()

    def reinitialize(self):
        """
        Reset Members.
        """
        self.StoredCertificates = None
        self.Deployment = None
        self.Incarnation = None
        self.Role = None
        self.HostingEnvironmentSettings = None
        self.ApplicationSettings = None
        self.Certificates = None
        self.ResourceReferences = None

    def Parse(self, xmlText):
        """
        Parse and create HostingEnvironmentConfig.xml.
        """
        self.reinitialize()
        SetFileContents("HostingEnvironmentConfig.xml", xmlText)
        dom = xml.dom.minidom.parseString(xmlText)
        for a in [ "HostingEnvironmentConfig", "Deployment", "Service",
                   "ServiceInstance", "Incarnation", "Role", ]:
            if not dom.getElementsByTagName(a):
                Error("HostingEnvironmentConfig.Parse: Missing " + a)
                return None
        node = dom.childNodes[0]
        if node.localName != "HostingEnvironmentConfig":
            Error("HostingEnvironmentConfig.Parse: root not HostingEnvironmentConfig")
            return None
        self.ApplicationSettings = dom.getElementsByTagName("Setting")
        self.Certificates = dom.getElementsByTagName("StoredCertificate")
        return self

    def DecryptPassword(self, e):
        """
        Return decrypted password.
        """
        SetFileContents("password.p7m",
            "MIME-Version: 1.0\n"
            + "Content-Disposition: attachment; filename=\"password.p7m\"\n"
            + "Content-Type: application/x-pkcs7-mime; name=\"password.p7m\"\n"
            + "Content-Transfer-Encoding: base64\n\n"
            + textwrap.fill(e, 64))
        return RunGetOutput(Openssl + " cms -decrypt -in password.p7m -inkey Certificates.pem -recip Certificates.pem")[1]

    def ActivateResourceDisk(self):
        return MyDistro.ActivateResourceDisk()

    def Process(self):
        """
        Execute ActivateResourceDisk in separate thread.
        Create the user account.
        Launch ConfigurationConsumer if specified in the config.
        """
        no_thread = False
        if DiskActivated == False:
            for m in inspect.getmembers(MyDistro):
                if 'ActivateResourceDiskNoThread' in m:
                    no_thread = True
                    break
            if no_thread == True :   
                MyDistro.ActivateResourceDiskNoThread()
            else :
                diskThread = threading.Thread(target = self.ActivateResourceDisk)
                diskThread.start()
        User = None
        Pass = None
        Expiration = None
        Thumbprint = None
        for b in self.ApplicationSettings:
            sname = b.getAttribute("name")
            svalue = b.getAttribute("value")
        if User != None and Pass != None:
            if User != "root" and User != "" and Pass != "":
                CreateAccount(User, Pass, Expiration, Thumbprint)
            else:
                Error("Not creating user account: " + User)
        for c in self.Certificates:
            csha1 = c.getAttribute("certificateId").split(':')[1].upper()
            if os.path.isfile(csha1 + ".prv"):
                Log("Private key with thumbprint: " + csha1 + " was retrieved.")
            if os.path.isfile(csha1 + ".crt"):
                Log("Public cert with thumbprint: " + csha1 + " was retrieved.")
        program = Config.get("Role.ConfigurationConsumer")
        if program != None:
            try:
                Children.append(subprocess.Popen([program, LibDir + "/HostingEnvironmentConfig.xml"]))
            except OSError, e :
                ErrorWithPrefix('HostingEnvironmentConfig.Process','Exception: '+ str(e) +' occured launching ' + program )

class GoalState(Util):
    """
    Primary container for all configuration except OvfXml.
    Encapsulates http communication with endpoint server.
    Initializes and populates:
    self.HostingEnvironmentConfig
    self.SharedConfig
    self.ExtensionsConfig
    self.Certificates
    """
    #
    # <GoalState xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="goalstate10.xsd">
    #   <Version>2010-12-15</Version>
    #   <Incarnation>1</Incarnation>
    #   <Machine>
    #     <ExpectedState>Started</ExpectedState>
    #     <LBProbePorts>
    #       <Port>16001</Port>
    #     </LBProbePorts>
    #   </Machine>
    #   <Container>
    #     <ContainerId>c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2</ContainerId>
    #     <RoleInstanceList>
    #       <RoleInstance>
    #         <InstanceId>MachineRole_IN_0</InstanceId>
    #         <State>Started</State>
    #         <Configuration>
    #           <HostingEnvironmentConfig>http://10.115.153.40:80/machine/c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2/MachineRole%5FIN%5F0?comp=config&amp;type=hostingEnvironmentConfig&amp;incarnation=1</HostingEnvironmentConfig>
    #           <SharedConfig>http://10.115.153.40:80/machine/c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2/MachineRole%5FIN%5F0?comp=config&amp;type=sharedConfig&amp;incarnation=1</SharedConfig>
    #           <Certificates>http://10.115.153.40:80/machine/c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2/MachineRole%5FIN%5F0?comp=certificates&amp;incarnation=1</Certificates>
    #          <ExtensionsConfig>http://100.67.238.230:80/machine/9c87aa94-3bda-45e3-b2b7-0eb0fca7baff/1552dd64dc254e6884f8d5b8b68aa18f.eg%2Dplug%2Dvm?comp=config&amp;type=extensionsConfig&amp;incarnation=2</ExtensionsConfig>
    #         <FullConfig>http://100.67.238.230:80/machine/9c87aa94-3bda-45e3-b2b7-0eb0fca7baff/1552dd64dc254e6884f8d5b8b68aa18f.eg%2Dplug%2Dvm?comp=config&amp;type=fullConfig&amp;incarnation=2</FullConfig>

    #         </Configuration>
    #       </RoleInstance>
    #     </RoleInstanceList>
    #   </Container>
    # </GoalState>
    #
    # There is only one Role for VM images.
    #
    # Of primary interest is:
    #  LBProbePorts -- an http server needs to run here
    #  We also note Container/ContainerID and RoleInstance/InstanceId to form the health report.
    #  And of course, Incarnation
    #
    def __init__(self, Agent):
        self.Agent = Agent
        self.Endpoint = Agent.Endpoint
        self.TransportCert = Agent.TransportCert
        self.reinitialize()

    def reinitialize(self):
        self.Incarnation = None # integer
        self.ExpectedState = None # "Started"
        self.HostingEnvironmentConfigUrl = None
        self.HostingEnvironmentConfigXml = None
        self.HostingEnvironmentConfig = None
        self.SharedConfigUrl = None
        self.SharedConfigXml = None
        self.SharedConfig = None
        self.CertificatesUrl = None
        self.CertificatesXml = None
        self.Certificates = None
        self.ExtensionsConfigUrl = None
        self.ExtensionsConfigXml = None
        self.ExtensionsConfig = None
        self.RoleInstanceId = None
        self.ContainerId = None
        self.LoadBalancerProbePort = None # integer, ?list of integers

    def Parse(self, xmlText):
        """
        Request configuration data from endpoint server.
        Parse and populate contained configuration objects.
        Calls Certificates().Parse()
        Calls SharedConfig().Parse
        Calls ExtensionsConfig().Parse
        Calls HostingEnvironmentConfig().Parse
        """
        self.reinitialize()
        LogIfVerbose(xmlText)
        node = xml.dom.minidom.parseString(xmlText).childNodes[0]
        if node.localName != "GoalState":
            Error("GoalState.Parse: root not GoalState")
            return None
        for a in node.childNodes:
            if a.nodeType == node.ELEMENT_NODE:
                if a.localName == "Incarnation":
                    self.Incarnation = GetNodeTextData(a)
                elif a.localName == "Machine":
                    for b in a.childNodes:
                        if b.nodeType == node.ELEMENT_NODE:
                            if b.localName == "ExpectedState":
                                self.ExpectedState = GetNodeTextData(b)
                                Log("ExpectedState: " + self.ExpectedState)
                            elif b.localName == "LBProbePorts":
                                for c in b.childNodes:
                                    if c.nodeType == node.ELEMENT_NODE and c.localName == "Port":
                                        self.LoadBalancerProbePort = int(GetNodeTextData(c))
                elif a.localName == "Container":
                    for b in a.childNodes:
                        if b.nodeType == node.ELEMENT_NODE:
                            if b.localName == "ContainerId":
                                self.ContainerId = GetNodeTextData(b)
                                Log("ContainerId: " + self.ContainerId)
                            elif b.localName == "RoleInstanceList":
                                for c in b.childNodes:
                                    if c.localName == "RoleInstance":
                                        for d in c.childNodes:
                                            if d.nodeType == node.ELEMENT_NODE:
                                                if d.localName == "InstanceId":
                                                    self.RoleInstanceId = GetNodeTextData(d)
                                                    Log("RoleInstanceId: " + self.RoleInstanceId)
                                                elif d.localName == "State":
                                                    pass
                                                elif d.localName == "Configuration":
                                                    for e in d.childNodes:
                                                        if e.nodeType == node.ELEMENT_NODE:
                                                            LogIfVerbose(e.localName)
                                                            if e.localName == "HostingEnvironmentConfig":
                                                                self.HostingEnvironmentConfigUrl = GetNodeTextData(e)
                                                                LogIfVerbose("HostingEnvironmentConfigUrl:" + self.HostingEnvironmentConfigUrl)
                                                                self.HostingEnvironmentConfigXml = self.HttpGetWithHeaders(self.HostingEnvironmentConfigUrl)
                                                                self.HostingEnvironmentConfig = HostingEnvironmentConfig().Parse(self.HostingEnvironmentConfigXml)
                                                            elif e.localName == "SharedConfig":
                                                                self.SharedConfigUrl = GetNodeTextData(e)
                                                                LogIfVerbose("SharedConfigUrl:" + self.SharedConfigUrl)
                                                                self.SharedConfigXml = self.HttpGetWithHeaders(self.SharedConfigUrl)
                                                                self.SharedConfig = SharedConfig().Parse(self.SharedConfigXml)
                                                            elif e.localName == "ExtensionsConfig":
                                                                self.ExtensionsConfigUrl = GetNodeTextData(e)
                                                                LogIfVerbose("ExtensionsConfigUrl:" + self.ExtensionsConfigUrl)
                                                                self.ExtensionsConfigXml = self.HttpGetWithHeaders(self.ExtensionsConfigUrl)
                                                            elif e.localName == "Certificates":
                                                                self.CertificatesUrl = GetNodeTextData(e)
                                                                LogIfVerbose("CertificatesUrl:" + self.CertificatesUrl)
                                                                self.CertificatesXml = self.HttpSecureGetWithHeaders(self.CertificatesUrl, self.TransportCert)
                                                                self.Certificates = Certificates().Parse(self.CertificatesXml)
        if self.Incarnation == None:
            Error("GoalState.Parse: Incarnation missing")
            return None
        if self.ExpectedState == None:
            Error("GoalState.Parse: ExpectedState missing")
            return None
        if self.RoleInstanceId == None:
            Error("GoalState.Parse: RoleInstanceId missing")
            return None
        if self.ContainerId == None:
            Error("GoalState.Parse: ContainerId missing")
            return None
        SetFileContents("GoalState." + self.Incarnation + ".xml", xmlText)
        return self

    def Process(self):
        """
        Calls HostingEnvironmentConfig.Process()
        """
        self.HostingEnvironmentConfig.Process()
        
class OvfEnv(object):
    """
    Read, and process provisioning info from provisioning file OvfEnv.xml
    """
    #
    # <?xml version="1.0" encoding="utf-8"?>
    # <Environment xmlns="http://schemas.dmtf.org/ovf/environment/1" xmlns:oe="http://schemas.dmtf.org/ovf/environment/1" xmlns:wa="http://schemas.microsoft.com/windowsazure" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
    #    <wa:ProvisioningSection>
    #      <wa:Version>1.0</wa:Version>
    #      <LinuxProvisioningConfigurationSet xmlns="http://schemas.microsoft.com/windowsazure" xmlns:i="http://www.w3.org/2001/XMLSchema-instance">
    #        <ConfigurationSetType>LinuxProvisioningConfiguration</ConfigurationSetType>
    #        <HostName>HostName</HostName>
    #        <UserName>UserName</UserName>
    #        <UserPassword>UserPassword</UserPassword>
    #        <DisableSshPasswordAuthentication>false</DisableSshPasswordAuthentication>
    #        <SSH>
    #          <PublicKeys>
    #            <PublicKey>
    #              <Fingerprint>EB0C0AB4B2D5FC35F2F0658D19F44C8283E2DD62</Fingerprint>
    #              <Path>$HOME/UserName/.ssh/authorized_keys</Path>
    #            </PublicKey>
    #          </PublicKeys>
    #          <KeyPairs>
    #            <KeyPair>
    #              <Fingerprint>EB0C0AB4B2D5FC35F2F0658D19F44C8283E2DD62</Fingerprint>
    #              <Path>$HOME/UserName/.ssh/id_rsa</Path>
    #            </KeyPair>
    #          </KeyPairs>
    #        </SSH>
    #      </LinuxProvisioningConfigurationSet>
    #    </wa:ProvisioningSection>
    # </Environment>
    #
    def __init__(self):
        self.reinitialize()

    def reinitialize(self):
        """
        Reset members.
        """
        self.WaNs = "http://schemas.microsoft.com/windowsazure"
        self.OvfNs = "http://schemas.dmtf.org/ovf/environment/1"
        self.MajorVersion = 1
        self.MinorVersion = 0
        self.ComputerName = None
        self.AdminPassword = None
        self.UserName = None
        self.UserPassword = None
        self.CustomData = None
        self.DisableSshPasswordAuthentication = True
        self.SshPublicKeys = []
        self.SshKeyPairs = []

    def Parse(self, xmlText):
        """
        Parse xml tree, retreiving user and ssh key information.
        Return self.
        """
        self.reinitialize()
        LogIfVerbose(xmlText)
        dom = xml.dom.minidom.parseString(xmlText)
        if len(dom.getElementsByTagNameNS(self.OvfNs, "Environment")) != 1:
            Error("Unable to parse OVF XML.")
        section = None
        newer = False
        for p in dom.getElementsByTagNameNS(self.WaNs, "ProvisioningSection"):
            for n in p.childNodes:
                if n.localName == "Version":
                    verparts = GetNodeTextData(n).split('.')
                    major = int(verparts[0])
                    minor = int(verparts[1])
                    if major > self.MajorVersion:
                        newer = True
                    if major != self.MajorVersion:
                        break
                    if minor > self.MinorVersion:
                        newer = True
                    section = p
        if newer == True:
            Warn("Newer provisioning configuration detected. Please consider updating waagent.")
        if section == None:
            Error("Could not find ProvisioningSection with major version=" + str(self.MajorVersion))
            return None
        self.ComputerName = GetNodeTextData(section.getElementsByTagNameNS(self.WaNs, "HostName")[0])
        self.UserName = GetNodeTextData(section.getElementsByTagNameNS(self.WaNs, "UserName")[0])
        try:
            self.UserPassword = GetNodeTextData(section.getElementsByTagNameNS(self.WaNs, "UserPassword")[0])
        except:
            pass
        CDSection=None
        try:
            CDSection=section.getElementsByTagNameNS(self.WaNs, "CustomData")
            if len(CDSection) > 0 :
                self.CustomData=GetNodeTextData(CDSection[0])
                if len(self.CustomData)>0:
                    SetFileContents(LibDir + '/CustomData',self.CustomData)
                    Log('Wrote ' + LibDir + '/CustomData')
                else :
                    Error('<CustomData> contains no data!')
        except Exception, e:
            Error( str(e)+' occured creating ' + LibDir + '/CustomData')
        disableSshPass = section.getElementsByTagNameNS(self.WaNs, "DisableSshPasswordAuthentication")
        if len(disableSshPass) != 0:
            self.DisableSshPasswordAuthentication = (GetNodeTextData(disableSshPass[0]).lower() == "true")
        for pkey in section.getElementsByTagNameNS(self.WaNs, "PublicKey"):
            LogIfVerbose(repr(pkey))
            fp = None
            path = None
            for c in pkey.childNodes:
                if c.localName == "Fingerprint":
                    fp = GetNodeTextData(c).upper()
                    LogIfVerbose(fp)
                if c.localName == "Path":
                    path = GetNodeTextData(c)
                    LogIfVerbose(path)
            self.SshPublicKeys += [[fp, path]]
        for keyp in section.getElementsByTagNameNS(self.WaNs, "KeyPair"):
            fp = None
            path = None
            LogIfVerbose(repr(keyp))
            for c in keyp.childNodes:
                if c.localName == "Fingerprint":
                    fp = GetNodeTextData(c).upper()
                    LogIfVerbose(fp)
                if c.localName == "Path":
                    path = GetNodeTextData(c)
                    LogIfVerbose(path)
            self.SshKeyPairs += [[fp, path]]
        return self

    def PrepareDir(self, filepath):
        """
        Create home dir for self.UserName
        Change owner and return path.
        """
        home = MyDistro.GetHome()
        # Expand HOME variable if present in path
        path = os.path.normpath(filepath.replace("$HOME", home))
        if (path.startswith("/") == False) or (path.endswith("/") == True):
            return None
        dir = path.rsplit('/', 1)[0]
        if dir != "":
            CreateDir(dir, "root", 0700)
            if path.startswith(os.path.normpath(home + "/" + self.UserName + "/")):
                ChangeOwner(dir, self.UserName)
        return path

    def NumberToBytes(self, i):
        """
        Pack number into bytes.  Retun as string.
        """
        result = []
        while i:
            result.append(chr(i & 0xFF))
            i >>= 8
        result.reverse()
        return ''.join(result)

    def BitsToString(self, a):
        """
        Return string representation of bits in a.
        """
        index=7
        s = ""
        c = 0
        for bit in a:
            c = c | (bit << index)
            index = index - 1
            if index == -1:
                s = s + struct.pack('>B', c)
                c = 0
                index = 7
        return s

    def OpensslToSsh(self, file):
        """
        Return base-64 encoded key appropriate for ssh.
        """
        from pyasn1.codec.der import decoder as der_decoder
        try:
            f = open(file).read().replace('\n','').split("KEY-----")[1].split('-')[0]
            k=der_decoder.decode(self.BitsToString(der_decoder.decode(base64.b64decode(f))[0][1]))[0]
            n=k[0]
            e=k[1]
            keydata=""
            keydata += struct.pack('>I',len("ssh-rsa"))
            keydata += "ssh-rsa"
            keydata += struct.pack('>I',len(self.NumberToBytes(e)))
            keydata += self.NumberToBytes(e)
            keydata += struct.pack('>I',len(self.NumberToBytes(n)) + 1)
            keydata += "\0"
            keydata += self.NumberToBytes(n)
        except Exception, e:
            print("OpensslToSsh: Exception " + str(e))
            return None
        return "ssh-rsa " + base64.b64encode(keydata) + "\n"

    def Process(self):
        """
        Process all certificate and key info.
        DisableSshPasswordAuthentication if configured.
        CreateAccount(user)
        Wait for WaAgent.EnvMonitor.IsHostnamePublished().
        Restart ssh service.
        """
        error = None
        if self.ComputerName == None :
            return "Error: Hostname missing"
        error=WaAgent.EnvMonitor.SetHostName(self.ComputerName)
        if error: return error
        if self.DisableSshPasswordAuthentication:
            filepath = "/etc/ssh/sshd_config"
            # Disable RFC 4252 and RFC 4256 authentication schemes.
            ReplaceFileContentsAtomic(filepath, "\n".join(filter(lambda a: not
                (a.startswith("PasswordAuthentication") or a.startswith("ChallengeResponseAuthentication")),
                GetFileContents(filepath).split('\n'))) + "PasswordAuthentication no\nChallengeResponseAuthentication no\n")
            Log("Disabled SSH password-based authentication methods.")
        if self.AdminPassword != None:
            MyDistro.changePass('root',self.AdminPassword)
        if self.UserName != None:
            error = MyDistro.CreateAccount(self.UserName, self.UserPassword, None, None)
        sel = MyDistro.isSelinuxRunning()
        if sel :
            MyDistro.setSelinuxEnforce(0)
        home = MyDistro.GetHome()
        for pkey in self.SshPublicKeys:
            if not os.path.isfile(pkey[0] + ".crt"):
                Error("PublicKey not found: " + pkey[0])
                error = "Failed to deploy public key (0x09)."
                continue
            path = self.PrepareDir(pkey[1])
            if path == None:
                Error("Invalid path: " + pkey[1] + " for PublicKey: " + pkey[0])
                error = "Invalid path for public key (0x03)."
                continue
            Run(Openssl + " x509 -in " + pkey[0] + ".crt -noout -pubkey > " + pkey[0] + ".pub")
            MyDistro.setSelinuxContext(pkey[0] + '.pub','unconfined_u:object_r:ssh_home_t:s0')
            MyDistro.sshDeployPublicKey(pkey[0] + '.pub',path)
            MyDistro.setSelinuxContext(path,'unconfined_u:object_r:ssh_home_t:s0')
            if path.startswith(os.path.normpath(home + "/" + self.UserName + "/")):
                ChangeOwner(path, self.UserName)
        for keyp in self.SshKeyPairs:
            if not os.path.isfile(keyp[0] + ".prv"):
                Error("KeyPair not found: " + keyp[0])
                error = "Failed to deploy key pair (0x0A)."
                continue
            path = self.PrepareDir(keyp[1])
            if path == None:
                Error("Invalid path: " + keyp[1] + " for KeyPair: " + keyp[0])
                error = "Invalid path for key pair (0x05)."
                continue
            SetFileContents(path, GetFileContents(keyp[0] + ".prv"))
            os.chmod(path, 0600)
            Run("ssh-keygen -y -f " + keyp[0] + ".prv > " + path + ".pub")
            MyDistro.setSelinuxContext(path,'unconfined_u:object_r:ssh_home_t:s0')
            MyDistro.setSelinuxContext(path + '.pub','unconfined_u:object_r:ssh_home_t:s0')
            if path.startswith(os.path.normpath(home + "/" + self.UserName + "/")):
                ChangeOwner(path, self.UserName)
                ChangeOwner(path + ".pub", self.UserName)
        if sel :
            MyDistro.setSelinuxEnforce(1)
        while not WaAgent.EnvMonitor.IsHostnamePublished():
            time.sleep(1)
        MyDistro.restartSshService()
        return error

class Agent(Util):
    """
    Primary object container for the provisioning process.
    
    """
    def __init__(self):
        self.GoalState = None
        self.Endpoint = None
        self.LoadBalancerProbeServer = None
        self.HealthReportCounter = 0
        self.TransportCert = ""
        self.EnvMonitor = None
        self.SendData = None
        self.DhcpResponse = None

    def CheckVersions(self):
        """
        Query endpoint server for wire protocol version.
        Fail if our desired protocol version is not seen.
        """
        #<?xml version="1.0" encoding="utf-8"?>
        #<Versions>
        #  <Preferred>
        #    <Version>2010-12-15</Version>
        #  </Preferred>
        #  <Supported>
        #    <Version>2010-12-15</Version>
        #    <Version>2010-28-10</Version>
        #  </Supported>
        #</Versions>
        global ProtocolVersion
        protocolVersionSeen = False
        node = xml.dom.minidom.parseString(self.HttpGetWithoutHeaders("/?comp=versions")).childNodes[0]
        if node.localName != "Versions":
            Error("CheckVersions: root not Versions")
            return False
        for a in node.childNodes:
            if a.nodeType == node.ELEMENT_NODE and a.localName == "Supported":
                for b in a.childNodes:
                    if b.nodeType == node.ELEMENT_NODE and b.localName == "Version":
                        v = GetNodeTextData(b)
                        LogIfVerbose("Fabric supported wire protocol version: " + v)
                        if v == ProtocolVersion:
                            protocolVersionSeen = True
            if a.nodeType == node.ELEMENT_NODE and a.localName == "Preferred":
                v = GetNodeTextData(a.getElementsByTagName("Version")[0])
                Log("Fabric preferred wire protocol version: " + v)
        if not protocolVersionSeen:
            Warn("Agent supported wire protocol version: " + ProtocolVersion + " was not advertised by Fabric.")
        else:
            Log("Negotiated wire protocol version: " + ProtocolVersion)
        return True

    def Unpack(self, buffer, offset, range):
        """
        Unpack bytes into python values.
        """
        result = 0
        for i in range:
            result = (result << 8) | Ord(buffer[offset + i])
        return result

    def UnpackLittleEndian(self, buffer, offset, length):
        """
        Unpack little endian bytes into python values.
        """
        return self.Unpack(buffer, offset, list(range(length - 1, -1, -1)))

    def UnpackBigEndian(self, buffer, offset, length):
        """
        Unpack big endian bytes into python values.
        """
        return self.Unpack(buffer, offset, list(range(0, length)))

    def HexDump3(self, buffer, offset, length):
        """
        Dump range of buffer in formatted hex.
        """
        return ''.join(['%02X' % Ord(char) for char in buffer[offset:offset + length]])

    def HexDump2(self, buffer):
        """
        Dump buffer in formatted hex.
        """
        return self.HexDump3(buffer, 0, len(buffer))

    def BuildDhcpRequest(self):
        """
        Build DHCP request string.
        """
        #
        # typedef struct _DHCP {
        #     UINT8   Opcode;                     /* op:     BOOTREQUEST or BOOTREPLY */
        #     UINT8   HardwareAddressType;        /* htype:  ethernet */
        #     UINT8   HardwareAddressLength;      /* hlen:   6 (48 bit mac address) */
        #     UINT8   Hops;                       /* hops:   0 */
        #     UINT8   TransactionID[4];           /* xid:    random */
        #     UINT8   Seconds[2];                 /* secs:   0 */
        #     UINT8   Flags[2];                   /* flags:  0 or 0x8000 for broadcast */
        #     UINT8   ClientIpAddress[4];         /* ciaddr: 0 */
        #     UINT8   YourIpAddress[4];           /* yiaddr: 0 */
        #     UINT8   ServerIpAddress[4];         /* siaddr: 0 */
        #     UINT8   RelayAgentIpAddress[4];     /* giaddr: 0 */
        #     UINT8   ClientHardwareAddress[16];  /* chaddr: 6 byte ethernet MAC address */
        #     UINT8   ServerName[64];             /* sname:  0 */
        #     UINT8   BootFileName[128];          /* file:   0  */
        #     UINT8   MagicCookie[4];             /*   99  130   83   99 */
        #                                         /* 0x63 0x82 0x53 0x63 */
        #     /* options -- hard code ours */
        #
        #     UINT8 MessageTypeCode;              /* 53 */
        #     UINT8 MessageTypeLength;            /* 1 */
        #     UINT8 MessageType;                  /* 1 for DISCOVER */
        #     UINT8 End;                          /* 255 */
        # } DHCP;
        #

        # tuple of 244 zeros
        # (struct.pack_into would be good here, but requires Python 2.5)
        sendData = [0] * 244

        transactionID = os.urandom(4)
        macAddress = MyDistro.GetMacAddress()

        # Opcode = 1
        # HardwareAddressType = 1 (ethernet/MAC)
        # HardwareAddressLength = 6 (ethernet/MAC/48 bits)
        for a in range(0, 3):
            sendData[a] = [1, 1, 6][a]

        # fill in transaction id (random number to ensure response matches request)
        for a in range(0, 4):
            sendData[4 + a] = Ord(transactionID[a])

        LogIfVerbose("BuildDhcpRequest: transactionId:%s,%04X" % (self.HexDump2(transactionID), self.UnpackBigEndian(sendData, 4, 4)))

        # fill in ClientHardwareAddress
        for a in range(0, 6):
            sendData[0x1C + a] = Ord(macAddress[a])

        # DHCP Magic Cookie: 99, 130, 83, 99
        # MessageTypeCode = 53 DHCP Message Type
        # MessageTypeLength = 1
        # MessageType = DHCPDISCOVER
        # End = 255 DHCP_END
        for a in range(0, 8):
            sendData[0xEC + a] = [99, 130, 83, 99, 53, 1, 1, 255][a]
        return array.array("B", sendData)

    def IntegerToIpAddressV4String(self, a):
        """
        Build DHCP request string.
        """
        return "%u.%u.%u.%u" % ((a >> 24) & 0xFF, (a >> 16) & 0xFF, (a >> 8) & 0xFF, a & 0xFF)

    def RouteAdd(self, net, mask, gateway):
        """
        Add specified route using /sbin/route add -net.
        """
        net = self.IntegerToIpAddressV4String(net)
        mask = self.IntegerToIpAddressV4String(mask)
        gateway = self.IntegerToIpAddressV4String(gateway)
        Run("/sbin/route add -net " + net + " netmask " + mask + " gw " + gateway,chk_err=False)

    def HandleDhcpResponse(self, sendData, receiveBuffer):
        """
        Parse DHCP response:
        Set default gateway.
        Set default routes.
        Retrieve endpoint server.
        Returns endpoint server or None on error.
        """
        LogIfVerbose("HandleDhcpResponse")
        bytesReceived = len(receiveBuffer)
        if bytesReceived < 0xF6:
            Error("HandleDhcpResponse: Too few bytes received " + str(bytesReceived))
            return None

        LogIfVerbose("BytesReceived: " + hex(bytesReceived))
        LogWithPrefixIfVerbose("DHCP response:", HexDump(receiveBuffer, bytesReceived))

        # check transactionId, cookie, MAC address
        # cookie should never mismatch
        # transactionId and MAC address may mismatch if we see a response meant from another machine

        for offsets in [list(range(4, 4 + 4)), list(range(0x1C, 0x1C + 6)), list(range(0xEC, 0xEC + 4))]:
            for offset in offsets:
                sentByte = Ord(sendData[offset])
                receivedByte = Ord(receiveBuffer[offset])
                if sentByte != receivedByte:
                    LogIfVerbose("HandleDhcpResponse: sent cookie:" + self.HexDump3(sendData, 0xEC, 4))
                    LogIfVerbose("HandleDhcpResponse: rcvd cookie:" + self.HexDump3(receiveBuffer, 0xEC, 4))
                    LogIfVerbose("HandleDhcpResponse: sent transactionID:" + self.HexDump3(sendData, 4, 4))
                    LogIfVerbose("HandleDhcpResponse: rcvd transactionID:" + self.HexDump3(receiveBuffer, 4, 4))
                    LogIfVerbose("HandleDhcpResponse: sent ClientHardwareAddress:" + self.HexDump3(sendData, 0x1C, 6))
                    LogIfVerbose("HandleDhcpResponse: rcvd ClientHardwareAddress:" + self.HexDump3(receiveBuffer, 0x1C, 6))
                    LogIfVerbose("HandleDhcpResponse: transactionId, cookie, or MAC address mismatch")
                    return None
        endpoint = None

        #
        # Walk all the returned options, parsing out what we need, ignoring the others.
        # We need the custom option 245 to find the the endpoint we talk to,
        # as well as, to handle some Linux DHCP client incompatibilities,
        # options 3 for default gateway and 249 for routes. And 255 is end.
        #

        i = 0xF0 # offset to first option
        while i < bytesReceived:
            option = Ord(receiveBuffer[i])
            length = 0
            if (i + 1) < bytesReceived:
                length = Ord(receiveBuffer[i + 1])
            LogIfVerbose("DHCP option " + hex(option) + " at offset:" + hex(i) + " with length:" + hex(length))
            if option == 255:
                LogIfVerbose("DHCP packet ended at offset " + hex(i))
                break
            elif option == 249:
                # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx
                LogIfVerbose("Routes at offset:" + hex(i) + " with length:" + hex(length))
                if length < 5:
                    Error("Data too small for option " + option)
                j = i + 2
                while j < (i + length + 2):
                    maskLengthBits = Ord(receiveBuffer[j])
                    maskLengthBytes = (((maskLengthBits + 7) & ~7) >> 3)
                    mask = 0xFFFFFFFF & (0xFFFFFFFF << (32 - maskLengthBits))
                    j += 1
                    net = self.UnpackBigEndian(receiveBuffer, j, maskLengthBytes)
                    net <<= (32 - maskLengthBytes * 8)
                    net &= mask
                    j += maskLengthBytes
                    gateway = self.UnpackBigEndian(receiveBuffer, j, 4)
                    j += 4
                    self.RouteAdd(net, mask, gateway)
                if j != (i + length + 2):
                    Error("HandleDhcpResponse: Unable to parse routes")
            elif option == 3 or option == 245:
                if i + 5 < bytesReceived:
                    if length != 4:
                        Error("HandleDhcpResponse: Endpoint or Default Gateway not 4 bytes")
                        return None
                    gateway = self.UnpackBigEndian(receiveBuffer, i + 2, 4)
                    IpAddress = self.IntegerToIpAddressV4String(gateway)
                    if option == 3:
                        self.RouteAdd(0, 0, gateway)
                        name = "DefaultGateway"
                    else:
                        endpoint = IpAddress
                        name = "Windows Azure wire protocol endpoint"
                    LogIfVerbose(name + ": " + IpAddress + " at " + hex(i))
                else:
                    Error("HandleDhcpResponse: Data too small for option " + option)
            else:
                LogIfVerbose("Skipping DHCP option " + hex(option) + " at " + hex(i) + " with length " + hex(length))
            i += length + 2
        return endpoint

    def DoDhcpWork(self):
        """
        Discover the wire server via DHCP option 245.
        And workaround incompatibility with Windows Azure DHCP servers.
        """
        ShortSleep = False # Sleep 1 second before retrying DHCP queries.
        ifname=None
        Run("iptables -D INPUT -p udp --dport 68 -j ACCEPT",chk_err=False)  # We supress error logging on error.
        Run("iptables -I INPUT -p udp --dport 68 -j ACCEPT",chk_err=False)  # We supress error logging on error.

        sleepDurations = [0, 10, 30, 60, 60]
        maxRetry = len(sleepDurations)
        lastTry = (maxRetry - 1)
        for retry in range(0, maxRetry):
            try:
                strRetry = str(retry)
                prefix = "DoDhcpWork: try=" + strRetry
                LogIfVerbose(prefix)
                sendData = self.BuildDhcpRequest()
                LogWithPrefixIfVerbose("DHCP request:", HexDump(sendData, len(sendData)))
                sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
                sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                missingDefaultRoute = True
                try:
                    if DistInfo()[0] == 'FreeBSD':
                        routes = RunGetOutput("netstat -nr")[1]
                    else:
                        routes = RunGetOutput("route -n")[1]
                    for line in routes.split('\n'):
                        if line.startswith("0.0.0.0 ") or line.startswith("default "):
                            missingDefaultRoute = False
                except:
                    pass
                if missingDefaultRoute:
                    # This is required because sending after binding to 0.0.0.0 fails with
                    # network unreachable when the default gateway is not set up.
                    ifname=MyDistro.GetInterfaceName()
                    Log("DoDhcpWork: Missing default route - adding broadcast route for DHCP.")
                    Run("route add 255.255.255.255 dev " + ifname,chk_err=False)   # We supress error logging on error.
                if MyDistro.dhcp_client_name == 'wickedd-dhcp4':
                    Run("service " + MyDistro.dhcp_client_name + " stop",chk_err=False)
                sock.bind(("0.0.0.0", 68)) 
                sock.sendto(sendData, ("<broadcast>", 67))
                sock.settimeout(10)
                Log("DoDhcpWork: Setting socket.timeout=10, entering recv")
                receiveBuffer = sock.recv(1024)
                endpoint = self.HandleDhcpResponse(sendData, receiveBuffer)
                if endpoint == None:
                    LogIfVerbose("DoDhcpWork: No endpoint found")
                if endpoint != None or retry == lastTry:
                    if endpoint != None:
                        self.SendData = sendData
                        self.DhcpResponse = receiveBuffer
                    if retry == lastTry:
                        LogIfVerbose("DoDhcpWork: try=" + strRetry)
                    return endpoint
                sleepDuration = [sleepDurations[retry % len(sleepDurations)], 1][ShortSleep]
                LogIfVerbose("DoDhcpWork: sleep=" + str(sleepDuration))
                time.sleep(sleepDuration)
            except Exception, e:
                ErrorWithPrefix(prefix, str(e))
                ErrorWithPrefix(prefix, traceback.format_exc())
            finally:
                sock.close()
                if missingDefaultRoute:
                    #We added this route - delete it
                    Run("route del 255.255.255.255 dev " + ifname,chk_err=False)  # We supress error logging on error.
                    Log("DoDhcpWork: Removing broadcast route for DHCP.")
                if MyDistro.dhcp_client_name == 'wickedd-dhcp4':
                    Run("service " + MyDistro.dhcp_client_name + " start",chk_err=False)
        return None

    def UpdateAndPublishHostName(self, name):
        """
        Set hostname locally and publish to iDNS
        """
        Log("Setting host name: " + name)
        MyDistro.publishHostname(name)
        ethernetInterface = MyDistro.GetInterfaceName()
        if DistInfo()[0] == 'FreeBSD':
            Run("service netif restart")
        else:
            Run("ifdown " + ethernetInterface + " && ifup " + ethernetInterface)
        self.RestoreRoutes()

    def RestoreRoutes(self):
        """
        If there is a DHCP response, then call HandleDhcpResponse.
        """
        if self.SendData != None and self.DhcpResponse != None:
            self.HandleDhcpResponse(self.SendData, self.DhcpResponse)

    def UpdateGoalState(self):
        """
        Retreive goal state information from endpoint server.
        Parse xml and initialize Agent.GoalState object.
        Return object or None on error.
        """
        goalStateXml = None
        maxRetry = 9
        log = NoLog
        for retry in range(1, maxRetry + 1):
            strRetry = str(retry)
            log("retry UpdateGoalState,retry=" + strRetry)
            goalStateXml = self.HttpGetWithHeaders("/machine/?comp=goalstate")
            if goalStateXml != None:
                break
            log = Log
            time.sleep(retry)
        if not goalStateXml:
            Error("UpdateGoalState failed.")
            return
        Log("Retrieved GoalState from Windows Azure Fabric.")
        self.GoalState = GoalState(self).Parse(goalStateXml)
        return self.GoalState

    def ReportReady(self):
        """
        Send health report 'Ready' to server.
        This signals the fabric that our provosion is completed,
        and the host is ready for operation.
        """
        counter = (self.HealthReportCounter + 1) % 1000000
        self.HealthReportCounter = counter
        healthReport = ("<?xml version=\"1.0\" encoding=\"utf-8\"?><Health xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\"><GoalStateIncarnation>"
                        + self.GoalState.Incarnation
                        + "</GoalStateIncarnation><Container><ContainerId>"
                        + self.GoalState.ContainerId
                        + "</ContainerId><RoleInstanceList><Role><InstanceId>"
                        + self.GoalState.RoleInstanceId
                        + "</InstanceId><Health><State>Ready</State></Health></Role></RoleInstanceList></Container></Health>")
        a = self.HttpPost("/machine?comp=health", healthReport)
        if a != None:
            return a.getheader("x-ms-latest-goal-state-incarnation-number")
        return None

    def ReportNotReady(self, status, desc):
        """
        Send health report 'Provisioning' to server.
        This signals the fabric that our provosion is starting.
        """
        healthReport = ("<?xml version=\"1.0\" encoding=\"utf-8\"?><Health xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\"><GoalStateIncarnation>"
                        + self.GoalState.Incarnation
                        + "</GoalStateIncarnation><Container><ContainerId>"
                        + self.GoalState.ContainerId
                        + "</ContainerId><RoleInstanceList><Role><InstanceId>"
                        + self.GoalState.RoleInstanceId
                        + "</InstanceId><Health><State>NotReady</State>"
                        + "<Details><SubStatus>" + status + "</SubStatus><Description>" + desc + "</Description></Details>"
                        + "</Health></Role></RoleInstanceList></Container></Health>")
        a = self.HttpPost("/machine?comp=health", healthReport)
        if a != None:
            return a.getheader("x-ms-latest-goal-state-incarnation-number")
        return None

    def ReportRoleProperties(self, thumbprint):
        """
        Send roleProperties and thumbprint to server. 
        """
        roleProperties = ("<?xml version=\"1.0\" encoding=\"utf-8\"?><RoleProperties><Container>"
                        + "<ContainerId>" + self.GoalState.ContainerId + "</ContainerId>"
                        + "<RoleInstances><RoleInstance>"
                        + "<Id>" + self.GoalState.RoleInstanceId + "</Id>"
                        + "<Properties><Property name=\"CertificateThumbprint\" value=\"" + thumbprint + "\" /></Properties>"
                        + "</RoleInstance></RoleInstances></Container></RoleProperties>")
        a = self.HttpPost("/machine?comp=roleProperties", roleProperties)
        Log("Posted Role Properties. CertificateThumbprint=" + thumbprint)
        return a

    def LoadBalancerProbeServer_Shutdown(self):
        """
        Shutdown the LoadBalancerProbeServer.
        """
        if self.LoadBalancerProbeServer != None:
            self.LoadBalancerProbeServer.shutdown()
            self.LoadBalancerProbeServer = None

    def GenerateTransportCert(self):
        """
        Create ssl certificate for https communication with endpoint server.
        """
        Run(Openssl + " req -x509 -nodes -subj /CN=LinuxTransport -days 32768 -newkey rsa:2048 -keyout TransportPrivate.pem -out TransportCert.pem")
        cert = ""
        for line in GetFileContents("TransportCert.pem").split('\n'):
            if not "CERTIFICATE" in line:
                cert += line.rstrip()
        return cert

    def DoVmmStartup(self):
        """
        Spawn the VMM startup script.
        """
        Log("Starting Microsoft System Center VMM Initialization Process")
        pid = subprocess.Popen(["/bin/bash","/mnt/cdrom/secure/"+VMM_STARTUP_SCRIPT_NAME,"-p /mnt/cdrom/secure/ "]).pid
        time.sleep(5)
        sys.exit(0)
        
    def TryUnloadAtapiix(self):
        """
        If global modloaded is True, then we loaded the ata_piix kernel module, unload it.
        """
        if modloaded:
            Run("rmmod ata_piix.ko",chk_err=False)
            Log("Unloaded ata_piix.ko driver for ATAPI CD-ROM")

    def TryLoadAtapiix(self):
        """
        Load the ata_piix kernel module if it exists.
        If successful, set global modloaded to True.
        If unable to load module leave modloaded False.
        """
        global modloaded
        modloaded=False
        retcode,krn=RunGetOutput('uname -r')
        krn_pth='/lib/modules/'+krn.strip('\n')+'/kernel/drivers/ata/ata_piix.ko'
        if Run("lsmod | grep ata_piix",chk_err=False) == 0 :
            Log("Module " + krn_pth + " driver for ATAPI CD-ROM is already present.")
            return 0
        if retcode:
            Error("Unable to provision: Failed to call uname -r")
            return "Unable to provision: Failed to call uname"
        if os.path.isfile(krn_pth):
            retcode,output=RunGetOutput("insmod " + krn_pth,chk_err=False)
        else:
            Log("Module " + krn_pth + " driver for ATAPI CD-ROM does not exist.")
            return 1
        if retcode != 0:
            Error('Error calling insmod for '+ krn_pth + ' driver for ATAPI CD-ROM')
            return retcode
        time.sleep(1)
        # check 3 times if the mod is loaded
        for i in range(3):
            if Run('lsmod | grep ata_piix'):
                continue
            else :
                modloaded=True
                break
        if not modloaded:
            Error('Unable to load '+ krn_pth + ' driver for ATAPI CD-ROM')
            return 1
        
        Log("Loaded " + krn_pth + " driver for ATAPI CD-ROM")
        
        # we have succeeded loading the ata_piix mod if it can be done.

    def SearchForVMMStartup(self):
        """
        Search for a DVD/CDROM containing VMM's VMM_CONFIG_FILE_NAME.
        Call TryLoadAtapiix in case we must load the ata_piix module first.

        If VMM_CONFIG_FILE_NAME is found, call DoVmmStartup.
        Else, return to Azure Provisioning process.
        """
        self.TryLoadAtapiix()
        if os.path.exists('/mnt/cdrom/secure') == False:
            CreateDir("/mnt/cdrom/secure", "root", 0700)
        mounted=False
        for dvds in [re.match(r'(sr[0-9]|hd[c-z]|cdrom[0-9]|cd[0-9]?)',x) for x in os.listdir('/dev/')]:
            if dvds == None:
                continue
            dvd = '/dev/'+dvds.group(0)
            if Run("LC_ALL=C fdisk -l " + dvd + " | grep Disk",chk_err=False):
                continue  # Not mountable
            else:
                for retry in range(1,6):
                    retcode,output=RunGetOutput("mount -v " + dvd + " /mnt/cdrom/secure")
                    Log(output[:-1])
                    if retcode == 0:
                        Log("mount succeeded on attempt #" + str(retry) )
                        mounted=True
                        break
                    if 'is already mounted on /mnt/cdrom/secure' in output:
                        Log("Device " + dvd + " is already mounted on /mnt/cdrom/secure." + str(retry) )
                        mounted=True
                        break
                    Log("mount failed on attempt #" + str(retry) )
                    Log("mount loop sleeping 5...")
                    time.sleep(5)
                if not mounted:
                    # unable to mount
                    continue
                if not os.path.isfile("/mnt/cdrom/secure/"+VMM_CONFIG_FILE_NAME):
                    #nope - mount the next drive
                    if mounted:
                        Run("umount "+dvd,chk_err=False)
                        mounted=False
                        continue
                else : # it is the vmm startup
                    self.DoVmmStartup()

        Log("VMM Init script not found.  Provisioning for Azure")
        return 
        
    def Provision(self):
        """
        Responible for:
        Regenerate ssh keys,
        Mount, read, and parse ovfenv.xml from provisioning dvd rom
        Process the ovfenv.xml info
        Call ReportRoleProperties
        If configured, delete root password.
        Return None on success, error string on error.
        """
        enabled = Config.get("Provisioning.Enabled")
        if enabled != None and enabled.lower().startswith("n"):
            return
        Log("Provisioning image started.")
        type = Config.get("Provisioning.SshHostKeyPairType")
        if type == None:
            type = "rsa"
        regenerateKeys = Config.get("Provisioning.RegenerateSshHostKeyPair")
        if regenerateKeys == None or regenerateKeys.lower().startswith("y"):
            Run("rm -f /etc/ssh/ssh_host_*key*")
            Run("ssh-keygen -N '' -t " + type + " -f /etc/ssh/ssh_host_" + type + "_key")
            MyDistro.restartSshService()
        #SetFileContents(LibDir + "/provisioned", "")
        dvd = None
        for dvds in [re.match(r'(sr[0-9]|hd[c-z]|cdrom[0-9]|cd[0-9]?)',x) for x in os.listdir('/dev/')]:
            if dvds == None :
                continue
            dvd = '/dev/'+dvds.group(0)
        if dvd == None:
            # No DVD device detected
            Error("No DVD device detected, unable to provision.")
            return "No DVD device detected, unable to provision."
        if MyDistro.mediaHasFilesystem(dvd) is False :
            out=MyDistro.load_ata_piix()
            if out:
                return out
            for i in range(10): # we may have to wait 
                if os.path.exists(dvd):
                    break
                Log("Waiting for DVD - sleeping 1 - "+str(i+1)+" try...")
                time.sleep(1)
        if os.path.exists('/mnt/cdrom/secure') == False:
            CreateDir("/mnt/cdrom/secure", "root", 0700)
        #begin mount loop - 5 tries - 5 sec wait between
        for retry in range(1,6):
            location='/mnt/cdrom/secure'
            retcode,output=MyDistro.mountDVD(dvd,location)
            Log(output[:-1])
            if retcode == 0:
                Log("mount succeeded on attempt #" + str(retry) )
                break
            if 'is already mounted on /mnt/cdrom/secure' in output:
                Log("Device " + dvd + " is already mounted on /mnt/cdrom/secure." + str(retry) )
                break
            Log("mount failed on attempt #" + str(retry) )
            Log("mount loop sleeping 5...")
            time.sleep(5)
        if not os.path.isfile("/mnt/cdrom/secure/ovf-env.xml"):
            Error("Unable to provision: Missing ovf-env.xml on DVD.")
            return "Failed to retrieve provisioning data (0x02)."
        ovfxml = (GetFileContents(u"/mnt/cdrom/secure/ovf-env.xml",asbin=False)) # use unicode here to ensure correct codec gets used.
        if ord(ovfxml[0]) > 128 and ord(ovfxml[1]) > 128 and ord(ovfxml[2]) > 128 :
            ovfxml = ovfxml[3:] # BOM is not stripped.  First three bytes are > 128 and not unicode chars so we ignore them.
        ovfxml=ovfxml.strip(chr(0x00)) # we may have NULLs.
        ovfxml=ovfxml[ovfxml.find('<?'):] # chop leading text if present
        SetFileContents("ovf-env.xml", re.sub("<UserPassword>.*?<", "<UserPassword>*<", ovfxml))
        Run("umount " + dvd,chk_err=False)
        MyDistro.unload_ata_piix()
        error = None
        if ovfxml != None:
            Log("Provisioning image using OVF settings in the DVD.")
            ovfobj = OvfEnv().Parse(ovfxml)
            if ovfobj != None:
                error = ovfobj.Process()
                if error :
                    Error ("Provisioning image FAILED " + error)
                    return ("Provisioning image FAILED " + error)
        # This is done here because regenerated SSH host key pairs may be potentially overwritten when processing the ovfxml
        fingerprint = RunGetOutput("ssh-keygen -lf /etc/ssh/ssh_host_" + type + "_key.pub")[1].rstrip().split()[1].replace(':','')
        self.ReportRoleProperties(fingerprint)
        delRootPass = Config.get("Provisioning.DeleteRootPassword")
        if delRootPass != None and delRootPass.lower().startswith("y"):
            MyDistro.deleteRootPassword()
        Log("Provisioning image completed.")
        return error

    def Run(self):
        """
        Called by 'waagent -daemon.'
        Main loop to process the goal state.  State is posted every 25 seconds
        when provisioning has been completed.
        
        Search for VMM enviroment, start VMM script if found.
        Perform DHCP and endpoint server discovery by calling DoDhcpWork().
        Check wire protocol versions.
        Set SCSI timeout on root device.
        Call GenerateTransportCert() to create ssl certs for server communication.
        Call UpdateGoalState().
        If not provisioned, call ReportNotReady("Provisioning", "Starting")
        Call Provision(), set global provisioned = True if successful.
        Call goalState.Process()
        Start LBProbeServer if indicated in waagent.conf.
        Start the StateConsumer if indicated in waagent.conf.
        ReportReady if provisioning is complete.
        If provisioning failed, call ReportNotReady("ProvisioningFailed", provisionError)
        """
        SetFileContents("/var/run/waagent.pid", str(os.getpid()) + "\n")

        # Determine if we are in VMM.  Spawn VMM_STARTUP_SCRIPT_NAME if found.
        self.SearchForVMMStartup()
        ipv4=''
        while ipv4 == '' or ipv4 == '0.0.0.0' :
            ipv4=MyDistro.GetIpv4Address()
            if ipv4 == '' or ipv4 == '0.0.0.0' :
                Log("Waiting for network.")
                time.sleep(10)

        Log("IPv4 address: " + ipv4)
        mac=''
        mac=MyDistro.GetMacAddress()
        if len(mac)>0 :
            Log("MAC  address: " + ":".join(["%02X" % Ord(a) for a in mac]))
        
        # Consume Entropy in ACPI table provided by Hyper-V
        try:
            SetFileContents("/dev/random", GetFileContents("/sys/firmware/acpi/tables/OEM0"))
        except:
            pass

        Log("Probing for Windows Azure environment.")
        self.Endpoint = self.DoDhcpWork()

        if self.Endpoint == None:
            Log("Windows Azure environment not detected.")
            while True:
                time.sleep(60)

        Log("Discovered Windows Azure endpoint: " + self.Endpoint)
        if not self.CheckVersions():
            Error("Agent.CheckVersions failed")
            sys.exit(1)

        self.EnvMonitor = EnvMonitor()

        # Set SCSI timeout on SCSI disks
        MyDistro.initScsiDiskTimeout()
        global provisioned
        global provisionError
        
        global Openssl
        Openssl = Config.get("OS.OpensslPath")
        if Openssl == None:
            Openssl = "openssl"

        self.TransportCert = self.GenerateTransportCert()

        incarnation = None # goalStateIncarnationFromHealthReport
        currentPort = None # loadBalancerProbePort
        goalState = None # self.GoalState, instance of GoalState
        provisioned = os.path.exists(LibDir + "/provisioned")
        program = Config.get("Role.StateConsumer")
        provisionError = None        
        lbProbeResponder = True
        setting = Config.get("LBProbeResponder")
        if setting != None and setting.lower().startswith("n"):
            lbProbeResponder = False
        while True:
            if (goalState == None) or (incarnation == None) or (goalState.Incarnation != incarnation):
                goalState = self.UpdateGoalState()
                if goalState == None :
                    continue
                if provisioned == False:
                    self.ReportNotReady("Provisioning", "Starting")

                goalState.Process()

                if provisioned == False:
                    provisionError = self.Provision()
                    if provisionError == None :
                        provisioned = True
                        SetFileContents(LibDir + "/provisioned", "")

                #
                # only one port supported
                # restart server if new port is different than old port
                # stop server if no longer a port
                #
                goalPort = goalState.LoadBalancerProbePort
                if currentPort != goalPort:
                    self.LoadBalancerProbeServer_Shutdown()
                    currentPort = goalPort
                    if currentPort != None and lbProbeResponder == True:
                        self.LoadBalancerProbeServer = LoadBalancerProbeServer(currentPort)
                        if self.LoadBalancerProbeServer == None :
                            lbProbeResponder = False
                            Log("Unable to create LBProbeResponder.")

                # Report SSH key fingerprint
                type = Config.get("Provisioning.SshHostKeyPairType")
                if type == None:
                    type = "rsa"
                fingerprint = RunGetOutput("ssh-keygen -lf /etc/ssh/ssh_host_" + type + "_key.pub")[1].rstrip().split()[1].replace(':','')
                self.ReportRoleProperties(fingerprint)

            if program != None and DiskActivated == True:
                try:
                    Children.append(subprocess.Popen([program, "Ready"]))
                except OSError, e :
                    ErrorWithPrefix('SharedConfig.Parse','Exception: '+ str(e) +' occured launching ' + program )
                program = None

            sleepToReduceAccessDenied = 3
            time.sleep(sleepToReduceAccessDenied)
            if provisionError != None:
                incarnation = self.ReportNotReady("ProvisioningFailed", provisionError)
            else:
                incarnation = self.ReportReady()
            # Process our extensions.
            if goalState.ExtensionsConfig == None and goalState.ExtensionsConfigXml != None :
                goalState.ExtensionsConfig = ExtensionsConfig().Parse(goalState.ExtensionsConfigXml)

            # report the status/heartbeat results of extension processing
            if goalState.ExtensionsConfig != None :
                goalState.ExtensionsConfig.ReportHandlerStatus()
                goalState.ExtensionsConfig.CheckHeartbeat()

            time.sleep(25 - sleepToReduceAccessDenied)	
            
WaagentLogrotate = """\
/var/log/waagent.log {
    monthly
    rotate 6
    notifempty
    missingok
}
"""

def FindInLinuxKernelCmdline(option):
    """
    Return match object if 'option' is present in the kernel boot options
    of the grub configuration.
    """
    m=None
    matchs=r'^.*?'+MyDistro.grubKernelBootOptionsLine+r'.*?'+option+r'.*$'
    try:
        m=FindStringInFile(MyDistro.grubKernelBootOptionsFile,matchs)
    except IOError, e:
        Error('FindInLinuxKernelCmdline: Exception opening ' + MyDistro.grubKernelBootOptionsFile + 'Exception:' + str(e))
        
    return m

def AppendToLinuxKernelCmdline(option):
    """
    Add 'option' to the kernel boot options of the grub configuration.
    """
    if not FindInLinuxKernelCmdline(option):
        src=r'^(.*?'+MyDistro.grubKernelBootOptionsLine+r')(.*?)("?)$'
        rep=r'\1\2 '+ option + r'\3'
        try:
            ReplaceStringInFile(MyDistro.grubKernelBootOptionsFile,src,rep)
        except IOError, e :
            Error('AppendToLinuxKernelCmdline: Exception opening ' + MyDistro.grubKernelBootOptionsFile + 'Exception:' + str(e))
            return 1
        Run("update-grub",chk_err=False)    
    return 0

def RemoveFromLinuxKernelCmdline(option):
    """
    Remove 'option' to the kernel boot options of the grub configuration.
    """
    if FindInLinuxKernelCmdline(option):
        src=r'^(.*?'+MyDistro.grubKernelBootOptionsLine+r'.*?)('+option+r')(.*?)("?)$'
        rep=r'\1\3\4'
        try:
            ReplaceStringInFile(MyDistro.grubKernelBootOptionsFile,src,rep)
        except IOError, e :
            Error('RemoveFromLinuxKernelCmdline: Exception opening ' + MyDistro.grubKernelBootOptionsFile + 'Exception:' + str(e))
            return 1
        Run("update-grub",chk_err=False)    
    return 0

def FindStringInFile(fname,matchs):
    """
    Return match object if found in file.
    """
    try:
        ms=re.compile(matchs)
        for l in (open(fname,'r')).readlines():
            m=re.search(ms,l)
            if m:
                return m
    except:
        raise
    
    return None

def ReplaceStringInFile(fname,src,repl):
    """
    Replace 'src' with 'repl' in file.
    """
    updated=''
    try:
        sr=re.compile(src)
        if FindStringInFile(fname,src):
            for l in (open(fname,'r')).readlines():
                n=re.sub(sr,repl,l)
                updated+=n
        ReplaceFileContentsAtomic(fname,updated)
    except :
        raise
    return

def ApplyVNUMAWorkaround():
    """
    If kernel version has NUMA bug, add 'numa=off' to
    kernel boot options.
    """
    VersionParts = platform.release().replace('-', '.').split('.')
    if int(VersionParts[0]) > 2:
        return
    if int(VersionParts[1]) > 6:
        return
    if int(VersionParts[2]) > 37:
        return
    if AppendToLinuxKernelCmdline("numa=off") == 0 :
        Log("Your kernel version " + platform.release() + " has a NUMA-related bug: NUMA has been disabled.")
    else :
        "Error adding 'numa=off'.  NUMA has not been disabled."
        
def RevertVNUMAWorkaround():
    """
    Remove 'numa=off' from kernel boot options.
    """
    if RemoveFromLinuxKernelCmdline("numa=off") == 0 :
        Log('NUMA has been re-enabled')
    else :
        Log('NUMA has not been re-enabled')

def Install():
    """
    Install the agent service.
    Check dependencies.
    Create /etc/waagent.conf and move old version to
    /etc/waagent.conf.old
    Copy RulesFiles to /var/lib/waagent
    Create /etc/logrotate.d/waagent
    Set /etc/ssh/sshd_config ClientAliveInterval to 180
    Call ApplyVNUMAWorkaround()
    """
    if MyDistro.checkDependencies():
        return 1
    os.chmod(sys.argv[0], 0755)
    SwitchCwd()
    for a in RulesFiles:
        if os.path.isfile(a):
            if os.path.isfile(GetLastPathElement(a)):
                os.remove(GetLastPathElement(a))
            shutil.move(a, ".")
            Warn("Moved " + a + " -> " + LibDir + "/" + GetLastPathElement(a) )
    MyDistro.registerAgentService()
    if os.path.isfile("/etc/waagent.conf"):
        try:
            os.remove("/etc/waagent.conf.old")
        except:
            pass
        try:
            os.rename("/etc/waagent.conf", "/etc/waagent.conf.old")
            Warn("Existing /etc/waagent.conf has been renamed to /etc/waagent.conf.old")
        except:
            pass
    SetFileContents("/etc/waagent.conf", MyDistro.waagent_conf_file)
    SetFileContents("/etc/logrotate.d/waagent", WaagentLogrotate)
    filepath = "/etc/ssh/sshd_config"
    ReplaceFileContentsAtomic(filepath, "\n".join(filter(lambda a: not
        a.startswith("ClientAliveInterval"),
        GetFileContents(filepath).split('\n'))) + "\nClientAliveInterval 180\n")
    Log("Configured SSH client probing to keep connections alive.")
    ApplyVNUMAWorkaround()
    return 0

def GetMyDistro(dist_class_name=''):
    """
    Return MyDistro object.
    NOTE: Logging is not initialized at this point.
    """
    if dist_class_name == '':
        if 'Linux' in platform.system():
            Distro=DistInfo()[0]
        else : # I know this is not Linux!
            if 'FreeBSD' in platform.system():
                Distro=platform.system()
        Distro=Distro.strip('"')
        Distro=Distro.strip(' ')
        dist_class_name=Distro+'Distro'
    else:
        Distro=dist_class_name
    if not globals().has_key(dist_class_name):
        print Distro+' is not a supported distribution.'
        return None
    return globals()[dist_class_name]() # the distro class inside this module.

def DistInfo(fullname=0):
    if 'FreeBSD' in platform.system():
        release = re.sub('\-.*\Z', '', str(platform.release()))
        distinfo = ['FreeBSD', release]
        return distinfo
    if 'linux_distribution' in dir(platform):
        distinfo = list(platform.linux_distribution(full_distribution_name=fullname))
        distinfo[0] = distinfo[0].strip() # remove trailing whitespace in distro name
        return distinfo
    else:
        return platform.dist()

def PackagedInstall(buildroot):
    """
    Called from setup.py for use by RPM.
    Generic implementation Creates directories and
    files /etc/waagent.conf, /etc/init.d/waagent, /usr/sbin/waagent,
    /etc/logrotate.d/waagent, /etc/sudoers.d/waagent under buildroot.
    Copies generated files waagent.conf, into place and exits.
    """
    MyDistro=GetMyDistro()
    if MyDistro == None :
        sys.exit(1)
    MyDistro.packagedInstall(buildroot)

def LibraryInstall(buildroot):
    pass

def Uninstall():
    """
    Uninstall the agent service.
    Copy RulesFiles back to original locations.
    Delete agent-related files.
    Call RevertVNUMAWorkaround().
    """
    SwitchCwd()
    for a in RulesFiles:
        if os.path.isfile(GetLastPathElement(a)):
            try:
                shutil.move(GetLastPathElement(a), a)
                Warn("Moved " + LibDir + "/" + GetLastPathElement(a) + " -> " + a )
            except:
                pass
    MyDistro.unregisterAgentService()
    MyDistro.uninstallDeleteFiles()
    RevertVNUMAWorkaround()
    return 0

def Deprovision(force, deluser):
    """
    Remove user accounts created by provisioning.
    Disables root password if Provisioning.DeleteRootPassword = 'y'
    Stop agent service.
    Remove SSH host keys if they were generated by the provision.
    Set hostname to 'localhost.localdomain'.
    Delete cached system configuration files in /var/lib and /var/lib/waagent.
    """
    SwitchCwd()
    ovfxml = GetFileContents(LibDir+"/ovf-env.xml")
    ovfobj = None
    if ovfxml != None:
        ovfobj = OvfEnv().Parse(ovfxml)

    print("WARNING! The waagent service will be stopped.")
    print("WARNING! All SSH host key pairs will be deleted.")
    print("WARNING! Cached DHCP leases will be deleted.")
    MyDistro.deprovisionWarnUser()
    delRootPass = Config.get("Provisioning.DeleteRootPassword")
    if delRootPass != None and delRootPass.lower().startswith("y"):
        print("WARNING! root password will be disabled. You will not be able to login as root.")

    if ovfobj != None and deluser == True:
        print("WARNING! " + ovfobj.UserName + " account and entire home directory will be deleted.")

    if force == False and not raw_input('Do you want to proceed (y/n)? ').startswith('y'):
        return 1

    MyDistro.stopAgentService()
    if deluser == True:
        MyDistro.DeleteAccount(ovfobj.UserName)

    # Remove SSH host keys
    regenerateKeys = Config.get("Provisioning.RegenerateSshHostKeyPair")
    if regenerateKeys == None or regenerateKeys.lower().startswith("y"):
        Run("rm -f /etc/ssh/ssh_host_*key*")

    # Remove root password
    if delRootPass != None and delRootPass.lower().startswith("y"):
        MyDistro.deleteRootPassword()
    # Remove distribution specific networking configuration

    MyDistro.publishHostname('localhost.localdomain')
    MyDistro.deprovisionDeleteFiles()
    return 0

def SwitchCwd():
    """
    Switch to cwd to /var/lib/waagent.
    Create if not present.
    """
    CreateDir(LibDir, "root", 0700)
    os.chdir(LibDir)

def Usage():
    """
    Print the arguments to waagent.
    """
    print("usage: " + sys.argv[0] + " [-verbose] [-force] [-help|-install|-uninstall|-deprovision[+user]|-version|-serialconsole|-daemon]")
    return 0

def main():
    """
    Instantiate MyDistro, exit if distro class is not defined.
    Parse command-line arguments, exit with usage() on error.
    Instantiate ConfigurationProvider.
    Call appropriate non-daemon methods and exit.
    If daemon mode, enter Agent.Run() loop.
    """
    if GuestAgentVersion == "":
        print("WARNING! This is a non-standard agent that does not include a valid version string.")
    
    if len(sys.argv) == 1:
        sys.exit(Usage())

    LoggerInit('/var/log/waagent.log','/dev/console')
    global LinuxDistro
    LinuxDistro=DistInfo()[0]
    global MyDistro
    MyDistro=GetMyDistro()
    if MyDistro == None :
        sys.exit(1)
    args = []
    global force
    force = False
    for a in sys.argv[1:]:
        if re.match("^([-/]*)(help|usage|\?)", a):
            sys.exit(Usage())
        elif re.match("^([-/]*)verbose", a):
            myLogger.verbose = True
        elif re.match("^([-/]*)force", a):
            force = True
        elif re.match("^([-/]*)(setup|install)", a):
            sys.exit(MyDistro.Install())
        elif re.match("^([-/]*)(uninstall)", a):
            sys.exit(Uninstall())
        else:
            args.append(a)
    global Config
    Config = ConfigurationProvider()
    
    verbose = Config.get("Logs.Verbose")
    if verbose != None and verbose.lower().startswith("y"):
        myLogger.verbose=True
    global daemon
    daemon = False
    for a in args:
        if re.match("^([-/]*)deprovision\+user", a):
            sys.exit(Deprovision(force, True))
        elif re.match("^([-/]*)deprovision", a):
            sys.exit(Deprovision(force, False))
        elif re.match("^([-/]*)daemon", a):
            daemon = True
        elif re.match("^([-/]*)version", a):
            print(GuestAgentVersion + " running on " + LinuxDistro)
            sys.exit(0)
        elif re.match("^([-/]*)serialconsole", a):
            AppendToLinuxKernelCmdline("console=ttyS0 earlyprintk=ttyS0")
            Log("Configured kernel to use ttyS0 as the boot console.")
            sys.exit(0)
        else:
            print("Invalid command line parameter:" + a)
            sys.exit(1)
    
    if daemon == False:
        sys.exit(Usage())
    global modloaded
    modloaded = False
    try:
        SwitchCwd()
        Log(GuestAgentLongName + " Version: " + GuestAgentVersion)
        if IsLinux():
            Log("Linux Distribution Detected      : " + LinuxDistro)
        global WaAgent
        WaAgent = Agent()
        WaAgent.Run()
    except Exception, e:
        Error(traceback.format_exc())
        Error("Exception: " + str(e))
        sys.exit(1)
    
if __name__ == '__main__' :
    main()