"""
Single interface for running commands via different transports (for example, SSH, Windows HCL, Unix HCL).
"""

import logging
import pipes
import os
import shutil
import socket
import struct
import pickle
import ssl
from contextlib import closing

from parallels.common.utils.steps_profiler import sleep
from parallels.common.utils import poa_utils, windows_utils, ssh_utils, unix_utils
from parallels.common.utils import windows_thirdparty
from parallels.common import local_command_exec
from parallels.common import MigrationError
from parallels.common.utils.windows_utils import check_windows_national_symbols_command
from parallels.utils import poll_data, polling_intervals, safe_string_repr, strip_multiline_spaces, cached
from parallels.common.windows_rsync import windows_rsync, RsyncInstallerPAExec
from parallels.common.utils.steps_profiler import get_default_steps_profiler

logger = logging.getLogger(__name__)
profiler = get_default_steps_profiler()

class HclRunnerException(MigrationError):
	def __init__(self, message, stdout='', stderr='', host_description='', cause=None):
		super(MigrationError, self).__init__(message)
		self.stdout = stdout
		self.stderr = stderr
		self.host_description = host_description
		self.cause = cause

class SSHRunnerException(MigrationError):
	def __init__(self, message, command_str=None, stdout=None, stderr=None):
		super(MigrationError, self).__init__(message)
		self.command_str = command_str
		self.stdout = stdout
		self.stderr = stderr

class BaseRunnerException(MigrationError):
	def __init__(self, message):
		super(MigrationError, self).__init__(message)

class ConnectionCheckError(MigrationError):
	pass

class WinexeRunnerException(MigrationError):
	def __init__(self, message, command_str=None, full_command_str=None, stdout=None, stderr=None):
		super(MigrationError, self).__init__(message)
		self.command_str = command_str
		self.full_command_str = full_command_str
		self.stdout = stdout
		self.stderr = stderr

class BaseRunner(object):
	def run_unchecked(self, cmd, args, stdin_content=None):
		"""Run command with specified args. Arguments are properly escaped."""
		raise NotImplementedError()

	def sh_unchecked(self, cmd_string, args=None, stdin_content=None):
		"""Safely substitute args as template varibales into cmd_string, then execute whole command string.
		
		For Unix: whole command is executed, then if you run "runner.sh('ls *')", '*' will be substituted by shell,
		while if you run "runner.sh('ls {file}', dict(file='*'))", '*' will be escaped, much like "run('ls', ['*'])".
		For Windows: there are no general command arguments parsing rules, and '*' is subsctituded 
		by each particular program, this function differs from run_unchecked only by a way to substitute arguments.

		If None is passed as args, no command formatting is performed, command run as is.

		Returns tuple (exit_code, stdout, stderr)
		"""
		raise NotImplementedError()

	def run(self, cmd, args=[], stdin_content=None):
		"""The same as run_unchecker(), but checks exit code and returns only stdout"""
		exit_code, stdout, stderr = self.run_unchecked(cmd, args, stdin_content)
		if exit_code != 0:
			raise BaseRunnerException(
				u"Command %s with arguments %r failed with exit code %d:\n"
				u"stdout: %s\nstderr: %s"
				% (cmd, args, exit_code, safe_string_repr(stdout), safe_string_repr(stderr))
			)
		return stdout 

	def sh(self, cmd_str, args=None, stdin_content=None):
		"""The same as sh_unchecked(), but checks exit code and returns only stdout"""
		exit_code, stdout, stderr = self.sh_unchecked(cmd_str, args, stdin_content)
		if exit_code != 0:
			raise BaseRunnerException(
				u"Command %s with arguments %r failed with exit code %d:\n"
				u"stdout: %s\nstderr: %s"
				% (cmd_str, args, exit_code, safe_string_repr(stdout), safe_string_repr(stderr))
			)
		return stdout

	def sh_multiple_attempts(self, cmd_str, args=None, stdin_content=None, max_attempts=5, interval_between_attempts=10):
		"""Run command with several attempts, checks exit code and returns only stdout"""
		for attempt in range(0, max_attempts):
			exit_code, stdout, stderr = self.sh_unchecked(cmd_str, args, stdin_content)

			if exit_code == 0:
				if attempt > 0:
					logger.info(
						'Command "{command}" was executed successfully after {attempts} attempt(s).'.format(
							command=self._get_command_str(cmd_str, args), attempts=attempt+1
						)
					)
				return stdout
			else:
				if attempt >= max_attempts - 1:
					self._raise_runner_exception(self._get_command_str(cmd_str, args), stdout, stderr)
				else:
					logger.error(
						'Failed to execute remote command, retry in {interval_between_attempts} seconds'.format(
							interval_between_attempts=interval_between_attempts
						)
					)
					sleep(interval_between_attempts, 'Retry executing command')

	def _raise_runner_exception(self, cmd_str, stdout, stderr):
		raise NotImplementedError()

	def upload_file_content(self, filename, content):
		raise NotImplementedError()

	def upload_file(self, local_filename, remote_filename):
		raise NotImplementedError()

	def get_file(self, remote_filename, local_filename):
		raise NotImplementedError()

	def get_file_contents(self, remote_filename):
		raise NotImplementedError()

	def remove_file(self, filename):
		raise NotImplementedError()

	def mkdir(self, dirname):
		raise NotImplementedError()

	def get_files_list(self, path):
		raise NotImplementedError()

	@staticmethod
	def _get_command_str(cmd_str, args=None):
		raise NotImplementedError()


class WindowsRunner(BaseRunner):
	"""Add codepage detection to migrator's Windows CMD interface classes."""
	@property
	@cached
	def codepage(self):
		"""Determine code page used by Windows CMD.

		'CHCP' utility returns Windows codepage, for example:
		"Active code page: 123"

		Python codec name is derived from 'chcp' output by adding 'cp' to code
		page number, for example: '866' -> 'cp866'
		
		Returns:
		    Python codec name to be used for decoding CMD output.
		"""

		code, stdout, stderr = self._run_command_and_decode_output(
				"cmd.exe /c CHCP", 'ascii', error_policy='ignore')
		if code != 0:
			raise BaseRunnerException(
				u"Unable to determine Windows CMD code page:\nstdout: %s\nstderr: %s"
					% (safe_string_repr(stdout), safe_string_repr(stderr))
			)
		else:
			codepage =  u"cp%s" % stdout.split()[-1].strip('.')
			logger.debug("Detected CMD codepage: %s" % codepage)
		return codepage

	def _run_command_and_decode_output(
			self, cmd_str, output_codepage=None, error_policy='strict'):
		"""Run a command, return its output decoded to UTF."""
		raise NotImplementedError()

class LocalUnixRunner(BaseRunner):
	"""Execute commands on local server"""

	def sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		if args is None:
			args = {}
		command = [
			'/bin/sh', '-c', 
			unix_utils.format_command(cmd_str, **args)
		]
		return local_command_exec(command, stdin_content)

	def mkdir(self, dirname):
		return self.sh('mkdir -p {dirname}', dict(dirname=dirname))

	def upload_file_content(self, filename, content):
		with open(filename, "wb") as fp:
			fp.write(content)

	def remove_file(self, filename):
		os.remove(filename)

	def get_files_list(self, path):
		return unix_utils.get_files_list(self, path)

class LocalWindowsRunner(WindowsRunner):
	"""Execute commands on local Windows server"""

	def __init__(self, disable_profiling=False):
		self._disable_profiling = disable_profiling

	def _run_command_and_decode_output(
			self, cmd_str, output_codepage=None, error_policy='strict'):
		return local_command_exec(cmd_str, '', output_codepage, error_policy)

	def sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		logger.debug("Execute command on local server")
		if args is not None:
			cmd_str = windows_utils.format_command(cmd_str, **args)

		if not self._disable_profiling:
			with profiler.measure_command_call(
				cmd_str, 'Local server'
			):
				return local_command_exec(
					cmd_str, stdin_content, self.codepage
				)
		else:
			return local_command_exec(cmd_str, stdin_content, self.codepage)

	def run_unchecked(self, cmd, args=[], stdin_content=None):
		return self.sh_unchecked(
			windows_utils.format_command_list(cmd, args)
		)

	def upload_file(self, local_filename, remote_filename):
		shutil.copy2(local_filename, remote_filename)

	def upload_file_content(self, filename, content):
		with open(filename, "wb") as fp:
			fp.write(content)

	def get_file_contents(self, remote_filename):
		with open(remote_filename, 'rb') as fp:
			return fp.read()

	def mkdir(self, dirname):
		self.sh('cmd /c if not exist {dirname} ( md {dirname} )', dict(dirname=dirname))

	def remove_file(self, filename):
		self.sh('cmd /c del {filename}', dict(filename=filename))

	def check(self, server_description):
		# No need to check local execution, it should never fail
		pass

class WindowsHclRunner(WindowsRunner):
	def __init__(self, ppa_ssh, poa_host_id, host_description):
		self.ppa_ssh = ppa_ssh
		self.poa_host_id = poa_host_id
		self.host_description = host_description

	def run(self, cmd, args=[], stdin_content=None):
		"""The same as run_unchecked(), but checks exit code and returns only stdout"""
		log_cmd_str = self._get_cmd_str_listargs(cmd, args)

		def run_func():
			exit_code, stdout, stderr = self._run_unchecked(cmd, args, stdin_content)
			if exit_code != 0:
				raise HclRunnerException(strip_multiline_spaces("""
					The command "{command}" executed at {host} returned a non-zero exit code.
					============================================================================
					Stderr:\n{stderr}
					============================================================================
					Stdout:\n{stdout}
					============================================================================
					1. If this happened because of a network issue, re-run the migration tool.
					2. To investigate the issue, execute an HCL request and look at the result:
					2.1. Log in to the PPA management node.
					2.2. Create an XML file with an HCL request (/tmp/tmp-hcl for example):
					{hcl_script}
					2.3. Run the command "/usr/local/pem/bin/pleskd_ctl -f /usr/local/pem/etc/pleskd.props processHCL /tmp/tmp-hcl {host_id}" 
					""").format(
						stderr=stderr,
						stdout=stdout,
						command=log_cmd_str,
						host=self._get_host_description(),
						host_id=self.poa_host_id,
						hcl_script=poa_utils.get_windows_hcl_script(log_cmd_str)
					),
					stdout,
					stderr
				)
			return stdout 

		return hcl_execute_multiple_attempts(run_func, self._get_host_description(), log_cmd_str)

	def run_unchecked(self, cmd, args=[], stdin_content=None):
		log_cmd_str = self._get_cmd_str_listargs(cmd, args)
		def run_func():
			return self._run_unchecked(cmd, args, stdin_content)
		log_cmd_str = " ".join([windows_utils.quote_arg(arg) for arg in [cmd] + args])
		return hcl_execute_multiple_attempts(run_func, self._get_host_description(), log_cmd_str)

	def _run_unchecked(self, cmd, args=[], stdin_content=None):
		if stdin_content is not None:
			raise NotImplementedError(u"This runner does not support stdin_content parameter")

		cmd_str = " ".join([windows_utils.quote_arg(arg) for arg in [cmd] + args])
		return self._run_command_and_decode_output(cmd_str, self.codepage)

	def sh(self, cmd_str, args=None, stdin_content=None):
		"""The same as sh_unchecked(), but checks exit code and returns only stdout"""
		log_cmd_str = self._get_cmd_str_formatargs(cmd_str, args)

		def run_func():
			exit_code, stdout, stderr = self.sh_unchecked(cmd_str, args, stdin_content)
			if exit_code != 0:
				logger.debug(u"Command '%s' failed with '%s' exit code, output: '%s', stderr: '%s'." % (log_cmd_str, exit_code, safe_string_repr(stdout), safe_string_repr(stderr)))
				raise HclRunnerException(strip_multiline_spaces("""
					The command "{command}" executed at {host} returned a non-zero exit code.
					============================================================================
					Stderr:\n{stderr}
					============================================================================
					Stdout:\n{stdout}
					============================================================================
					1. If this happened because of a network issue, re-run the migration tool.
					2. To investigate the issue, execute an HCL request and look at the result:
					2.1. Log in to the PPA management node.
					2.2. Create an XML file with an HCL request (/tmp/tmp-hcl for example):
					{hcl_script}
					2.3. Run the command "/usr/local/pem/bin/pleskd_ctl -f /usr/local/pem/etc/pleskd.props processHCL /tmp/tmp-hcl {host_id}" 
					""").format(
						stderr=stderr,
						stdout=stdout,
						command=log_cmd_str,
						host=self._get_host_description(),
						host_id=self.poa_host_id,
						hcl_script=poa_utils.get_windows_hcl_script(log_cmd_str)
					),
					stdout,
					stderr
				)
			return stdout

		return hcl_execute_multiple_attempts(run_func, self._get_host_description(), log_cmd_str)

	def sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		def run_func():
			return self._sh_unchecked(cmd_str, args, stdin_content)
		return hcl_execute_multiple_attempts(run_func, self._get_host_description(), self._get_cmd_str_formatargs(cmd_str, args))

	def _sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		if stdin_content is not None:
			raise NotImplementedError(u"This runner does not support stdin_content parameter")

		if args is not None:
			cmd_str = windows_utils.format_command(cmd_str, **args)
		return self._run_command_and_decode_output(cmd_str, self.codepage)

	def _run_command_and_decode_output(self, cmd_str, codepage, error_policy = 'strict'):
		"""Run a command, return its output decoded from 'codepage' to UTF."""
		check_windows_national_symbols_command(cmd_str)
		try:
			with profiler.measure_command_call(
				cmd_str, self._get_host_description()
			):
				return poa_utils.windows_exec_unchecked(self.ppa_ssh, self.poa_host_id, cmd_str, codepage=codepage, error_policy=error_policy)
		except poa_utils.HclPleskdCtlNonZeroExitCodeError as e:
			logger.debug(u'Exception:', exc_info=e)
			raise HclRunnerException(strip_multiline_spaces("""
				Failed to execute the command "{command}" at {host}: pleskd_ctl returned a non-zero exit code when executing HCL request. 
				============================================================================
				Stderr:\n{stderr}
				============================================================================
				Stdout:\n{stdout}
				============================================================================
				1. If this happened because of a network issue, re-run the migration tool.
				2. To investigate the issue, execute an HCL request and look at the result:
				2.1. Log in to the PPA management node.
				2.2. Create an XML file with an HCL request (/tmp/tmp-hcl for example):
				{hcl_script}
				2.3. Run the command "/usr/local/pem/bin/pleskd_ctl -f /usr/local/pem/etc/pleskd.props processHCL /tmp/tmp-hcl {host_id}" 
				""").format(
					stderr=e.stderr,
					stdout=e.stdout,
					command=cmd_str,
					host=self._get_host_description(),
					host_id=self.poa_host_id,
					hcl_script=poa_utils.get_windows_hcl_script(cmd_str)
				),
				'',
				str(e),
				cause=e,
				host_description=self._get_host_description()
			)

	def upload_file(self, local_filename, remote_filename):
		with open(local_filename, 'rb') as fp:
			return poa_utils.windows_upload(
				self.ppa_ssh, self.poa_host_id, 
				remote_filename, fp.read()
			)
	
	def upload_file_content(self, filename, content):
		return poa_utils.windows_upload(self.ppa_ssh, self.poa_host_id, filename, content)

	def mkdir(self, dirname):
		self.sh('cmd /c if not exist {dirname} ( md {dirname} )', dict(dirname=dirname))

	def remove_file(self, filename):
		self.sh('cmd /c del {filename}', dict(filename=filename))

	def _get_host_description(self):
		return self.host_description if self.host_description is not None else "PPA service node #%s" % (self.poa_host_id,)

	@staticmethod
	def _get_cmd_str_listargs(cmd, args):
		return " ".join([windows_utils.quote_arg(arg) for arg in [cmd] + args])

	@staticmethod
	def _get_cmd_str_formatargs(cmd_str, args):
		if args is not None:
			return windows_utils.format_command(cmd_str, **args)
		else:
			return cmd_str

class UnixHclRunner(BaseRunner):
	def __init__(self, ppa_ssh, poa_host_id, host_description=None):
		self.ppa_ssh = ppa_ssh
		self.poa_host_id = poa_host_id
		self.host_description = host_description

	def run(self, cmd, args=[], stdin_content=None):
		"""The same as run_unchecker(), but checks exit code and returns only stdout"""
		def run_func():
			log_cmd_str = self._get_cmd_str_listargs(cmd, args)
			exit_code, stdout, stderr = self._run_unchecked(cmd, args, stdin_content)
			if exit_code != 0:
				logger.debug(u"Command '%s' failed with '%s' exit code, output: '%s', stderr: '%s'." % (log_cmd_str, exit_code, safe_string_repr(stdout), safe_string_repr(stderr)))
				raise HclRunnerException(strip_multiline_spaces("""
					The command "{log_command}" executed at {host} returned a non-zero exit code. 
					============================================================================
					Stderr:\n{stderr}
					============================================================================
					Stdout:\n{stdout}
					============================================================================
					1. If this happened because of a network issue, re-run the migration tool.
					2. To investigate the issue, execute an HCL request and look at the result:
					2.1. Log in to the PPA management node.
					2.2. Create an XML file with an HCL request (/tmp/tmp-hcl for example):
					{hcl_script}
					2.3. Run the command "/usr/local/pem/bin/pleskd_ctl -f /usr/local/pem/etc/pleskd.props processHCL /tmp/tmp-hcl {host_id}"
					""").format(
						stdout=stdout,
						stderr=stderr,
						log_command=log_cmd_str,
						host=self._get_host_description(),
						host_id=self.poa_host_id,
						hcl_script=poa_utils.get_unix_hcl_script(cmd, args)
					),
					stdout,
					stderr
				)
			return stdout 

		return hcl_execute_multiple_attempts(run_func, self._get_host_description(), self._get_cmd_str_listargs(cmd, args))

	def run_unchecked(self, cmd, args=[], stdin_content=None):
		def run_func():
			return self._run_unchecked(cmd, args, stdin_content)

		return hcl_execute_multiple_attempts(run_func, self._get_host_description(), self._get_cmd_str_listargs(cmd, args))

	def _run_unchecked(self, cmd, args=[], stdin_content=None):
		if stdin_content is not None:
			raise NotImplementedError(u"This runner does not support stdin_content parameter")
		log_cmd_str = self._get_cmd_str_listargs(cmd, args)

		try:
			with profiler.measure_command_call(
				log_cmd_str, self._get_host_description()
			):
				return poa_utils.unix_exec_unchecked(self.ppa_ssh, self.poa_host_id, cmd, args)
		except poa_utils.HclPleskdCtlNonZeroExitCodeError as e:
			logger.debug(u'Exception:', exc_info=e)
			raise HclRunnerException(strip_multiline_spaces("""
				Failed to execute the command "{command}" at {host}. 
				============================================================================
				Stderr:\n{stderr}
				============================================================================
				Stdout:\n{stdout}
				============================================================================
				1. If this happened because of a network issue, re-run the migration tool.
				2. To investigate the issue, execute an HCL request and look at the result:
				2.1. Log in to the PPA management node.
				2.2. Create an XML file with an HCL request (/tmp/tmp-hcl for example):
				{hcl_script}
				2.3. Run the command "/usr/local/pem/bin/pleskd_ctl -f /usr/local/pem/etc/pleskd.props processHCL /tmp/tmp-hcl {host_id}"
				""").format(
					stderr=e.stderr,
					stdout=e.stdout,
					command=log_cmd_str,
					host=self._get_host_description(),
					host_id=self.poa_host_id,
					hcl_script=poa_utils.get_unix_hcl_script(cmd, args)
				),
				'',
				str(e),
				cause=e,
				host_description=self._get_host_description()
			)

	def sh(self, cmd_str, args=None, stdin_content=None):
		"""The same as sh_unchecked(), but checks exit code and returns only stdout"""
		log_cmd_str = self._get_cmd_str_formatargs(cmd_str, args)

		def run_func():
			exit_code, stdout, stderr = self._sh_unchecked(cmd_str, args, stdin_content)
			if exit_code != 0:
				logger.debug(u"Command '%s' failed with '%s' exit code, output: '%s', stderr: '%s'." % (log_cmd_str, exit_code, safe_string_repr(stdout), safe_string_repr(stderr)))
				raise HclRunnerException(strip_multiline_spaces("""
					The command "{log_command}" executed at {host} returned a non-zero exit code.
					============================================================================
					Stderr:\n{stderr}
					============================================================================
					Stdout:\n{stdout}
					============================================================================
					1. If this happened because of a network issue, re-run the migration tool.
					2. To investigate the issue, execute an HCL request and look at the result:
					2.1. Log in to the PPA management node.
					2.2. Create an XML file with an HCL request (/tmp/tmp-hcl for example):
					{hcl_script}
					2.3. Run the command "/usr/local/pem/bin/pleskd_ctl -f /usr/local/pem/etc/pleskd.props processHCL /tmp/tmp-hcl {host_id}" 
					""").format(
						stdout=stdout,
						stderr=stderr,
						log_command=log_cmd_str,
						host=self._get_host_description(),
						host_id=self.poa_host_id,
						hcl_script=poa_utils.get_unix_hcl_script(log_cmd_str, [])
					),
					stdout,
					stderr
				)
			return stdout

		return hcl_execute_multiple_attempts(run_func, self._get_host_description(), log_cmd_str)

	def sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		def run_func():
			return self._sh_unchecked(cmd_str, args, stdin_content)

		return hcl_execute_multiple_attempts(run_func, self._get_host_description(), self._get_cmd_str_formatargs(cmd_str, args))

	def _sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		run_cmd, run_args = self._get_command_list(cmd_str, args)
		if stdin_content is not None:
			raise NotImplementedError(u"This runner does not support stdin_content parameter")

		return self._run_unchecked(run_cmd, run_args)

	def _get_command_list(self, cmd_str, args):
		if args is not None:
			cmd_str = unix_utils.format_command(cmd_str, **args)
		return '/bin/sh', ['-c', cmd_str]

	def get_file_contents(self, remote_filename):
		return self.run('/bin/cat', [remote_filename,])

	def remove_file(self, filename):
		return self.run('/bin/rm', [filename])

	def mkdir(self, dirname):
		return self.run('/bin/mkdir', ['-p', dirname])

	def get_files_list(self, path):
		return unix_utils.get_files_list(self, path)

	def _get_host_description(self):
		return self.host_description if self.host_description is not None else "PPA service node #%s" % (self.poa_host_id,)

	@staticmethod
	def _get_cmd_str_listargs(cmd, args):
		return unix_utils.format_command_list(cmd, args)

	def _get_cmd_str_formatargs(self, cmd_str, args):
		run_cmd, run_args = self._get_command_list(cmd_str, args)
		return unix_utils.format_command_list(run_cmd, run_args)

def hcl_execute_multiple_attempts(run_function, host, command, max_attempts=5, interval_between_attempts=10):
	for attempt in range(0, max_attempts):
		try:
			result = run_function()
			if attempt > 0:
				logger.info(
					'Command "{command}" was executed successfully at {host} after {attempts} attempt(s).'.format(
						command=command, host=host, attempts=attempt+1
					)
				)
			return result
		except HclRunnerException as e:
			logger.debug("Exception: ", exc_info=True)
			if attempt >= max_attempts - 1:
				raise e
			else:
				logger.error(
					'Failed to execute remote command, retry in {interval_between_attempts} seconds'.format(
						interval_between_attempts=interval_between_attempts
					)
				)
				sleep(interval_between_attempts, 'Retry executing command')

def hcl_is_windows(ppa_runner, host_id, host_description=None):
	"""Hack to detect service node platform"""

	host_description = host_description if host_description is not None else "PPA service node #%s" % (host_id,)
	windows_pattern = "User '' does not exist"
	command = 'cmd /c REM'

	def run_func():
		# REM - command used for comments in batch file; does nothing, has no output.
		try:
			exit_code, stdout, stderr = poa_utils.windows_exec_unchecked(ppa_runner.ssh, host_id, command, 'utf-8')
			if exit_code != 0: # looks like Windows, but there are some problems executing the command, so raise HclRunnerException which means - try again
				raise HclRunnerException(strip_multiline_spaces("""
					Failed to detect OS of the node. The command "{command}" executed at {host_description} returned a non-zero exit code. 
					============================================================================
					Stderr:\n{stderr}
					============================================================================
					Stdout:\n{stdout}
					============================================================================
					1. If this happened because of a network issue, re-run the migration tool.
					2. To investigate the issue, execute an HCL request and look at the result:
					2.1. Log in to the PPA management node.
					2.2. Create an XML file with an HCL request (/tmp/tmp-hcl for example):
					{hcl_script}
					2.3. Run the command "/usr/local/pem/bin/pleskd_ctl -f /usr/local/pem/etc/pleskd.props processHCL /tmp/tmp-hcl {host_id}"
					2.4. If node OS is Windows, expect error message '{windows_pattern}', otherwise expect no error messages
					""").format(
						stderr=stderr,
						stdout=stdout,
						command=command,
						host_id=host_id,
						host_description=host_description,
						hcl_script=poa_utils.get_windows_hcl_script(command),
						windows_pattern=windows_pattern
					),
					stdout, stderr,
					host_description=host_description,
				)
			return True # no errors occured, so that is Windows
		except poa_utils.HclPleskdCtlNonZeroExitCodeError as e:
			if windows_pattern in e.stdout or windows_pattern in e.stderr:
				return False
			else:
				# looks like a problem communicating the node, or something else unexpected - try again
				raise HclRunnerException(strip_multiline_spaces("""
					Failed to detect OS of the node. The command "{command}" executed at {host_description} returned a non-zero exit code.
					============================================================================
					Stderr:\n{stderr}
					============================================================================
					Stdout:\n{stdout}
					============================================================================
					1. If this happened because of a network issue, re-run the migration tool.
					2. To investigate the issue, execute an HCL request and look at the result:
					2.1. Log in to the PPA management node.
					2.2. Create an XML file with an HCL request (/tmp/tmp-hcl for example):
					{hcl_script}
					2.3. Run the command "/usr/local/pem/bin/pleskd_ctl -f /usr/local/pem/etc/pleskd.props processHCL /tmp/tmp-hcl {host_id}"
					2.4. If node OS is Windows, expect error message '{windows_pattern}', otherwise expect no error messages
					""").format(
						stderr=e.stderr,
						stdout=e.stdout,
						command=command,
						host_id=host_id,
						host_description=host_description,
						hcl_script=poa_utils.get_windows_hcl_script(command),
						windows_pattern=windows_pattern
					),
					e.stdout, e.stderr,
					host_description=host_description,
					cause=e
				)

	return hcl_execute_multiple_attempts(run_func, host_description, command)

class SSHRunner(BaseRunner):
	def __init__(self, ssh, host_description=None):
		self.ssh = ssh
		self.host_description = host_description

	def run(self, cmd, args=[], stdin_content=None):
		"""The same as run_unchecker(), but checks exit code and returns only stdout"""
		exit_code, stdout, stderr = self.run_unchecked(cmd, args, stdin_content)
		if exit_code != 0:
			self._raise_runner_exception(unix_utils.format_command_list(cmd, args), stdout, stderr)

		return stdout 
	
	def run_unchecked(self, cmd, args=[], stdin_content=None):
		command_str = unix_utils.format_command_list(cmd, args)
		with profiler.measure_command_call(
			command_str, self.host_description
		):
			return ssh_utils.run_unchecked(
				self.ssh, command_str, stdin_content=stdin_content
			)

	def sh(self, cmd_str, args=None, stdin_content=None):
		"""The same as sh_unchecked(), but checks exit code and returns only stdout"""
		return self.sh_multiple_attempts(cmd_str, args, stdin_content, max_attempts = 1)

	def sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		command_str = self._get_command_str(cmd_str, args)
		with profiler.measure_command_call(
			command_str, self.host_description
		):
			return ssh_utils.run_unchecked(
				self.ssh, command_str, stdin_content=stdin_content
			)

	def upload_file(self, local_filename, remote_filename):
		with closing(self.ssh.open_sftp()) as sftp:
			sftp.put(local_filename, remote_filename)

	def get_file(self, remote_filename, local_filename):
		with closing(self.ssh.open_sftp()) as sftp:
			sftp.get(remote_filename, local_filename)

	def get_file_contents(self, remote_filename):
		return self.run('/bin/cat', [remote_filename,])

	def remove_file(self, filename):
		return self.run('/bin/rm', [filename])

	def mkdir(self, dirname):
		return self.run('/bin/mkdir', ['-p', dirname])

	def get_files_list(self, path):
		return unix_utils.get_files_list(self, path)

	def _raise_runner_exception(self, command, stdout, stderr):
		if self.host_description is not None:
			where = " executed at %s" % (self.host_description,)
		else:
			where = ""

		raise SSHRunnerException(
			u"""The command "{command}"{where} returned a non-zero exit code.\n"""
			u"============================================================================\n"
			u"Stderr:\n{stderr}\n"
			u"============================================================================\n"
			u"Stdout:\n{stdout}\n"
			u"============================================================================\n\n"
			u"1. If this happened because of a network issue, re-run the migration tool.\n"
			u"2. Ensure that the host is up and there are no firewall rules that may block SSH connection to the host.\n"
			u"3. To investigate the issue, login to the host by SSH, run the command and look at the result.\n".format(
				command=command, where=where, stderr=safe_string_repr(stderr), stdout=safe_string_repr(stdout)
			),
			command_str=command,
			stdout=stdout,
			stderr=stderr
		)

	@staticmethod
	def _get_command_str(cmd_str, args=None):
		if args is not None:
			cmd_str = unix_utils.format_command(cmd_str, **args)
		return cmd_str

class WindowsAgentRunner(BaseRunner):
	def __init__(self, settings):
		self.remote = RemoteObject(settings)
		self.settings = settings

		if not self.remote.try_connect():
			self.remote.deploy()
			self.remote.start()
		if not self.remote.try_connect():
			self._print_agent_installation_instructions()

	def _print_agent_installation_instructions(self):
		raise Exception(
			u"Migration agent was not installed on server '%s'. "
			u"Upload, run agent-installer.exe to the server."
			u"Then go to C:\ParallelsPanelTransferAgent "
			u"and run start.bat there." % (
			self.settings.ip
			)
		)

	def check(self, server_description):
		pass

	def sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		logger.debug("Execute remote command %s", cmd_str)
		exit_code, stdout, stderr = self.remote.sh_unchecked(cmd_str, args, stdin_content) 
		logger.debug("Exit code: %s. Stdout: %s. Stderr: %s.", exit_code, stdout, stderr)
		return exit_code, stdout, stderr

	def run_unchecked(self, cmd, args=[], stdin_content=None):
		return self.sh_unchecked(
			windows_utils.format_command_list(cmd, args)
		)

	def get_file(self, remote_filename, local_filename):
		with open(local_filename, 'wb') as fp:
			fp.write(self.get_file_contents(remote_filename))

	def upload_file(self, local_filename, remote_filename):
		with open(local_filename, 'rb') as fp:
			self.upload_file_content(remote_filename, fp.read())

	def _raise_runner_exception(self, cmd_str, stdout, stderr):
		raise BaseRunnerException("Failed to execute command '%s'. Stdout: %s. Stderr: %s." % (
			cmd_str, stdout, stderr
		))

	def upload_file_content(self, filename, content):
		self.remote.upload_file_content(filename, content)

	def get_file_contents(self, remote_filename):
		return self.remote.get_file_contents(remote_filename)

	def remove_file(self, filename):
		self.remote.remove_file(filename)

	def mkdir(self, dirname):
		self.remote.mkdir(dirname)

	@staticmethod
	def _get_command_str(cmd_str, args=None):
		return windows_utils.format_command(cmd_str, **args)

class RemoteObject(object):
	def __init__(self, settings):
		self.settings = settings
		self.socket = None

	def deploy(self):
		local_runner = LocalWindowsRunner()
		logger.info("Pack agent into self-extractable archive")
		local_runner.sh(windows_utils.cmd_command(
				r"{thirdparty}\7zip\7za.exe a -sfx "
				r"{thirdparty}\agent-installer.exe "
				r"{thirdparty}\python27 "
				r"{thirdparty}\..\agent\server.py {thirdparty}\..\agent\config.ini "
				r"{thirdparty}\..\agent\logging.config {thirdparty}\..\agent\start.bat".format(
					thirdparty=windows_thirdparty.get_thirdparty_dir()
				)
		))
		logger.info("Upload and install agent")
		local_runner.sh(windows_utils.cmd_command(
			r'cd %s &&%s \\%s -u %s -p %s -c %s -o%s -y' % (
				windows_thirdparty.get_thirdparty_dir(),
				windows_thirdparty.get_paexec_bin(),
				self.settings.ip,
				self.settings.windows_auth.username,
				self.settings.windows_auth.password,
				".\\agent-installer.exe",
				'c:\\ParallelsPanelTransferAgent'
			)
		))

	def start(self):
		logger.info("Start agent")
		local_runner = LocalWindowsRunner()
		local_runner.sh(windows_utils.cmd_command(
			r'%s \\%s -u %s -p %s -d c:\\ParallelsPanelTransferAgent\\start.bat' % (
				windows_thirdparty.get_paexec_bin(),
				self.settings.ip,
				self.settings.windows_auth.username,
				self.settings.windows_auth.password,
			)
		))
		# XXX hack and possible race condition there
		sleep(10, 'Wait for agent start')

	def try_connect(self):
		try:
			self.connect()
			return True
		except socket.error:
			return False

	def connect(self):
		self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		if self.settings.agent_settings.use_ssl:
			self.socket = ssl.wrap_socket(
				self.socket, 
				keyfile=self.settings.agent_settings.client_key, 
				certfile=self.settings.agent_settings.client_cert, 
				cert_reqs=ssl.CERT_REQUIRED, 
				ca_certs=self.settings.agent_settings.server_cert
			)
		self.socket.connect((
			self.settings.ip, self.settings.agent_settings.port
		))

	def reconnect(self):
		# close socket
		if self.socket is not None:
			try:
				self.socket.close()
			except:
				logger.debug("Exception when closing socket: ", exc_info=True)
		# try to connect
		self.connect()

	def __getattr__(self, attr):
		remote_function_name = attr

		def run_remote_function(*args, **kwargs):
			def run():
				command = pickle.dumps(
					(remote_function_name, args, kwargs)
				)
				self.socket.sendall(struct.pack("I", len(command)))
				self.socket.sendall(command)
				length = self._receive(4)
				length, = struct.unpack("I", length)
				result = pickle.loads(self._receive(length))
				return result

			return self._run_multiple_attempts(run)

		return run_remote_function

	def _run_multiple_attempts(self, run_function, max_attempts=5, interval_between_attempts=10):
		for attempt in range(0, max_attempts):
			try:
				if attempt > 0:
					# reconnect on 2nd and later attempts
					self.reconnect()

				result = run_function()
				if attempt > 0:
					logger.info(
						'Remote operation executed successfully at {host} after {attempts} attempt(s).'.format(
							host=self.settings.ip, attempts=attempt+1
						)
					)
				return result
			except socket.error as e:
				logger.debug("Exception: ", exc_info=True)
				if attempt >= max_attempts - 1:
					raise e
				else:
					logger.error(
						'Failed to run remote operation, retry in {interval_between_attempts} seconds'.format(
							interval_between_attempts=interval_between_attempts
						)
					)
					sleep(interval_between_attempts, 'Retry running remote command')

	def _receive(self, size):
		b = ''
		while len(b) < size:
			r = self.socket.recv(size - len(b))
			if not r:
				return b
			b += r
		return b

class PAExecRunner(BaseRunner):
	"""Execute commands on remote Windows server with paexec utility"""

	def __init__(self, settings, migrator_server, source_server):
		self.settings = settings
		self.migrator_server = migrator_server
		self.source_server = source_server

	def sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		logger.debug("Execute command on remote server with paexec")
		if stdin_content is not None:
			raise NotImplementedError(u"This runner does not support stdin_content parameter")

		if args is not None:
			cmd_str = windows_utils.format_command(cmd_str, **args)

		with profiler.measure_command_call(
			cmd_str, self.settings.ip
		):
			passwords_file = self.migrator_server.get_session_file_path(
				'node-password-%s' % self.settings.ip
			)
			with open(passwords_file, 'wb') as f:
				f.write(self.settings.windows_auth.password)
			return LocalWindowsRunner(disable_profiling=True).sh_unchecked(
				r'%s \\%s -u %s -p@ %s -p@d ' % (
					windows_thirdparty.get_paexec_bin(),
					self.settings.ip,
					self.settings.windows_auth.username,
					passwords_file	
				) + cmd_str
			)

	def run_unchecked(self, cmd, args=[], stdin_content=None):
		return self.sh_unchecked(
			windows_utils.format_command_list(cmd, args)
		)

	def get_file(self, remote_filename, local_filename):
		with windows_rsync(
			self.source_server, self.migrator_server, 
			rsync_installer_source=RsyncInstallerPAExec(
				self.migrator_server, self.source_server
			)
		) as rsync:
			self.sh(r'cmd /c copy "%s" c:\migrator\download' % remote_filename)
			rsync.sync(
				source_path='migrator/download', 
				target_path=windows_utils.convert_path_to_cygwin(local_filename)
			)
			self.remove_file(r'c:\migrator\download')

	def get_file_contents(self, remote_filename):
		temp_filename = self.migrator_server.get_session_file_path('paexec_download')
		try:
			self.get_file(remote_filename, temp_filename)
			with open(temp_filename, 'rb') as f:
				return f.read()
		finally:
			if os.path.isfile(temp_filename):
				os.remove(temp_filename)

	def upload_file_content(self, filename, content):
		local_temp_file = self.migrator_server.get_session_file_path('upload')
		with open(local_temp_file, "wb") as fp:
			fp.write(content)
		self.upload_file(local_temp_file, filename)

	def upload_file(self, local_filename, remote_filename):
		with windows_rsync(
			self.source_server, self.migrator_server,
			rsync_installer_source=RsyncInstallerPAExec(
				self.migrator_server, self.source_server
			)
		) as rsync:
			rsync.upload(
				source_path=windows_utils.convert_path_to_cygwin(local_filename),
				target_path='migrator/upload'
			)
		self.sh(r'cmd /c move c:\migrator\upload "%s"' % remote_filename)

	def mkdir(self, dirname):
		self.sh('cmd /c if not exist {dirname} ( md {dirname} )', dict(dirname=dirname))

	def remove_file(self, filename):
		self.sh('cmd /c del {filename}', dict(filename=filename))

	def check(self, server_description):
		try:
			self.sh('cmd /c echo 1')
		except Exception:
			logger.debug('Exception: ', exc_info=True)
			raise ConnectionCheckError(
				"Failed to connect to server %s with paexec utility\n"
				"Check that:\n"
				"1) Correct Windows login and password are specified in config.ini.\n"
				"2) Samba service is working on the Windows server." % (
					server_description
				)
			)

class WinexeRunner(WindowsRunner):
	"""Remotely execute Windows CMD commands on Linux using 'winexe' tool."""
	def __init__(self, settings, runas = False, local_temp_dir = None):
		self.settings = settings
		self.runas = runas
		self.local_temp_dir = local_temp_dir

	def run_unchecked(self, cmd, args=[], stdin_content=None):
		if stdin_content is not None:
			raise NotImplementedError(u"This runner does not support stdin_content parameter")

		command_str = windows_utils.format_command_list(cmd, args)
		with profiler.measure_command_call(
			command_str, self.settings.ip
		):
			return self._run_command_and_decode_output(command_str)

	def sh_unchecked(self, cmd_str, args=None, stdin_content=None):
		if stdin_content is not None:
			raise NotImplementedError(u"This runner does not support stdin_content parameter")

		if args is not None:
			cmd_str = windows_utils.format_command(cmd_str, **args)

		with profiler.measure_command_call(
			cmd_str, self.settings.ip
		):
			return self._run_command_and_decode_output(cmd_str)

	def run(self, cmd, args=[], stdin_content=None):
		"""The same as run_unchecked(), but checks exit code and returns only stdout"""
		exit_code, stdout, stderr = self.run_unchecked(cmd, args, stdin_content)
		if exit_code != 0:
			self._raise_runner_exception(windows_utils.format_command_list(cmd, args), stdout, stderr)
		return stdout 

	def sh(self, cmd_str, args=None, stdin_content=None):
		"""The same as sh_unchecked(), but checks exit code and returns only stdout"""
		return self.sh_multiple_attempts(cmd_str, args, stdin_content, max_attempts=1)

	def _run_command_and_decode_output(
			self, cmd_str, default_codepage=None, error_policy='strict'):
		"""Run a command, return its output decoded from 'codepage' to UTF."""
		check_windows_national_symbols_command(cmd_str)
		formatted_command = self._construct_winexe_command(cmd_str)
		if not default_codepage:
			effective_codepage = self.codepage
		else:
			effective_codepage = default_codepage

		errors = []
		def run_winexe_command():
			exit_code, stdout, stderr = local_command_exec(
				formatted_command, stdin_content=None,
				output_codepage=effective_codepage)

			# remove winexe warning with no impact on us
			stderr = stderr.replace(
				"dos charset 'CP850' unavailable - using ASCII\n", ''
			)

			if stderr.find(u"Failed to open connection - NT_STATUS_LOGON_FAILURE") != -1:
				errors.append(u"Command '%s' failed to execute with '%s' error. Please check that credentials for server '%s' are correct in config.ini" % (
					u" ".join([pipes.quote(c) for c in formatted_command]),
					stderr.strip(),
					self.settings.ip
				))
				return None
			elif stderr.find(u"Failed to open connection") != -1:
				errors.append(u"Command '%s' failed to execute with '%s' error. Please check that connection settings for server '%s' are correct in config.ini" % (
					u" ".join([pipes.quote(c) for c in formatted_command]),
					stderr.strip(),
					self.settings.ip
				))
				return None
			elif stderr.find(u"NT_STATUS_") != -1:
				errors.append(u"Command '%s' failed to execute with '%s' error" % (
					u" ".join([pipes.quote(c) for c in formatted_command]),
					stderr.strip()
				))
				return None
			else:
				return exit_code, stdout, stderr 
		
		result = poll_data(run_winexe_command, polling_intervals(starting=3, max_attempt_interval=5, max_total_time=30))
		if result is not None:
			return result
		else:
			logger.debug(u"Errors while executing command:\n%s" % "\n".join(errors))
			raise MigrationError(errors[len(errors)-1])

	def upload_file(self, local_filename, remote_filename):
		# implementation note: winexe runner utilizes smbclient to upload files
		windows_utils.upload_file(local_filename, remote_filename, self.settings)

	def upload_file_content(self, filename, content):
		if self.local_temp_dir is None:
			raise Exception("Unable to upload file contents to %s: directory for temporary files was not specified to WinexeRunner" % (filename,))
		# implementation note: winexe runner utilizes smbclient to upload files
		temp_filename = os.path.join(self.local_temp_dir, 'winexe_upload')	
		try:
			with open(temp_filename, "w") as f:
				f.write(content)

			windows_utils.upload_file(temp_filename, filename, self.settings)
		finally:
			if os.path.isfile(temp_filename):
				os.remove(temp_filename)

	def get_file(self, remote_filename, local_filename):
		# implementation note: winexe runner utilizes smbclient to download files
		windows_utils.download_file(remote_filename, local_filename, self.settings)

	def get_file_contents(self, remote_filename):
		if self.local_temp_dir is None:
			raise Exception(
				"Unable to download file contents to %s: "
				"directory for temporary files was not "
				"specified to WinexeRunner" % (remote_filename,)
			)
		temp_filename = os.path.join(self.local_temp_dir, 'winexe_download')
		try:
			windows_utils.download_file(
				remote_filename, temp_filename, self.settings
			)
			with open(temp_filename, 'rb') as f:
				return f.read()
		finally:
			if os.path.isfile(temp_filename):
				os.remove(temp_filename)

	def remove_file(self, filename):
		self.sh('cmd /c del {filename}', dict(filename=filename))

	def mkdir(self, dirname):
		self.sh('cmd /c if not exist {dirname} ( md {dirname} )', dict(dirname=dirname))

	def check(self, server_description):
		try:
			logger.debug(u"Try to connect to samba share C$ on %s." % server_description)
			windows_utils.run_smbclient_cmd(
				 # no command - just connect to the share and exit
				"", "C$", self.settings
			)
		except Exception as err:
			logger.debug(u"Exception:", exc_info=err)
			raise ConnectionCheckError(strip_multiline_spaces(
			u"""Failed to connect to samba share C$ on server '%s': %s. 
				Check that:
				1) Correct Windows login and password are specified in config.ini.
				2) Samba service is working on the Windows server.
				""" % (server_description, err)
			))

	def _construct_winexe_command(self, cmd_str):
		formated_command = [
			u"winexe", 
			u"--debug-stderr",
			u"--reinstall",
			u"-U", "%s%%%s" % (self.settings.windows_auth.username, self.settings.windows_auth.password),
			u"//%s" % (self.settings.ip,),
			cmd_str
		]

		if self.runas:
			formated_command += [u"--runas", "%s%%%s" % (self.settings.windows_auth.username, self.settings.windows_auth.password)]

		return formated_command

	@staticmethod
	def _get_cmd_str_listargs(cmd, args):
		return " ".join([windows_utils.quote_arg(arg) for arg in [cmd] + args])

	@staticmethod
	def _get_cmd_str_formatargs(cmd_str, args):
		if args is not None:
			return windows_utils.format_command(cmd_str, **args)
		else:
			return cmd_str

	def _raise_runner_exception(self, command_str, stdout, stderr):
		full_command = self._construct_winexe_command(command_str)
		full_cmd = full_command[0]
		full_args = full_command[1:]
		full_command_str = unix_utils.format_command_list(full_cmd, full_args)
		where = " executed at %s" % (self.settings.ip,)
		raise WinexeRunnerException(strip_multiline_spaces("""
				The command "{command_str}"{where} returned a non-zero exit code.
				============================================================================
				Stderr:\n{stderr}
				============================================================================
				Stdout:\n{stdout}
				============================================================================
				1. If this happened because of a network issue, re-run the migration tool.
				2. Ensure that the host is up and there are no firewall rules that may block winexe connection to the host.
				3. To investigate the issue:
				- login to the host and run the command:
				{command_str}
				- try to run the command with winexe from the current host:
				{full_command_str}
			""").format(
				command_str=command_str, full_command_str=full_command_str,
				where=where, stderr=safe_string_repr(stderr), stdout=safe_string_repr(stdout)
			), 
			command_str=command_str, full_command_str=full_command_str,
			stderr=stderr, stdout=stdout
		)

	@staticmethod
	def _get_command_str(cmd_str, args=None):
		if args is not None:
			cmd_str = windows_utils.format_command(cmd_str, **args)
		return cmd_str
