/*
 * jETeL/CloverETL - Java based ETL application framework.
 * Copyright (c) Javlin, a.s. ([email protected])
 *  
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package org.jetel.component;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.Charset;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.HttpVersion;
import org.apache.http.NameValuePair;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.AuthenticationException;
import org.apache.http.auth.Credentials;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpOptions;
import org.apache.http.client.methods.HttpPatch;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.methods.HttpTrace;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.URIUtils;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContextBuilder;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.cookie.Cookie;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.InputStreamBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.TargetAuthenticationStrategy;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.log4j.Level;
import org.jetel.data.ByteDataField;
import org.jetel.data.DataField;
import org.jetel.data.DataRecord;
import org.jetel.data.DataRecordFactory;
import org.jetel.data.Defaults;
import org.jetel.data.StringDataField;
import org.jetel.exception.AttributeNotFoundException;
import org.jetel.exception.ComponentNotReadyException;
import org.jetel.exception.ConfigurationStatus;
import org.jetel.exception.JetelRuntimeException;
import org.jetel.exception.TempFileCreationException;
import org.jetel.exception.XMLConfigurationException;
import org.jetel.graph.InputPort;
import org.jetel.graph.Node;
import org.jetel.graph.OutputPort;
import org.jetel.graph.Result;
import org.jetel.graph.TransformationGraph;
import org.jetel.graph.runtime.tracker.ComponentTokenTracker;
import org.jetel.graph.runtime.tracker.ReformatComponentTokenTracker;
import org.jetel.metadata.DataFieldContainerType;
import org.jetel.metadata.DataFieldMetadata;
import org.jetel.metadata.DataFieldType;
import org.jetel.metadata.DataRecordMetadata;
import org.jetel.util.CTLMapping;
import org.jetel.util.CTLTransformUtils.Field;
import org.jetel.util.ExceptionUtils;
import org.jetel.util.SynchronizeUtils;
import org.jetel.util.bytes.SystemOutByteChannel;
import org.jetel.util.file.FileURLParser;
import org.jetel.util.file.FileUtils;
import org.jetel.util.property.ComponentXMLAttributes;
import org.jetel.util.property.PropertyRefResolver;
import org.jetel.util.property.RefResFlag;
import org.jetel.util.protocols.UserInfo;
import org.jetel.util.stream.StreamUtils;
import org.jetel.util.string.StringUtils;
import org.w3c.dom.Element;

import oauth.signpost.OAuthConsumer;
import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;

/**
 * * <h3>HttpConnector Component</h3>
 * 
 * <table border="1">
 * <th>Component:</th>
 * <tr>
 * <td>
 * <h4><i>Name:</i></h4></td>
 * <td>HttpConnector</td>
 * </tr>
 * <tr>
 * <td>
 * <h4><i>Category:</i></h4></td>
 * <td>Others</td>
 * </tr>
 * <tr>
 * <td>
 * <h4><i>Description:</i></h4></td>
 * <td>Component provides a functionality for sending http requests. URL can be set or retrieved from input port.
 * Implemented http methods are GET and POST. Place holders can be used at URL when input port is connected. The format
 * of placeholder is *{placeholder_name}. Input fields not used for substitution of placeholders can be added to the URL
 * as parameters (they can be add to the query string or method body if the POST method is used. Ignored fields specify
 * which input fields can't be add as parameters. If POST method is used input fields can be added as multipart
 * entities. Component allows HTTP Authentication (basic and digest). Authentication is proceed if username and password
 * is set.)</td>
 * </tr>
 * <tr>
 * <td></td>
 * </tr>
 * </table>
 * <br>
 * <table border="1">
 * <tr>
 * <td><b>type</b></td>
 * <td>"HTTP_CONNECTOR"</td>
 * </tr>
 * <tr>
 * <td><b>url</b></td>
 * <td>URL for http request. Place holders can be used when input port is connected. Place holder format is *{}</td>
 * <tr>
 * <td><b>urlInputField</b></td>
 * <td>URL for http request from metadata field. Place holders can be used.</td>
 * <tr>
 * <td><b>requestMethod</b></td>
 * <td>Http request method. Can be DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT, TRACE.</td>
 * <tr>
 * <td><b>addInputFieldsAsParametres</b></td>
 * <td>Specifies whether parameters are added to the URL. (values: true/false)</td>
 * <tr>
 * <td><b>addInputFieldsAsParametresTo</b></td>
 * <td>Specifies whether input fields should be add to the query string or method body. Parameters can be added to the
 * method body in case that POST method is used. (values: Query/Body)</td>
 * <tr>
 * <td><b>ignoredFields</b></td>
 * <td>Specifies which input fields aren't added as parameters. List of input fields separated by ; is expected.</td>
 * <tr>
 * <td><b>multipartEntities</b></td>
 * <td>Specifies which input fields are added to the request as multipart entities. Multipart entities can be added to
 * the request in case that POST method is used. List of input fields separated by ; is expected.</td>
 * <tr>
 * <td><b>headerProperties</b></td>
 * <td>Additional http header properties.</td>
 * <tr>
 * <td><b>charset</b></td>
 * <td>Character encoding of the output file (if not specified, then UTF-8 is used)</td>
 * <tr>
 * <td><b>inputField</b></td>
 * <td>The input field whose content is sent as the request.</td>
 * <tr>
 * <td><b>requestContent</b></td>
 * <td>The text field whose content is sent as the request.</td>
 * <tr>
 * <td><b>inFileUrl</b></td>
 * <td>Input file.</td>
 * <tr>
 * <td><b>outFileUrl</b></td>
 * <td>Output file.</td>
 * <tr>
 * <td><b>append</b></td>
 * <td>Whether to append data at the end if output file exists or replace it (values: true/false)</td>
 * <tr>
 * <td><b>authenticationMethod</b></td>
 * <td>HTTP Authentication method. Authentication is done if username and password is entered. (values:
 * BASIC/DIGEST/ANY)</td>
 * <tr>
 * <td><b>username</b></td>
 * <td>Username for http authentication</td>
 * <tr>
 * <td><b>password</b></td>
 * <td>Password for http authentication</td>
 * <tr>
 * <td><b>consumerKey</b></td>
 * <td>Consumer key to be used with OAuth authentication</td>
 * <tr>
 * <td><b>consumerSecret</b></td>
 * <td>Consumer secret to be used with OAuth authentication</td>
 * <tr>
 * <td><b>responseAsFileName</b></td>
 * <td>If specified, the component will write response to a temporary file and send the file name in output field User
 * can then read with "indirect" reading. If not specified, the response is passed by value. (values true/false)</td>
 * <tr>
 * <td><b>responseDirectory</b></td>
 * <td>Directory for response files.</td>
 * <tr>
 * <td><b>responseFilePrefix</b></td>
 * <td>Prefix for response files</td>
 * </tr>
 * </table>
 * 
 * @author Martin Zatopek ([email protected]) (c) Javlin Consulting (www.javlinconsulting.cz),
 * 
 * @created 24.6.2009
 * @version 15.3.2010
 */

public class HttpConnector extends Node {

	private final static Log logger = LogFactory.getLog(HttpConnector.class);

	/**
	 * The port index used for data record input
	 */
	private static final int INPUT_PORT_NUMBER = 0;

	/**
	 * The port index used for data record output for tokens with successful status
	 */
	private static final int STANDARD_OUTPUT_PORT_NUMBER = 0;

	/**
	 * The port index used for optional data record output for tokens with non-successful status
	 */
	private static final int ERROR_OUTPUT_PORT_NUMBER = 1;

	/**
	 * Valid authentication methods.
	 */
	private static final Set<String> SUPPORTED_AUTHENTICATION_METHODS = new HashSet<String>();
	static {
		SUPPORTED_AUTHENTICATION_METHODS.add("BASIC");
		SUPPORTED_AUTHENTICATION_METHODS.add("DIGEST");
		SUPPORTED_AUTHENTICATION_METHODS.add("ANY");
	}

	/**
	 * Supported HTTP request methods.
	 */
	private static final Set<String> ENTITY_ENCLOSING_REQUEST_METHODS = new HashSet<String>();
	static {
		ENTITY_ENCLOSING_REQUEST_METHODS.add(HttpPatch.METHOD_NAME);
		ENTITY_ENCLOSING_REQUEST_METHODS.add(HttpPost.METHOD_NAME);
		ENTITY_ENCLOSING_REQUEST_METHODS.add(HttpPut.METHOD_NAME);
	}

	/**
	 * Supported HTTP request methods.
	 */
	private static final Set<String> PLAIN_REQUEST_METHODS = new HashSet<String>();
	static {
		PLAIN_REQUEST_METHODS.add(HttpDelete.METHOD_NAME);
		PLAIN_REQUEST_METHODS.add(HttpGet.METHOD_NAME);
		PLAIN_REQUEST_METHODS.add(HttpHead.METHOD_NAME);
		PLAIN_REQUEST_METHODS.add(HttpOptions.METHOD_NAME);
		PLAIN_REQUEST_METHODS.add(HttpTrace.METHOD_NAME);
	}

	/**
	 * The name of an XML attribute representing target URL.
	 */
	private final static String XML_URL_ATTRIBUTE = "url";

	/**
	 * The name of an XML attribute representing request method.
	 */
	private final static String XML_REQUEST_METHOD_ATTRIBUTE = "requestMethod";

	/**
	 * The name of an XML attribute representing request content.
	 */
	private final static String XML_REQUEST_CONTENT_ATTRIBUTE = "requestContent";

	/**
	 * The name of an XML attribute representing input file URL.
	 */
	private final static String XML_INPUT_FILEURL_ATTRIBUTE = "inFileUrl";

	/**
	 * The name of an XML attribute representing output file URL.
	 */
	private final static String XML_OUTPUT_FILEURL_ATTRIBUTE = "outFileUrl";

	/**
	 * The name of an XML attribute representing if the output should be appended.
	 */
	private final static String XML_APPEND_OUTPUT_ATTRIBUTE = "appendOutput";

	/**
	 * The name of an XML attribute representing additional header parameters.
	 */
	public final static String XML_ADDITIONAL_HTTP_HEADERS_ATTRIBUTE = "headerProperties";

	/**
	 * The name of an XML attribute representing additional header parameters.
	 */
	public final static String XML_REQUEST_PARAMETERS_ATTRIBUTE = "requestParameters";

	/**
	 * The name of an XML attribute representing input field name, holding request content.
	 */
	private final static String XML_INPUT_PORT_FIELD_NAME = "inputField";

	/**
	 * The name of an XML attribute representing output field name, where the response should be written.
	 */
	private final static String XML_OUTPUT_PORT_FIELD_NAME = "outputField";

	/**
	 * The name of an XML attribute representing character set to use.
	 */
	private final static String XML_CHARSET_ATTRIBUTE = "charset";

	/**
	 * The name of an XML attribute representing authentication method.
	 */
	private final static String XML_AUTHENTICATION_METHOD_ATTRIBUTE = "authenticationMethod";

	/**
	 * The name of an XML attribute representing user name.
	 */
	private final static String XML_USERNAME_ATTRIBUTE = "username";

	/**
	 * The name of an XML attribute representing password.
	 */
	private final static String XML_PASSWORD_ATTRIBUTE = "password";

	/**
	 * The name of an XML attribute representing flag indicating, whether fields of the input metadata should be added
	 * as parameters to the query.
	 */
	private final static String XML_ADD_INPUT_FIELDS_AS_PARAMETERS_ATTRIBUTE = "addInputFieldsAsParameters";

	/**
	 * The name of an XML attribute representing a way in which the fields of the input metadataq are added as
	 * parameters to the request.
	 */
	private final static String XML_ADD_INPUT_FIELDS_AS_PARAMETERS_TO_ATTRIBUTE = "addInputFieldsAsParametersTo";

	/**
	 * The name of an XML attribute representing fields, that shouldn't be added as parameters to the request.
	 */
	private final static String XML_IGNORED_FIELDS_ATTRIBUTE = "ignoredFields";

	/**
	 * The name of an XML attribute representing multipart entities.
	 */
	public final static String XML_MULTIPART_ENTITIES_FIELDS_LIST_ATTRIBUTE = "multipartEntities";

	/**
	 * The name of an XML attribute representing field holding a target URL.
	 */
	private final static String XML_URL_FROM_INPUT_FIELD_ATTRIBUTE = "urlInputField";

	/**
	 * The name of an XML attribute representing flag indicating, whether the response should be stored in temporary
	 * file.
	 */
	private final static String XML_STORE_RESPONSE_TO_TEMP_FILE = "responseAsFileName";

	/**
	 * The name of an XML attribute representing prefix of the temporary files, used to store responses
	 */
	private final static String XML_TEMPORARY_FILE_PREFIX = "responseFilePrefix";

	/**
	 * The name of an XML attribute used to define input mapping.
	 */
	public static final String XML_INPUT_MAPPING_ATTRIBUTE = "inputMapping";

	/**
	 * The name of an XML attribute used to define output mapping to the first standard output port
	 */
	public static final String XML_STANDARD_OUTPUT_MAPPING_ATTRIBUTE = "standardOutputMapping";

	/**
	 * The name of an XML attribute used to define output mapping to the second error output port
	 */
	public static final String XML_ERROR_OUTPUT_MAPPING_ATTRIBUTE = "errorOutputMapping";

	/** the name of an XML attribute for error output redirection */
	public static final String XML_REDIRECT_ERROR_OUTPUT = "redirectErrorOutput";

	/**
	 * The name of an XML attribute representing consumer key, used for OAuth authentication
	 */
	private final static String XML_CONSUMER_KEY_ATTRIBUTE = "consumerKey";
	/**
	 * The name of an XML attribute representing consumer secret, used for OAuth authentication
	 */
	private final static String XML_CONSUMER_SECRET_ATTRIBUTE = "consumerSecret";

	private final static String XML_OAUTH_ACCESS_TOKEN_ATTRIBUTE = "oAuthAccessToken";

	private final static String XML_OAUTH_ACCESS_TOKEN_SECRET_ATTRIBUTE = "oAuthAccessTokenSecret";

	private final static String XML_RAW_HTTP_HEADERS_ATTRIBUTE = "rawHeaders";

	public final static String XML_REQUEST_COOKIES_ATTRIBUTE = "requestCookies";

	public final static String XML_RESPONSE_COOKIES_ATTRIBUTE = "responseCookies";

	public final static String XML_STREAMING_ATTRIBUTE = "streaming";
	
	private static final String XML_DISABLE_SSL_CERT_VALIDATION = "disableSSLCertValidation";

	private static final String XML_TIMEOUT_ATTRIBUTE = "timeout";
	
	private static final String XML_RETRY_COUNT_ATTRIBUTE = "retryCount";

	/**
	 * Default value of the 'append output' flag
	 */
	private final static boolean DEFAULT_APPEND_OUTPUT = false;

	/**
	 * Input record name.
	 */
	private static final String INPUT_RECORD_NAME = "Input record";

	/**
	 * Output record name.
	 */
	private static final String STANDARD_OUTPUT_RECORD_NAME = "Standard output record";

	/**
	 * Output record name.
	 */
	private static final String ERROR_OUTPUT_RECORD_NAME = "Error output record";

	/* === Result metadata === */

	/**
	 * Name of the result record
	 */
	private static final String RESULT_RECORD_NAME = "Response";

	/**
	 * Result field representing the content of the response
	 */
	private static final int RP_CONTENT_INDEX = 0;
	private static final String RP_CONTENT_NAME = "content";

	/**
	 * Result field representing the BYTE content of the response
	 */
	private static final int RP_CONTENT_BYTE_INDEX = 1;
	private static final String RP_CONTENT_BYTE_NAME = "contentByte";

	/**
	 * Result field representing the URL of an output file
	 */
	private static final int RP_OUTPUTFILE_INDEX = 2;
	private static final String RP_OUTPUTFILE_NAME = "outputFilePath";

	/**
	 * Result field representing the status code of the response
	 */
	private static final int RP_STATUS_CODE_INDEX = 3;
	private static final String RP_STATUS_CODE_NAME = "statusCode";

	/**
	 * Result field representing the header of the response
	 */
	private static final int RP_HEADER_INDEX = 4;
	private static final String RP_HEADER_NAME = "header";

	/**
	 * Result field representing HTTP headers sent by server in a raw format
	 */
	private static final int RP_RAW_HTTP_HAEDERS_INDEX = 5;
	private static final String RP_RAW_HTTP_HAEDERS_NAME = "rawHeaders";

	/**
	 * Result field representing the error message (can have a value only if error port is redirected to std port)
	 */
	private static final int RP_MESSAGE_INDEX = 6;
	private static final String RP_MESSAGE_NAME = "errorMessage";

	/* === Error metadata === */

	/**
	 * Name of the error record
	 */
	private static final String ERROR_RECORD_NAME = "Error";

	/**
	 * Error field representing the error message
	 */
	private static final int EP_MESSAGE_INDEX = 0;
	private static final String EP_MESSAGE_NAME = RP_MESSAGE_NAME;
	
	private static final String INPUT_FILE_ATTRIBUTE_WARN = "'Input file URL' will be ignored because 'Input field' attribute is set.";
	private static final String INPUT_FILE_FIELD_WARN = "'Input file URL' will be ignored because field '%s' is mapped.";

	private static String MULTIPART_CONTENT = "EntityContent";
	private static String MULTIPART_CONTENT_BYTE = "EntityContentByte";
	private static String MULTIPART_SOURCE_FILE = "EntitySourceFile";
	private static String MULTIPART_FILENAME = "EntityFileNameAttribute";
	private static String MULTIPART_CHARSET = "EntityCharsetAttribute";
	private static String MULTIPART_CONTENTTYPE = "EntityMimeTypeAttribute";

	private interface ResponseWriter {
		public void writeResponse(HttpResponse response) throws IOException;
	}

	/**
	 * Writer that sends response directly stored in String field
	 */
	private class ResponseByValueWriter implements ResponseWriter {

		private final DataField outputField;
		private final DataField outputFieldByte;

		public ResponseByValueWriter(DataField outputField, DataField outputFieldByte) {
			this.outputField = outputField;
			this.outputFieldByte = outputFieldByte;
		}

		@Override
		public void writeResponse(HttpResponse response) throws IOException {
			if (outputField == null && outputFieldByte == null) {
				return;
			}

			InputStream responseInputStream = null;
			HttpEntity entity = response.getEntity();
			if (entity != null) {
				responseInputStream = entity.getContent();
			}

			if (outputFieldByte != null) {
				if (responseInputStream != null) {
					byte[] responseBytes = getResponseContentAsByteArray(responseInputStream);
					outputFieldByte.setValue(responseBytes);
					responseInputStream = new ByteArrayInputStream(responseBytes); // original responseInputStream
																					// cannot be read for 2nd time
				} else {
					outputFieldByte.setNull(true);
				}
			}

			if (outputField != null) {
				if (responseInputStream != null) {
					outputField.setValue(getResponseContentAsString(responseInputStream));
				} else {
					outputField.setNull(true);
				}
			}
		}
	}

	private abstract class AbstractResponseFileWriter implements ResponseWriter {

		private final DataField fileURLField;

		public AbstractResponseFileWriter(DataField fileURLField) {
			this.fileURLField = fileURLField;
		}

		/**
		 * Returns a file output channel. The method is guaranteed to be called before the {@link #getFileOutputPath()},
		 * so that the implementation may cache the data.
		 * 
		 * @return a file output channel
		 */
		abstract protected WritableByteChannel getFileOutputChannel() throws IOException;

		/**
		 * Returns an output file path. The method is guaranteed to be called after the {@link #getFileOutputChannel()},
		 * so that the implementation may cache the data.
		 * 
		 * @return an output file path
		 */
		abstract protected String getFileOutputPath() throws IOException;

		@Override
		public void writeResponse(HttpResponse response) throws IOException {
			WritableByteChannel outputChannel = getFileOutputChannel();

			if (outputChannel == null) {
				outputChannel = new SystemOutByteChannel();
			}

			HttpEntity entity = response.getEntity();
			if (entity != null) {
				InputStream inputStream = entity.getContent();

				if (inputStream != null) {
					try {
						StreamUtils.copy(Channels.newChannel(inputStream), outputChannel);
					} finally {
						try {
							FileUtils.close(inputStream);
						} catch (IOException e) {
							logger.warn("Failed to close HTTP response input channel");
						}
						try {
							FileUtils.close(outputChannel);
						} catch (IOException e) {
							logger.warn("Failed to close HTTP response output channel");
						}
					}
				}
			}

			// populate output field with the temporary file name
			if (fileURLField != null) {
				fileURLField.setValue(getFileOutputPath());
			}
		}
	}

	private class ResponseTempFileWriter extends AbstractResponseFileWriter {

		private final String prefix;
		private File tempFile = null;

		public ResponseTempFileWriter(DataField fileURLField, String prefix) {
			super(fileURLField);
			this.prefix = prefix;
		}

		@Override
		protected WritableByteChannel getFileOutputChannel() throws IOException {
			tempFile = null;
			try {
				tempFile = getGraph().getAuthorityProxy().newTempFile(prefix, ".tmp", -1);

				return Channels.newChannel(new FileOutputStream(tempFile, appendOutputToUse));
			} catch (TempFileCreationException e) {
				throw new IOException(e);
			}
		}

		@Override
		protected String getFileOutputPath() throws IOException {
			if (tempFile != null) {
				return tempFile.getAbsolutePath();
			}

			return null;
		}
	}

	private class ResponseFileWriter extends AbstractResponseFileWriter {

		private final String fileName;

		public ResponseFileWriter(DataField fileURLField, String fileName) {
			super(fileURLField);

			this.fileName = fileName;
		}

		@Override
		protected WritableByteChannel getFileOutputChannel() throws IOException {
			return FileUtils.getWritableChannel(getContextURL(), fileName, appendOutputToUse);
		}

		@Override
		protected String getFileOutputPath() throws IOException {
			File file = FileUtils.getJavaFile(getContextURL(), fileName);
			if (file != null) {
				return file.getAbsolutePath();
			}

			return null;
		}
	}

	public static class HTTPConnectorException extends Exception {
		private static final long serialVersionUID = 1L;

		/**
		 * 
		 */
		public HTTPConnectorException() {
			super();
		}

		/**
		 * @param message
		 * @param cause
		 */
		public HTTPConnectorException(String message, Throwable cause) {
			super(message, cause);
		}

		/**
		 * @param message
		 */
		public HTTPConnectorException(String message) {
			super(message);
		}

		/**
		 * @param cause
		 */
		public HTTPConnectorException(Throwable cause) {
			super(cause);
		}
	}

	/**
	 * Class representing a result of the request.
	 * 
	 * @author Tomas Laurincik ([email protected]) (c) Javlin, a.s. (www.cloveretl.com)
	 * 
	 * @created 8.6.2012
	 */
	protected static class RequestResult {
		private HttpResponse response;
		private Exception exception;

		public HttpResponse getResponse() {
			return response;
		}

		public void setResponse(HttpResponse response) {
			this.response = response;
		}

		public Exception getException() {
			return exception;
		}

		public void setException(Exception exception) {
			this.exception = exception;
		}
	}

	/* === Component context === */

	/**
	 * only optional input port or <code>null</code> if no input edge is assigned
	 */
	private InputPort inputPort;

	/**
	 * standard output port or <code>null</code> if no output edge is assigned
	 */
	private OutputPort standardOutputPort;

	/**
	 * optional error output port or <code>null</code> if no error output edge is assigned
	 */
	private OutputPort errorOutputPort;

	/**
	 * Is the optional input port attached.
	 */
	private boolean hasInputPort;

	/**
	 * Is the optional standard output port attached.
	 */
	private boolean hasStandardOutputPort;

	/**
	 * Is the optional error output port attached.
	 */
	protected boolean hasErrorOutputPort;

	/**
	 * Record which is used for input mapping as other configuration attributes.
	 */
	protected DataRecord inputParamsRecord;

	/**
	 * Record with run status information.
	 */
	protected DataRecord resultRecord;

	/**
	 * Record with run status information.
	 */
	protected DataRecord errorRecord;

	/**
	 * Input port record or null if input port is not attached.
	 */
	protected DataRecord inputRecord;

	protected DataRecord additionalHeadersRecord;

	protected DataRecord requestParametersRecord;

	protected DataRecord requestCookiesRecord;

	protected DataRecord responseCookiesRecord;

	protected DataRecord multipartRequestPropertiesRecord;

	/**
	 * Input records for input mapping transformation.
	 */
	protected DataRecord[] inputMappingInRecords;

	/**
	 * Output records for input mapping transformation.
	 */
	protected DataRecord[] inputMappingOutRecords;

	/**
	 * Input records for output mapping transformation - error output mapping.
	 */
	protected DataRecord[] errorOutputMappingInRecords;

	/**
	 * Output records for standard output mapping transformation.
	 */
	protected DataRecord[] standardOutputMappingOutRecords;

	/**
	 * Output records for error output mapping transformation.
	 */
	protected DataRecord[] errorOutputMappingOutRecords;

	/**
	 * Use Transfer-Encoding: chunked for Input file.
	 */
	private boolean streaming;

	/* === Component base properties === */

	/**
	 * Name of the run configuration record
	 */
	private static final String ATTRIBUTES_RECORD_NAME = "Attributes";

	public static final String ADDITIONAL_HTTP_HEADERS_RECORD_NAME = "AdditionalHTTPHeaders";

	public static final String REQUEST_PARAMETERS_RECORD_NAME = "RequestParameters";

	public static final String REQUEST_COOKIES_RECORD_NAME = "RequestCookies";

	private static final String RESPONSE_COOKIES_RECORD_NAME = "ResponseCookies";

	private static final String RESPONSE_COOKIES_SEPARATOR = ";";

	public static final String MULTIPART_ENTITIES_RECORD_NAME = "MultipartEntities";

	public static final String MULTIPART_ENTITIES_SEPARATOR = ";";
	/**
	 * URL to which the HTTPConnector should connect.
	 */
	private String rawUrl;
	private String rawUrlToUse;
	private static final int IP_URL_INDEX = 0;
	private static final String IP_URL_NAME = "URL";

	/**
	 * Field of the input metadata containing the target URL.
	 */
	private String urlInputField;
	// private String urlInputFieldToUse;
	// private static final int IP_URL_FIELD_INDEX = 1;
	// private static final String IP_URL_FIELD_NAME = "urlInputField";

	/**
	 * Request method to be used.
	 */
	private String requestMethod;
	private String requestMethodToUse;
	private static final int IP_REQUEST_METHOD_INDEX = 1;
	private static final String IP_REQUEST_METHOD_NAME = "requestMethod";

	/**
	 * <code>true</code> if the fields of the input metadata should be added as parameters to the request.
	 */
	private boolean addInputFieldsAsParameters;
	private Boolean addInputFieldsAsParametersToUse;
	private static final int IP_ADD_INPUT_FIELDS_AS_PARAMETERS_INDEX = 2;
	private static final String IP_ADD_INPUT_FIELDS_AS_PARAMETERS_NAME = "addInputFieldsAsParameters";

	/**
	 * String representing a way, the parameters are passed (QUERY, BODY, ...).
	 */
	private String addInputFieldsAsParametersTo;
	private String addInputFieldsAsParametersToToUse;
	private static final int IP_ADD_INPUT_FIELDS_AS_PARAMETERS_TO_INDEX = 3;
	private static final String IP_ADD_INPUT_FIELDS_AS_PARAMETERS_TO_NAME = "addInputFieldsAsParametersTo";

	/**
	 * Fields that should not be added as parameters to the request.
	 */
	private String ignoredFields;
	private String ignoredFieldsToUse;
	private static final int IP_IGNORED_FIELDS_INDEX = 4;
	private static final String IP_IGNORED_FIELDS_NAME = "ignoredFields";

	/**
	 * String representing properties, that should be used as additional HTTP headers.
	 */
	private String additionalRequestHeadersStr;
	private static final int IP_ADDITIONAL_REQUEST_HEADERS_INDEX = 5;
	private static final String IP_ADDITIONAL_REQUEST_HEADERS_NAME = "additionalHTTPHeaders";

	/**
	 * Character set that should be used.
	 */
	private String charset;
	private String charsetToUse;
	private static final int IP_CHARSET_INDEX = 6;
	private static final String IP_CHARSET_NAME = "charset";

	/**
	 * Name of the field of the input metadata, that contains the request content.
	 */
	private String inputFieldName;
	// private String inputFieldNameToUse;
	// private static final int IP_INPUT_FIELD_NAME_INDEX = 5;
	// private static final String IP_INPUT_FIELD_NAME_NAME = "inputFieldName";

	/**
	 * Name of field of the output metadata, that should be populated with the response.
	 */
	private String outputFieldName;
	// private String outputFieldNameToUse;
	// private static final int IP_OUTPUT_FIELD_NAME_INDEX = 6;
	// private static final String IP_OUTPUT_FIELD_NAME_NAME = "outputFieldName";

	/**
	 * String representing request content.
	 */
	private String requestContent;
	private String requestContentToUse;
	private static final int IP_REQUEST_CONTENT_INDEX = 7;
	private static final String IP_REQUEST_CONTENT_NAME = "requestContent";

	/**
	 * Raw request content bytes.
	 */
	private byte[] requestContentByteToUse;
	private static final int IP_REQUEST_CONTENT_BYTE_INDEX = 8;
	private static final String IP_REQUEST_CONTENT_BYTE_NAME = "requestContentByte";

	/**
	 * URL of the file containing request content.
	 */
	private String inputFileUrl;
	private String inputFileUrlToUse;
	private static final int IP_INPUT_FILE_URL_INDEX = 9;
	private static final String IP_INPUT_FILE_URL_NAME = "inputFileUrl";

	/**
	 * URL of the file where the response should be stored.
	 */
	private String outputFileUrl;
	private String outputFileUrlToUse;
	private static final int IP_OUTPUT_FILE_URL_INDEX = 10;
	private static final String IP_OUTPUT_FILE_URL_NAME = "outputFileUrl";

	/**
	 * <code>True</code>, if the response should be appended to the output file, <code>false</code> if the file should
	 * be overwritten.
	 */
	private boolean appendOutput;
	private Boolean appendOutputToUse;
	private static final int IP_APPEND_OUTPUT_INDEX = 11;
	private static final String IP_APPEND_OUTPUT_NAME = "appendOutput";

	/**
	 * Authentication method to be used (BASIC, DIGEST, ...).
	 */
	private String authenticationMethod;
	private String authenticationMethodToUse;
	private static final int IP_AUTHENTICATION_METHOD_INDEX = 12;
	private static final String IP_AUTHENTICATION_METHOD_NAME = "authenticationMethod";

	/**
	 * User name to be used for authentication.
	 */
	private String username;
	private String usernameToUse;
	private static final int IP_USERNAME_INDEX = 13;
	private static final String IP_USERNAME_NAME = "username";

	/**
	 * Password to be used for authentication.
	 */
	private String password;
	private String passwordToUse;
	private static final int IP_PASSWORD_INDEX = 14;
	private static final String IP_PASSWORD_NAME = "password";

	/**
	 * Consumer key used for OAuth authentication
	 */
	private String consumerKey;
	private String consumerKeyToUse;
	private static final int IP_CONSUMER_KEY_INDEX = 15;
	private static final String IP_CONSUMER_KEY_NAME = "consumerKey";

	/**
	 * Consumer secret used for OAuth authentication
	 */
	private String consumerSecret;
	private String consumerSecretToUse;
	private static final int IP_CONSUMER_SECRET_INDEX = 16;
	private static final String IP_CONSUMER_SECRET_NAME = "consumerSecret";

	private String oAuthAccessToken;
	private String oAuthAccessTokenToUse;
	private static final int IP_OATUH_TOKEN_INDEX = 17;
	private static final String IP_OATUH_TOKEN_NAME = "oAuthAccessToken";

	private String oAuthAccessTokenSecret;
	private String oAuthAccessTokenSecretToUse;
	private static final int IP_OATUH_TOKEN_SECRET_INDEX = 18;
	private static final String IP_OATUH_TOKEN_SECRET_NAME = "oAuthAccessTokenSecret";
	/**
	 * <code>true</code> if the responses should be stored to temporary files, <code>false</code> otherwise.
	 */
	private boolean storeResponseToTempFile;
	private Boolean storeResponseToTempFileToUse;
	private static final int IP_STORE_RESPONSE_TO_TEMP_INDEX = 19;
	private static final String IP_STORE_RESPONSE_TO_TEMP_NAME = "storeResponseToTempFile";

	/**
	 * Prefix used for the temporary files with responses.
	 */
	private String temporaryFilePrefix;
	private String temporaryFilePrefixToUse;
	private static final int IP_TEMP_FILE_PREFIX_INDEX = 20;
	private static final String IP_TEMP_FILE_PREFIX_NAME = "temporaryFilePrefix";

	/**
	 * String representing fields used as multipart entities.
	 */
	private String multipartEntities;
	private String multipartEntitiesToUse;
	private static final int IP_MULTIPART_ENTITIES_INDEX = 21;
	private static final String IP_MULTIPART_ENTITIES_NAME = "multipartEntities";

	/**
	 * String representing raw HTTP headers to be directly injected into HTTP request.
	 */
	private String rawHttpHeaders;
	private List<CharSequence> rawHttpHeadersToUse;
	private static final int IP_RAW_HTTP_HEADERS_INDEX = 22;
	private static final String IP_RAW_HTTP_HEADERS_NAME = "rawHTTPHeaders";

	/**
	 * String representing properties, that should be used as explicitly mapped request parameters.
	 */
	private String requestParametersStr;
	private static final int IP_REQUEST_PARAMETERS_INDEX = 23;
	private static final String IP_REQUEST_PARAMETERS_NAME = "requestParameters";
	
	
	/**
	 * String representing fields used as multipart entities.
	 */
	private String requestCookiesStr;
	private Properties requestCookies;
	private String responseCookies;

	/* === Job flow related properties === */

	/**
	 * Source code of the input mapping.
	 */
	private String inputMapping;

	/**
	 * Source code of the error output mapping.
	 */
	private String errorOutputMapping;

	/**
	 * Source code of the output mapping.
	 */
	private String standardOutputMapping;

	/**
	 * Enables redirection of error output to standard output. Disabled by default.
	 */
	private boolean redirectErrorOutput;

	/* === Processed properties === */

	/**
	 * Structure holding the user authentication details.
	 */
	private UsernamePasswordCredentials creds;

	/**
	 * Standard output field.
	 */
	private StringDataField outField;

	/**
	 * Parsed request properties - used as the additional HTTP headers.
	 */
	private Properties additionalRequestHeaders;
	private Map<String, CharSequence> additionalRequestHeadersToUse; // additionalRequestHeaders modified by input
																		// mapping

	private Properties requestParameters;
	private Map<String, CharSequence> requestParametersToUse; // additionalRequestHeaders modified by input

	/**
	 * Fields that should be ignored
	 */
	private Set<String> ignoredFieldsSet = new HashSet<String>();

	/**
	 * Runtime for input mapping.
	 */
	protected CTLMapping inputMappingTransformation;

	/**
	 * Runtime for input mapping.
	 */
	protected CTLMapping standardOutputMappingTransformation;

	/**
	 * Runtime for input mapping.
	 */
	protected CTLMapping errorOutputMappingTransformation;

	/* === Component state properties per record === */

	/**
	 * Standard output port record or null if standard output port is not attached.
	 */
	private DataRecord standardOutputRecord;

	/**
	 * Error output port record or null if error output port is not attached.
	 */
	private DataRecord errorOutputRecord;

	/**
	 * Result of the processing of one record.
	 */
	private RequestResult result;
	
	/**
	 * Toggle for disabling verification of certificates.
	 */
	private boolean disableSSLCertValidation;
	
	private long timeout = -1;
	private long timeoutToUse = -1;
	private static final int IP_TIMEOUT_INDEX = 24;
	private static final String IP_TIMEOUT_NAME = "timeout";
	
	private int retryCount = 0;
	private int retryCountToUse = 0;
	private static final int IP_RETRY_COUNT_INDEX = 25;
	private static final String IP_RETRY_COUNT_NAME = "retryCount";

	/* === Tools used === */

	/**
	 * Resolver for substituting field references.
	 */
	private PropertyRefResolver refResolver = new PropertyRefResolver();;

	/**
	 * Utility class for writing the response content to file or output port.
	 */
	private ResponseWriter responseWriter;

	/**
	 * HttpClient used for HTTP communication.
	 */
	private CloseableHttpClient httpClient;

	private RequestResponseCookieStore cookieStore;

	private HttpContext httpContext;
	
//	private DefaultHttpRequestRetryHandler retryHandler;
	

	/**
	 * OAuth consumer used to sign requests when OAuth is used.
	 */
	private OAuthConsumer oauthConsumer;

	/**
	 * Request interceptor used for logging.
	 */
	private HttpRequestInterceptor requestLoggingInterceptor = new HttpRequestInterceptor() {

		@Override
		public void process(HttpRequest paramHttpRequest, HttpContext paramHttpContext) throws HttpException,
				IOException {
			logger.debug("Sending HTTP request:\n\n" + buildRequestLogString(paramHttpRequest));
		}
	};

	/**
	 * Response interceptor used for logging.
	 */
	private HttpResponseInterceptor responseLoggingInterceptor = new HttpResponseInterceptor() {

		@Override
		public void process(HttpResponse paramHttpResponse, HttpContext paramHttpContext) throws HttpException,
				IOException {
			logger.debug("Received HTTP response:\n\n" + buildResponseLogString(paramHttpResponse));
		}
	};

	/**
	 * Helper class holding a name and body part for multipart message.
	 * 
	 * @author Tomas Laurincik ([email protected]) (c) Javlin, a.s. (www.cloveretl.com)
	 * 
	 * @created 5.6.2012
	 */
	protected static class PartWithName {
		public String name;
		public ContentBody value;

		public PartWithName(String name, ContentBody value) {
			super();
			this.name = name;
			this.value = value;
		}
	}

	/**
	 * Bean representing configuration of one HTTP request
	 * 
	 * @author Tomas Laurincik ([email protected]) (c) Javlin, a.s. (www.cloveretl.com)
	 * 
	 * @created 4.6.2012
	 */
	protected static class HTTPRequestConfiguration {
		private String target;
		private String proxy;
		private Map<String, String> parameters = new LinkedHashMap<String, String>();
		private Map<String, HttpConnectorMutlipartEntity> multipartEntities = new LinkedHashMap<String, HttpConnectorMutlipartEntity>();
		private Object content;

		private URL targetURL;
		private URL proxyURL;

		public HTTPRequestConfiguration() {
			super();
		}

		public void prepare() throws ComponentNotReadyException {
			if (getTarget() != null) {
				try {
					targetURL = new URL(getTarget());
				} catch (MalformedURLException e) {
					throw new ComponentNotReadyException("Given target URL '" + getTarget() + "' is invalid.");
				}
			}

			if (getProxy() != null) {
				try {
					proxyURL = FileUtils.getFileURL(getProxy());
				} catch (MalformedURLException e) {
					throw new ComponentNotReadyException("Given proxy URL '" + getProxy() + "' is invalid.");
				}
			}
		}

		public String getTarget() {
			return target;
		}

		public void setTarget(String target) {
			this.target = target;
		}

		public String getProxy() {
			return proxy;
		}

		public void setProxy(String proxy) {
			this.proxy = proxy;
		}

		public Map<String, String> getParameters() {
			return parameters;
		}

		public void setParameters(Map<String, String> parameters) {
			this.parameters.clear();
			this.parameters.putAll(parameters);
		}

		public Map<String, HttpConnectorMutlipartEntity> getMultipartEntities() {
			return multipartEntities;
		}

		public void setMultipartEntities(Map<String, HttpConnectorMutlipartEntity> multipartEntities) {
			this.multipartEntities.clear();
			this.multipartEntities.putAll(multipartEntities);
		}

		public Object getContent() {
			return content;
		}

		private void setContent(Object content) {
			this.content = content;
		}

		public void setContent(String content) {
			this.content = content;
		}

		public void setContent(byte[] content) {
			this.content = content;
		}

		public void setContent(InputStream content) {
			this.content = content;
		}

		public URL getTargetURL() {
			return targetURL;
		}

		public void setTargetURL(URL targetURL) {
			this.targetURL = targetURL;
		}

		public URL getProxyURL() {
			return proxyURL;
		}

		public void setProxyURL(URL proxyURL) {
			this.proxyURL = proxyURL;
		}

		public void copyTo(HTTPRequestConfiguration configuration) {
			configuration.setTarget(target);
			configuration.setProxy(proxy);
			configuration.setContent(content);
			configuration.setParameters(new LinkedHashMap<String, String>(parameters));
			configuration.setMultipartEntities(new LinkedHashMap<String, HttpConnectorMutlipartEntity>(multipartEntities));

			configuration.setProxyURL(proxyURL);
			configuration.setTargetURL(targetURL);
		}
	}

	/**
	 * Creates new instance of the HTTPConnector with given id
	 * 
	 * @param id
	 */
	public HttpConnector(String id) {
		super(id);
	}

	@Override
	public void init() throws ComponentNotReadyException {
		if (isInitialized())
			return;

		super.init();

		// if charset isn't filled, we use the engine default, see CLO-10204
		if (StringUtils.isEmpty(this.charset)) {
			this.charset = Defaults.DataParser.DEFAULT_CHARSET_DECODER;
		}

		tryToInit(null);

		// create response writer based on the configuration XXX is this really needed? Happens in preProcessForRecord()
		// too
		if (getOutputFileUrl() != null) {
			responseWriter = new ResponseFileWriter(outField, outputFileUrl);

		} else if (getStoreResponseToTempFile()) {
			responseWriter = new ResponseTempFileWriter(outField, temporaryFilePrefix);

		} else {
			responseWriter = new ResponseByValueWriter(outField, null);
		}
	}

	/**
	 * Extracts a set of ignored field names from given string representation.
	 * 
	 * @param rawIgnoredFields
	 * @return a set of ignored field names from given string representation.
	 */
	private Set<String> extractIgnoredFields(String rawIgnoredFields) {
		Set<String> result = new HashSet<String>();

		// find out which metadata fields shoud be ignored
		if (!StringUtils.isEmpty(ignoredFieldsToUse)) {
			StringTokenizer tokenizer = new StringTokenizer(ignoredFieldsToUse, ";");
			while (tokenizer.hasMoreTokens()) {
				result.add(tokenizer.nextToken());
			}
		}

		return result;
	}

	/**
	 * Returns a Boolean value of given field in input record.
	 * 
	 * @param index
	 * @return a Boolean value of given field in input record.
	 */
	private Boolean getBooleanInputParameterValue(int index, Boolean defaultValue) {
		Object value = inputParamsRecord.getField(index).getValue();

		if (Boolean.FALSE.equals(value)) {
			return false;
		} else if (Boolean.TRUE.equals(value)) {
			return true;
		}

		return defaultValue;
	}

	/**
	 * Returns a string value of the given field in input record.
	 * 
	 * @param index
	 * @return a string value of the given field.
	 */
	private String getStringInputParameterValue(int index) {
		return getStringInputParameterValue(index, null);
	}
	
	private String getStringInputParameterValue(int index, String defaultValue) {
		Object value = inputParamsRecord.getField(index).getValue();
		if (value != null) {
			return value.toString();
		}

		return defaultValue;
	}
	
	private int getIntInputParameterValue(int index, int defaultValue) {
		Object value = inputParamsRecord.getField(index).getValue();
		if (value != null) {
			return Integer.parseInt(value.toString());
		}

		return defaultValue;
	}

	private long getLongInputParameterValue(int index, long defaultValue) {
		Object value = inputParamsRecord.getField(index).getValue();
		if (value != null) {
			return Long.parseLong(value.toString());
		}

		return defaultValue;
	}

	/**
	 * Returns a byte[] value of the given field in input record.
	 * 
	 * @param index
	 * @return a byte[] value of the given field.
	 */
	private byte[] getByteInputParameterValue(int index) {
		Object value = inputParamsRecord.getField(index).getValue();
		if (value != null) {
			return (byte[]) value;
		}

		return null;
	}

	/**
	 * Processes the input records. The values contained in the fields of the record are set as a properties of this
	 * components.
	 * 
	 */
	@SuppressWarnings("unchecked")
	protected void processInputParamsRecord() {
		rawUrlToUse = getStringInputParameterValue(IP_URL_INDEX);
		// urlInputFieldToUse = getStringInputParameterValue(IP_URL_FIELD_INDEX);
		requestMethodToUse = getStringInputParameterValue(IP_REQUEST_METHOD_INDEX);
		requestContentToUse = getStringInputParameterValue(IP_REQUEST_CONTENT_INDEX);
		requestContentByteToUse = getByteInputParameterValue(IP_REQUEST_CONTENT_BYTE_INDEX);
		inputFileUrlToUse = getStringInputParameterValue(IP_INPUT_FILE_URL_INDEX);
		// inputFieldNameToUse = getStringInputParameterValue(IP_INPUT_FIELD_NAME_INDEX);
		// outputFieldNameToUse = getStringInputParameterValue(IP_OUTPUT_FIELD_NAME_INDEX);
		outputFileUrlToUse = getStringInputParameterValue(IP_OUTPUT_FILE_URL_INDEX);
		appendOutputToUse = getBooleanInputParameterValue(IP_APPEND_OUTPUT_INDEX, Boolean.FALSE);
		charsetToUse = getStringInputParameterValue(IP_CHARSET_INDEX, Defaults.DataParser.DEFAULT_CHARSET_DECODER);
		addInputFieldsAsParametersToUse = getBooleanInputParameterValue(IP_ADD_INPUT_FIELDS_AS_PARAMETERS_INDEX, Boolean.FALSE);
		addInputFieldsAsParametersToToUse = getStringInputParameterValue(IP_ADD_INPUT_FIELDS_AS_PARAMETERS_TO_INDEX);
		ignoredFieldsToUse = getStringInputParameterValue(IP_IGNORED_FIELDS_INDEX);
		authenticationMethodToUse = getStringInputParameterValue(IP_AUTHENTICATION_METHOD_INDEX);
		usernameToUse = getStringInputParameterValue(IP_USERNAME_INDEX);
		passwordToUse = getStringInputParameterValue(IP_PASSWORD_INDEX);
		storeResponseToTempFileToUse = getBooleanInputParameterValue(IP_STORE_RESPONSE_TO_TEMP_INDEX, Boolean.FALSE);
		temporaryFilePrefixToUse = getStringInputParameterValue(IP_TEMP_FILE_PREFIX_INDEX);
		multipartEntitiesToUse = getStringInputParameterValue(IP_MULTIPART_ENTITIES_INDEX);
		consumerKeyToUse = getStringInputParameterValue(IP_CONSUMER_KEY_INDEX);
		consumerSecretToUse = getStringInputParameterValue(IP_CONSUMER_SECRET_INDEX);
		oAuthAccessTokenToUse = getStringInputParameterValue(IP_OATUH_TOKEN_INDEX);
		oAuthAccessTokenSecretToUse = getStringInputParameterValue(IP_OATUH_TOKEN_SECRET_INDEX);
		rawHttpHeadersToUse = (List<CharSequence>) inputParamsRecord.getField(IP_RAW_HTTP_HEADERS_INDEX).getValue();

		additionalRequestHeadersToUse = (Map<String, CharSequence>) inputParamsRecord.getField(IP_ADDITIONAL_REQUEST_HEADERS_INDEX).getValue();

		if (additionalHeadersRecord != null) {
			for (DataField field : additionalHeadersRecord) {
				if (inputMappingTransformation.isOutputOverridden(additionalHeadersRecord, field)) {
					String labelOrName = field.getMetadata().getLabelOrName();
					if (!field.isNull()) {
						additionalRequestHeadersToUse.put(labelOrName, field.getValue().toString());
					} else {
						additionalRequestHeadersToUse.remove(labelOrName);
					}
				}
			}
		}
		
		requestParametersToUse = (Map<String, CharSequence>) inputParamsRecord.getField(IP_REQUEST_PARAMETERS_INDEX).getValue();

		if (requestParametersRecord != null) {
			for (DataField field : requestParametersRecord) {
				if (inputMappingTransformation.isOutputOverridden(requestParametersRecord, field)) {
					String labelOrName = field.getMetadata().getLabelOrName();
					if (!field.isNull()) {
						requestParametersToUse.put(labelOrName, field.getValue().toString());
					} else {
						requestParametersToUse.remove(labelOrName);
					}
				}
			}
		}
		
		timeoutToUse = getLongInputParameterValue(IP_TIMEOUT_INDEX, timeout);
		retryCountToUse = getIntInputParameterValue(IP_RETRY_COUNT_INDEX, retryCount);
		//
		// if(this.multipartRequestPropertiesRecord != null) {
		// if(multipartRequestMappingToUse==null) {
		// multipartRequestMappingToUse = new HashMap<String, String>();
		// }else{
		// this.multipartRequestMappingToUse.clear();
		// }
		// for (DataField field : multipartRequestPropertiesRecord) {
		// if (inputMappingTransformation.isOutputOverridden(multipartRequestPropertiesRecord, field)) {
		// String labelOrName = field.getMetadata().getLabelOrName();
		// if (!field.isNull()) {
		// multipartRequestMappingToUse.put(labelOrName, field.getValue().toString());
		// }
		// }
		// }
		// }

	}

	/**
	 * Resets the records in a given array.
	 * 
	 * @param records
	 */
	private static void resetRecords(DataRecord... records) {
		for (DataRecord record : records) {
			if (record != null) {
				record.reset();
			}
		}
	}

	/**
	 * Returns <code>true</code> if the record does not have a field with the given name, <code>false</code> otherwise.
	 * 
	 * @param record
	 * @return <code>true</code> if the record does not have a field with the given name, <code>false</code> otherwise.
	 */
	private DataField getFieldSafe(DataRecord record, String name) {
		if (name == null) {
			return null;
		}

		for (String fieldName : record.getMetadata().getFieldNamesArray()) {
			if (name.equals(fieldName)) {
				return record.getField(name);
			}
		}

		return null;
	}

	/**
	 * Pre-process the record.
	 * 
	 * @throws ComponentNotReadyException
	 */
	private void preProcessForRecord() throws ComponentNotReadyException {
		ignoredFieldsSet = extractIgnoredFields(ignoredFieldsToUse);

		// if (inputFieldNameToUse != null) {
		// inField = (StringDataField) getFieldSafe(inputRecord, inputFieldNameToUse);
		// if (inField == null) {
		// throw new ComponentNotReadyException("Unknown input field name '" + inputFieldNameToUse + "'.");
		// }
		// }
		//
		// // BACKWARD COMPATIBILITY: get the output field, where the content should be sent.
		// if (standardOutputRecord != null) {
		// if (outputFieldNameToUse == null) {
		// outField = (StringDataField) standardOutputRecord.getField(0);
		// } else {
		// outField = (StringDataField) getFieldSafe(standardOutputRecord, outputFieldNameToUse);
		// if (outField == null) {
		// throw new ComponentNotReadyException("Unknown output field name '" + outputFieldNameToUse + "'.");
		// }
		// }
		// }

		// create response writer based on the configuration
		if (outputFileUrlToUse != null) {
			responseWriter = new ResponseFileWriter(resultRecord != null ? resultRecord.getField(RP_OUTPUTFILE_INDEX) : null, outputFileUrlToUse);

		} else if (storeResponseToTempFileToUse) {
			responseWriter = new ResponseTempFileWriter(resultRecord != null ? resultRecord.getField(RP_OUTPUTFILE_INDEX) : null, temporaryFilePrefixToUse);

		} else {
			responseWriter = new ResponseByValueWriter(resultRecord != null ? resultRecord.getField(RP_CONTENT_INDEX) : null, resultRecord != null ? resultRecord.getField(RP_CONTENT_BYTE_INDEX) : null);
		}

		// filter multipart entities (ignored fields are removed from multipart entities record)
		if (!ignoredFieldsSet.isEmpty() && !StringUtils.isEmpty(multipartEntities)) {
			List<String> multipartEntitiesList = new ArrayList<String>();
			StringTokenizer tokenizer = new StringTokenizer(multipartEntities, ";");
			while (tokenizer.hasMoreElements()) {
				multipartEntitiesList.add(tokenizer.nextToken());
			}

			// remove ignored fields
			multipartEntitiesList.removeAll(ignoredFieldsSet);

			if (!multipartEntitiesList.isEmpty()) {
				multipartEntities = "";
				for (String entity : multipartEntitiesList) {
					multipartEntities += entity + ";";
				}
				multipartEntities = multipartEntities.substring(0, multipartEntities.length() - 1);
			} else {
				multipartEntities = null;
			}
		}
	}

	/**
	 * Builds a string from given request.
	 * 
	 * @param message
	 * @return a string from given request.
	 */
	private String buildRequestLogString(HttpRequest message) {
		if (message == null) {
			return "<null>";
		}

		StringBuilder result = new StringBuilder();
		if (message.getRequestLine() != null) {
			result.append(message.getRequestLine().toString()).append("\n");
		}

		if (message.getAllHeaders() != null) {
			for (Header header : message.getAllHeaders()) {
				result.append(header.toString()).append("\n");
			}
		}

		if (message instanceof HttpEntityEnclosingRequest) {
			HttpEntity entity = ((HttpEntityEnclosingRequest) message).getEntity();
			if (entity != null) {
				if (entity.isRepeatable()) {
					try {
						ByteArrayOutputStream entityBytes = new ByteArrayOutputStream();
						entity.writeTo(entityBytes);
						result.append(new String(entityBytes.toByteArray(), entity.getContentEncoding() == null ? charsetToUse : entity.getContentEncoding().getValue()));
					} catch (IOException e) {
						// ignore
					}
				} else {
					result.append("\n<content not repeatable>\n");
				}
			}
		}

		return result.toString();
	}

	/**
	 * Builds a string from given request.
	 * 
	 * @param message
	 * @return a string from given request.
	 */
	private String buildResponseLogString(HttpResponse message) {
		if (message == null) {
			return "<null>";
		}

		StringBuilder result = new StringBuilder();

		if (message.getStatusLine() != null) {
			result.append(message.getStatusLine().toString()).append("\n");
		}

		if (message.getAllHeaders() != null) {
			for (Header header : message.getAllHeaders()) {
				result.append(header.toString()).append("\n");
			}
		}

		if (message instanceof BasicHttpResponse) {
			HttpEntity entity = ((BasicHttpResponse) message).getEntity();
			if (entity != null) {
				if (entity.isRepeatable()) {
					try {
						ByteArrayOutputStream entityBytes = new ByteArrayOutputStream();
						entity.writeTo(entityBytes);
						result.append(new String(entityBytes.toByteArray(), entity.getContentEncoding() == null ? charsetToUse : entity.getContentEncoding().getValue()));
					} catch (IOException e) {
						// ignore
					}
				} else {
					result.append("\n<content not repeatable>\n");
				}
			}
		}

		return result.toString();
	}

	/**
	 * Prepares the request content. The content is read from the input field, component property or input file.
	 * 
	 * @return the request content.
	 */
	private Object prepareRequestContent() {
		DataField inField = getFieldSafe(inputRecord, inputFieldName);

		if (inField != null) {
			return inField.toString();

		} else if (requestContentByteToUse != null) {
			return requestContentByteToUse;

		} else if (!StringUtils.isEmpty(requestContentToUse)) {
			return requestContentToUse;

		} else if (!StringUtils.isEmpty(inputFileUrlToUse)) {
			// input file with http request is entered
			InputStream is = null;
			try {
				is = FileUtils.getInputStream(getContextURL(), inputFileUrlToUse);
				return is;
			} catch (IOException ioe) {
				FileUtils.closeQuietly(is);
				throw new JetelRuntimeException("Can't open input stream", ioe);
			}
		}

		return null;
	}

	/**
	 * Try to initialize the component.
	 * 
	 * @throws ComponentNotReadyException
	 */
	protected void tryToInit(ConfigurationStatus status) throws ComponentNotReadyException {
		boolean runningFromCheckConfig = (status != null);
		// find the attached ports (input and output)
		inputPort = getInputPortDirect(INPUT_PORT_NUMBER);
		standardOutputPort = getOutputPort(STANDARD_OUTPUT_PORT_NUMBER);
		errorOutputPort = getOutputPort(ERROR_OUTPUT_PORT_NUMBER);

		// which ports are attached?
		hasInputPort = (inputPort != null);
		hasStandardOutputPort = (standardOutputPort != null);
		hasErrorOutputPort = (errorOutputPort != null);

		if (redirectErrorOutput && hasErrorOutputPort) {
			throw new ComponentNotReadyException("Error output is redirected to standard output port, but error port has an edge connected", XML_REDIRECT_ERROR_OUTPUT);
		}

		// create input mapping transformation
		inputMappingTransformation = new CTLMapping("Input mapping", this);
		inputMappingTransformation.setTransformation(inputMapping);

		// create standard output mapping transformation
		standardOutputMappingTransformation = new CTLMapping("Standard output mapping", this);
		standardOutputMappingTransformation.setTransformation(standardOutputMapping);

		// create error output mapping transformation
		errorOutputMappingTransformation = new CTLMapping("Error output mapping", this);
		errorOutputMappingTransformation.setTransformation(errorOutputMapping);

		DataRecordMetadata paramsMetadata = createInputParametersMetadata();
		inputParamsRecord = DataRecordFactory.newRecord(paramsMetadata);

		if (requestCookiesStr != null) {
			requestCookies = new Properties();
			try {
				requestCookies.load(new StringReader(requestCookiesStr));
			} catch (IOException e) {
				throw new ComponentNotReadyException("Failed to load request cookies", e);
			}

			requestCookiesRecord = DataRecordFactory.newRecord(createMetadataFromProperties(requestCookies, REQUEST_COOKIES_RECORD_NAME));
		}

		// build properties from request headers
		if (!StringUtils.isEmpty(additionalRequestHeadersStr)) {
			additionalRequestHeaders = new Properties();
			try {
				additionalRequestHeaders.load(new StringReader(additionalRequestHeadersStr));

				additionalHeadersRecord = DataRecordFactory.newRecord(createMetadataFromProperties(additionalRequestHeaders, ADDITIONAL_HTTP_HEADERS_RECORD_NAME));
			} catch (Exception e) {
				throw new ComponentNotReadyException(this, "Unexpected exception during request headers reading.", e);
			}
		}

		// build record for multipart entities
		if (!StringUtils.isEmpty(this.multipartEntities)) {
			try {
				this.multipartRequestPropertiesRecord = DataRecordFactory.newRecord(createMultipartMetadataFromString(multipartEntities, MULTIPART_ENTITIES_RECORD_NAME, MULTIPART_ENTITIES_SEPARATOR));
			} catch (Exception e) {
				throw new ComponentNotReadyException(this, "Unexpected exception during request headers reading.", e);
			}
		}

		// create internal result data record
		// build properties from request headers
		if (!StringUtils.isEmpty(requestParametersStr)) {
			requestParameters = new Properties();
			try {
				requestParameters.load(new StringReader(requestParametersStr));

				requestParametersRecord = DataRecordFactory.newRecord(createMetadataFromProperties(requestParameters, REQUEST_PARAMETERS_RECORD_NAME));
			} catch (Exception e) {
				throw new ComponentNotReadyException(this, "Unexpected exception during request headers reading.", e);
			}
		}

		// create internal result data record
		if (hasStandardOutputPort) {
			DataRecordMetadata resultMetadata = createResultMetadata();
			resultRecord = DataRecordFactory.newRecord(resultMetadata);

			if (responseCookies != null) {
				responseCookiesRecord = DataRecordFactory.newRecord(createResponseCookiesMetadata(responseCookies));
			}
		}

		// create internal error data record
		if (hasErrorOutputPort || redirectErrorOutput) {
			DataRecordMetadata errorMetadata = createErrorMetadata();
			errorRecord = DataRecordFactory.newRecord(errorMetadata);
		}

		// create input data record, if necessary
		if (hasInputPort) {
			inputRecord = DataRecordFactory.newRecord(inputPort.getMetadata());
		}

		// create output data records, if necessary
		if (hasStandardOutputPort) {
			standardOutputRecord = DataRecordFactory.newRecord(standardOutputPort.getMetadata());
			if (outputFieldName != null) {
				outField = (StringDataField) standardOutputRecord.getField(outputFieldName);
			} else if (standardOutputMapping == null) {
				outField = (StringDataField) standardOutputRecord.getField(0);
			}
		}

		if (hasErrorOutputPort) {
			errorOutputRecord = DataRecordFactory.newRecord(errorOutputPort.getMetadata());
		}

		// create input records for input mapping
		if (hasInputPort) {
			inputMappingInRecords = new DataRecord[] { inputRecord };
			inputMappingTransformation.addInputRecord(INPUT_RECORD_NAME, inputRecord);
		} else {
			inputMappingInRecords = new DataRecord[] {};
		}

		// create output records for input mapping
		List<DataRecord> inputMappingOutRecordsList = new ArrayList<DataRecord>();
		inputMappingOutRecordsList.add(inputParamsRecord);
		inputMappingOutRecordsList.add(additionalHeadersRecord);
		inputMappingOutRecordsList.add(requestCookiesRecord);
		inputMappingOutRecordsList.add(requestParametersRecord);
		inputMappingOutRecordsList.add(multipartRequestPropertiesRecord);

		inputMappingOutRecords = inputMappingOutRecordsList.toArray(new DataRecord[0]);

		inputMappingTransformation.addOutputRecord(ATTRIBUTES_RECORD_NAME, inputParamsRecord);
		inputMappingTransformation.addOutputRecord(ADDITIONAL_HTTP_HEADERS_RECORD_NAME, additionalHeadersRecord);
		inputMappingTransformation.addOutputRecord(REQUEST_COOKIES_RECORD_NAME, requestCookiesRecord);
		inputMappingTransformation.addOutputRecord(REQUEST_PARAMETERS_RECORD_NAME, requestParametersRecord);
		inputMappingTransformation.addOutputRecord(MULTIPART_ENTITIES_RECORD_NAME, multipartRequestPropertiesRecord);

		// create input records for standard output mapping
		standardOutputMappingTransformation.addInputRecord(INPUT_RECORD_NAME, inputRecord);
		standardOutputMappingTransformation.addInputRecord(RESULT_RECORD_NAME, resultRecord);
		standardOutputMappingTransformation.addInputRecord(ATTRIBUTES_RECORD_NAME, inputParamsRecord);
		standardOutputMappingTransformation.addInputRecord(RESPONSE_COOKIES_RECORD_NAME, responseCookiesRecord);

		// create input records for error output mapping
		errorOutputMappingInRecords = new DataRecord[] { inputRecord, errorRecord, inputParamsRecord };
		errorOutputMappingTransformation.addInputRecord(INPUT_RECORD_NAME, inputRecord);
		errorOutputMappingTransformation.addInputRecord(ERROR_RECORD_NAME, errorRecord);
		errorOutputMappingTransformation.addInputRecord(ATTRIBUTES_RECORD_NAME, inputParamsRecord);

		// create output records for standard output mapping
		if (hasStandardOutputPort) {
			standardOutputMappingOutRecords = new DataRecord[] { standardOutputRecord };
			standardOutputMappingTransformation.addOutputRecord(STANDARD_OUTPUT_RECORD_NAME, standardOutputRecord);
		} else {
			standardOutputMappingOutRecords = new DataRecord[] {};
		}

		// create output records for error output mapping
		if (hasErrorOutputPort) {
			errorOutputMappingOutRecords = new DataRecord[] { null, errorOutputRecord };
			errorOutputMappingTransformation.addOutputRecord(STANDARD_OUTPUT_RECORD_NAME, null);
			errorOutputMappingTransformation.addOutputRecord(ERROR_OUTPUT_RECORD_NAME, errorOutputRecord);
		} else {
			errorOutputMappingOutRecords = new DataRecord[] {};
		}

		// preset default values into runtime variables
		if (!runningFromCheckConfig) {
			initExecutionParametersFromComponentAttributes();
		}

		standardOutputMappingTransformation.addAutoMapping(INPUT_RECORD_NAME, STANDARD_OUTPUT_RECORD_NAME);
		standardOutputMappingTransformation.addAutoMapping(RESULT_RECORD_NAME, STANDARD_OUTPUT_RECORD_NAME);

		errorOutputMappingTransformation.addAutoMapping(INPUT_RECORD_NAME, ERROR_OUTPUT_RECORD_NAME);
		errorOutputMappingTransformation.addAutoMapping(ERROR_RECORD_NAME, ERROR_OUTPUT_RECORD_NAME);
		
		inputMappingTransformation.init(status, XML_INPUT_MAPPING_ATTRIBUTE);
		standardOutputMappingTransformation.init(status, XML_STANDARD_OUTPUT_MAPPING_ATTRIBUTE);
		errorOutputMappingTransformation.init(status, XML_ERROR_OUTPUT_MAPPING_ATTRIBUTE);
	}

	private void initExecutionParametersFromComponentAttributes() throws ComponentNotReadyException {
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_URL_NAME, rawUrl);
		// inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_URL_FIELD_NAME, urlInputField);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_REQUEST_METHOD_NAME, requestMethod);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_REQUEST_CONTENT_NAME, requestContent);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_INPUT_FILE_URL_NAME, inputFileUrl);
		// inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_INPUT_FIELD_NAME_NAME,
		// inputFieldName);
		// inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_OUTPUT_FIELD_NAME_NAME,
		// outputFieldName);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_OUTPUT_FILE_URL_NAME, outputFileUrl);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_APPEND_OUTPUT_NAME, appendOutput);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_CHARSET_NAME, charset);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_ADDITIONAL_REQUEST_HEADERS_NAME, additionalRequestHeaders != null ? new LinkedHashMap<Object, Object>(additionalRequestHeaders) : new LinkedHashMap<String, Object>());
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_ADD_INPUT_FIELDS_AS_PARAMETERS_NAME, addInputFieldsAsParameters);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_ADD_INPUT_FIELDS_AS_PARAMETERS_TO_NAME, addInputFieldsAsParametersTo);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_IGNORED_FIELDS_NAME, ignoredFields);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_AUTHENTICATION_METHOD_NAME, authenticationMethod);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_USERNAME_NAME, username);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_PASSWORD_NAME, password);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_STORE_RESPONSE_TO_TEMP_NAME, storeResponseToTempFile);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_TEMP_FILE_PREFIX_NAME, temporaryFilePrefix);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_MULTIPART_ENTITIES_NAME, multipartEntities);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_CONSUMER_KEY_NAME, consumerKey);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_CONSUMER_SECRET_NAME, consumerSecret);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_OATUH_TOKEN_NAME, oAuthAccessToken);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_OATUH_TOKEN_SECRET_NAME, oAuthAccessTokenSecret);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_REQUEST_PARAMETERS_NAME, requestParameters != null ? new LinkedHashMap<Object, Object>(requestParameters) : new LinkedHashMap<String, Object>());
		
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_TIMEOUT_NAME, timeout);
		inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_RETRY_COUNT_NAME, retryCount);
		
		if (!StringUtils.isEmpty(rawHttpHeaders)) {
			inputMappingTransformation.setDefaultOutputValue(ATTRIBUTES_RECORD_NAME, IP_RAW_HTTP_HEADERS_NAME, parseRawHttpHeadersItems());
		}

		if (requestCookies != null) {
			for (Entry<Object, Object> cookieEntry : requestCookies.entrySet()) {
				String name = (String) cookieEntry.getKey();
				String value = (String) cookieEntry.getValue();
				inputMappingTransformation.setDefaultOutputValue(REQUEST_COOKIES_RECORD_NAME, StringUtils.normalizeName(name), value);
			}
		}
	}

	private List<String> parseRawHttpHeadersItems() {
		return Arrays.asList(rawHttpHeaders.split("\\n|\\r\\n|\\r"));
	}

	/**
	 * Maps values in <code>toRecord</code> from records contained in <code>fromRecords</code> by name.
	 * 
	 * @param fromRecords
	 * @param toRecord
	 */
	private static void mapByName(DataRecord[] fromRecords, DataRecord toRecord) {
		if (fromRecords == null) {
			return;
		}

		for (DataRecord record : fromRecords) {
			toRecord.copyFieldsByName(record);
		}
	}

	/**
	 * Processes given request configuration. New request is built based on given configuration, the request is then
	 * sent to the target URL and the response is processed.
	 * 
	 * @param configuration
	 *            - configuration specifying the request
	 * @throws Exception
	 */
	private void process() throws Exception {
		HTTPRequestConfiguration configuration = prepareConfigurationForRecord();

		HttpResponse response = buildAndSendRequest(configuration);

		if (response != null) {
			int statusCode = response.getStatusLine().getStatusCode();
			if (statusCode != 200) {
				String message = "Returned code for http request " + configuration.getTarget() + " is " + statusCode;
				if (statusCode >= 300) { // CLO-6152
					// warn if the HTTP response code isn't 2xx
					logger.warn(message);
				} else {
					// log 1xx and 2xx status codes as info (except for 200)
					logger.info(message);
				}
			}
		}

		result.setResponse(response);

		// process the response
		processResponse();
	}

	/**
	 * Builds a request based on the given configuration and sends it, using a HTTP client
	 * 
	 * @param configuration
	 * @return response retrieved
	 * 
	 * @throws ComponentNotReadyException
	 * @throws IOException
	 * @throws InterruptedException
	 */
	private HttpResponse buildAndSendRequest(HTTPRequestConfiguration configuration) throws Exception {
		initHTTPClient(configuration);

		HttpRequestBase method = prepareMethod(configuration);
		
		
		// sign the request before sending it
		if (oauthConsumer != null) {
			oauthConsumer.sign(method);
		}
		HttpResponse response = httpClient.execute(method, this.httpContext);
		return response;
	}

	/**
	 * Processes the given response. The response is written to the output port (if connected) or to an output file (if
	 * specified). Otherwise, the content of the response is written to the standard output.
	 * 
	 * @param response
	 * 
	 * @throws ComponentNotReadyException
	 * @throws IOException
	 * @throws InterruptedException
	 */
	private void processResponse() throws ComponentNotReadyException, IOException, InterruptedException {
		// output port is connected - write the response there
		logProcessing();

		HttpResponse response = result.getResponse();
		responseWriter.writeResponse(response);

		if (resultRecord != null) {
			resultRecord.getField(RP_STATUS_CODE_INDEX).setValue(response.getStatusLine().getStatusCode());

			Header[] headers = response.getAllHeaders();
			Map<String, String> headersMap = new LinkedHashMap<String, String>(headers.length);
			List<String> rawHeaders = new ArrayList<String>(headers.length);
			for (Header header : headers) {
				headersMap.put(header.getName(), header.getValue());
				rawHeaders.add(header.getName() + ": " + header.getValue());
			}
			resultRecord.getField(RP_HEADER_INDEX).setValue(headersMap);
			resultRecord.getField(RP_RAW_HTTP_HAEDERS_INDEX).setValue(rawHeaders);
		}

		if (responseCookiesRecord != null) {
			responseCookiesRecord.reset();
			processCookies();
		}
	}

	private void processCookies() {
		List<Cookie> cookies = this.cookieStore.getCookies();
		for (Cookie cookie : cookies) {
			DataField field = getRecordFieldByLabelOrName(responseCookiesRecord, cookie.getName());
			if (field != null) {
				field.setValue(cookie.getValue());
			}
		}
	}

	private static DataField getRecordFieldByLabelOrName(DataRecord record, String labelOrName) {
		DataRecordMetadata metadata = record.getMetadata();
		DataFieldMetadata field = metadata.getFieldByLabel(labelOrName); // getField*() directly on DataRecord throws
																			// IndexOutOfBoundsException :-(
		if (field == null) {
			field = metadata.getField(labelOrName);
		}
		return field == null ? null : record.getField(field.getNumber());
	}

	@Override
	public Result execute() throws Exception {
		if (hasInputPort) {
			while (inputPort.readRecord(inputRecord) != null && runIt) {
				executeForRecord();
				SynchronizeUtils.cloverYield();
			}
		} else {
			if (tokenTracker != null) {
				// in case no input port is attached - dummy token is prepared and initialized and all logging messages
				// are tracked on this dummy token
				inputRecord = DataRecordFactory.newToken("input");
				tokenTracker.initToken(inputRecord);
			}

			// no input port connected - only one request will be sent
			executeForRecord();
		}

		broadcastEOF();

		return (runIt ? Result.FINISHED_OK : Result.ABORTED);
	}

	/**
	 * Logs success.
	 * 
	 */
	private void logSuccess() {
		tokenTracker.logMessage(inputRecord, Level.INFO, "Executed sucessfully.", null);
	}

	/**
	 * Logs progress.
	 * 
	 */
	private void logProcessing() {
		tokenTracker.logMessage(inputRecord, Level.INFO, "Processing response...", null);
	}

	/**
	 * Method that send the request based on the given record on input (or dummy record, if input port is not connected)
	 * 
	 * @return
	 * @throws Exception
	 */
	protected void executeForRecord() throws Exception {
		initForRecord();

		try {
			mapInput();

			process();

			logSuccess();

		} catch (Exception e) {
			// FIXME: UnknownHostException("bla.bla") can be thrown here, where there's no message like "unknown host"
			// we need to somehow tell the user that this is an "Unknown host" situation
			// elsewhere in this class we call e.toString() to work around that
			result.setException(e);
		}

		if (!mapOutput()) {
			// no error mapping, fail here
			throw result.getException();
		} else {
			if (result.getException() != null) {
				// log exception only briefly as INFO as it got sent to error port by now
				String msg = "Request error: " + ExceptionUtils.getMessage(result.getException());
				tokenTracker.logMessage(inputRecord, Level.INFO, msg, null);
			}
		}
	}

	/**
	 * Prepares the state of the component for processing of a next record.
	 * 
	 */
	private void initForRecord() {
		// reset all the records used
		resetRecords(inputParamsRecord, resultRecord, errorRecord, responseCookiesRecord);
		resetRecords(standardOutputMappingOutRecords);
		resetRecords(errorOutputMappingOutRecords);

		result = new RequestResult();
	}

	@Override
	public void preExecute() throws ComponentNotReadyException {
		super.preExecute();
		inputMappingTransformation.preExecute();
		standardOutputMappingTransformation.preExecute();
		errorOutputMappingTransformation.preExecute();
	}

	@Override
	public void postExecute() throws ComponentNotReadyException {
		super.postExecute();
		inputMappingTransformation.postExecute();
		standardOutputMappingTransformation.postExecute();
		errorOutputMappingTransformation.postExecute();
	}

	/**
	 * Method that fills the parameter record based on the given transformation (or by name if no explicit
	 * transformation specified)
	 * 
	 */
	protected void mapInput() {
		resetRecords(inputMappingOutRecords);
		inputMappingTransformation.execute();
		processInputParamsRecord();
	}

	/**
	 * Prepares configuration based on record read from the input port.
	 * 
	 * @return a configuration based on record read from the input port.
	 * @throws ComponentNotReadyException
	 */
	private HTTPRequestConfiguration prepareConfigurationForRecord() throws ComponentNotReadyException {
		preProcessForRecord();

		// get the URL template
		String urlFromField = rawUrlToUse;
		if (!StringUtils.isEmpty(urlInputField) && !attributeMapped(IP_URL_NAME)) {
			// urlInputField is set so it will be used (url field is ignored)
			urlFromField = inputRecord.getField(urlInputField).toString();
		}

		Set<String> fieldsToIgnore = new HashSet<String>(ignoredFieldsSet);

		// prepare the URL (substitute variables) and fill the fields to ignore set
		String finalURL = prepareURL(urlFromField, fieldsToIgnore);

		// now build the configuration
		HTTPRequestConfiguration configuration = new HTTPRequestConfiguration();

		configuration.setTarget(FileURLParser.getOuterAddress(finalURL));
		configuration.setProxy(FileURLParser.getInnerAddress(finalURL));

		// prepare the parameters, ignore fields that are set as ignored + fields used to resolve URL placeholders
		configuration.setParameters(prepareRequestParameters(fieldsToIgnore));
		// prepare the multipart entities
		configuration.setMultipartEntities(prepareMultipartEntities());

		configuration.setContent(prepareRequestContent());

		configuration.prepare();

		return configuration;
	}

	/**
	 * Maps the output.
	 * 
	 * @return success
	 * @throws Exception
	 */
	private boolean mapOutput() {
		if (result.getException() == null) {

			mapStandardOutput();
			return true;

		} else {

			if (isLegacyErrorHandling() || (!redirectErrorOutput && !hasErrorOutputPort) || (redirectErrorOutput && !hasStandardOutputPort)) {
				return false;
			}

			if (redirectErrorOutput) {
				mapStandardOutput();
			} else {
				mapErrorOutput();
			}
			return true;

		}
	}

	private boolean isLegacyErrorHandling() {
		return standardOutputMapping == null && errorOutputMapping == null && !redirectErrorOutput && !hasErrorOutputPort;
	}

	/**
	 * Method that fill the standard output record based on the given output mapping
	 * 
	 * @throws Exception
	 */
	private void mapStandardOutput() {
		if (hasStandardOutputPort) {
			populateInputParamsRecord();
			populateResultRecordError();

			if (outField != null) {
				outField.setNull(true);
			}

			standardOutputMappingTransformation.execute();

			mapOverridingOutput();

			try {
				standardOutputPort.writeRecord(standardOutputRecord);
			} catch (InterruptedException e) {
				throw new JetelRuntimeException(e);
			} catch (IOException e) {
				throw new JetelRuntimeException(e);
			}
		}
	}

	/**
	 * Maps fields based on the configuration to preserve compatibility with old output field property.
	 * 
	 * @throws Exception
	 */
	private void mapOverridingOutput() {
		// User defined the output field explicitly - set the value there, if the value was
		// not set by a mapping
		if (outputFieldName != null && outField != null && outField.isNull()) {
			outField.setValue(resultRecord.getField(RP_CONTENT_INDEX));
		}
	}

	/**
	 * Method that fill the error output record based on the given error output mapping (or by name, if no explicit
	 * transformation specified)
	 * 
	 * @throws Exception
	 */
	private void mapErrorOutput() {
		if (hasErrorOutputPort) {
			populateErrorRecord();
			populateInputParamsRecord();
			if (errorOutputMapping != null) {
				errorOutputMappingTransformation.execute();
			} else {
				// output transformation is not specified - default star mapping is performed
				mapByName(errorOutputMappingInRecords, errorOutputRecord);
			}
			try {
				errorOutputPort.writeRecord(errorOutputRecord);
			} catch (IOException e) {
				throw new JetelRuntimeException(e);
			} catch (InterruptedException e) {
				throw new JetelRuntimeException(e);
			}
		} else if (hasStandardOutputPort) {
			mapStandardOutput();
		}
	}

	/**
	 * Creates an instance of the HTTPConnector based on the given XML definition.
	 * 
	 * @param graph
	 * @param xmlElement
	 * @return an instance of the HTTPConnector based on the given XML definition.
	 * @throws XMLConfigurationException
	 * @throws AttributeNotFoundException
	 */
	public static HttpConnector fromXML(TransformationGraph graph, Element xmlElement)
			throws XMLConfigurationException, AttributeNotFoundException {
		ComponentXMLAttributes xattribs = new ComponentXMLAttributes(xmlElement, graph);

		HttpConnector httpConnector = new HttpConnector(xattribs.getString(XML_ID_ATTRIBUTE));

		/** base properties */
		httpConnector.setUrl(xattribs.getStringEx(XML_URL_ATTRIBUTE, null, RefResFlag.URL));
		httpConnector.setRequestMethod(xattribs.getString(XML_REQUEST_METHOD_ATTRIBUTE, HttpGet.METHOD_NAME));
		httpConnector.setInputFileUrl(xattribs.getStringEx(XML_INPUT_FILEURL_ATTRIBUTE, null, RefResFlag.URL));
		httpConnector.setOutputFileUrl(xattribs.getStringEx(XML_OUTPUT_FILEURL_ATTRIBUTE, null, RefResFlag.URL));
		httpConnector.setAppendOutput(xattribs.getBoolean(XML_APPEND_OUTPUT_ATTRIBUTE, DEFAULT_APPEND_OUTPUT));
		httpConnector.setAdditionalRequestHeaders(xattribs.getString(XML_ADDITIONAL_HTTP_HEADERS_ATTRIBUTE, null));
		httpConnector.setRequestContent(xattribs.getString(XML_REQUEST_CONTENT_ATTRIBUTE, null));
		httpConnector.setInputFieldName(xattribs.getString(XML_INPUT_PORT_FIELD_NAME, null));
		httpConnector.setOutputFieldName(xattribs.getString(XML_OUTPUT_PORT_FIELD_NAME, null));
		httpConnector.setCharset(xattribs.getString(XML_CHARSET_ATTRIBUTE, null));
		httpConnector.setStoreResponseToTempFile(xattribs.getBoolean(XML_STORE_RESPONSE_TO_TEMP_FILE, false));
		httpConnector.setTemporaryFilePrefix(xattribs.getString(XML_TEMPORARY_FILE_PREFIX, "http-response-"));
		httpConnector.setAuthenticationMethod(xattribs.getString(XML_AUTHENTICATION_METHOD_ATTRIBUTE, "BASIC"));
		httpConnector.setUsername(xattribs.getString(XML_USERNAME_ATTRIBUTE, null));
		httpConnector.setPassword(xattribs.getStringEx(XML_PASSWORD_ATTRIBUTE, null, RefResFlag.PASSWORD));
		httpConnector.setAddInputFieldsAsParameters(xattribs.getBoolean(XML_ADD_INPUT_FIELDS_AS_PARAMETERS_ATTRIBUTE, false));
		httpConnector.setAddInputFieldsAsParametersTo(xattribs.getString(XML_ADD_INPUT_FIELDS_AS_PARAMETERS_TO_ATTRIBUTE, "QUERY"));
		httpConnector.setIgnoredFields(xattribs.getString(XML_IGNORED_FIELDS_ATTRIBUTE, null));
		httpConnector.setMultipartEntities(xattribs.getString(XML_MULTIPART_ENTITIES_FIELDS_LIST_ATTRIBUTE, null));
		httpConnector.setUrlInputField(xattribs.getString(XML_URL_FROM_INPUT_FIELD_ATTRIBUTE, null));
		httpConnector.setConsumerKey(xattribs.getString(XML_CONSUMER_KEY_ATTRIBUTE, null));
		httpConnector.setConsumerSecret(xattribs.getString(XML_CONSUMER_SECRET_ATTRIBUTE, null));
		httpConnector.setoAuthAccessToken(xattribs.getString(XML_OAUTH_ACCESS_TOKEN_ATTRIBUTE, null));
		httpConnector.setoAuthAccessTokenSecret(xattribs.getString(XML_OAUTH_ACCESS_TOKEN_SECRET_ATTRIBUTE, null));
		httpConnector.setRawHttpHeaders(xattribs.getString(XML_RAW_HTTP_HEADERS_ATTRIBUTE, null));
		httpConnector.setRequestCookies(xattribs.getString(XML_REQUEST_COOKIES_ATTRIBUTE, null));
		httpConnector.setResponseCookies(xattribs.getString(XML_RESPONSE_COOKIES_ATTRIBUTE, null));
		httpConnector.setStreaming(xattribs.getBoolean(XML_STREAMING_ATTRIBUTE, true));
		httpConnector.setRequestParametersStr(xattribs.getString(XML_REQUEST_PARAMETERS_ATTRIBUTE, null));
		httpConnector.setDisableSSLCertValidation(xattribs.getBoolean(XML_DISABLE_SSL_CERT_VALIDATION, false));
		httpConnector.setTimeout(xattribs.getTimeInterval(XML_TIMEOUT_ATTRIBUTE, -1));
		httpConnector.setRetryCount(xattribs.getInteger(XML_RETRY_COUNT_ATTRIBUTE, 0));

		/** job flow related properties */
		httpConnector.setInputMapping(xattribs.getStringEx(XML_INPUT_MAPPING_ATTRIBUTE, null, RefResFlag.SPEC_CHARACTERS_OFF));
		httpConnector.setStandardOutputMapping(xattribs.getStringEx(XML_STANDARD_OUTPUT_MAPPING_ATTRIBUTE, null, RefResFlag.SPEC_CHARACTERS_OFF));
		httpConnector.setErrorOutputMapping(xattribs.getStringEx(XML_ERROR_OUTPUT_MAPPING_ATTRIBUTE, null, RefResFlag.SPEC_CHARACTERS_OFF));
		httpConnector.setRedirectErrorOutput(xattribs.getBoolean(XML_REDIRECT_ERROR_OUTPUT, false));

		return httpConnector;
	}

	private boolean attributeMappedFromInput(String attributeName) {
		if (inputMapping == null) {
			return false;
		}

		return inputMapping.contains("$out.0." + attributeName + " = $in.");
	}

	private boolean attributeMapped(String attributeName) {
		if (inputMapping == null) {
			return false;
		}

		if (inputMapping.contains("$out.0." + attributeName + " =")) {
			return true;
		}

		InputPort inputPort = getInputPort(INPUT_PORT_NUMBER);
		if (inputMapping.contains("$out.0.* =")) {
			if (inputPort != null) {
				DataRecordMetadata metadata = inputPort.getMetadata();
				if (metadata != null && metadata.getField(attributeName) != null) {
					return true;
				}
			}
		}

		return false;
	}

	@Override
	public ConfigurationStatus checkConfig(ConfigurationStatus status) {
		super.checkConfig(status);

		// check number of input/output ports
		if (!checkInputPorts(status, 0, 1) || !checkOutputPorts(status, 0, 2)) {
			return status;
		}

		// check character set
		if (charset != null && !Charset.isSupported(charset)) {
			status.addError(this, XML_CHARSET_ATTRIBUTE, "Charset " + charset + " not supported!");
		}

		InputPort inputPort = getInputPort(INPUT_PORT_NUMBER);
		if (inputPort == null) {
			// no input port, but URL from field defined
			if (!StringUtils.isEmpty(urlInputField)) {
				status.addWarning(this, null, "'URL from input field' attribute is set, but no input port is connected.");
			}

			// no input port, but URL mapped from input
			if (attributeMappedFromInput(IP_URL_NAME)) {
				status.addWarning(this, null, "URL is mapped from input field, but no input port is connected.");
			}

			// no URL defined (in the situation without an input port, the URL can be only set using 'URL' attribute or
			// mapped in Input Mapping)
			if (StringUtils.isEmpty(rawUrl) && !attributeMapped(IP_URL_NAME)) {
				status.addError(this, null, "No URL defined - please set the 'URL' attribute or map URL in Input Mapping.");
			}
		}

		// input mapping does not define the URL
		if (!attributeMapped(IP_URL_NAME)) {

			// check whether both URL and input field URL isn't entered (note: the case when no input port is connected
			// is handled above)
			if (!StringUtils.isEmpty(rawUrl) && !StringUtils.isEmpty(urlInputField)) {
				status.addWarning(this, null, "Both URL and URL from input field is entered. URL from input field will be used.");
			}

			// no URL defined - check only if the input port is not null (the case of no input port is checked above)
			if (inputPort != null && StringUtils.isEmpty(rawUrl) && StringUtils.isEmpty(urlInputField)) {
				status.addError(this, null, "No URL defined - please set the 'URL' attribute or map URL in Input Mapping.");
			}

		} else {
			// URL field set and URL mapped - mapping will be used
			if (!StringUtils.isEmpty(urlInputField)) {
				status.addWarning(this, null, "URL is mapped in 'Input Mapping' and 'URL from input field' attribute is set. 'URL from input field' will be ignored.");
			}

		}

		// Unknown request method
		if (!PLAIN_REQUEST_METHODS.contains(requestMethod) && !ENTITY_ENCLOSING_REQUEST_METHODS.contains(requestMethod)) {
			status.addError(this, XML_REQUEST_METHOD_ATTRIBUTE, "Unsupported request method: " + requestMethod);
		}

		// Unknown authentication method
		if (!SUPPORTED_AUTHENTICATION_METHODS.contains(authenticationMethod)) {
			status.addError(this, XML_AUTHENTICATION_METHOD_ATTRIBUTE,
					"Unsupported authentication method: " + authenticationMethod);
		}

		DataFieldMetadata inField = null;
		if (inputPort != null) {
			if (!StringUtils.isEmpty(inputFieldName)) {
				inField = inputPort.getMetadata().getField(inputFieldName);
			} else {
				inField = inputPort.getMetadata().getField(0);
			}

			// Input field is set, but does not exist
			if (inputFieldName != null && inField == null) {
				status.addError(this, XML_INPUT_PORT_FIELD_NAME,
						"Input field name '" + inputFieldName + "' does not exist in input metadata.");

				// Input field is set, but has incompatible type
			} else if (inputFieldName != null && inField.getDataType() != DataFieldType.STRING) {
				status.addError(this, XML_INPUT_PORT_FIELD_NAME,
						"Input field '" + inputFieldName + "' has incompatible type '" + inField.getDataType().toString() + "'. Field has to be String.");
			}

			// check whether multipart entities and ignored list is entered just when possible or reasonable
			if (!addInputFieldsAsParameters) {
				if (!StringUtils.isEmpty(multipartEntities)) {
					status.addWarning(this, null,
							"'Multipart entities' attribute will be ignored, because 'Add input fields as parameters' attribute is set to 'false'.");
				}
				if (!StringUtils.isEmpty(ignoredFields)) {
					status.addWarning(this, null,
							"'Ignored fields' attribute will be ignored, because 'Add input fields as parameters' attribute is set to 'false'.");
				}
			} else {
				if (!StringUtils.isEmpty(multipartEntities) && !ENTITY_ENCLOSING_REQUEST_METHODS.contains(requestMethod)) {
					status.addWarning(this, null,
							"Multipart entities cannot be used with a " + requestMethod + " request method.");
				}
			}

//			// check whether multipart entities list contains just values from metadata
//			if (!StringUtils.isEmpty(multipartEntities)) {
////				List<String> multipartEntitiesList = new ArrayList<String>();
////				StringTokenizer tokenizer = new StringTokenizer(multipartEntities, ";");
////				while (tokenizer.hasMoreElements()) {
////					multipartEntitiesList.add(tokenizer.nextToken());
////				}
////				String[] metadataNames = inputPort.getMetadata().getFieldNamesArray();
////				for (String metadataName : metadataNames) {
////					multipartEntitiesList.remove(metadataName);
////				}
////
////				if (!multipartEntitiesList.isEmpty()) {
////					status.add(new ConfigurationProblem("Given multipart entities list contains value not defined at metadata: " + multipartEntitiesList.get(0), Severity.ERROR, this, Priority.NORMAL));
////				}
//			}

			// check whether ignored fields list contains just values from metadata
			if (!StringUtils.isEmpty(ignoredFields)) {
				List<String> ignoredList = new ArrayList<String>();
				StringTokenizer tokenizer = new StringTokenizer(ignoredFields, ";");
				while (tokenizer.hasMoreElements()) {
					ignoredList.add(tokenizer.nextToken());
				}
				String[] metadataNames = inputPort.getMetadata().getFieldNamesArray();
				for (String metadataName : metadataNames) {
					ignoredList.remove(metadataName);
				}
				if (!ignoredList.isEmpty()) {
					status.addError(this, null, "Given ignored fields list contains value not defined at metadata: " + ignoredList.get(0));
				}
			}

			// test whether it is specified where the input fields should be added (query or body)
			if (addInputFieldsAsParameters) {
				if (StringUtils.isEmpty(addInputFieldsAsParametersTo)) {
					if (ENTITY_ENCLOSING_REQUEST_METHODS.contains(requestMethod)) {
						status.addError(this, null, "Add input fields as parameters must be specified.");
					}
				} else if (addInputFieldsAsParametersTo.equals("BODY") && PLAIN_REQUEST_METHODS.contains(requestMethod)) {
					status.addError(this, null,
							"Cannot add input fields as parameters to body of HTTP request when " + requestMethod + " method is used.");
				}
				
				if(!StringUtils.isEmpty(this.requestParametersStr)) {
					status.addWarning(this, null,
							"Request parameters are defined but HTTPConnector will use input fields as parameters. Set 'Add input fields as parameters' to false if you want to use specified request parameters.");
				}
			}else if(!StringUtils.isEmpty(this.requestParametersStr) && addInputFieldsAsParametersTo.equals("BODY") && PLAIN_REQUEST_METHODS.contains(requestMethod)){
				status.addError(this, null,
						"Cannot add request parameters to body of HTTP request when " + requestMethod + " method is used.");
			}
		}

		OutputPort outputPort = getOutputPort(STANDARD_OUTPUT_PORT_NUMBER);
		if (outputPort != null) {
			DataFieldMetadata outField = null;
			if (!StringUtils.isEmpty(outputFieldName)) {
				outField = outputPort.getMetadata().getField(outputFieldName);
			} else if (standardOutputMapping == null) {
				outField = outputPort.getMetadata().getField(0);
			}

			if (outputFieldName != null && outField == null) {
				status.addError(this, XML_OUTPUT_PORT_FIELD_NAME,
						"Output field name '" + outputFieldName + "' does not exist in output metadata.");

			}
			if (outField != null && outField.getDataType() != DataFieldType.STRING) {
				if (outputFieldName != null) {
					status.addError(this, XML_OUTPUT_PORT_FIELD_NAME,
							"Output field '" + outputFieldName + "' has incompatible type '" + outField.getDataType().toString() + "'. The field has to be 'string'.");
				} else {
					status.addError(this, XML_OUTPUT_PORT_FIELD_NAME, 
							"'Output field' not specified -> HTTP response content will be filled in the first field of output metadata, but the field has incompatible type '" + outField.getDataType().toString() + "'. It has to be 'string'.");
				}
			}
		}

		if (inputPort == null && outputPort == null && StringUtils.isEmpty(outputFileUrl)) {
			status.addWarning(this, null, "Output port isn't connected and output file is not set.");
		}

		if (storeResponseToTempFile && outputFileUrl != null) {
			status.addError(this, null, "Only one of 'Output file URL' and 'Store response file URL to output field' may be used.");
		}

		if (!StringUtils.isEmpty(requestContent) && !StringUtils.isEmpty(inputFileUrl)) {
			status.addError(this, null, "You can set either 'Request content' or 'Input file URL'.");
		}

		// check existence of the temporary directory; if specified
		if (getStoreResponseToTempFile()) {

			// output port must be connected so that we can write file names into
			if (outputPort == null) {
				status.addError(this, null,
						"An output port must be connected in order to write response temporary file names.");
			}
		}

		// check whether both user name and password for BASIC http auth are entered or none of them
		if ((!StringUtils.isEmpty(getUsername()) && StringUtils.isEmpty(getPassword())) || (StringUtils.isEmpty(getUsername()) && !StringUtils.isEmpty(getPassword()))) {
			status.addError(this, null, "Both username and password must be entered or none of them.");
		}

		// check restrictions of the GET-like methods
		if (PLAIN_REQUEST_METHODS.contains(requestMethod)) {
			// no content allowed in GET request (actually, it is not invalid for GET request to contain content
			// according to standard, but the HTTPClient library does not support it
			// as it is not used in practice)
			if (!StringUtils.isEmpty(requestContent)) {
				status.addError(this, XML_REQUEST_CONTENT_ATTRIBUTE,
						"Request content not allowed when " + requestMethod + " method is used.");
			}
			if (!StringUtils.isEmpty(inputFieldName)) {
				status.addError(this, XML_INPUT_PORT_FIELD_NAME,
						"Input field not allowed when " + requestMethod + " method is used.");
			}
			if (!StringUtils.isEmpty(inputFileUrl)) {
				status.addError(this, XML_INPUT_FILEURL_ATTRIBUTE, 
						"Input file URL not allowed when " + requestMethod + " method is used.");
			}
		}

		if (!StringUtils.isEmpty(rawHttpHeaders)) {
			for (CharSequence rawHeader : parseRawHttpHeadersItems()) {
				try {
					parseRawHeaderItem(rawHeader);
				} catch (IllegalArgumentException e) {
					status.addWarning(this, XML_RAW_HTTP_HEADERS_ATTRIBUTE,
							"Missing ':' semicolon character in \"raw HTTP header\" item: \"" + rawHeader + "\"");
				}
			}
		}
		
		if (disableSSLCertValidation) {
			status.addWarning(this, XML_DISABLE_SSL_CERT_VALIDATION,
					"Certificate validation is disabled. Connection will not be secure.");
		}

		try {
			tryToInit(status);
		} catch (Exception e) {
			status.addError(this, null, "Initialization failed.", e);
		}
		
		if (inputPort != null && !StringUtils.isEmpty(inputFileUrl)) {
			if (inputFieldName != null && inField != null) {
				status.addWarning(this, null, INPUT_FILE_ATTRIBUTE_WARN);
			} else if (inputMappingTransformation != null) {
				Set<Field> usedOutputFields = inputMappingTransformation.getUsedOutputFields();
				
				if (containsField(usedOutputFields, IP_REQUEST_CONTENT_NAME)) {
					status.addWarning(this, null, String.format(INPUT_FILE_FIELD_WARN, IP_REQUEST_CONTENT_NAME));
				} else if (containsField(usedOutputFields, IP_REQUEST_CONTENT_BYTE_NAME)) {
					status.addWarning(this, null, String.format(INPUT_FILE_FIELD_WARN, IP_REQUEST_CONTENT_BYTE_NAME));
				}
			}
		}
		
		return status;
	}
	
	private boolean containsField(Set<Field> fields, String fieldName) {
		if (fields == null || fields.isEmpty()) {
			return false;
		}
		
		for (Field field : fields) {
			if (fieldName.equals(field.name)) {
				return true;
			}
		}
		
		return false;
	}

	/**
	 * Validates given prepared HTTP configuration.
	 * 
	 * @param configuration
	 * @return prepared configuration
	 * @throws ComponentNotReadyException
	 */
	private void validateConfiguration(HTTPRequestConfiguration configuration) throws ComponentNotReadyException {
		// check if the target URL is not empty
		if (StringUtils.isEmpty(configuration.getTarget())) {
			throw new ComponentNotReadyException("Invalid target URL - no URL provided");
		}

		// check if the protocol is HTTP or HTTPS
		String protocol = configuration.getTargetURL().getProtocol();
		if (!protocol.equals("http") && !protocol.equals("https")) {
			throw new ComponentNotReadyException("Given URL has incompatible protocol: " + protocol);
		}
	}

	/**
	 * Prepares a POST-like (entity enclosing) method for given configuration.
	 * 
	 * @param configuration
	 * @return a POST-like method for given configuration.
	 * @throws IOException
	 */
	private HttpEntityEnclosingRequestBase prepareEntityEnclosingMethod(String method,
			HTTPRequestConfiguration configuration) throws IOException {
		if (logger.isDebugEnabled()) {
			logger.debug("Creating " + method + " request to " + configuration.getTarget());
		}

		HttpEntityEnclosingRequestBase httpMethod = new HttpEntityEnclosingRequest(method, configuration.getTarget());

		// set multipart request entity if any
		if (!configuration.getMultipartEntities().isEmpty()) {
			MultipartEntity entity = new MultipartEntity();
			for (PartWithName stringPart : buildMultiPart(configuration.getMultipartEntities())) {
				entity.addPart(stringPart.name, stringPart.value);
			}

			httpMethod.setEntity(entity);
		}

		// process parameters
		if ("BODY".equals(addInputFieldsAsParametersToToUse)) {
			// set request body if any
			// FIXME: this replaces the multipart entity set in the previous step
			// we leave it as-is to make the behavior compatible with older version using httpClient 3.x
			// which also cleared the previously set multipart entity.
			httpMethod.setEntity(new UrlEncodedFormEntity(Arrays.asList(buildNameValuePairs(configuration.getParameters())), charsetToUse));

		} else { // if (addInputFieldsAsParametersTo.equals("QUERY")) {
			addQuery(configuration, httpMethod);
		}

		// process content
		Object content = configuration.getContent();
		if (content != null) {
			HttpEntity entity = null;
			ContentType contentType = ContentType.create(ContentType.TEXT_PLAIN.getMimeType(), charsetToUse);
			if (content instanceof String) {
				entity = new StringEntity((String) content, contentType);
			} else if (content instanceof byte[]) {
				entity = new ByteArrayEntity((byte[]) content, contentType);
			} else if (content instanceof InputStream) {
				if (!isStreaming() || httpMethod.getRequestLine().getProtocolVersion().lessEquals(HttpVersion.HTTP_1_0)) {
					// chunked transfer encoding is not supported in HTTP/1.0
					try {
						ByteArrayOutputStream os = new ByteArrayOutputStream();
						StreamUtils.copy((InputStream) content, os, true, true);
						entity = new ByteArrayEntity(os.toByteArray(), contentType);
					} catch (IOException ioe) {
						throw new JetelRuntimeException("Reading from input stream failed", ioe);
					}
				} else {
					// the length is set to -1 (unknown), which enforces Transfer-Encoding: chunked
					try {
						entity = new BufferedHttpEntity(new InputStreamEntity((InputStream) content, -1, contentType));
					} catch (IOException e) {
						entity = new InputStreamEntity((InputStream) content, -1, contentType);
					}
				}
			}
			if (entity != null) {
				httpMethod.setEntity(entity);
			}
		}

		return httpMethod;
	}

	/**
	 * Prepares a GET-like method for given configuration.
	 * 
	 * @param configuration
	 * @return a GET-like method for given configuration.
	 * 
	 * @throws UnsupportedEncodingException
	 */
	private HttpRequestBase preparePlainMethod(final String method, HTTPRequestConfiguration configuration)
			throws UnsupportedEncodingException {
		if (logger.isDebugEnabled()) {
			logger.debug("Creating " + method + " request to " + configuration.getTarget());
		}

		HttpRequestBase httpMethod = new HttpPlainRequest(method, configuration.getTarget());

		addQuery(configuration, httpMethod);

		return httpMethod;
	}

	private void addQuery(HTTPRequestConfiguration configuration, HttpRequestBase httpMethod)
			throws UnsupportedEncodingException {
		// process parameters
		String preparedQuery = buildQueryString(configuration.getParameters());

		// set query string if any
		if (!StringUtils.isEmpty(preparedQuery)) {
			addQuery(preparedQuery, httpMethod);
		}
	}

	/** Serves as generic replacement of all direct non-abstract child classes of Apache HttpRequestBase (like HttpGet) */
	private static class HttpPlainRequest extends HttpRequestBase {

		private final String method;

		private HttpPlainRequest(String method, String uri) {
			super();
			this.method = method;
			setURI(URI.create(uri));
		}

		@Override
		public String getMethod() {
			return method;
		}

	}

	/** Serves as generic replacement of all child classes of Apache HttpEntityEnclosingRequestBase (like HttpPost) */
	private static class HttpEntityEnclosingRequest extends HttpEntityEnclosingRequestBase {

		private final String method;

		private HttpEntityEnclosingRequest(String method, String uri) {
			super();
			this.method = method;
			setURI(URI.create(uri));
		}

		@Override
		public String getMethod() {
			return method;
		}

	}

	/**
	 * Prepares a HTTP method (based on the given configuration) to be used for a request.
	 * 
	 * @param configuration
	 * @return a HTTP method (based on the given configuration) to be used for a request.
	 * @throws UnsupportedEncodingException
	 * @throws ComponentNotReadyException
	 * @throws IOException
	 */
	private HttpRequestBase prepareMethod(HTTPRequestConfiguration configuration) throws ComponentNotReadyException,
			IOException {
		HttpRequestBase method = null;

		if (requestMethodToUse != null) {
			requestMethodToUse = requestMethodToUse.toUpperCase(Locale.ENGLISH);
		}

		
		// configure the request method
		if (PLAIN_REQUEST_METHODS.contains(requestMethodToUse)) {
			method = preparePlainMethod(requestMethodToUse, configuration);

		} else if (ENTITY_ENCLOSING_REQUEST_METHODS.contains(requestMethodToUse)) {
			method = prepareEntityEnclosingMethod(requestMethodToUse, configuration);

		} else {
			// another request method than get or post
			throw new ComponentNotReadyException(this, "Unsupported request method: " + requestMethodToUse);
		}

		// add resolved header parameters
		if ((additionalRequestHeadersToUse != null && !additionalRequestHeadersToUse.isEmpty()) || (rawHttpHeadersToUse != null && !rawHttpHeadersToUse.isEmpty())) {
			addHeaderParameters(method);
		}

		addRequestCookies(method);

		if(this.timeoutToUse>0) {
			RequestConfig reqConfig = RequestConfig.custom().setConnectTimeout((int)this.timeoutToUse).
					setConnectionRequestTimeout((int)this.timeoutToUse).setSocketTimeout((int)this.timeoutToUse).build();
			method.setConfig(reqConfig);
		}
		
		try {
			if (!StringUtils.isEmpty(this.usernameToUse) && !StringUtils.isEmpty(this.passwordToUse) && (("BASIC".equals(this.authenticationMethodToUse) || "ANY".equals(this.authenticationMethodToUse)))) {
				Header[] headers = method.getHeaders("Authorization");

				if (headers == null || headers.length == 0) {
					logger.info("No Authorization HTTP header found. Initializing preemptive basic authentication.");

					UsernamePasswordCredentials creds = new UsernamePasswordCredentials(this.usernameToUse, this.passwordToUse);
					method.addHeader(new BasicScheme().authenticate(creds, method, this.httpContext));
				}
			}
		} catch (AuthenticationException e) {
			logger.warn("Preemptive authentication. Authorization header generation failed.", e);
		}
		
		//CLO-6504 - put Accept header to request, because some servers interprets missing accept header wrongly
		Header[] headers = method.getHeaders("Accept");
		if((headers==null) || headers.length == 0) {
			method.addHeader("Accept", "*/*");
		}
				
		return method;
	}

	/**
	 * Adds a query string to a given method URI.
	 * 
	 * @param query
	 *            - a query string to be added
	 * @param method
	 *            - a method to be updated
	 */
	private void addQuery(String query, HttpRequestBase method) {
		if (!StringUtils.isEmpty(method.getURI().getQuery())) {
			method.setURI(changeQueryString(method.getURI(), method.getURI().getRawQuery() + "&" + query));
		} else {
			method.setURI(changeQueryString(method.getURI(), query));
		}
	}

	/**
	 * Builds a query string from given parameters map.
	 * 
	 * @param parameters
	 * @return a query string built from given parameters map.
	 * 
	 * @throws UnsupportedEncodingException
	 */
	private String buildQueryString(Map<String, String> parameters) throws UnsupportedEncodingException {
		if (parameters == null) {
			return "";
		}

		StringBuilder result = new StringBuilder();
		Iterator<Entry<String, String>> it = parameters.entrySet().iterator();
		while (it.hasNext()) {
			Entry<String, String> parameter = it.next();

			result.append(URLEncoder.encode(parameter.getKey(), "UTF-8")).append('=').append(URLEncoder.encode(parameter.getValue(), "UTF-8"));
			if (it.hasNext()) {
				result.append('&');
			}
		}

		return result.toString();
	}

	/**
	 * Builds a name-value pairs from given parameters map.
	 * 
	 * @param parameters
	 * @return a name-value pairs built from given parameters map.
	 */
	private NameValuePair[] buildNameValuePairs(Map<String, String> parameters) {
		if (parameters == null) {
			return null;
		}

		int index = 0;
		NameValuePair[] result = new NameValuePair[parameters.size()];
		for (Entry<String, String> parameter : parameters.entrySet()) {
			result[index] = new BasicNameValuePair(parameter.getKey(), parameter.getValue());
			index++;
		}

		return result;
	}

	/**
	 * Builds an array of content parts for given entity map.
	 * 
	 * @param multipartEntities
	 * @return an array of content parts for given entity map.
	 * @throws IOException
	 */
	protected PartWithName[] buildMultiPart(Map<String, HttpConnectorMutlipartEntity> multipartEntities)
			throws IOException {
		if (multipartEntities == null) {
			return null;
		}

		int index = 0;
		PartWithName[] result = new PartWithName[multipartEntities.size()];
		for (Entry<String, HttpConnectorMutlipartEntity> parameter : multipartEntities.entrySet()) {
			if(parameter.getValue().sourceFile!=null) {
				//input stream
				InputStream inputStream = FileUtils.getInputStream(this.getContextURL(), parameter.getValue().sourceFile);
				String fileName = null;
				File file = FileUtils.getJavaFile(this.getContextURL(), parameter.getValue().sourceFile);
				if (file != null) {
					fileName = file.getName();
				} else {
					fileName = parameter.getKey();
				}
				if(parameter.getValue().fileNameAttribute!=null) {
					fileName = parameter.getValue().fileNameAttribute;
				}
				InputStreamBody body = (parameter.getValue().conentType != null) ? new InputStreamBody(inputStream, parameter.getValue().conentType, fileName) : new InputStreamBody(inputStream, fileName);
				result[index] = new PartWithName(parameter.getKey(), body);

				if (logger.isDebugEnabled()) {
					if (parameter.getValue().charset != null) {
						logger.debug("Charset specified to " + parameter.getValue().charset + ". But charset cannot be specified for file multipart entity. Charset will be ignored.");
					}
				}
			}else if(parameter.getValue().contentByte!=null) {
				//input stream
				ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(parameter.getValue().contentByte);
				String fileName = parameter.getValue().name; 
				if(parameter.getValue().fileNameAttribute!=null) {
					fileName = parameter.getValue().fileNameAttribute;
				}
				InputStreamBody body = (parameter.getValue().conentType != null) ? new InputStreamBody(byteArrayInputStream, parameter.getValue().conentType, fileName) : new InputStreamBody(byteArrayInputStream, fileName);
				result[index] = new PartWithName(parameter.getKey(), body);
				if (logger.isDebugEnabled()) {
					if (parameter.getValue().charset != null) {
						logger.debug("Charset specified to " + parameter.getValue().charset + ". But charset cannot be specified for file multipart entity. Charset will be ignored.");
					}
				}
			}else{ 
				//string content
				final String fileName = parameter.getValue().fileNameAttribute;
				
				String value = parameter.getValue().content != null ? parameter.getValue().content : "";
				String contentTypeString = parameter.getValue().conentType != null ? parameter.getValue().conentType : ContentType.TEXT_PLAIN.getMimeType();
				Charset charset = Charset.forName(parameter.getValue().charset != null ? parameter.getValue().charset : Defaults.DataParser.DEFAULT_CHARSET_DECODER);

				ContentType contentType = ContentType.create(contentTypeString, charset);
				
				StringBody stringBody = new StringBody(value, contentType) {
					@Override
					public String getFilename() {
						return fileName;
					}
					
				};
						
				
				result[index] = new PartWithName(parameter.getKey(), stringBody);
			}
			
//			if (parameter.getValue().fileNameAttribute == null) {
//				Charset charset = parameter.getValue().charset != null ? Charset.forName(parameter.getValue().charset) : Charset.defaultCharset();
//				String contentType = parameter.getValue().conentType != null ? parameter.getValue().conentType : ContentType.TEXT_PLAIN.toString();
//
//				String value = parameter.getValue().content != null ? parameter.getValue().content : "";
//				result[index] = new PartWithName(parameter.getKey(), new StringBody(value, contentType, charset));
//			} else if (parameter.getValue().content != null) {
//				Charset charset = parameter.getValue().charset != null ? Charset.forName(parameter.getValue().charset) : Charset.defaultCharset();
//				String contentType = parameter.getValue().conentType != null ? parameter.getValue().conentType : ContentType.TEXT_PLAIN.toString();
//
//				final String fileName = parameter.getValue().fileNameAttribute;
//				String value = parameter.getValue().content != null ? parameter.getValue().content : "";
//				result[index] = new PartWithName(parameter.getKey(), new StringBody(value, contentType, charset) {
//					@Override
//					public String getFilename() {
//						return fileName;
//					}
//				});
//
//			} else {

	//		}
			index++;
		}

		return result;
	}

	/**
	 * Initializes the HTTP client for a new request. Only properties that don't depend on the record should be used
	 * here.
	 * 
	 * @throws ComponentNotReadyException
	 * @throws InterruptedException
	 * @throws IOException
	 */
	@SuppressWarnings("deprecation")
	private void initHTTPClient(HTTPRequestConfiguration configuration) throws ComponentNotReadyException, IOException,
			InterruptedException {
		validateConfiguration(configuration);

		HttpClientBuilder builder = HttpClientBuilder.create();

		builder = builder.useSystemProperties();
		HttpRequestRetryHandler retryHandler;
		retryHandler = new HttpRequestRetryHandler() {
			@Override
			public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
				if(executionCount <= HttpConnector.this.retryCountToUse) {
					return true;
				}
				return false;
			}
		};
		builder.setRetryHandler(retryHandler);

		
		cookieStore = new RequestResponseCookieStore();

		builder.addInterceptorLast(requestLoggingInterceptor);
		builder.addInterceptorLast(cookieStore);
		builder.addInterceptorLast(responseLoggingInterceptor);

		BasicCredentialsProvider credentialProvider = new BasicCredentialsProvider();
		builder.setDefaultCredentialsProvider(credentialProvider);

		// configure proxy
		if (configuration.getProxyURL() != null) {
			String proxyHost = configuration.getProxyURL().getHost();
			int proxyPort = configuration.getProxyURL().getPort();
			HttpHost proxy = new HttpHost(proxyHost, proxyPort);
			builder.setProxy(proxy);

			String userInfo = configuration.getProxyURL().getUserInfo();
			if (!StringUtils.isEmpty(userInfo)) {
				UserInfo proxyCredentials = new UserInfo(userInfo);

				String username = proxyCredentials.getUser();
				String password = proxyCredentials.getPassword();
				if ((username != null) && (password != null)) {
					Credentials credentials = new UsernamePasswordCredentials(username, password);
					AuthScope authScope = new AuthScope(proxyHost, proxyPort, AuthScope.ANY_REALM);

					credentialProvider.setCredentials(authScope, credentials);
				}
			}
		}

		// configure authentication
		if (usernameToUse != null && passwordToUse != null) {
			// create credentials
			creds = new UsernamePasswordCredentials(usernameToUse, passwordToUse);
			credentialProvider.setCredentials(AuthScope.ANY, creds);

			// set authentication method
			String authMethod = null;
			if ("BASIC".equals(authenticationMethodToUse)) {
				// basic http authentication
				authMethod = AuthSchemes.BASIC;
			} else if ("DIGEST".equals(authenticationMethodToUse)) {
				// digest http authentication
				authMethod = AuthSchemes.DIGEST;
			} else if ("ANY".equals(authenticationMethodToUse)) {
				// one of the possible authentication method will be used
				authMethod = "ANY";
			} else {
				throw new JetelRuntimeException("Unknown auth method '" + authenticationMethodToUse + "', only BASIC, DIGEST and ANY are supported.");
			}
			final List<String> authPrefs = new ArrayList<String>();
			if (authMethod.equals("ANY")) {
				authPrefs.add(AuthSchemes.BASIC);
				authPrefs.add(AuthSchemes.DIGEST);
			} else {
				authPrefs.add(authMethod);
			}
			authPrefs.add(authMethod);

			builder.setTargetAuthenticationStrategy(TargetAuthenticationStrategy.INSTANCE);

			// httpClient.setTargetAuthenticationHandler(new DefaultTargetAuthenticationHandler() {
			//
			// @Override
			// protected List<String> getAuthPreferences(HttpResponse response, HttpContext context) {
			// return authPrefs;
			// }
			//
			// });
		}

		if (disableSSLCertValidation) {
			// THIS MAKES THE SSL CONNECTION UNSECURE
			try {
				SSLContextBuilder contextBuilder = new SSLContextBuilder();
				contextBuilder.loadTrustMaterial(null, new TrustStrategy() {
					@Override
					public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
						// trust ALL certificates
						return true;
					}
				});
				SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(
						contextBuilder.build(),
						SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
				
				builder.setSSLSocketFactory(sslFactory);
			} catch (Exception e) {
				throw new ComponentNotReadyException(this, "Problem with HTTPS connection.", e);
			}
		}

		// configure OAuth authentication
		if (!StringUtils.isEmpty(consumerKeyToUse) && !StringUtils.isEmpty(consumerSecretToUse)) {
			// Consumer instance that should be used (used for OAuth authentication)
			oauthConsumer = new CommonsHttpOAuthConsumer(consumerKeyToUse, consumerSecretToUse);

			if (!StringUtils.isEmpty(this.oAuthAccessTokenToUse) && !StringUtils.isEmpty(this.oAuthAccessTokenSecretToUse)) {
				oauthConsumer.setTokenWithSecret(oAuthAccessTokenToUse, oAuthAccessTokenSecretToUse);
			}
		}

		this.httpContext = new BasicHttpContext();

		httpContext.setAttribute(HttpClientContext.COOKIE_STORE, this.cookieStore);

		httpClient = builder.build();

	}

	/**
	 * Changes the query string in given URI and returns the updated URI.
	 * 
	 * @param uri
	 * @param newQuery
	 * @return the updated URI.
	 */
	private static URI changeQueryString(URI uri, String newQuery) {
		try {
			URI newURI = URIUtils.createURI(uri.getScheme(), uri.getHost(), uri.getPort(), uri.getPath(), newQuery, uri.getFragment());
			return newURI;
		} catch (URISyntaxException e) {
		}
		return null;
	}

	/**
	 * Sets the additional header parameters and resolving references in context of given record.
	 * 
	 * @param record
	 *            - record to be used to resolve references
	 * @throws ComponentNotReadyException
	 */
	private void addHeaderParameters(HttpRequestBase method) throws ComponentNotReadyException {
		if (inputRecord != null) {
			Properties fieldValues = new Properties();

			Iterator<DataField> it = inputRecord.iterator();
			while (it.hasNext()) {
				DataField field = it.next();
				fieldValues.setProperty(field.getMetadata().getName(), field.toString());
			}

			setRefProperties(fieldValues);
		}

		if (rawHttpHeadersToUse != null && !rawHttpHeadersToUse.isEmpty()) {
			for (CharSequence rawHeader : rawHttpHeadersToUse) {
				try {
					RawHeader header = parseRawHeaderItem(rawHeader);
					if (header != null) {
						method.addHeader(header.name, header.value);
					}
				} catch (IllegalArgumentException e) {
					logger.debug("Ignoring invalid \"raw HTTP header\" item: \"" + rawHeader + "\"");
				}
			}
		}

		if (additionalRequestHeadersToUse != null && !additionalRequestHeadersToUse.isEmpty()) {
			// pass request properties to the http connection
			for (Entry<String, CharSequence> entry : additionalRequestHeadersToUse.entrySet()) {
				String value = refResolver.resolveRef(entry.getValue().toString());

				// check if the value is fully resolved
				if (PropertyRefResolver.containsProperty(value)) {
					throw new ComponentNotReadyException(this, "Could not resolve all references in additional HTTP header: '" + entry.getValue() + "' (resolved as '" + value + "')");
				}

				method.getParams().setParameter((String) entry.getKey(), value);
				method.addHeader((String) entry.getKey(), value);
			}
		}
	}

	private static class RawHeader {
		public final String name;
		public final String value;

		private RawHeader(String name, String value) {
			this.name = name;
			this.value = value;
		}
	}

	private RawHeader parseRawHeaderItem(CharSequence rawHeader) {
		if (rawHeader != null) {
			String rawHeaderStr = rawHeader.toString().trim();
			if (!rawHeaderStr.isEmpty()) {
				rawHeaderStr = refResolver.resolveRef(rawHeaderStr);
				int separatorIndex = rawHeaderStr.indexOf(':');
				if (separatorIndex > 0) {
					String name = rawHeaderStr.substring(0, separatorIndex).trim();
					String value = rawHeaderStr.substring(separatorIndex + 1).trim();
					return new RawHeader(name, value);
				} else {
					throw new IllegalArgumentException();
				}
			}
		}
		return null;
	}

	private void addRequestCookies(HttpRequestBase method) throws ComponentNotReadyException {
		if (requestCookies == null) {
			return;
		}

		for (DataField field : requestCookiesRecord) {
			if (!field.isNull()) {
				String name = field.getMetadata().getLabelOrName();
				String value = field.getValue().toString();
				BasicClientCookie basicClientCookie = new BasicClientCookie(name, value);
				if (method.getURI() != null) {
					basicClientCookie.setDomain(method.getURI().getHost());
				}
				basicClientCookie.setPath("/");
				cookieStore.addCookie(basicClientCookie);
			}
		}
	
	}

	/**
	 * Creates a map representing request parameters.
	 * 
	 * @param fieldsToIgnore
	 * @return a map representing request parameters.
	 * @throws ComponentNotReadyException 
	 */
	private Map<String, String> prepareRequestParameters(Set<String> fieldsToIgnore) throws ComponentNotReadyException {
		Map<String, String> parameters = new LinkedHashMap<String, String>();

		// there are some input fields which should be added to the request
		if (addInputFieldsAsParametersToUse) {
			// find out which metadata fields weren't used for substitution of placeholders
			List<String> unusedMetadata = new ArrayList<String>();
			DataRecordMetadata metadata = inputPort.getMetadata();
			String[] metadataNames = metadata.getFieldNamesArray();
			for (String metadataName : metadataNames) {
				if (!fieldsToIgnore.contains(metadataName)) {
					unusedMetadata.add(metadataName);
				}
			}

			for (String property : unusedMetadata) {
				parameters.put(property, inputRecord.getField(property).toString());
			}
		} else { // if not addInputFieldsAsParametersToUse, is there any parameter explicitly mapped
			if (requestParametersToUse != null) {
				for (Entry<String, CharSequence> entry : requestParametersToUse.entrySet()) {
					if (entry.getValue() != null) {
						String value = refResolver.resolveRef(entry.getValue().toString());

						// check if the value is fully resolved
						if (PropertyRefResolver.containsProperty(value)) {
							throw new ComponentNotReadyException(this, "Could not resolve all references in additional HTTP header: '" + entry.getValue() + "' (resolved as '" + value + "')");
						}
						parameters.put(entry.getKey(), value);
					}
				}
			}

		}

		return parameters;
	}

	/**
	 * Build a map representing multi-part entities.
	 * 
	 * @return a map representing multi-part entities.
	 */
	protected Map<String, HttpConnectorMutlipartEntity> prepareMultipartEntities() {
		Map<String, HttpConnectorMutlipartEntity> multipartEntitiesMap = new LinkedHashMap<String, HttpConnectorMutlipartEntity>();

		if (StringUtils.isEmpty(multipartEntitiesToUse) && !StringUtils.isEmpty(this.multipartEntities)) {
			this.multipartEntitiesToUse = this.multipartEntities;
		}

		// parse multipart entities
		if (multipartEntitiesToUse != null) {
			StringTokenizer parts = new StringTokenizer(this.multipartEntitiesToUse, ";");
			boolean multipartExists = false;
			while (parts.hasMoreTokens()) {
				String token = parts.nextToken();
				String value = null;
				if(inputRecord.hasField(token)) {
					value = inputRecord.getField(token).toString();
					multipartExists = true;
				}

				HttpConnectorMutlipartEntity entity = new HttpConnectorMutlipartEntity();

				String contentValue = null;
				if (multipartRequestPropertiesRecord != null) {
					DataField field = multipartRequestPropertiesRecord.getField(token + "_" + MULTIPART_CONTENT);
					if (field != null && this.inputMappingTransformation.isOutputOverridden(this.multipartRequestPropertiesRecord, field)) {
						multipartExists = true;
						contentValue = field.getValue() != null ? field.getValue().toString() : "";
					}

					field = multipartRequestPropertiesRecord.getField(token + "_" + MULTIPART_CHARSET);
					if (field != null && this.inputMappingTransformation.isOutputOverridden(this.multipartRequestPropertiesRecord, field)) {
						entity.charset = field.getValue() != null ? field.getValue().toString() : null;
						multipartExists = true;
					}

					field = multipartRequestPropertiesRecord.getField(token + "_" + MULTIPART_CONTENTTYPE);
					if (field != null && this.inputMappingTransformation.isOutputOverridden(this.multipartRequestPropertiesRecord, field)) {
						entity.conentType = field.getValue() != null ? field.getValue().toString() : null;
						multipartExists = true;
					}

					field = multipartRequestPropertiesRecord.getField(token + "_" + MULTIPART_FILENAME);
					if (field != null && this.inputMappingTransformation.isOutputOverridden(this.multipartRequestPropertiesRecord, field)) {
						entity.fileNameAttribute = field.getValue() != null ? field.getValue().toString() : null;
						multipartExists = true;
					}

					field = multipartRequestPropertiesRecord.getField(token + "_" + MULTIPART_SOURCE_FILE);
					if (field != null && this.inputMappingTransformation.isOutputOverridden(this.multipartRequestPropertiesRecord, field)) {
						entity.sourceFile = field.getValue() != null ? field.getValue().toString() : null;
						multipartExists = true;
					}

					field = multipartRequestPropertiesRecord.getField(token + "_" + MULTIPART_CONTENT_BYTE);
					if (field != null && this.inputMappingTransformation.isOutputOverridden(this.multipartRequestPropertiesRecord, field)) {
						if(field instanceof ByteDataField) {
							entity.contentByte = ((ByteDataField)field).getByteArray();
							multipartExists = true;
						}else {
							entity.contentByte = null;
						}
					}
				}
				entity.name = token;

				if (contentValue != null) {
					entity.content = contentValue;
				} else if (entity.fileNameAttribute == null) {
					entity.content = value; // use not mapped input field value only if file is not mapped to
				}
				if(multipartExists) {
				  multipartEntitiesMap.put(token, entity);
				}
			}
		}

		return multipartEntitiesMap;
	}

	private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\\*\\{(" + StringUtils.OBJECT_NAME_PATTERN + ")\\}");

	/**
	 * Prepares URL based on the given URL template. The substituted placeholders are added to the set given as
	 * parameter.
	 * 
	 * @throws ComponentNotReadyException
	 * @throws InterruptedException
	 * @throws IOException
	 */
	private String prepareURL(String urlTemplate, Set<String> substituedPlaceHolders) throws ComponentNotReadyException {
		if (urlTemplate == null) {
			throw new ComponentNotReadyException("Invalid URL: null");
		}

		// parse URL
		// placeholder format: *{placeholder_name}
		Matcher matcher = PLACEHOLDER_PATTERN.matcher(urlTemplate);

		StringBuffer sb = new StringBuffer(urlTemplate.length());
		while (matcher.find()) {
			String fieldName = matcher.group(1);
			DataField field = inputRecord.getField(fieldName);
			if (field != null) {
				matcher.appendReplacement(sb, Matcher.quoteReplacement(field.toString()));
				substituedPlaceHolders.add(fieldName);
			} else {
				// some placeholder wasn't substituted. This should never happen.
				throw new ComponentNotReadyException("Invalid URL - no such field in the input metadata: '" + fieldName + "'");
			}
		}
		matcher.appendTail(sb);

		return sb.toString();
	}

	/**
	 * Sets the properties to be used by reference resolver
	 * 
	 * @param props
	 */
	private void setRefProperties(Properties props) {
		refResolver.clear();
		refResolver.addProperties(props);
	}

	/**
	 * Retrieves a string representation of result of the HTTP request
	 * 
	 * @return String representation of result of the HTTP request
	 * 
	 * @throws IOException
	 */
	private String getResponseContentAsString(InputStream responseInputStream) throws IOException {
		try {
			return IOUtils.toString(responseInputStream, charsetToUse);
		} finally {
			closeStreamSilent(responseInputStream);
		}
	}

	private byte[] getResponseContentAsByteArray(InputStream responseInputStream) throws IOException {
		try {
			return IOUtils.toByteArray(responseInputStream);
		} finally {
			closeStreamSilent(responseInputStream);
		}
	}

	private void closeStreamSilent(InputStream inputStream) {
		try {
			inputStream.close();
		} catch (IOException e) {
			logger.warn("Could not close response input stream", e);
		}
	}

	/**
	 * Populates the input parameter record with the parameters used.
	 * 
	 */
	protected void populateInputParamsRecord() {
		inputParamsRecord.getField(IP_URL_INDEX).setValue(rawUrlToUse);
		// inputParamsRecord.getField(IP_URL_FIELD_INDEX).setValue(urlInputFieldToUse);
		inputParamsRecord.getField(IP_REQUEST_METHOD_INDEX).setValue(requestMethodToUse);
		inputParamsRecord.getField(IP_REQUEST_CONTENT_INDEX).setValue(requestContentToUse);
		// inputParamsRecord.getField(IP_REQUEST_CONTENT_BYTE_INDEX).setValue(requestContentByteToUse);
		inputParamsRecord.getField(IP_INPUT_FILE_URL_INDEX).setValue(inputFileUrlToUse);
		// inputParamsRecord.getField(IP_INPUT_FIELD_NAME_INDEX).setValue(inputFieldNameToUse);
		// inputParamsRecord.getField(IP_OUTPUT_FIELD_NAME_INDEX).setValue(outputFieldNameToUse);
		inputParamsRecord.getField(IP_OUTPUT_FILE_URL_INDEX).setValue(outputFileUrlToUse);
		inputParamsRecord.getField(IP_APPEND_OUTPUT_INDEX).setValue(appendOutputToUse);
		inputParamsRecord.getField(IP_CHARSET_INDEX).setValue(charsetToUse);
		inputParamsRecord.getField(IP_ADDITIONAL_REQUEST_HEADERS_INDEX).setValue(additionalRequestHeadersToUse);
		inputParamsRecord.getField(IP_ADD_INPUT_FIELDS_AS_PARAMETERS_INDEX).setValue(addInputFieldsAsParametersToUse);
		inputParamsRecord.getField(IP_ADD_INPUT_FIELDS_AS_PARAMETERS_TO_INDEX).setValue(addInputFieldsAsParametersToToUse);
		inputParamsRecord.getField(IP_IGNORED_FIELDS_INDEX).setValue(ignoredFieldsToUse);
		inputParamsRecord.getField(IP_AUTHENTICATION_METHOD_INDEX).setValue(authenticationMethodToUse);
		inputParamsRecord.getField(IP_USERNAME_INDEX).setValue(usernameToUse);
		inputParamsRecord.getField(IP_PASSWORD_INDEX).setValue(passwordToUse);
		inputParamsRecord.getField(IP_STORE_RESPONSE_TO_TEMP_INDEX).setValue(storeResponseToTempFileToUse);
		inputParamsRecord.getField(IP_TEMP_FILE_PREFIX_INDEX).setValue(temporaryFilePrefixToUse);
		inputParamsRecord.getField(IP_MULTIPART_ENTITIES_INDEX).setValue(multipartEntitiesToUse);
		inputParamsRecord.getField(IP_CONSUMER_KEY_INDEX).setValue(consumerKeyToUse);
		inputParamsRecord.getField(IP_CONSUMER_SECRET_INDEX).setValue(consumerSecretToUse);
		inputParamsRecord.getField(IP_OATUH_TOKEN_INDEX).setValue(oAuthAccessTokenToUse);
		inputParamsRecord.getField(IP_OATUH_TOKEN_SECRET_INDEX).setValue(oAuthAccessTokenSecretToUse);
		inputParamsRecord.getField(IP_RAW_HTTP_HEADERS_INDEX).setValue(rawHttpHeadersToUse);
		inputParamsRecord.getField(IP_REQUEST_PARAMETERS_INDEX).setValue(requestParametersToUse);
		inputParamsRecord.getField(IP_TIMEOUT_INDEX).setValue(timeoutToUse);
		inputParamsRecord.getField(IP_RETRY_COUNT_INDEX).setValue(retryCountToUse);
	}

	/**
	 * Populates the error record.
	 * 
	 */
	protected void populateErrorRecord() {
		populateErrorField(errorRecord, EP_MESSAGE_INDEX);
	}

	private void populateResultRecordError() {
		populateErrorField(resultRecord, RP_MESSAGE_INDEX);
	}

	private void populateErrorField(DataRecord record, int errorFieldIndex) {
		Exception ex = result.getException();
		if (ex != null) {
			// FIXME: UnknownHostException("bla.bla") can be thrown here, where there's no message like "unknown host"
			// we need to somehow tell the user that this is an "Unknown host" situation
			// that's why we call ex.toString() instead of ExceptionUtils
			record.getField(errorFieldIndex).setValue(ex.toString());
		}
	}

	/**
	 * Creates input parameters metadata for mapping dialogs.
	 * 
	 * @return input parameter metadata used by the component's mapping dialogs
	 */
	public static DataRecordMetadata createUIInputParametersMetadata() {
		DataRecordMetadata metadata = new DataRecordMetadata(ATTRIBUTES_RECORD_NAME);

		metadata.addField(new DataFieldMetadata(IP_URL_NAME, DataFieldType.STRING, null));
		metadata.addField(new DataFieldMetadata(IP_REQUEST_METHOD_NAME, DataFieldType.STRING, null));
		metadata.addField(new DataFieldMetadata(IP_REQUEST_CONTENT_NAME, DataFieldType.STRING, null));
		metadata.addField(new DataFieldMetadata(IP_INPUT_FILE_URL_NAME, DataFieldType.STRING, null));

		return metadata;
	}

	/**
	 * Creates input parameters metadata.
	 * 
	 * @return input parameter metadata used by this component
	 */
	public static DataRecordMetadata createInputParametersMetadata() {
		DataRecordMetadata metadata = new DataRecordMetadata(ATTRIBUTES_RECORD_NAME);

		metadata.addField(IP_URL_INDEX, new DataFieldMetadata(IP_URL_NAME, DataFieldType.STRING, null));
		// metadata.addField(IP_URL_FIELD_INDEX, new DataFieldMetadata(IP_URL_FIELD_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_REQUEST_METHOD_INDEX, new DataFieldMetadata(IP_REQUEST_METHOD_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_ADD_INPUT_FIELDS_AS_PARAMETERS_INDEX, new DataFieldMetadata(IP_ADD_INPUT_FIELDS_AS_PARAMETERS_NAME, DataFieldType.BOOLEAN, null));
		metadata.addField(IP_ADD_INPUT_FIELDS_AS_PARAMETERS_TO_INDEX, new DataFieldMetadata(IP_ADD_INPUT_FIELDS_AS_PARAMETERS_TO_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_IGNORED_FIELDS_INDEX, new DataFieldMetadata(IP_IGNORED_FIELDS_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_ADDITIONAL_REQUEST_HEADERS_INDEX, new DataFieldMetadata(IP_ADDITIONAL_REQUEST_HEADERS_NAME, DataFieldType.STRING, null, DataFieldContainerType.MAP));
		metadata.addField(IP_CHARSET_INDEX, new DataFieldMetadata(IP_CHARSET_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_REQUEST_CONTENT_INDEX, new DataFieldMetadata(IP_REQUEST_CONTENT_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_REQUEST_CONTENT_BYTE_INDEX, new DataFieldMetadata(IP_REQUEST_CONTENT_BYTE_NAME, DataFieldType.BYTE, null));
		metadata.addField(IP_INPUT_FILE_URL_INDEX, new DataFieldMetadata(IP_INPUT_FILE_URL_NAME, DataFieldType.STRING, null));
		// metadata.addField(IP_INPUT_FIELD_NAME_INDEX, new DataFieldMetadata(IP_INPUT_FIELD_NAME_NAME,
		// DataFieldType.STRING, null));
		// metadata.addField(IP_OUTPUT_FIELD_NAME_INDEX, new DataFieldMetadata(IP_OUTPUT_FIELD_NAME_NAME,
		// DataFieldType.STRING, null));
		metadata.addField(IP_OUTPUT_FILE_URL_INDEX, new DataFieldMetadata(IP_OUTPUT_FILE_URL_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_APPEND_OUTPUT_INDEX, new DataFieldMetadata(IP_APPEND_OUTPUT_NAME, DataFieldType.BOOLEAN, null));
		metadata.addField(IP_AUTHENTICATION_METHOD_INDEX, new DataFieldMetadata(IP_AUTHENTICATION_METHOD_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_USERNAME_INDEX, new DataFieldMetadata(IP_USERNAME_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_PASSWORD_INDEX, new DataFieldMetadata(IP_PASSWORD_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_CONSUMER_KEY_INDEX, new DataFieldMetadata(IP_CONSUMER_KEY_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_CONSUMER_SECRET_INDEX, new DataFieldMetadata(IP_CONSUMER_SECRET_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_OATUH_TOKEN_INDEX, new DataFieldMetadata(IP_OATUH_TOKEN_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_OATUH_TOKEN_SECRET_INDEX, new DataFieldMetadata(IP_OATUH_TOKEN_SECRET_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_STORE_RESPONSE_TO_TEMP_INDEX, new DataFieldMetadata(IP_STORE_RESPONSE_TO_TEMP_NAME, DataFieldType.BOOLEAN, null));
		metadata.addField(IP_TEMP_FILE_PREFIX_INDEX, new DataFieldMetadata(IP_TEMP_FILE_PREFIX_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_MULTIPART_ENTITIES_INDEX, new DataFieldMetadata(IP_MULTIPART_ENTITIES_NAME, DataFieldType.STRING, null));
		metadata.addField(IP_RAW_HTTP_HEADERS_INDEX, new DataFieldMetadata(IP_RAW_HTTP_HEADERS_NAME, DataFieldType.STRING, null, DataFieldContainerType.LIST));
		metadata.addField(IP_REQUEST_PARAMETERS_INDEX, new DataFieldMetadata(IP_REQUEST_PARAMETERS_NAME, DataFieldType.STRING, null, DataFieldContainerType.MAP));
		metadata.addField(IP_TIMEOUT_INDEX, new DataFieldMetadata(IP_TIMEOUT_NAME, DataFieldType.LONG, null));
		metadata.addField(IP_RETRY_COUNT_INDEX, new DataFieldMetadata(IP_RETRY_COUNT_NAME, DataFieldType.INTEGER, null));

		return metadata;
	}

	public static DataRecordMetadata createMetadataFromString(String inputString, String metadataName, String separator) {
		DataRecordMetadata metadata = new DataRecordMetadata(metadataName);

		String[] splitted = inputString.split(separator);

		for (String part : splitted) {
			String partTrimmed = part.trim();
			if (!partTrimmed.isEmpty()) {
				DataFieldMetadata field = new DataFieldMetadata("xxx", DataFieldType.STRING, null);
				field.setLabel(partTrimmed);
				metadata.addField(field);
			}
		}

		metadata.normalize();
		return metadata;
	}

	public static DataRecordMetadata createMultipartMetadataFromString(String multipart, String metadataName,
			String separator) {
		if (multipart == null || StringUtils.isEmpty(multipart)) {
			return null;
		}

		DataRecordMetadata metadata = new DataRecordMetadata(metadataName);

		String[] splitted = multipart.split(separator);

		for (String part : splitted) {
			String partTrimmed = part.trim();
			if (!partTrimmed.isEmpty()) {
				DataFieldMetadata field;

				field = new DataFieldMetadata("xxx", DataFieldType.STRING, null);
				field.setLabel(partTrimmed + "_" + MULTIPART_CONTENT);
				metadata.addField(field);
				
				field = new DataFieldMetadata("xxx", DataFieldType.BYTE, null);
				field.setLabel(partTrimmed + "_" + MULTIPART_CONTENT_BYTE);
				metadata.addField(field);

				field = new DataFieldMetadata("xxx", DataFieldType.STRING, null);
				field.setLabel(partTrimmed + "_" + MULTIPART_SOURCE_FILE);
				metadata.addField(field);

				field = new DataFieldMetadata("xxx", DataFieldType.STRING, null);
				field.setLabel(partTrimmed + "_" + MULTIPART_FILENAME);
				metadata.addField(field);

				field = new DataFieldMetadata("xxx", DataFieldType.STRING, null);
				field.setLabel(partTrimmed + "_" + MULTIPART_CHARSET);
				metadata.addField(field);

				field = new DataFieldMetadata("xxx", DataFieldType.STRING, null);
				field.setLabel(partTrimmed + "_" + MULTIPART_CONTENTTYPE);
				metadata.addField(field);

			}
		}

		metadata.normalize();
		return metadata;
	}

	public static DataRecordMetadata createMetadataFromProperties(Properties requestCookies, String metadataName) {
		DataRecordMetadata metadata = new DataRecordMetadata(metadataName);

		for (Object variableName : requestCookies.keySet()) {
			DataFieldMetadata field = new DataFieldMetadata("xxx", DataFieldType.STRING, null);
			field.setLabel((String) variableName);
			metadata.addField(field);
		}

		metadata.normalize();
		return metadata;
	}

	public static DataRecordMetadata createResponseCookiesMetadata(String responseCookies) {
		return createMetadataFromString(responseCookies, RESPONSE_COOKIES_RECORD_NAME, RESPONSE_COOKIES_SEPARATOR);
	}

	/**
	 * Creates result metadata.
	 * 
	 * @return result metadata used by this component
	 */
	public static DataRecordMetadata createResultMetadata() {
		DataRecordMetadata metadata = new DataRecordMetadata(HttpConnector.RESULT_RECORD_NAME);

		metadata.addField(RP_CONTENT_INDEX, new DataFieldMetadata(RP_CONTENT_NAME, DataFieldType.STRING, null));
		metadata.addField(RP_CONTENT_BYTE_INDEX, new DataFieldMetadata(RP_CONTENT_BYTE_NAME, DataFieldType.BYTE, null));
		metadata.addField(RP_OUTPUTFILE_INDEX, new DataFieldMetadata(RP_OUTPUTFILE_NAME, DataFieldType.STRING, null));
		metadata.addField(RP_STATUS_CODE_INDEX, new DataFieldMetadata(RP_STATUS_CODE_NAME, DataFieldType.INTEGER, null));
		metadata.addField(RP_HEADER_INDEX, new DataFieldMetadata(RP_HEADER_NAME, DataFieldType.STRING, null, DataFieldContainerType.MAP));
		metadata.addField(RP_RAW_HTTP_HAEDERS_INDEX, new DataFieldMetadata(RP_RAW_HTTP_HAEDERS_NAME, DataFieldType.STRING, null, DataFieldContainerType.LIST));
		metadata.addField(RP_MESSAGE_INDEX, new DataFieldMetadata(RP_MESSAGE_NAME, DataFieldType.STRING, null));

		return metadata;
	}

	/**
	 * Creates error metadata.
	 * 
	 * @return error metadata used by this component
	 */
	public static DataRecordMetadata createErrorMetadata() {
		DataRecordMetadata metadata = new DataRecordMetadata(ERROR_RECORD_NAME);

		metadata.addField(EP_MESSAGE_INDEX, new DataFieldMetadata(EP_MESSAGE_NAME, DataFieldType.STRING, null));

		return metadata;
	}

	public void setUrl(String rawUrl) {
		this.rawUrl = rawUrl;
	}

	public void setRequestMethod(String requestMethod) {
		this.requestMethod = requestMethod;
	}

	public String getRequestMethod() {
		return requestMethod;
	}

	public void setInputFileUrl(String inputFileUrl) {
		this.inputFileUrl = inputFileUrl;
	}

	public String getInputFileUrl() {
		return inputFileUrl;
	}

	public void setOutputFileUrl(String outputFileUrl) {
		this.outputFileUrl = outputFileUrl;
	}

	public String getOutputFileUrl() {
		return outputFileUrl;
	}

	public void setAppendOutput(boolean appendOutput) {
		this.appendOutput = appendOutput;
	}

	public boolean isAppendOutput() {
		return appendOutput;
	}

	public void setAdditionalRequestHeaders(String additionalRequestHeadersAsProperties) {
		this.additionalRequestHeadersStr = additionalRequestHeadersAsProperties;
	}

	public String getRequestContent() {
		return requestContent;
	}

	public void setRequestContent(String requestContent) {
		this.requestContent = requestContent;
	}

	public String getInputFieldName() {
		return inputFieldName;
	}

	public void setInputFieldName(String inputFieldName) {
		this.inputFieldName = inputFieldName;
	}

	public String getOutputFieldName() {
		return outputFieldName;
	}

	public void setOutputFieldName(String outputFieldName) {
		this.outputFieldName = outputFieldName;
	}

	public String getCharset() {
		return charset;
	}

	public void setCharset(String charset) {
		this.charset = charset;
	}

	public void setTemporaryFilePrefix(String temporaryFilePrefix) {
		this.temporaryFilePrefix = temporaryFilePrefix;
	}

	public String getTemporaryFilePrefix() {
		return temporaryFilePrefix;
	}

	public void setStoreResponseToTempFile(boolean storeResponseToTempFile) {
		this.storeResponseToTempFile = storeResponseToTempFile;
	}

	public boolean getStoreResponseToTempFile() {
		return storeResponseToTempFile;
	}

	public String getInputMapping() {
		return inputMapping;
	}

	public void setInputMapping(String inputMapping) {
		this.inputMapping = inputMapping;
	}

	public String getStandardOutputMapping() {
		return standardOutputMapping;
	}

	public void setStandardOutputMapping(String standardOutputMapping) {
		this.standardOutputMapping = standardOutputMapping;
	}

	public String getErrorOutputMapping() {
		return errorOutputMapping;
	}

	public void setErrorOutputMapping(String errorOutputMapping) {
		this.errorOutputMapping = errorOutputMapping;
	}

	public void setUsername(String username) {
		this.username = username;
	}

	public String getUsername() {
		return username;
	}

	public void setPassword(String password) {
		this.password = password;
	}

	public String getPassword() {
		return password;
	}

	public void setAddInputFieldsAsParameters(boolean addInputFieldsAsParameters) {
		this.addInputFieldsAsParameters = addInputFieldsAsParameters;
	}

	public boolean getAddInputFieldsAsParameters() {
		return addInputFieldsAsParameters;
	}

	public void setIgnoredFields(String ignoredFields) {
		this.ignoredFields = ignoredFields;
	}

	public String getIgnoredFields() {
		return ignoredFields;
	}

	public void setMultipartEntities(String multipartEntities) {
		this.multipartEntities = multipartEntities;
	}

	public String getMultipartEntities() {
		return multipartEntities;
	}

	public void setUrlInputField(String urlInputField) {
		this.urlInputField = urlInputField;
	}

	public String getUrlInputField() {
		return urlInputField;
	}

	public void setAddInputFieldsAsParametersTo(String addInputFieldsAsParametersTo) {
		this.addInputFieldsAsParametersTo = addInputFieldsAsParametersTo;
	}

	public String getAddInputFieldsAsParametersTo() {
		return addInputFieldsAsParametersTo;
	}

	public void setAuthenticationMethod(String authenticationMethod) {
		this.authenticationMethod = authenticationMethod;
	}

	public String getAuthenticationMethod() {
		return authenticationMethod;
	}

	public String getConsumerKey() {
		return consumerKey;
	}

	public void setConsumerKey(String consumerKey) {
		this.consumerKey = consumerKey;
	}

	public String getConsumerSecret() {
		return consumerSecret;
	}

	public void setConsumerSecret(String consumerSecret) {
		this.consumerSecret = consumerSecret;
	}

	
	
	/**
	 * @return the requestParametersStr
	 */
	public String getRequestParametersStr() {
		return requestParametersStr;
	}

	/**
	 * @param requestParametersStr the requestParametersStr to set
	 */
	public void setRequestParametersStr(String requestParametersStr) {
		this.requestParametersStr = requestParametersStr;
	}

	public void setRawHttpHeaders(String rawHttpHeaders) {
		this.rawHttpHeaders = rawHttpHeaders;
	}

	/**
	 * @return the oAuthAccessToken
	 */
	public String getoAuthAccessToken() {
		return oAuthAccessToken;
	}

	/**
	 * @param oAuthAccessToken
	 *            the oAuthAccessToken to set
	 */
	public void setoAuthAccessToken(String oAuthAccessToken) {
		this.oAuthAccessToken = oAuthAccessToken;
	}

	/**
	 * @return the oAuthAccessTokenSecret
	 */
	public String getoAuthAccessTokenSecret() {
		return oAuthAccessTokenSecret;
	}

	/**
	 * @param oAuthAccessTokenSecret
	 *            the oAuthAccessTokenSecret to set
	 */
	public void setoAuthAccessTokenSecret(String oAuthAccessTokenSecret) {
		this.oAuthAccessTokenSecret = oAuthAccessTokenSecret;
	}

	public boolean isStreaming() {
		return streaming;
	}

	public void setStreaming(boolean streaming) {
		this.streaming = streaming;
	}

	private void setRedirectErrorOutput(boolean redirectErrorOutput) {
		this.redirectErrorOutput = redirectErrorOutput;
	}

	public void setRequestCookies(String requestCookies) {
		this.requestCookiesStr = requestCookies;
	}

	public void setResponseCookies(String responseCookies) {
		this.responseCookies = responseCookies;
	}
	
	public boolean isDisableSSLCertValidation() {
		return disableSSLCertValidation;
	}

	public void setDisableSSLCertValidation(boolean disableSSLCertValidation) {
		this.disableSSLCertValidation = disableSSLCertValidation;
	}
	
	

	/**
	 * @return the timeout
	 */
	public long getTimeout() {
		return timeout;
	}

	/**
	 * @param timeout the timeout to set
	 */
	public void setTimeout(long timeout) {
		this.timeout = timeout;
	}

	/**
	 * @return the retryCount
	 */
	public int getRetryCount() {
		return retryCount;
	}

	/**
	 * @param retryCount the retryCount to set
	 */
	public void setRetryCount(int retryCount) {
		this.retryCount = retryCount;
	}

	@Override
	protected ComponentTokenTracker createComponentTokenTracker() {
		return new ReformatComponentTokenTracker(this);
	}

	/**
	 * CookieStore for HttpClient allowing to separate request and response cookies.
	 * 
	 * @author tkramolis ([email protected]) (c) Javlin, a.s. (www.cloveretl.com)
	 * 
	 * @created 9.8.2012
	 */
	private static class RequestResponseCookieStore implements CookieStore, HttpRequestInterceptor {

		private BasicCookieStore requestCookieStore = new BasicCookieStore();
		private BasicCookieStore responseCookieStore = new BasicCookieStore();
		private boolean responseState = false;

		private CookieStore getCurrentStore() {
			return responseState ? responseCookieStore : requestCookieStore;
		}

		@Override
		public void addCookie(Cookie cookie) {
			getCurrentStore().addCookie(cookie);
		}

		@Override
		public List<Cookie> getCookies() {
			return getCurrentStore().getCookies();
		}

		@Override
		public boolean clearExpired(Date date) {
			// Note: no reson for this particular implemention, just a suggestion; this method is not used anyway (at
			// least now)
			return requestCookieStore.clearExpired(date) || responseCookieStore.clearExpired(date);
		}

		@Override
		public void clear() {
			requestCookieStore.clear();
			responseCookieStore.clear();
		}

		@Override
		public void process(HttpRequest request, HttpContext context) throws HttpException, IOException {
			responseState = true;
		}

	}

}

class HttpConnectorMutlipartEntity {
	public String name = null;
	public String fileNameAttribute = null;
	public String sourceFile = null;
	public String content = null;
	public byte[] contentByte = null;
	public String conentType = null; // MIME type
	public String charset = null;
}