/****************************************************************
 * Copyright (C) 2005 LAMS Foundation (http://lamsfoundation.org)
 * =============================================================
 * License Information: http://lamsfoundation.org/licensing/lams/2.0/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2.0
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
 * USA
 *
 * http://www.gnu.org/licenses/gpl.txt
 * ****************************************************************
 */

package org.lamsfoundation.lams.util;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Properties;

import javax.mail.internet.MimeUtility;
import javax.servlet.http.HttpServletRequest;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.hibernate.id.Configurable;
import org.hibernate.id.IdentifierGenerator;
import org.hibernate.id.UUIDGenerator;
import org.hibernate.type.StringType;
import org.lamsfoundation.lams.learningdesign.service.ToolContentVersionFilter;
import org.lamsfoundation.lams.util.zipfile.ZipFileUtilException;
import org.w3c.dom.Document;

import com.thoughtworks.xstream.XStream;
import com.thoughtworks.xstream.converters.ConversionException;
import com.thoughtworks.xstream.io.xml.StaxDriver;
import com.thoughtworks.xstream.security.AnyTypePermission;

/**
 * General File Utilities
 */
public class FileUtil {
    private static Logger log = Logger.getLogger(FileUtil.class);

    public static final String ENCODING_UTF_8 = "UTF8";
    public static final SimpleDateFormat EXPORT_TO_SPREADSHEET_TITLE_DATE_FORMAT = new SimpleDateFormat(
	    "dd/MM/yyyy HH:mm:ss");
    public static final SimpleDateFormat EXPORT_TO_SPREADSHEET_CELL_DATE_FORMAT = new SimpleDateFormat("dd/MM/yyyy");

    public static final String LAMS_WWW_SECURE_DIR = "secure";
    public static final String LAMS_WWW_DIR = "lams-www.war";
    public static final String LAMS_RUNTIME_CONTENT_DIR = "runtime";
    private static final long numMilliSecondsInADay = 24 * 60 * 60 * 1000;

    public static final String ALLOWED_EXTENSIONS_FLASH = ".swf,.fla";
    public static final String ALLOWED_EXTENSIONS_IMAGE = ".jpg,.gif,.jpeg,.png,.bmp";
    public static final String ALLOWED_EXTENSIONS_MEDIA = ".3gp,.avi,.flv,.m4v,.mkv,.mov,.mp3,.mp4,.mpe,.mpeg,.mpg,.mpv,.mts,.m2ts,ogg,.wma,.wmv";

    protected static final String prefix = "lamstmp_"; // protected rather than private to suit junit test

    private static Transformer xmlTransformer = null;

    static {
	TransformerFactory tf = TransformerFactory.newInstance();
	try {
	    FileUtil.xmlTransformer = tf.newTransformer();
	    FileUtil.xmlTransformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
	    // a bit of beautification
	    FileUtil.xmlTransformer.setOutputProperty(OutputKeys.INDENT, "yes");
	    FileUtil.xmlTransformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
	} catch (Exception e) {
	    FileUtil.log.error("Error while initialising XML transformer", e);
	}
    }

    /**
     * Deleting a directory using File.delete() only works if the directory is empty. This method deletes a directory
     * and all of its contained files.
     *
     * This method is not transactional - if it fails to delete some contained files or directories, it will continue
     * deleting all the other files in the directory. If only a partial deletion is done, then the files and directories
     * that could not be deleted are listed in the log file, and the method returns false.
     *
     * This method has not been tested in Linux or Unix systems, so the behaviour across symbolic links is unknown.
     */
    public static boolean deleteDirectory(File directory) {
	boolean retValue = true;

	File[] files = directory.listFiles();
	if (files != null) {
	    for (int i = 0; i < files.length; i++) {
		File file = files[i];
		if (file.isDirectory()) {
		    FileUtil.deleteDirectory(file);
		} else if (!file.delete()) {
		    FileUtil.log.error("Unable to delete file " + file.getName());
		    retValue = false;
		}
	    }
	}
	if (directory.delete()) {
	    return retValue;
	} else {
	    return false;
	}
    }

    public static boolean deleteDirectory(String directoryName) throws FileUtilException {
	boolean isDeleted = false;
	if ((directoryName == null) || (directoryName.length() == 0)) {
	    throw new FileUtilException("A directory name must be specified");
	}

	File dir = new File(directoryName);
	isDeleted = FileUtil.deleteDirectory(dir);

	return isDeleted;

    }

    /**
     * Check if this directory is empty. If checkSubdirectories = true, then it also checks its subdirectories to make
     * sure they aren't empty. If checkSubdirectories = true and the directory contains empty subdirectories it will
     * return true. If checkSubdirectories = false and the directory contains empty subdirectories it will return false.
     */
    public static boolean isEmptyDirectory(String directoryName, boolean checkSubdirectories) throws FileUtilException {

	if ((directoryName == null) || (directoryName.length() == 0)) {
	    throw new FileUtilException("A directory name must be specified");
	}

	return FileUtil.isEmptyDirectory(new File(directoryName), checkSubdirectories);

    }

    private static boolean isEmptyDirectory(File directory, boolean checkSubdirectories) throws FileUtilException {

	if (directory.exists()) {
	    File files[] = directory.listFiles();

	    if (files.length > 0) {
		if (!checkSubdirectories) {
		    return false;
		} else {
		    boolean isEmpty = true;
		    for (int i = 0; (i < files.length) && isEmpty; i++) {
			File file = files[i];
			isEmpty = file.isDirectory() ? FileUtil.isEmptyDirectory(file, true) : false;
		    }
		    return isEmpty;
		}

	    } else {
		return true;
	    }
	}

	return true;
    }

    /**
     * Create a temporary directory with the name in the form lamstmp_timestamp_suffix inside the default temporary-file
     * directory for the system. This method is protected (rather than private) so that it may be called by the junit
     * tests for this class.
     *
     * @param zipFileName
     * @return name of the new directory
     * @throws ZipFileUtilException
     *             if the java io temp directory is not defined, or we are unable to calculate a unique name for the
     *             expanded directory, or an IOException occurs.
     */
    public static String createTempDirectory(String suffix) throws FileUtilException {

	String tempSysDirName = FileUtil.getTempDir();
	if (tempSysDirName == null) {
	    throw new FileUtilException(
		    "No temporary directory known to the server. [System.getProperty( \"java.io.tmpdir\" ) returns null. ]\n Cannot upload package.");
	}

	if (!(new File(tempSysDirName)).canWrite()) {
	    String javaTemp = System.getProperty("java.io.tmpdir");
	    if (!(new File(javaTemp).canWrite())) {
		throw new FileUtilException("Do not have write permissions for temporary directory: " + tempSysDirName
			+ " or java temp dir: " + javaTemp);
	    }
	    tempSysDirName = javaTemp;
	}

	String tempDirName = tempSysDirName + File.separator + FileUtil.prefix
		+ FileUtil.generateUniqueContentFolderID() + "_" + suffix;
	File tempDir = new File(tempDirName);

	// try 100 different variations. If I can't find a unique
	// one in 100 tries, then give up.
	int i = 0;
	while (tempDir.exists() && (i < 100)) {
	    tempDirName = tempSysDirName + File.separator + FileUtil.prefix + FileUtil.generateUniqueContentFolderID()
		    + "_" + suffix;
	    tempDir = new File(tempDirName);
	    i++;
	}
	if (tempDir.exists()) {
	    throw new FileUtilException(
		    "Unable to create temporary directory. The temporary filename/directory that we would use to extract files already exists: "
			    + tempDirName);
	}

	tempDir.mkdirs();
	return tempDirName;
    }

    /**
     * This method creates a directory with the name <code>directoryName</code>. Also creates any necessary parent
     * directories that may not yet exist.
     *
     * If the directoryname is null or an empty string, a FileUtilException is thrown
     *
     * @param directoryName
     *            the name of the directory to create
     * @return boolean. Returns true if the directory is created and false otherwise
     * @throws FileUtilException
     *             if the directory name is null or an empty string
     */
    public static boolean createDirectory(String directoryName) throws FileUtilException {
	boolean isCreated = false;
	// check directoryName to see if its empty or null
	if ((directoryName == null) || (directoryName.length() == 0)) {
	    throw new FileUtilException("A directory name must be specified");
	}

	File dir = new File(directoryName);
	isCreated = dir.exists() ? false : dir.mkdirs();

	return isCreated;
    }

    /**
     * Creates a subdirectory under the parent directory <code>parentDirName</code> If the parent or child directory is
     * null, FileUtilException is thrown.
     *
     * If the parent directory has not been created yet, it will be created.
     *
     *
     * @param parentDirName
     *            The name of the parent directory in which the subdirectory should be created in
     * @param subDirName
     *            The name of the subdirectory to create
     * @return boolean. Returns true if the subdirectory was created and false otherwise
     * @throws FileUtilException
     *             if the parent/child directory name is null or empty.
     */
    public static boolean createDirectory(String parentDirName, String subDirName) throws FileUtilException {
	boolean isSubDirCreated = false;
	boolean isParentDirCreated;

	if ((parentDirName == null) || (parentDirName.length() == 0) || (subDirName == null)
		|| (subDirName.length() == 0)) {
	    throw new FileUtilException("A parent or subdirectory name must be specified");
	}

	File parentDir = new File(parentDirName);
	if (!parentDir.exists()) {
	    isParentDirCreated = FileUtil.createDirectory(parentDirName);
	} else {
	    isParentDirCreated = true; // parent directory already exists
	}

	if (FileUtil.trailingForwardSlashPresent(parentDirName)) {
	    parentDirName = FileUtil.removeTrailingForwardSlash(parentDirName);
	}

	// concatenate the two together
	String combinedDirName = parentDirName + File.separator + subDirName;

	isSubDirCreated = FileUtil.createDirectory(combinedDirName);

	return isSubDirCreated && isParentDirCreated;
    }

    /**
     * If the directory name specified has a slash at the end of it such as "directoryName/", then the slash will be
     * removed and "directoryName" will be returned. The createDirectory(parentdir, childdir) method requires that there
     * is no slash at the end of the directory name.
     *
     * @param stringToModify
     * @return
     */
    public static String removeTrailingForwardSlash(String stringToModify) {
	String stringWithoutSlashAtEnd = stringToModify.substring(0, stringToModify.length() - 1);
	return stringWithoutSlashAtEnd;
    }

    /**
     * Checks to see if there is a slash at the end of the string.
     *
     * @param stringToCheck
     *            the directoryName to check
     * @return boolean. Returns true if there is a slash at the end and false if not.
     */
    public static boolean trailingForwardSlashPresent(String stringToCheck) {
	int indexOfSlash = stringToCheck.lastIndexOf("/");
	if (indexOfSlash == (stringToCheck.length() - 1)) {
	    return true;
	} else {
	    return false;
	}
    }

    public static boolean directoryExist(String directoryToCheck) {
	File dir = new File(directoryToCheck);
	return dir.exists();
    }

    /**
     * get file name from a string which may include directory information. For example : "c:\\dir\\ndp\\pp.txt"; will
     * return pp.txt.? If file has no path infomation, then just return input fileName.
     *
     */
    public static String getFileName(String fileName) {
	if (fileName == null) {
	    return "";
	}

	fileName = fileName.trim();

	int dotPos = fileName.lastIndexOf("/");
	int dotPos2 = fileName.lastIndexOf("\\");
	dotPos = Math.max(dotPos, dotPos2);
	if (dotPos == -1) {
	    return fileName;
	}
	return fileName.substring(dotPos + 1, fileName.length());

    }

    /**
     * Get file directory info.
     *
     * @param fileName
     *            with path info.
     * @return return only path info with the given fileName
     */
    public static String getFileDirectory(String fileName) {
	if (fileName == null) {
	    return "";
	}

	fileName = fileName.trim();

	int dotPos = fileName.lastIndexOf("/");
	int dotPos2 = fileName.lastIndexOf("\\");
	dotPos = Math.max(dotPos, dotPos2);
	if (dotPos == -1) {
	    return "";
	}
	// return the last char is '/'
	return fileName.substring(0, dotPos + 1);

    }

    /**
     * Merge two input parameter into full path and adjust File.separator to OS default separator as well.
     *
     * @param path
     * @param file
     *            could be file name,or sub directory path.
     * @return
     */
    public static String getFullPath(String path, String file) {
	String fullpath;
	if (path.endsWith(File.separator)) {
	    fullpath = path + file;
	} else {
	    fullpath = path + File.separator + file;
	}

	return FileUtil.makeCanonicalPath(fullpath);

    }

    public static String makeCanonicalPath(String pathfile) {
	if (File.separator.indexOf("\\") != -1) {
	    pathfile = pathfile.replaceAll("\\/", "\\\\");
	} else {
	    pathfile = pathfile.replaceAll("\\\\", File.separator);
	}

	return pathfile;
    }

    /**
     * get file extension name from a String, such as from "textabc.doc", return "doc" fileName also can contain
     * directory infomation.
     */
    public static String getFileExtension(String fileName) {
	if (fileName == null) {
	    return "";
	}

	fileName = fileName.trim();
	int dotPos = fileName.lastIndexOf(".");
	if (dotPos == -1) {
	    return "";
	}
	return fileName.substring(dotPos + 1, fileName.length());
    }

    /**
     * Check whether file is executable according to its extenstion and executable extension name list from LAMS
     * configuaration.
     *
     * @param filename
     * @return
     */
    public static boolean isExecutableFile(String filename) {
	String extname = FileUtil.getFileExtension(filename);
	FileUtil.log.debug("Check executable file for extension name " + extname);

	if (StringUtils.isBlank(extname)) {
	    return false;
	}
	extname = "." + extname;

	String exeListStr = Configuration.get(ConfigurationKeys.EXE_EXTENSIONS);
	String[] extList = StringUtils.split(exeListStr, ',');
	boolean executable = false;
	for (String ext : extList) {
	    if (StringUtils.equalsIgnoreCase(ext, extname)) {
		executable = true;
		break;
	    }
	}

	return executable;
    }

    /**
     * Verify if a file with such extension is allowed to be uploaded.
     *
     * @param fileType
     *            file type can be of the following values:File, Image, Flash, Media
     * @param fileName
     */
    public static boolean isExtensionAllowed(String fileType, String fileName) {
	String ext = UploadFileUtil.getFileExtension(fileName);
	ext = "." + ext;
	String allowedExtensions;

	if ("File".equals(fileType)) {
	    // executables are not allowed
	    return !FileUtil.isExecutableFile(fileName);

	} else if ("Image".equals(fileType)) {
	    allowedExtensions = FileUtil.ALLOWED_EXTENSIONS_IMAGE;

	} else if ("Flash".equals(fileType)) {
	    allowedExtensions = FileUtil.ALLOWED_EXTENSIONS_FLASH;

	} else if ("Media".equals(fileType)) {
	    allowedExtensions = FileUtil.ALLOWED_EXTENSIONS_MEDIA;

	} else {
	    // unknown fileType
	    return false;
	}

	String[] allowedExtensionsList = StringUtils.split(allowedExtensions, ',');
	for (String allowedExtension : allowedExtensionsList) {
	    if (StringUtils.equalsIgnoreCase(ext, allowedExtension)) {
		return true;
	    }
	}

	return false;
    }

    /**
     * Clean up any old directories in the java tmp directory, where the directory name starts with lamszip_ or lamstmp_
     * and is <numdays> days old or older. This has the potential to be a heavy call - it has to do complete directory
     * listing and then recursively delete the files and directories as needed.
     *
     * Note: this method has not been tested as it is rather hard to write a junit test for!
     *
     * @param directories
     * @return number of directories deleted
     */
    public static int cleanupOldFiles(File[] directories) {
	int numDeleted = 0;
	if (directories != null) {
	    for (int i = 0; i < directories.length; i++) {
		if (FileUtil.deleteDirectory(directories[i])) {
		    FileUtil.log.info("Directory " + directories[i].getPath() + " deleted.");
		} else {
		    FileUtil.log.info("Directory " + directories[i].getPath()
			    + " partially deleted - some directories/files could not be deleted.");
		}
		numDeleted++;
	    }
	}
	return numDeleted;
    }

    /**
     * List files in temp directory older than numDays.
     *
     * @param numDays
     *            Number of days old that the directory should be to be deleted. Must be greater than 0
     * @return array of files older than input date
     * @throws FileUtilException
     *             if numDays <= 0
     */
    public static File[] getOldTempFiles(int numDays) throws FileUtilException {
	// Contract checking
	if (numDays < 0) {
	    throw new FileUtilException("Invalid getOldTempFiles call - the parameter numDays is " + numDays
		    + ". Must not be less than 0.");
	}

	// calculate comparison date
	long newestDateToKeep = System.currentTimeMillis() - (numDays * FileUtil.numMilliSecondsInADay);
	Date date = new Date(newestDateToKeep);
	FileUtil.log.info("Getting all temp zipfile expanded directories before " + date.toString() + " (server time) ("
		+ newestDateToKeep + ")");

	File tempSysDir = new File(FileUtil.getTempDir());
	File candidates[] = tempSysDir.listFiles(new TempDirectoryFilter(newestDateToKeep, FileUtil.log));
	return candidates;
    }

    /**
     * Recursively calculates size in bytes of given file or directory.
     *
     * @param file
     * @return Size in bytes.
     */
    public static long calculateFileSize(File file) {
	if (file != null) {
	    if (file.isFile()) {
		return file.length();
	    } else if (file.isDirectory()) {
		File[] fileList = file.listFiles();
		long totalSize = 0;
		if (fileList != null) {
		    for (int i = 0; i < fileList.length; i++) {
			totalSize += FileUtil.calculateFileSize(fileList[i]);
		    }
		    return totalSize;
		} else {
		    return 0;
		}
	    }
	} else {
	    return 0;
	}
	return 0;
    }

    /**
     * Remove chars from a file name that may be invalid on a file system.
     *
     * @param name
     * @return a filename that can be saved to a file system.
     */
    public static String stripInvalidChars(String name) {
	name = name.replaceAll("\\\\", "");
	name = name.replaceAll("\\/", "");
	name = name.replaceAll("\\:", "");
	name = name.replaceAll("\\*", "");
	name = name.replaceAll("\\?", "");
	name = name.replaceAll("\\>", "");
	name = name.replaceAll("\\<", "");
	name = name.replaceAll("\\|", "");
	name = name.replaceAll("\\#", "");
	name = name.replaceAll("\\%", "");
	name = name.replaceAll("\\$", "");
	name = name.replaceAll("\\;", "");
	return name;
    }

    /**
     * Encode a filename in such a way that the UTF-8 characters won't be munged during the download by a browser. Need
     * the request to work out the user's browser type
     *
     * @return encoded filename
     * @throws UnsupportedEncodingException
     */
    public static String encodeFilenameForDownload(HttpServletRequest request, String unEncodedFilename)
	    throws UnsupportedEncodingException {

	// Different browsers handle stream downloads differently LDEV-1243
	String agent = request.getHeader("USER-AGENT");
	String filename = null;

	if ((null != agent) && (-1 != agent.indexOf("MSIE"))) {
	    // if MSIE then urlencode it
	    filename = URLEncoder.encode(unEncodedFilename, FileUtil.ENCODING_UTF_8);

	} else if ((null != agent) && (-1 != agent.indexOf("Mozilla"))) {
	    // if Mozilla then base64 url_safe encoding
	    filename = MimeUtility.encodeText(unEncodedFilename, FileUtil.ENCODING_UTF_8, "B");

	} else {
	    // any others use same filename.
	    filename = unEncodedFilename;

	}

	// wrap filename in quotes as if it contains comma character Chrome can throw a multiple headers error
	filename = "\"" + filename + "\"";

	return filename;
    }

    public static String generateUniqueContentFolderID() {
	IdentifierGenerator uuidGen = new UUIDGenerator();
	((Configurable) uuidGen).configure(StringType.INSTANCE, new Properties(), null);

//	Serializable generate(SharedSessionContractImplementor session, Object object)
	
	// lowercase to resolve OS issues
	return ((String) uuidGen.generate(null, null)).toLowerCase();
    }

    /**
     * Return content folder (unique to each learner and lesson) which is used for storing user generated content. It's
     * been used by CKEditor.
     *
     * @param toolSessionId
     * @param userId
     * @return
     */
    public static String getLearnerContentFolder(Long lessonId, Long userId) {
	return FileUtil.LAMS_RUNTIME_CONTENT_DIR + "/" + lessonId + "/" + userId;
    }

    /**
     * Return lesson's content folder which is used for storing user generated content. It's been used by CKEditor.
     *
     * @param toolSessionId
     * @param userId
     * @return
     */
    public static String getLearnerContentFolder(Long lessonId) {
	return FileUtil.LAMS_RUNTIME_CONTENT_DIR + "/" + lessonId;
    }

    /**
     * Call xstream to get the POJOs from the XML file. To make it backwardly compatible we catch any exceptions due to
     * added fields, remove the field using the ToolContentVersionFilter functionality and try to reparse. We can't
     * nominate the problem fields in advance as we are making XML created by newer versions of LAMS compatible with an
     * older version.
     *
     * This logic depends on the exception message containing the text. When we upgrade xstream, we must check that this
     * message doesn't change.
     *
     * <pre>
     * 	com.thoughtworks.xstream.converters.ConversionException: unknownField : unknownField
     * 	---- Debugging information ----
     * 	required-type       : org.lamsfoundation.lams.learningdesign.dto.LearningDesignDTO
     * 	cause-message       : unknownField : unknownField
     * 	class               : org.lamsfoundation.lams.learningdesign.dto.LearningDesignDTO
     * 	message             : unknownField : unknownField
     * 	line number         : 15
     * 	path                : /org.lamsfoundation.lams.learningdesign.dto.LearningDesignDTO/unknownField
     * 	cause-exception     : com.thoughtworks.xstream.alias.CannotResolveClassException
     * 	-------------------------------
     * </pre>
     */
    public static Object getObjectFromXML(XStream xStream, String fullFilePath) throws IOException {

	Reader file = null;
	XStream conversionXml = xStream != null ? xStream : new XStream(new StaxDriver());
	// allow parsing all classes
	conversionXml.addPermission(AnyTypePermission.ANY);
	conversionXml.ignoreUnknownElements();
	ConversionException finalException = null;
	String lastFieldRemoved = "";
	ToolContentVersionFilter contentFilter = null;
	// cap the maximum number of retries to 30 - if we add more than 30 new
	// fields then we need to rethink our
	// strategy
	int maxRetries = 30;
	int numTries = 0;

	while (true) {
	    try {

		if (numTries > maxRetries) {
		    break;
		}
		numTries++;

		file = new InputStreamReader(new FileInputStream(fullFilePath), FileUtil.ENCODING_UTF_8);
		return conversionXml.fromXML(file);

	    } catch (ConversionException ce) {
		FileUtil.log.debug("Failed import", ce);
		finalException = ce;
		file.close();

		if (ce.getMessage() == null) {
		    // can't retry, so get out of here!
		    break;

		} else {
		    // try removing the field from our XML and retry
		    String message = ce.getMessage();
		    String classname = FileUtil.extractValue(message, "required-type");
		    String fieldname = FileUtil.extractValue(message, "message");
		    /*
		     * alternative for field extraction
		     * String path = FileUtil.extractValue(message, "path");
		     * int classFieldDelimiter = path.indexOf('/', 1);
		     * String classname = path.substring(1, classFieldDelimiter);
		     * String fieldname = path.substring(classFieldDelimiter + 1);
		     */
		    if ((fieldname == null) || fieldname.equals("")
			    || lastFieldRemoved.equals(classname + "." + fieldname)) {
			// can't retry, so get out of here!
			break;
		    } else {
			if (contentFilter == null) {
			    contentFilter = new ToolContentVersionFilter();
			}

			Class problemClass = FileUtil.getClass(classname);
			if (problemClass == null) {
			    // can't retry, so get out of here!
			    break;
			}

			contentFilter.removeField(problemClass, fieldname);
			contentFilter.transformXML(fullFilePath);
			lastFieldRemoved = classname + "." + fieldname;
			FileUtil.log.debug("Retrying import after removing field " + fieldname);
			continue;
		    }
		}
	    } finally {
		if (file != null) {
		    file.close();
		}
	    }
	}
	throw finalException;
    }

    /**
     * Extract the class name or field name from a ConversionException message
     */
    private static String extractValue(String message, String fieldToLookFor) {
	try {
	    int startIndex = message.indexOf(fieldToLookFor);
	    if (startIndex > -1) {
		startIndex = message.indexOf(":", startIndex + 1);
		if ((startIndex > -1) && ((startIndex + 2) < message.length())) {
		    startIndex = startIndex + 2;
		    int endIndex = Math.min(message.indexOf(" ", startIndex), message.indexOf("\n", startIndex));
		    String value = message.substring(startIndex, endIndex);
		    return value.trim();
		}
	    }
	} catch (ArrayIndexOutOfBoundsException e) {
	}
	return "";
    }

    private static Class getClass(String classname) {
	try {
	    return Class.forName(classname);
	} catch (ClassNotFoundException e) {
	    FileUtil.log.error("Trying to remove unwanted fields from import but we can't find the matching class "
		    + classname + ". Aborting retry.", e);
	    return null;
	}
    }

    /**
     * Gets the temp dir, creates if not exists, returns java system temp dir if inaccessible
     *
     * @return
     */
    public static String getTempDir() {

	String ret = Configuration.get(ConfigurationKeys.LAMS_TEMP_DIR);
	File tempDir = new File(ret);

	// Create if not exists
	if (!tempDir.exists()) {
	    boolean success = tempDir.mkdirs();
	    if (!success) {
		FileUtil.log.error("Could not create temp directory: " + ret);
		return System.getProperty("java.io.tmpdir");
	    }
	}

	// Return java temp dir if not accessible
	if (!tempDir.canWrite()) {
	    return System.getProperty("java.io.tmpdir");
	} else {
	    return ret;
	}
    }

    public static void writeXMLtoFile(Document doc, File file) throws IOException {
	StreamResult streamResult = new StreamResult(new FileOutputStream(file));
	DOMSource domSource = new DOMSource(doc);
	try {
	    FileUtil.xmlTransformer.transform(domSource, streamResult);
	} catch (TransformerException e) {
	    throw new IOException("Error while writing out XML document to file", e);
	}
    }

    public static String writeXMLtoString(Document doc) {
	try (StringWriter writer = new StringWriter()) {
	    StreamResult streamResult = new StreamResult(writer);
	    DOMSource domSource = new DOMSource(doc);
	    FileUtil.xmlTransformer.transform(domSource, streamResult);
	    return writer.toString();
	} catch (Exception e) {
	    FileUtil.log.error("Error while writing out XML document to string", e);
	    return null;
	}
    }
}