/*
* Copyright (C) 2014 ddecap
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package be.ugent.intec.halvade.utils;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import htsjdk.samtools.SAMSequenceDictionary;
import htsjdk.samtools.SAMSequenceRecord;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
*
* @author ddecap
*/
public class HalvadeConf {
/*
* Custom configuration
* helps to set some fixed parameters in shared configuration
*/
private static final String java = "Java";
public static void setJava(Configuration conf, String val) {
conf.set(java, val);
}
public static String getJava(Configuration conf) {
return conf.get(java);
}
private static final String filterDBSnp = "filterSNPs";
public static void setFilterDBSnp(Configuration conf, boolean filter) {
if(filter)
conf.set(filterDBSnp, "true");
else
conf.set(filterDBSnp, "false");
}
public static boolean getFilterDBSnp(Configuration conf) {
String s = conf.get(filterDBSnp);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String useGenotyper = "usegeno";
public static void setUseUnifiedGenotyper(Configuration conf, boolean use) {
if(use)
conf.set(useGenotyper, "true");
else
conf.set(useGenotyper, "false");
}
public static boolean getUseUnifiedGenotyper(Configuration conf) {
String s = conf.get(useGenotyper);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String keepFiles = "keepFiles";
public static void setKeepFiles(Configuration conf, boolean use) {
if(use)
conf.set(keepFiles, "true");
else
conf.set(keepFiles, "false");
}
public static boolean getKeepFiles(Configuration conf) {
String s = conf.get(keepFiles);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String useElPrep = "useiprep";
public static void setUseElPrep(Configuration conf, boolean use) {
if(use)
conf.set(useElPrep, "true");
else
conf.set(useElPrep, "false");
}
public static boolean getUseElPrep(Configuration conf) {
String s = conf.get(useElPrep);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String redist = "redistribute";
public static void setRedistribute(Configuration conf, boolean val) {
if(val)
conf.set(redist, "true");
else
conf.set(redist, "false");
}
public static boolean getRedistribute(Configuration conf) {
String s = conf.get(redist);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String paired = "ispaired";
public static void setIsPaired(Configuration conf, boolean isPaired) {
if(isPaired)
conf.set(paired, "true");
else
conf.set(paired, "false");
}
public static boolean getIsPaired(Configuration conf) {
String s = conf.get(paired);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String outdir = "outputdir";
public static void setOutDir(Configuration conf, String val) {
if(!val.endsWith("/"))
conf.set(outdir, val + "/");
else
conf.set(outdir, val);
}
public static String getOutDir(Configuration conf) {
return conf.get(outdir);
}
private static final String refSize = "refsize";
private static final long HUMAN_REF_SIZE = 3137161264L; // based on ucsc.hg19.fasta (see gatk bundle)
public static void setRefSize(Configuration conf, long val) {
conf.setLong(refSize, val);
}
public static long getRefSize(Configuration conf) {
return conf.getLong(refSize, HUMAN_REF_SIZE);
}
private static final String vCores = "vcores";
public static void setVcores(Configuration conf, int val) {
conf.setInt(vCores, val);
}
public static int getVcores(Configuration conf) {
return conf.getInt(vCores, 1);
}
private static final String pass2Suffix = "pass2suffix";
public static void setPass2Suffix(Configuration conf, String val) {
conf.set(pass2Suffix, val);
Logger.DEBUG("setting 2 pass suffix to: " + val);
}
public static String getPass2Suffix(Configuration conf) {
return conf.get(pass2Suffix);
}
private static final String mapThreads = "mapthreads";
public static void setMapThreads(Configuration conf, int val) {
conf.setInt(mapThreads, val);
}
public static int getMapThreads(Configuration conf) {
return conf.getInt(mapThreads, 1);
}
private static final String reduceThreads = "reducethreads";
public static void setReducerThreads(Configuration conf, int val) {
conf.setInt(reduceThreads, val);
}
public static int getReducerThreads(Configuration conf) {
return conf.getInt(reduceThreads, 1);
}
private static final String scratchTempDirName = "tempdir";
public static void setScratchTempDir(Configuration conf, String val) {
if(!val.endsWith("/"))
conf.set(scratchTempDirName, val + "/");
else
conf.set(scratchTempDirName, val);
}
public static String getScratchTempDir(Configuration conf) {
return conf.get(scratchTempDirName);
}
private static final String gff = "gff";
public static String getGff(Configuration conf) {
return conf.get(gff);
}
public static void setGff(Configuration conf, String val) {
conf.set(gff, val);
}
private static final String readgroup = "readgroup";
public static void setReadGroup(Configuration conf, String val) {
conf.set(readgroup, val);
}
public static String getReadGroup(Configuration conf) {
return conf.get(readgroup);
}
private static final String refOnScratchName = "scratchref";
public static void setRefDirOnScratch(Configuration conf, String val) {
conf.set(refOnScratchName, val);
}
public static String getRefDirOnScratch(Configuration conf) {
return conf.get(refOnScratchName);
}
private static final String sitesOnHDFSName = "hdfssites";
private static final String numberOfSites = "numsites";
public static void setKnownSitesOnHDFS(Configuration conf, String[] val) throws IOException, URISyntaxException {
conf.setInt(numberOfSites, val.length);
FileSystem fs;
for(int i = 0; i < val.length;i ++) {
// check if dir add all files!
fs = FileSystem.get(new URI(val[i]), conf);
if(fs.isFile(new Path(val[i]))) {
conf.set(sitesOnHDFSName + i, val[i]);
} else {
FileStatus[] files = fs.listStatus(new Path(val[i]));
for(FileStatus file : files) {
if (!file.isDir()) {
conf.set(sitesOnHDFSName + i, file.getPath().toString());
}
}
}
}
}
public static String[] getKnownSitesOnHDFS(Configuration conf) {
int size = conf.getInt(numberOfSites, 0);
String[] sites = new String[size];
for(int i = 0; i < size;i ++) {
sites[i] = conf.get(sitesOnHDFSName + i);
}
return sites;
}
private static final String tasksDone = "tasksDone/";
public static void clearTaskFiles(Configuration conf) throws IOException, URISyntaxException {
String filepath = conf.get(outdir) + tasksDone;
FileSystem fs = FileSystem.get(new URI(filepath), conf);
fs.delete(new Path(filepath), true);
}
public static boolean addTaskRunning(Configuration conf, String val) throws IOException, URISyntaxException {
val = val.substring(0, val.lastIndexOf("_")); // rewrite file if second attempt
String filepath = conf.get(outdir) + tasksDone + val;
FileSystem fs = FileSystem.get(new URI(filepath), conf);
return fs.createNewFile(new Path(filepath));
}
private static final String totalContainers = "containers";
public static void setMapContainerCount(Configuration conf, int val) {
conf.setInt(totalContainers, val);
}
public static int getMapContainerCount(Configuration conf) {
return conf.getInt(totalContainers, 1);
}
public static int getMapTasksLeft(Configuration conf) throws IOException, URISyntaxException {
int containers = conf.getInt(totalContainers, 1);
int tasks = 0;
String filedir = conf.get(outdir) + tasksDone;
FileSystem fs = FileSystem.get(new URI(filedir), conf);
FileStatus[] files = fs.listStatus(new Path(filedir));
for(FileStatus file : files) {
if (!file.isDirectory()) {
tasks++;
}
}
Logger.DEBUG("containers left: " + (Integer.parseInt(conf.get("mapred.map.tasks")) - tasks));
return Integer.parseInt(conf.get("mapred.map.tasks")) - tasks;
}
public static boolean allTasksCompleted(Configuration conf) throws IOException, URISyntaxException {
int tasks = 0;
String filedir = conf.get(outdir) + tasksDone;
FileSystem fs = FileSystem.get(new URI(filedir), conf);
FileStatus[] files = fs.listStatus(new Path(filedir));
for(FileStatus file : files) {
if (!file.isDirectory()) {
tasks++;
}
}
Logger.DEBUG("tasks started: " + tasks);
return tasks >= Integer.parseInt(conf.get("mapred.map.tasks"));
}
private static final String refOnHDFSName = "hdfsref";
public static void setRefOnHDFS(Configuration conf, String val) {
conf.set(refOnHDFSName, val);
}
public static String getRefOnHDFS(Configuration conf) {
return conf.get(refOnHDFSName);
}
private static final String starDirOnHDFSName = "hdfsSTARref";
public static void setStarDirOnHDFS(Configuration conf, String val) {
conf.set(starDirOnHDFSName, val);
}
public static String getStarDirOnHDFS(Configuration conf) {
return conf.get(starDirOnHDFSName);
}
private static final String starDirPass2HDFSName = "hdfsSTARrefPass2";
private static final String pass2GenomeDirName = "pass2STARGenome/";
public static void setStarDirPass2HDFS(Configuration conf, String val) {
if(!val.endsWith("/"))
conf.set(starDirPass2HDFSName, val + "/" + pass2GenomeDirName);
else
conf.set(starDirPass2HDFSName, val + pass2GenomeDirName);
}
public static String getStarDirPass2HDFS(Configuration conf) {
return conf.get(starDirPass2HDFSName);
}
public static int getNumberOfFiles(Configuration conf) {
return Integer.parseInt(conf.get("mapred.map.tasks"));
}
private static final String dictionarySequenceName = "seqdictionary_";
private static final String dictionarySequenceLength = "seqdictionarylength_";
private static final String dictionaryCount = "seqcount";
public static void setSequenceDictionary(Configuration conf, SAMSequenceDictionary dict) throws IOException, URISyntaxException {
int counter = 0;
for(SAMSequenceRecord seq : dict.getSequences()) {
conf.set(dictionarySequenceName + counter, seq.getSequenceName());
conf.setInt(dictionarySequenceLength + counter, seq.getSequenceLength());
counter++;
}
conf.setInt(dictionaryCount, counter);
}
public static SAMSequenceDictionary getSequenceDictionary(Configuration conf) throws IOException {
int counter = conf.getInt(dictionaryCount, 0);
SAMSequenceDictionary dict = new SAMSequenceDictionary();
for(int i = 0; i < counter; i++) {
String seqName = conf.get(dictionarySequenceName + i);
int seqLength = conf.getInt(dictionarySequenceLength + i, 0);
SAMSequenceRecord seq = new SAMSequenceRecord(seqName, seqLength);
dict.addSequence(seq);
}
return dict;
}
private static final String chrList = "chrlist";
public static void setChrList(Configuration conf, String val) {
conf.set(chrList, val);
}
public static String getChrList(Configuration conf) {
return conf.get(chrList);
}
private static final String minChrSize = "minchrSize";
private static final int DEFAULT_MIN_CHR_SIZE = 63025520;
public static void setMinChrLength(Configuration conf, int val) {
Logger.DEBUG("min chr size set to " + val, 3);
conf.setInt(minChrSize, val);
}
public static int getMinChrLength(Configuration conf) {
return conf.getInt(minChrSize, DEFAULT_MIN_CHR_SIZE);
}
private static final String rna = "isRNA";
public static void setIsRNA(Configuration conf, boolean isRNA) {
if(isRNA)
conf.set(rna, "true");
else
conf.set(rna, "false");
}
public static boolean getIsRNA(Configuration conf) {
String s = conf.get(rna);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String mergebam = "mergeBam";
public static void setMergeBam(Configuration conf, boolean mergeBam) {
if(mergeBam)
conf.set(mergebam, "true");
else
conf.set(mergebam, "false");
}
public static boolean getMergeBam(Configuration conf) {
String s = conf.get(mergebam);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String scc = "scc";
private static final float DEFAULT_DNA_SCC = 30.0f;
private static final float DEFAULT_RNA_SCC = 20.0f;
public static void setSCC(Configuration conf, double val) {
conf.setFloat(scc, (float) val);
}
public static double getSCC(Configuration conf, boolean isRNA) {
if(isRNA)
return conf.getFloat(scc, DEFAULT_RNA_SCC);
else
return conf.getFloat(scc, DEFAULT_DNA_SCC);
}
private static final String sec = "sec";
private static final float DEFAULT_DNA_SEC = 30.0f;
private static final float DEFAULT_RNA_SEC = 20.0f;
public static void setSEC(Configuration conf, double val) {
conf.setFloat(sec, (float)val);
}
public static double getSEC(Configuration conf, boolean isRNA) {
if(isRNA)
return conf.getFloat(sec, DEFAULT_RNA_SEC);
else
return conf.getFloat(sec, DEFAULT_DNA_SEC);
}
private static final String fbed = "filterBed";
public static void setFilterBed(Configuration conf, String bed_) {
conf.set(fbed, bed_);
}
public static String getFilterBed(Configuration conf) {
return conf.get(fbed);
}
private static final String bed = "fullBed";
public static void setBed(Configuration conf, String bed_) {
conf.set(bed, bed_);
}
public static String getBed(Configuration conf) {
return conf.get(bed);
}
private static final String bedRegions = "regionsBed";
public static void setBedRegions(Configuration conf, String bed) {
conf.set(bedRegions, bed);
}
public static String getBedRegions(Configuration conf) {
return conf.get(bedRegions);
}
private static final String reportAllVariant = "reportAllVariant";
public static void setReportAllVariant(Configuration conf, boolean val) {
if(val)
conf.set(reportAllVariant, "true");
else
conf.set(reportAllVariant, "false");
}
public static boolean getReportAllVariant(Configuration conf) {
String s = conf.get(reportAllVariant, "false");
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String keepChrSplitPairs = "keepChrSplitPairs";
public static void setkeepChrSplitPairs(Configuration conf, boolean val) {
if(val)
conf.set(keepChrSplitPairs, "true");
else
conf.set(keepChrSplitPairs, "false");
}
public static boolean getkeepChrSplitPairs(Configuration conf) {
String s = conf.get(keepChrSplitPairs);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String inputDir = "InputDir";
public static void setInputDir(Configuration conf, String val) {
conf.set(inputDir, val);
}
public static String getInputDir(Configuration conf) {
return conf.get(inputDir);
}
private static final String headerFile = "headerf";
public static void setHeaderFile(Configuration conf, String val) {
conf.set(headerFile, val);
}
public static String getHeaderFile(Configuration conf) {
return conf.get(headerFile);
}
private static final String starGtf = "stargtf";
public static void setStarGtf(Configuration conf, String val) {
conf.set(starGtf, val);
}
public static String getStarGtf(Configuration conf) {
return conf.get(starGtf);
}
/*
* Custom Arguments for all commands used in Halvade
*
*/
private static final String customArgs = "ca_";
public static void setCustomArgs(Configuration conf, String programName, String toolName, String val) {
conf.set(customArgs + programName.toLowerCase() + "_" + toolName.toLowerCase(), val);
}
public static String getCustomArgs(Configuration conf, String programName, String toolName) {
return conf.get(customArgs + programName.toLowerCase() + "_" + toolName.toLowerCase());
}
private static final String starPass2 = "starPass2";
public static void setIsPass2(Configuration conf, boolean val) {
if(val)
conf.set(starPass2, "true");
else
conf.set(starPass2, "false");
}
public static boolean getIsPass2(Configuration conf) {
String s = conf.get(starPass2);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String inputIsBam = "inputIsBam";
public static void setInputIsBam(Configuration conf, boolean val) {
if(val)
conf.set(inputIsBam, "true");
else
conf.set(inputIsBam, "false");
}
public static boolean inputIsBam(Configuration conf) {
String s = conf.get(inputIsBam);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String updateRG = "updateRG";
public static void setUpdateReadGroup(Configuration conf, boolean val) {
if(val)
conf.set(updateRG, "true");
else
conf.set(updateRG, "false");
}
public static boolean getUpdateReadGroup(Configuration conf) {
String s = conf.get(updateRG);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String keepDups = "keepDups";
public static void setKeepDups(Configuration conf, boolean val) {
if(val)
conf.set(keepDups, "true");
else
conf.set(keepDups, "false");
}
public static boolean getKeepDups(Configuration conf) {
String s = conf.get(keepDups);
if(s.equalsIgnoreCase("true"))
return true;
else
return false;
}
private static final String refDirIsSet = "refDirIsSet";
public static void setRefDirIsSet(Configuration conf, boolean val) {
conf.setBoolean(refDirIsSet, val);
}
public static boolean getRefDirIsSet(Configuration conf) {
return conf.getBoolean(refDirIsSet, false);
}
private static final String fixQualEnc = "fixQualEnc";
public static void setFixQualEnc(Configuration conf, boolean val) {
conf.setBoolean(fixQualEnc, val);
}
public static boolean getFixQualEnc(Configuration conf) {
return conf.getBoolean(fixQualEnc, false);
}
}
