/******************************************************************************* * In the Hi-WAY project we propose a novel approach of executing scientific * workflows processing Big Data, as found in NGS applications, on distributed * computational infrastructures. The Hi-WAY software stack comprises the func- * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster * for Apache Hadoop 2.x (YARN). * * List of Contributors: * * Marc Bux (HU Berlin) * Jörgen Brandt (HU Berlin) * Hannes Schuh (HU Berlin) * Ulf Leser (HU Berlin) * * Jörgen Brandt is funded by the European Commission through the BiobankCloud * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through * research training group SOAMED (GRK 1651). * * Copyright 2014 Humboldt-Universität zu Berlin * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. ******************************************************************************/ package de.huberlin.wbi.hiway.am.cuneiforme; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; import java.util.Map; import org.apache.hadoop.yarn.api.records.ContainerId; import org.json.JSONException; import org.json.JSONObject; import de.huberlin.wbi.cfjava.cuneiform.HaltMsg; import de.huberlin.wbi.cfjava.cuneiform.RemoteWorkflow; import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; import de.huberlin.wbi.hiway.am.WorkflowDriver; import de.huberlin.wbi.hiway.common.Data; import de.huberlin.wbi.hiway.common.HiWayConfiguration; import de.huberlin.wbi.hiway.common.TaskInstance; public class CuneiformEApplicationMaster extends WorkflowDriver { public static void main(String[] args) { WorkflowDriver.launch(new CuneiformEApplicationMaster(), args); } private RemoteWorkflow workflow; private Map<CuneiformETaskInstance, JSONObject> requests; public CuneiformEApplicationMaster() { super(); requests = new HashMap<>(); setDetermineFileSizes(); } @Override public Collection<TaskInstance> parseWorkflow() { try (BufferedReader reader = new BufferedReader(new FileReader(getWorkflowFile().getLocalPath().toString()))) { WorkflowDriver.writeToStdout("Parsing Cuneiform workflow " + getWorkflowFile()); StringBuilder sb = new StringBuilder(); String line; while ((line = reader.readLine()) != null) { sb.append(line).append("\n"); } String ip = getConf().get(HiWayConfiguration.HIWAY_WORKFLOW_LANGUAGE_CUNEIFORME_SERVER_IP); WorkflowDriver.writeToStdout("Connecting to Cuneiform server at " + ip); workflow = new RemoteWorkflow(sb.toString(), ip); } catch (IOException e) { e.printStackTrace(System.out); System.exit(-1); } return getNextTasks(); } @Override public boolean isDone() { return !workflow.isRunning(); } @Override protected void loop() { try { workflow.update(); } catch (IOException e) { e.printStackTrace(System.out); } getScheduler().addTasks(getNextTasks()); super.loop(); } @Override protected void finish() { HaltMsg haltMsg = workflow.getHaltMsg(); if (haltMsg.isOk()) { } else if (haltMsg.isErrorTask()) { WorkflowDriver.writeToStdout("Workflow execution halted due to task failure."); WorkflowDriver.writeToStdout("app line: " + haltMsg.getAppLine()); WorkflowDriver.writeToStdout("lam name: " + haltMsg.getLamName()); WorkflowDriver.writeToStdout("output: " + haltMsg.getOutput()); WorkflowDriver.writeToStdout("script: " + haltMsg.getScript()); } else if (haltMsg.isErrorWorkflow()) { WorkflowDriver.writeToStdout("Workflow execution halted due to workflow failure."); WorkflowDriver.writeToStdout("line: " + haltMsg.getLine()); WorkflowDriver.writeToStdout("module: " + haltMsg.getModule()); WorkflowDriver.writeToStdout("reason: " + haltMsg.getReason()); } else { throw new UnsupportedOperationException(); } super.finish(); } @Override protected Collection<String> getOutput() { Collection<String> outputs = new LinkedList<>(); HaltMsg haltMsg = workflow.getHaltMsg(); if (haltMsg.isOk()) { for (String output : haltMsg.getResult()) { outputs.add(files.containsKey(output) ? files.get(output).getHdfsPath().toString() : output); } } return outputs; } private Collection<TaskInstance> getNextTasks() { Collection<TaskInstance> tasks = new LinkedList<>(); while (workflow.hasNextRequest()) { JSONObject request = workflow.nextRequest(); // TODO: parse request String taskName = RemoteWorkflow.getLamName(request); CuneiformETaskInstance task = new CuneiformETaskInstance(getRunId(), taskName); requests.put(task, request); for (String fileName : RemoteWorkflow.getInputSet(request)) { Data file = files.get(fileName); if (file == null) { file = new Data(fileName); files.put(fileName, file); } task.addInputData(file); } try (BufferedWriter writer = new BufferedWriter(new FileWriter(task.getId() + "_request"))) { writer.write(request.toString()); } catch (IOException e) { e.printStackTrace(System.out); System.exit(-1); } task.setCommand("effi -r true " + task.getId() + "_request " + task.getId() + "_reply"); tasks.add(task); writeEntryToLog(new JsonReportEntry(task.getWorkflowId(), task.getTaskId(), task.getTaskName(), task.getLanguageLabel(), Long.valueOf(task.getId()), null, JsonReportEntry.KEY_INVOC_SCRIPT, task.getCommand())); writeEntryToLog(new JsonReportEntry(task.getWorkflowId(), task.getTaskId(), task.getTaskName(), task.getLanguageLabel(), Long.valueOf(task.getId()), null, JsonReportEntry.KEY_INVOC_EXEC, request.toString())); } return tasks; } @Override public void taskSuccess(TaskInstance task, ContainerId containerId) { try { (new Data(task.getId() + "_reply", containerId.toString())).stageIn(); JSONObject reply = parseEffiFile(task.getId() + "_reply"); workflow.addReply(reply); writeEntryToLog(new JsonReportEntry(task.getWorkflowId(), task.getTaskId(), task.getTaskName(), task.getLanguageLabel(), Long.valueOf(task.getId()), null, JsonReportEntry.KEY_INVOC_OUTPUT, reply.toString())); for (String fileName : RemoteWorkflow.getOutputSet(requests.get(task), reply)) { files.put(fileName, new Data(fileName, containerId.toString())); } } catch (IOException | JSONException e) { e.printStackTrace(System.out); System.exit(-1); } } public static JSONObject parseEffiFile(String fileName) throws JSONException { StringBuilder sb = new StringBuilder(); try (BufferedReader reader = new BufferedReader(new FileReader(fileName))) { String line; while ((line = reader.readLine()) != null) { sb.append(line).append("\n"); } } catch (IOException e) { e.printStackTrace(System.out); System.exit(-1); } return new JSONObject(sb.toString()); } }