org.apache.zeppelin.interpreter.Interpreter Java Examples

The following examples show how to use org.apache.zeppelin.interpreter.Interpreter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: YarnInterpreterLauncherIntegrationTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testLaunchShellInYarn() throws YarnException, InterpreterException, InterruptedException {
  InterpreterSetting shellInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("sh");
  shellInterpreterSetting.setProperty("zeppelin.interpreter.launcher", "yarn");
  shellInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());

  Interpreter shellInterpreter = interpreterFactory.getInterpreter("sh", new ExecutionContextBuilder().setUser("user1").setNoteId("note1").setDefaultInterpreterGroup("sh").createExecutionContext());

  InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").build();
  InterpreterResult interpreterResult = shellInterpreter.interpret("pwd", context);
  assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
  assertTrue(interpreterResult.toString(), interpreterResult.message().get(0).getData().contains("/usercache/"));

  Thread.sleep(1000);
  // 1 yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(1, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 
Example #2
Source File: RemoteInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertDynamicForms() throws InterpreterException {
  GUI gui = new GUI();
  OptionInput.ParamOption[] paramOptions = {
      new OptionInput.ParamOption("value1", "param1"),
      new OptionInput.ParamOption("value2", "param2")
  };
  List<Object> defaultValues = new ArrayList();
  defaultValues.add("default1");
  defaultValues.add("default2");
  gui.checkbox("checkbox_id", paramOptions, defaultValues);
  gui.select("select_id", paramOptions, "default");
  gui.textbox("textbox_id");
  Map<String, Input> expected = new LinkedHashMap<>(gui.getForms());
  Interpreter interpreter = interpreterSetting.getDefaultInterpreter("user1", "note1");
  InterpreterContext context = createDummyInterpreterContext();

  interpreter.interpret("text", context);
  assertArrayEquals(expected.values().toArray(), gui.getForms().values().toArray());
}
 
Example #3
Source File: RemoteInterpreterServer.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
protected Interpreter getInterpreter(String sessionId, String className) throws TException {
  if (interpreterGroup == null) {
    throw new TException(
        new InterpreterException("Interpreter instance " + className + " not created"));
  }
  synchronized (interpreterGroup) {
    List<Interpreter> interpreters = interpreterGroup.get(sessionId);
    if (interpreters == null) {
      throw new TException(
          new InterpreterException("Interpreter " + className + " not initialized"));
    }
    for (Interpreter inp : interpreters) {
      if (inp.getClassName().equals(className)) {
        return inp;
      }
    }
  }
  throw new TException(new InterpreterException("Interpreter instance "
      + className + " not found"));
}
 
Example #4
Source File: RemoteInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailToLaunchInterpreterProcess_InvalidRunner() {
  try {
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName(), "invalid_runner");
    final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
    final InterpreterContext context1 = createDummyInterpreterContext();
    // run this dummy interpret method first to launch the RemoteInterpreterProcess to avoid the
    // time overhead of launching the process.
    try {
      interpreter1.interpret("1", context1);
      fail("Should not be able to launch interpreter process");
    } catch (InterpreterException e) {
      assertTrue(ExceptionUtils.getStackTrace(e).contains("No such file or directory"));
    }
  } finally {
    System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName());
  }
}
 
Example #5
Source File: RemoteInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiInterpreterSession() {
  interpreterSetting.getOption().setPerUser(InterpreterOption.SCOPED);
  Interpreter interpreter1_user1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
  Interpreter interpreter2_user1 = interpreterSetting.getInterpreter("user1", "note1", "echo");
  assertEquals(interpreter1_user1.getInterpreterGroup(), interpreter2_user1.getInterpreterGroup());
  assertEquals(interpreter1_user1.getScheduler(), interpreter2_user1.getScheduler());

  Interpreter interpreter1_user2 = interpreterSetting.getInterpreter("user2", "note1", "sleep");
  Interpreter interpreter2_user2 = interpreterSetting.getInterpreter("user2", "note1", "echo");
  assertEquals(interpreter1_user2.getInterpreterGroup(), interpreter2_user2.getInterpreterGroup());
  assertEquals(interpreter1_user2.getScheduler(), interpreter2_user2.getScheduler());

  // scheduler is shared in session but not across session
  assertNotEquals(interpreter1_user1.getScheduler(), interpreter1_user2.getScheduler());
}
 
Example #6
Source File: RemoteInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailToLaunchInterpreterProcess_ErrorInRunner() {
  try {
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName(),
             zeppelinHome.getAbsolutePath() + "/zeppelin-zengine/src/test/resources/bin/interpreter_invalid.sh");
    final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
    final InterpreterContext context1 = createDummyInterpreterContext();
    // run this dummy interpret method first to launch the RemoteInterpreterProcess to avoid the
    // time overhead of launching the process.
    try {
      interpreter1.interpret("1", context1);
      fail("Should not be able to launch interpreter process");
    } catch (InterpreterException e) {
      assertTrue(ExceptionUtils.getStackTrace(e).contains("invalid_command: command not found"));
    }
  } finally {
    System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName());
  }
}
 
Example #7
Source File: RemoteInterpreterServer.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
public void cancel(String sessionId,
                   String className,
                   RemoteInterpreterContext interpreterContext) throws TException {
  LOGGER.info("cancel {} {}", className, interpreterContext.getParagraphId());
  Interpreter intp = getInterpreter(sessionId, className);
  String jobId = interpreterContext.getParagraphId();
  Job job = intp.getScheduler().getJob(jobId);

  if (job != null && job.getStatus() == Status.PENDING) {
    job.setStatus(Status.ABORT);
  } else {
    Thread thread = new Thread( ()-> {
      try {
        intp.cancel(convert(interpreterContext, null));
      } catch (InterpreterException e) {
        LOGGER.error("Fail to cancel paragraph: " + interpreterContext.getParagraphId());
      }
    });
    thread.start();
  }
}
 
Example #8
Source File: RemoteInterpreterServer.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
public int getProgress(String sessionId, String className,
                       RemoteInterpreterContext interpreterContext)
    throws TException {
  Integer manuallyProvidedProgress = progressMap.get(interpreterContext.getParagraphId());
  if (manuallyProvidedProgress != null) {
    return manuallyProvidedProgress;
  } else {
    Interpreter intp = getInterpreter(sessionId, className);
    if (intp == null) {
      throw new TException("No interpreter {} existed for session {}".format(
          className, sessionId));
    }
    try {
      return intp.getProgress(convert(interpreterContext, null));
    } catch (InterpreterException e) {
      throw new TException("Fail to getProgress", e);
    }
  }
}
 
Example #9
Source File: RemoteInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailToLaunchInterpreterProcess_Timeout() {
  try {
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName(),
            zeppelinHome.getAbsolutePath() + "/zeppelin-zengine/src/test/resources/bin/interpreter_timeout.sh");
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "10000");
    final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
    final InterpreterContext context1 = createDummyInterpreterContext();
    // run this dummy interpret method first to launch the RemoteInterpreterProcess to avoid the
    // time overhead of launching the process.
    try {
      interpreter1.interpret("1", context1);
      fail("Should not be able to launch interpreter process");
    } catch (InterpreterException e) {
      assertTrue(ExceptionUtils.getStackTrace(e).contains("Interpreter Process creation is time out"));
    }
  } finally {
    System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName());
    System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName());
  }
}
 
Example #10
Source File: RInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testInvalidR() throws InterpreterException {
  tearDown();

  Properties properties = new Properties();
  properties.setProperty("zeppelin.R.cmd", "invalid_r");
  properties.setProperty("spark.master", "local");
  properties.setProperty("spark.app.name", "test");

  InterpreterGroup interpreterGroup = new InterpreterGroup();
  Interpreter rInterpreter = new LazyOpenInterpreter(new RInterpreter(properties));
  interpreterGroup.addInterpreterToSession(rInterpreter, "session_1");
  rInterpreter.setInterpreterGroup(interpreterGroup);

  InterpreterContext context = getInterpreterContext();
  InterpreterContext.set(context);

  try {
    rInterpreter.interpret("1+1", getInterpreterContext());
    fail("Should fail to open SparkRInterpreter");
  } catch (InterpreterException e) {
    String stacktrace = ExceptionUtils.getStackTrace(e);
    assertTrue(stacktrace, stacktrace.contains("No such file or directory"));
  }
}
 
Example #11
Source File: LocalRecoveryStorageTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleInterpreterProcess() throws InterpreterException, IOException {
  InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test");
  interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);

  Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
  RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
  InterpreterContext context1 = InterpreterContext.builder()
          .setNoteId("noteId")
          .setParagraphId("paragraphId")
          .build();
  remoteInterpreter1.interpret("hello", context1);

  assertEquals(1, interpreterSettingManager.getRecoveryStorage().restore().size());

  interpreterSetting.close();
  assertEquals(0, interpreterSettingManager.getRecoveryStorage().restore().size());
}
 
Example #12
Source File: RemoteInterpreterServer.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
public String getStatus(String sessionId, String jobId)
    throws TException {
  if (interpreterGroup == null) {
    return Status.UNKNOWN.name();
  }

  synchronized (interpreterGroup) {
    List<Interpreter> interpreters = interpreterGroup.get(sessionId);
    if (interpreters == null) {
      return Status.UNKNOWN.name();
    }

    for (Interpreter intp : interpreters) {
      Scheduler scheduler = intp.getScheduler();
      if (scheduler != null) {
        Job job = scheduler.getJob(jobId);
        if (job != null) {
          return job.getStatus().name();
        }
      }
    }
  }
  return Status.UNKNOWN.name();
}
 
Example #13
Source File: CassandraInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
@Ignore
//TODO(n.a.) activate test when using Java 8 and C* 3.x
public void should_describe_aggregate() throws Exception {
  //Given
  Properties properties = new Properties();
  properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1");
  properties.setProperty(CASSANDRA_PORT,  "9042");
  Interpreter interpreter = new CassandraInterpreter(properties);
  interpreter.open();

  final String query = "DESCRIBE AGGREGATES;";

  //When
  final InterpreterResult actual = interpreter.interpret(query, intrContext);

  //Then
  assertThat(actual.code()).isEqualTo(Code.SUCCESS);
}
 
Example #14
Source File: CassandraInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
@Ignore
//TODO(n.a.) activate test when using Java 8 and C* 3.x
public void should_describe_materialized_view() throws Exception {
  //Given
  Properties properties = new Properties();
  properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1");
  properties.setProperty(CASSANDRA_PORT,  "9042");
  Interpreter interpreter = new CassandraInterpreter(properties);
  interpreter.open();

  final String query = "DESCRIBE MATERIALIZED VIEWS;";

  //When
  final InterpreterResult actual = interpreter.interpret(query, intrContext);

  //Then
  assertThat(actual.code()).isEqualTo(Code.SUCCESS);
}
 
Example #15
Source File: HeliumApplicationFactoryTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testInterpreterUnbindOfNullReplParagraph() throws IOException {
  // create note
  Note note1 = notebook.createNote("note1", anonymous);

  // add paragraph with invalid magic
  Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
  p1.setText("%fake ");

  // make sure that p1's repl is null
  Interpreter intp = null;
  try {
    intp = p1.getBindedInterpreter();
    fail("Should throw InterpreterNotFoundException");
  } catch (InterpreterNotFoundException e) {

  }

  // remove note
  notebook.removeNote(note1, anonymous);
}
 
Example #16
Source File: PigQueryInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws InterpreterException {
  Properties properties = new Properties();
  properties.put("zeppelin.pig.execType", "local");
  properties.put("zeppelin.pig.maxResult", "20");

  pigInterpreter = new LazyOpenInterpreter(new PigInterpreter(properties));
  pigQueryInterpreter = new LazyOpenInterpreter(new PigQueryInterpreter(properties));
  List<Interpreter> interpreters = new ArrayList();
  interpreters.add(pigInterpreter);
  interpreters.add(pigQueryInterpreter);
  InterpreterGroup group = new InterpreterGroup();
  group.put("note_id", interpreters);
  pigInterpreter.setInterpreterGroup(group);
  pigQueryInterpreter.setInterpreterGroup(group);
  pigInterpreter.open();
  pigQueryInterpreter.open();

  context = InterpreterContext.builder().setParagraphId("paragraphId").build();
}
 
Example #17
Source File: PythonInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws InterpreterException {

  intpGroup = new InterpreterGroup();

  Properties properties = new Properties();
  properties.setProperty("zeppelin.python.maxResult", "3");
  properties.setProperty("zeppelin.python.useIPython", "false");
  properties.setProperty("zeppelin.python.gatewayserver_address", "127.0.0.1");

  interpreter = new LazyOpenInterpreter(new PythonInterpreter(properties));

  intpGroup.put("note", new LinkedList<Interpreter>());
  intpGroup.get("note").add(interpreter);
  interpreter.setInterpreterGroup(intpGroup);

  InterpreterContext.set(getInterpreterContext());
  interpreter.open();
}
 
Example #18
Source File: Notebook.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
public List<InterpreterSetting> getBindedInterpreterSettings(String noteId) throws IOException {
  Note note  = getNote(noteId);
  if (note == null) {
    return new ArrayList<>();
  }
  Set<InterpreterSetting> settings = new HashSet<>();
  for (Paragraph p : note.getParagraphs()) {
    try {
      Interpreter intp = p.getBindedInterpreter();
      settings.add((
              (ManagedInterpreterGroup) intp.getInterpreterGroup()).getInterpreterSetting());
    } catch (InterpreterNotFoundException e) {
      // ignore this
    }
  }
  // add the default interpreter group
  InterpreterSetting defaultIntpSetting =
          interpreterSettingManager.getByName(note.getDefaultInterpreterGroup());
  if (defaultIntpSetting != null) {
    settings.add(defaultIntpSetting);
  }
  return new ArrayList<>(settings);
}
 
Example #19
Source File: FileSystemRecoveryStorageTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleInterpreterProcess() throws InterpreterException, IOException {
  InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test");
  interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);

  Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
  RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
  InterpreterContext context1 = InterpreterContext.builder()
      .setNoteId("noteId")
      .setParagraphId("paragraphId")
      .build();
  remoteInterpreter1.interpret("hello", context1);

  assertEquals(1, interpreterSettingManager.getRecoveryStorage().restore().size());

  interpreterSetting.close();
  assertEquals(0, interpreterSettingManager.getRecoveryStorage().restore().size());
}
 
Example #20
Source File: PythonInterpreterMatplotlibTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  Properties p = new Properties();
  p.setProperty("zeppelin.python", "python");
  p.setProperty("zeppelin.python.maxResult", "100");
  p.setProperty("zeppelin.python.useIPython", "false");

  intpGroup = new InterpreterGroup();

  python = new PythonInterpreter(p);
  python.setInterpreterGroup(intpGroup);

  List<Interpreter> interpreters = new LinkedList<>();
  interpreters.add(python);
  intpGroup.put("note", interpreters);

  out = new InterpreterOutput(this);

  context = InterpreterContext.builder()
      .setInterpreterOut(out)
      .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null))
      .build();
  InterpreterContext.set(context);

  python.open();
}
 
Example #21
Source File: IPyFlinkInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
protected void startInterpreter(Properties properties) throws InterpreterException {
  InterpreterContext context = getInterpreterContext();
  context.setIntpEventClient(mockIntpEventClient);
  InterpreterContext.set(context);

  this.flinkScalaInterpreter = new LazyOpenInterpreter(
      new FlinkInterpreter(properties));
  intpGroup = new InterpreterGroup();
  intpGroup.put("session_1", new ArrayList<Interpreter>());
  intpGroup.get("session_1").add(flinkScalaInterpreter);
  flinkScalaInterpreter.setInterpreterGroup(intpGroup);

  LazyOpenInterpreter pyFlinkInterpreter =
      new LazyOpenInterpreter(new PyFlinkInterpreter(properties));
  intpGroup.get("session_1").add(pyFlinkInterpreter);
  pyFlinkInterpreter.setInterpreterGroup(intpGroup);

  interpreter = new LazyOpenInterpreter(new IPyFlinkInterpreter(properties));
  intpGroup.get("session_1").add(interpreter);
  interpreter.setInterpreterGroup(intpGroup);

  interpreter.open();

  angularObjectRegistry = new AngularObjectRegistry("flink", null);
}
 
Example #22
Source File: HeliumApplicationFactory.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
  try {
    // get interpreter process
    Interpreter intp = paragraph.getBindedInterpreter();
    ManagedInterpreterGroup intpGroup = (ManagedInterpreterGroup) intp.getInterpreterGroup();
    RemoteInterpreterProcess intpProcess = intpGroup.getRemoteInterpreterProcess();
    if (intpProcess == null) {
      throw new ApplicationException("Target interpreter process is not running");
    }

    // load application
    load(intpProcess, appState);

    // run application
    RunApplication runTask = new RunApplication(paragraph, appState.getId());
    runTask.run();
  } catch (Exception e) {
    logger.error(e.getMessage(), e);

    if (appState != null) {
      appStatusChange(paragraph, appState.getId(), ApplicationState.Status.ERROR);
      appState.setOutput(e.getMessage());
    }
  }
}
 
Example #23
Source File: PySparkInterpreterMatplotlibTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  intpGroup = new InterpreterGroup();
  intpGroup.put("note", new LinkedList<Interpreter>());
  context = InterpreterContext.builder()
      .setNoteId("note")
      .setInterpreterOut(new InterpreterOutput(null))
      .setIntpEventClient(mock(RemoteInterpreterEventClient.class))
      .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null))
      .build();
  InterpreterContext.set(context);

  sparkInterpreter = new SparkInterpreter(getPySparkTestProperties());
  intpGroup.get("note").add(sparkInterpreter);
  sparkInterpreter.setInterpreterGroup(intpGroup);
  sparkInterpreter.open();

  pyspark = new AltPySparkInterpreter(getPySparkTestProperties());
  intpGroup.get("note").add(pyspark);
  pyspark.setInterpreterGroup(intpGroup);
  pyspark.open();
}
 
Example #24
Source File: PySubmarineInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws InterpreterException {
  intpGroup = new InterpreterGroup();
  Properties properties = new Properties();
  properties.setProperty(ZEPPELIN_SUBMARINE_AUTH_TYPE, "simple");
  properties.setProperty("zeppelin.python.useIPython", "false");
  properties.setProperty("zeppelin.python.gatewayserver_address", "127.0.0.1");
  properties.setProperty(SubmarineConstants.SUBMARINE_HADOOP_PRINCIPAL, "user");

  pySubmarineIntp = new PySubmarineInterpreter(properties);

  intpGroup.put("note", new LinkedList<Interpreter>());
  intpGroup.get("note").add(pySubmarineIntp);
  pySubmarineIntp.setInterpreterGroup(intpGroup);

  InterpreterContext.set(getIntpContext());
  pySubmarineIntp.open();
}
 
Example #25
Source File: IPySparkInterpreterTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
protected void startInterpreter(Properties properties) throws InterpreterException {
  InterpreterContext context = getInterpreterContext();
  context.setIntpEventClient(mockIntpEventClient);
  InterpreterContext.set(context);

  LazyOpenInterpreter sparkInterpreter = new LazyOpenInterpreter(
      new SparkInterpreter(properties));
  intpGroup = new InterpreterGroup();
  intpGroup.put("session_1", new ArrayList<Interpreter>());
  intpGroup.get("session_1").add(sparkInterpreter);
  sparkInterpreter.setInterpreterGroup(intpGroup);

  LazyOpenInterpreter pySparkInterpreter =
      new LazyOpenInterpreter(new PySparkInterpreter(properties));
  intpGroup.get("session_1").add(pySparkInterpreter);
  pySparkInterpreter.setInterpreterGroup(intpGroup);

  interpreter = new LazyOpenInterpreter(new IPySparkInterpreter(properties));
  intpGroup.get("session_1").add(interpreter);
  interpreter.setInterpreterGroup(intpGroup);

  interpreter.open();
}
 
Example #26
Source File: FlinkIntegrationTest.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
private void testInterpreterBasics() throws IOException, InterpreterException {
  // test FlinkInterpreter
  Interpreter flinkInterpreter = interpreterFactory.getInterpreter("flink", new ExecutionContextBuilder().setUser("user1").setNoteId("note1").setDefaultInterpreterGroup("flink").createExecutionContext());

  InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").build();
  InterpreterResult interpreterResult = flinkInterpreter.interpret("1+1", context);
  assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
  assertTrue(interpreterResult.message().get(0).getData().contains("2"));

  interpreterResult = flinkInterpreter.interpret("val data = benv.fromElements(1, 2, 3)\ndata.collect()", context);
  assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
  assertTrue(interpreterResult.message().get(0).getData().contains("1, 2, 3"));
}
 
Example #27
Source File: IPyFlinkInterpreterTest.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
public static void testStreamPyFlink(Interpreter interpreter, Interpreter flinkScalaInterpreter) throws InterpreterException, IOException {
  InterpreterContext context = createInterpreterContext();
  InterpreterResult result = interpreter.interpret(
          "import tempfile\n" +
          "import os\n" +
          "import shutil\n" +
          "sink_path = tempfile.gettempdir() + '/streaming.csv'\n" +
          "if os.path.exists(sink_path):\n" +
          "    if os.path.isfile(sink_path):\n" +
          "      os.remove(sink_path)\n" +
          "    else:\n" +
          "      shutil.rmtree(sink_path)\n" +
          "s_env.set_parallelism(1)\n" +
          "t = st_env.from_elements([(1, 'hi', 'hello'), (2, 'hi', 'hello')], ['a', 'b', 'c'])\n" +
          "st_env.connect(FileSystem().path(sink_path)) \\\n" +
          "    .with_format(OldCsv()\n" +
          "      .field_delimiter(',')\n" +
          "      .field(\"a\", DataTypes.BIGINT())\n" +
          "      .field(\"b\", DataTypes.STRING())\n" +
          "      .field(\"c\", DataTypes.STRING())) \\\n" +
          "    .with_schema(Schema()\n" +
          "      .field(\"a\", DataTypes.BIGINT())\n" +
          "      .field(\"b\", DataTypes.STRING())\n" +
          "      .field(\"c\", DataTypes.STRING())) \\\n" +
          "    .create_temporary_table(\"stream_sink\")\n" +
          "t.select(\"a + 1, b, c\").insert_into(\"stream_sink\")\n" +
          "st_env.execute(\"stream_job\")"
          , context);
  assertEquals(context.out.toString(), InterpreterResult.Code.SUCCESS, result.code());
}
 
Example #28
Source File: RemoteInterpreterServer.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
@Override
public List<InterpreterCompletion> completion(String sessionId,
                                              String className,
                                              String buf,
                                              int cursor,
                                              RemoteInterpreterContext remoteInterpreterContext)
    throws TException {
  Interpreter intp = getInterpreter(sessionId, className);
  try {
    return intp.completion(buf, cursor, convert(remoteInterpreterContext, null));
  } catch (InterpreterException e) {
    throw new TException("Fail to get completion", e);
  }
}
 
Example #29
Source File: PySparkInterpreterTest.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
@Override
@Test
public void testFailtoLaunchPythonProcess() throws InterpreterException {
  tearDown();

  intpGroup = new InterpreterGroup();

  Properties properties = new Properties();
  properties.setProperty(SparkStringConstants.APP_NAME_PROP_NAME, "Zeppelin Test");
  properties.setProperty("spark.pyspark.python", "invalid_python");
  properties.setProperty("zeppelin.python.useIPython", "false");
  properties.setProperty("zeppelin.python.gatewayserver_address", "127.0.0.1");
  properties.setProperty("zeppelin.spark.maxResult", "3");

  interpreter = new LazyOpenInterpreter(new PySparkInterpreter(properties));
  interpreter.setInterpreterGroup(intpGroup);
  Interpreter sparkInterpreter =
          new LazyOpenInterpreter(new SparkInterpreter(properties));
  sparkInterpreter.setInterpreterGroup(intpGroup);
  LazyOpenInterpreter iPySparkInterpreter =
          new LazyOpenInterpreter(new IPySparkInterpreter(properties));
  iPySparkInterpreter.setInterpreterGroup(intpGroup);

  intpGroup.put("note", new LinkedList<Interpreter>());
  intpGroup.get("note").add(interpreter);
  intpGroup.get("note").add(sparkInterpreter);
  intpGroup.get("note").add(iPySparkInterpreter);


  InterpreterContext.set(getInterpreterContext());

  try {
    interpreter.interpret("1+1", getInterpreterContext());
    fail("Should fail to open PySparkInterpreter");
  } catch (InterpreterException e) {
    String stacktrace = ExceptionUtils.getStackTrace(e);
    assertTrue(stacktrace, stacktrace.contains("No such file or directory"));
  }
}
 
Example #30
Source File: PythonInterpreterTest.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
@Test
public void testFailtoLaunchPythonProcess() throws InterpreterException {
  tearDown();

  intpGroup = new InterpreterGroup();

  Properties properties = new Properties();
  properties.setProperty("zeppelin.python", "invalid_python");
  properties.setProperty("zeppelin.python.useIPython", "false");
  properties.setProperty("zeppelin.python.gatewayserver_address", "127.0.0.1");

  interpreter = new LazyOpenInterpreter(new PythonInterpreter(properties));

  intpGroup.put("note", new LinkedList<Interpreter>());
  intpGroup.get("note").add(interpreter);
  interpreter.setInterpreterGroup(intpGroup);

  InterpreterContext.set(getInterpreterContext());

  try {
    interpreter.interpret("1+1", getInterpreterContext());
    fail("Should fail to open PythonInterpreter");
  } catch (InterpreterException e) {
    String stacktrace = ExceptionUtils.getStackTrace(e);
    assertTrue(stacktrace, stacktrace.contains("No such file or directory"));
  }
}